From f77cb1be47837b566e4391c395b1b3cb09ed1ca9 Mon Sep 17 00:00:00 2001 From: Arpad Borsos Date: Sat, 3 Oct 2020 18:10:54 +0200 Subject: [PATCH] merge the registry caches together --- README.md | 43 ++++++++++++------------ dist/restore/index.js | 48 ++++++++++----------------- dist/save/index.js | 76 ++++++++++++++++++------------------------- src/common.ts | 53 ++++++++++++------------------ src/restore.ts | 8 ++--- src/save.ts | 40 +++++++++++------------ 6 files changed, 112 insertions(+), 156 deletions(-) diff --git a/README.md b/README.md index cbf0869..5779434 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Rust Cache Action -A GitHub Action that implements smart caching for rust/cargo projects +A GitHub Action that implements smart caching for rust/cargo projects with +sensible defaults. ## Example usage @@ -8,33 +9,29 @@ A GitHub Action that implements smart caching for rust/cargo projects - uses: Swatinem/rust-cache@v1 ``` -## Specifics +### Registry Cache -This action tries to be better than just caching the following directories: +- `~/.cargo/registry/index` +- `~/.cargo/registry/cache` -``` -~/.cargo/registry -~/.cargo/git -target -``` +This cache is automatically keyed by hashing the `Cargo.lock` / `Cargo.toml` +files. Before persisting, the cache is cleaned of intermediate artifacts and +unneeded dependencies. -It disables incremental compilation and only caches dependencies. The -assumption is that we will likely recompile our own crate(s) anyway. +**TODO**: The `~/.cargo/git/db` database is not yet persisted, support will be +added at a later point. -It also separates the cache into 3 groups, each treated differently: +### Target Cache -- Registry Index: `~/.cargo/registry/index/`: +- `./target` - This is always restored from its latest snapshot, and persisted based on the - most recent revision. +This cache is automatically keyed by: -- Registry Cache: `~/.cargo/registry/cache/`: +- the github `job`, +- the rustc release / host / hash, and +- a hash of the `Cargo.lock` / `Cargo.toml` files. - Automatically keyed by the lockfile/toml hash, and is being pruned to only - persist the dependencies that are being used. - -- target: `./target` - - Automatically keyed by the lockfile, toml hash and job, and is being pruned - to only persist the dependencies that are being used. This is especially - throwing away any intermediate artifacts. +Before persisting, the cache is cleaned of anything that is not a needed +dependency. In particular, no caching of workspace crates will be done. For +this reason, this action will automatically set `CARGO_INCREMENTAL=0` to +disable incremental compilation. diff --git a/dist/restore/index.js b/dist/restore/index.js index cdbba47..11c07c5 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -54616,7 +54616,7 @@ const home = external_os_default().homedir(); const paths = { index: external_path_default().join(home, ".cargo/registry/index"), cache: external_path_default().join(home, ".cargo/registry/cache"), - // git: path.join(home, ".cargo/git/db"), + git: external_path_default().join(home, ".cargo/git/db"), target: "target", }; const RefKey = "GITHUB_REF"; @@ -54638,30 +54638,21 @@ async function getCaches() { if (job) { targetKey = `${job}-${targetKey}`; } - const registryIndex = `v0-registry-index`; - const registryCache = `v0-registry-cache`; + const registry = `v0-registry`; const target = `v0-target-${targetKey}${rustKey}`; return { - index: { - name: "Registry Index", - path: paths.index, - key: `${registryIndex}-`, - restoreKeys: [registryIndex], + registry: { + name: "Registry", + paths: [ + paths.index, + paths.cache, + ], + key: `${registry}-`, + restoreKeys: [registry], }, - cache: { - name: "Registry Cache", - path: paths.cache, - key: `${registryCache}-${lockHash}`, - restoreKeys: [registryCache], - }, - // git: { - // name: "Git Dependencies", - // path: paths.git, - // key: "git-db", - // }, target: { name: "Target", - path: paths.target, + paths: [paths.target], key: `${target}-${lockHash}`, restoreKeys: [target], }, @@ -54669,7 +54660,7 @@ async function getCaches() { } async function getRustKey() { const rustc = await getRustVersion(); - return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; + return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`; } async function getRustVersion() { const stdout = await getCmdOutput("rustc", ["-vV"]); @@ -54693,12 +54684,9 @@ async function getRegistryName() { const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); const files = await globber.glob(); if (files.length > 1) { - core.debug(`got multiple registries: "${files.join('", "')}"`); + core.warning(`got multiple registries: "${files.join('", "')}"`); } const first = files.shift(); - if (!first) { - return; - } return external_path_default().basename(external_path_default().dirname(first)); } async function getLockfileHash() { @@ -54722,7 +54710,7 @@ async function getLockfileHash() { finally { if (e_1) throw e_1.error; } } } - return hasher.digest("hex"); + return hasher.digest("hex").slice(0, 20); } // CONCATENATED MODULE: ./src/restore.ts @@ -54736,16 +54724,16 @@ async function run() { try { core.exportVariable("CARGO_INCREMENTAL", 0); const caches = await getCaches(); - for (const [type, { name, path, key, restoreKeys }] of Object.entries(caches)) { + for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) { const start = Date.now(); core.startGroup(`Restoring ${name}…`); - core.info(`Restoring to path "${path}".`); + core.info(`Restoring paths:\n ${paths.join("\n ")}.`); core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`); try { - const restoreKey = await cache.restoreCache([path], key, restoreKeys); + const restoreKey = await cache.restoreCache(paths, key, restoreKeys); if (restoreKey) { core.info(`Restored from cache key "${restoreKey}".`); - core.saveState(type, restoreKey); + core.saveState(`CACHEKEY-${type}`, restoreKey); } else { core.info("No cache found."); diff --git a/dist/save/index.js b/dist/save/index.js index 81bd5fe..d314c71 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -54619,7 +54619,7 @@ const home = external_os_default().homedir(); const paths = { index: external_path_default().join(home, ".cargo/registry/index"), cache: external_path_default().join(home, ".cargo/registry/cache"), - // git: path.join(home, ".cargo/git/db"), + git: external_path_default().join(home, ".cargo/git/db"), target: "target", }; const RefKey = "GITHUB_REF"; @@ -54641,30 +54641,21 @@ async function getCaches() { if (job) { targetKey = `${job}-${targetKey}`; } - const registryIndex = `v0-registry-index`; - const registryCache = `v0-registry-cache`; + const registry = `v0-registry`; const target = `v0-target-${targetKey}${rustKey}`; return { - index: { - name: "Registry Index", - path: paths.index, - key: `${registryIndex}-`, - restoreKeys: [registryIndex], + registry: { + name: "Registry", + paths: [ + paths.index, + paths.cache, + ], + key: `${registry}-`, + restoreKeys: [registry], }, - cache: { - name: "Registry Cache", - path: paths.cache, - key: `${registryCache}-${lockHash}`, - restoreKeys: [registryCache], - }, - // git: { - // name: "Git Dependencies", - // path: paths.git, - // key: "git-db", - // }, target: { name: "Target", - path: paths.target, + paths: [paths.target], key: `${target}-${lockHash}`, restoreKeys: [target], }, @@ -54672,7 +54663,7 @@ async function getCaches() { } async function getRustKey() { const rustc = await getRustVersion(); - return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; + return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`; } async function getRustVersion() { const stdout = await getCmdOutput("rustc", ["-vV"]); @@ -54696,12 +54687,9 @@ async function getRegistryName() { const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); const files = await globber.glob(); if (files.length > 1) { - core.debug(`got multiple registries: "${files.join('", "')}"`); + core.warning(`got multiple registries: "${files.join('", "')}"`); } const first = files.shift(); - if (!first) { - return; - } return external_path_default().basename(external_path_default().dirname(first)); } async function getLockfileHash() { @@ -54725,7 +54713,7 @@ async function getLockfileHash() { finally { if (e_1) throw e_1.error; } } } - return hasher.digest("hex"); + return hasher.digest("hex").slice(0, 20); } // CONCATENATED MODULE: ./src/save.ts @@ -54749,33 +54737,35 @@ async function run() { } try { const caches = await getCaches(); + let upToDate = true; + for (const [type, { key }] of Object.entries(caches)) { + if (core.getState(`CACHEKEY-${type}`) !== key) { + upToDate = false; + break; + } + } + if (upToDate) { + core.info(`All caches up-to-date`); + return; + } const registryName = await getRegistryName(); const packages = await getPackages(); // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands await macOsWorkaround(); + await io.rmRF(external_path_default().join(paths.index, registryName, ".cache")); + await pruneRegistryCache(registryName, packages); await pruneTarget(packages); - if (registryName) { - // save the index based on its revision - const indexRef = await getIndexRef(registryName); - caches.index.key += indexRef; - await io.rmRF(external_path_default().join(paths.index, registryName, ".cache")); - await pruneRegistryCache(registryName, packages); - } - else { - delete caches.index; - delete caches.cache; - } - for (const [type, { name, path, key }] of Object.entries(caches)) { - if (core.getState(type) === key) { + for (const [type, { name, path: paths, key }] of Object.entries(caches)) { + if (core.getState(`CACHEKEY-${type}`) === key) { core.info(`${name} up-to-date.`); continue; } const start = Date.now(); core.startGroup(`Saving ${name}…`); - core.info(`Saving path "${path}".`); + core.info(`Saving paths:\n ${paths.join("\n ")}.`); core.info(`Using key "${key}".`); try { - await cache.saveCache([path], key); + await cache.saveCache(paths, key); } catch (e) { core.info(`[warning] ${e.message}`); @@ -54792,10 +54782,6 @@ async function run() { } } run(); -async function getIndexRef(registryName) { - const cwd = external_path_default().join(paths.index, registryName); - return (await getCmdOutput("git", ["rev-parse", "--short", "origin/master"], { cwd })).trim(); -} async function getPackages() { const cwd = process.cwd(); const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"])); diff --git a/src/common.ts b/src/common.ts index 9b31c08..f6bcce0 100644 --- a/src/common.ts +++ b/src/common.ts @@ -10,21 +10,19 @@ const home = os.homedir(); export const paths = { index: path.join(home, ".cargo/registry/index"), cache: path.join(home, ".cargo/registry/cache"), - // git: path.join(home, ".cargo/git/db"), + git: path.join(home, ".cargo/git/db"), target: "target", }; interface CacheConfig { name: string; - path: string; + paths: Array; key: string; restoreKeys?: Array; } interface Caches { - index: CacheConfig; - cache: CacheConfig; - // git: CacheConfig; + registry: CacheConfig; target: CacheConfig; } @@ -50,30 +48,22 @@ export async function getCaches(): Promise { targetKey = `${job}-${targetKey}`; } - const registryIndex = `v0-registry-index`; - const registryCache = `v0-registry-cache`; + const registry = `v0-registry`; const target = `v0-target-${targetKey}${rustKey}`; return { - index: { - name: "Registry Index", - path: paths.index, - key: `${registryIndex}-`, - restoreKeys: [registryIndex], + registry: { + name: "Registry", + paths: [ + paths.index, + paths.cache, + // TODO: paths.git, + ], + key: `${registry}-${lockHash}`, + restoreKeys: [registry], }, - cache: { - name: "Registry Cache", - path: paths.cache, - key: `${registryCache}-${lockHash}`, - restoreKeys: [registryCache], - }, - // git: { - // name: "Git Dependencies", - // path: paths.git, - // key: "git-db", - // }, target: { name: "Target", - path: paths.target, + paths: [paths.target], key: `${target}-${lockHash}`, restoreKeys: [target], }, @@ -82,7 +72,7 @@ export async function getCaches(): Promise { async function getRustKey(): Promise { const rustc = await getRustVersion(); - return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; + return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`; } interface RustVersion { @@ -119,21 +109,18 @@ export async function getCmdOutput( return stdout; } -export async function getRegistryName() { +export async function getRegistryName(): Promise { const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); const files = await globber.glob(); if (files.length > 1) { - core.debug(`got multiple registries: "${files.join('", "')}"`); + core.warning(`got multiple registries: "${files.join('", "')}"`); } - const first = files.shift(); - if (!first) { - return; - } + const first = files.shift()!; return path.basename(path.dirname(first)); } -async function getLockfileHash() { +async function getLockfileHash(): Promise { const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false }); const files = await globber.glob(); files.sort((a, b) => a.localeCompare(b)); @@ -144,5 +131,5 @@ async function getLockfileHash() { hasher.update(chunk); } } - return hasher.digest("hex"); + return hasher.digest("hex").slice(0, 20); } diff --git a/src/restore.ts b/src/restore.ts index 59ecb7b..10d3c57 100644 --- a/src/restore.ts +++ b/src/restore.ts @@ -11,16 +11,16 @@ async function run() { core.exportVariable("CARGO_INCREMENTAL", 0); const caches = await getCaches(); - for (const [type, { name, path, key, restoreKeys }] of Object.entries(caches)) { + for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) { const start = Date.now(); core.startGroup(`Restoring ${name}…`); - core.info(`Restoring to path "${path}".`); + core.info(`Restoring paths:\n ${paths.join("\n ")}.`); core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`); try { - const restoreKey = await cache.restoreCache([path], key, restoreKeys); + const restoreKey = await cache.restoreCache(paths, key, restoreKeys); if (restoreKey) { core.info(`Restored from cache key "${restoreKey}".`); - core.saveState(type, restoreKey); + core.saveState(`CACHEKEY-${type}`, restoreKey); } else { core.info("No cache found."); } diff --git a/src/save.ts b/src/save.ts index 06e5bde..99d23a1 100644 --- a/src/save.ts +++ b/src/save.ts @@ -13,37 +13,40 @@ async function run() { try { const caches = await getCaches(); + let upToDate = true; + for (const [type, { key }] of Object.entries(caches)) { + if (core.getState(`CACHEKEY-${type}`) !== key) { + upToDate = false; + break; + } + } + if (upToDate) { + core.info(`All caches up-to-date`); + return; + } + const registryName = await getRegistryName(); const packages = await getPackages(); // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands await macOsWorkaround(); + await io.rmRF(path.join(paths.index, registryName, ".cache")); + await pruneRegistryCache(registryName, packages); + await pruneTarget(packages); - if (registryName) { - // save the index based on its revision - const indexRef = await getIndexRef(registryName); - caches.index.key += indexRef; - await io.rmRF(path.join(paths.index, registryName, ".cache")); - - await pruneRegistryCache(registryName, packages); - } else { - delete (caches as any).index; - delete (caches as any).cache; - } - - for (const [type, { name, path, key }] of Object.entries(caches)) { - if (core.getState(type) === key) { + for (const [type, { name, path: paths, key }] of Object.entries(caches)) { + if (core.getState(`CACHEKEY-${type}`) === key) { core.info(`${name} up-to-date.`); continue; } const start = Date.now(); core.startGroup(`Saving ${name}…`); - core.info(`Saving path "${path}".`); + core.info(`Saving paths:\n ${paths.join("\n ")}.`); core.info(`Using key "${key}".`); try { - await cache.saveCache([path], key); + await cache.saveCache(paths, key); } catch (e) { core.info(`[warning] ${e.message}`); } @@ -60,11 +63,6 @@ async function run() { run(); -async function getIndexRef(registryName: string) { - const cwd = path.join(paths.index, registryName); - return (await getCmdOutput("git", ["rev-parse", "--short", "origin/master"], { cwd })).trim(); -} - interface PackageDefinition { name: string; version: string;