From e8e3c57b3bd33f2a5909701928eab8863e3f123b Mon Sep 17 00:00:00 2001 From: Arpad Borsos Date: Sat, 3 Oct 2020 18:39:38 +0200 Subject: [PATCH] merge all the caches and simplify --- README.md | 24 ++++---- action.yml | 4 +- dist/restore/index.js | 93 ++++++++++++----------------- dist/save/index.js | 132 ++++++++++++++++++------------------------ package.json | 2 +- src/common.ts | 77 ++++++++++-------------- src/restore.ts | 42 +++++++------- src/save.ts | 82 +++++++++++++------------- 8 files changed, 197 insertions(+), 259 deletions(-) diff --git a/README.md b/README.md index 5779434..a81149c 100644 --- a/README.md +++ b/README.md @@ -9,20 +9,12 @@ sensible defaults. - uses: Swatinem/rust-cache@v1 ``` -### Registry Cache +### Cache Details + +The cache currently caches the following directories: - `~/.cargo/registry/index` - `~/.cargo/registry/cache` - -This cache is automatically keyed by hashing the `Cargo.lock` / `Cargo.toml` -files. Before persisting, the cache is cleaned of intermediate artifacts and -unneeded dependencies. - -**TODO**: The `~/.cargo/git/db` database is not yet persisted, support will be -added at a later point. - -### Target Cache - - `./target` This cache is automatically keyed by: @@ -31,7 +23,13 @@ This cache is automatically keyed by: - the rustc release / host / hash, and - a hash of the `Cargo.lock` / `Cargo.toml` files. -Before persisting, the cache is cleaned of anything that is not a needed -dependency. In particular, no caching of workspace crates will be done. For +An additional input `key` can be provided if the builtin keys are not sufficient. + +Before persisting, the cache is cleaned of intermediate artifacts and +anything that is not a workspace dependency. +In particular, no caching of workspace crates will be done. For this reason, this action will automatically set `CARGO_INCREMENTAL=0` to disable incremental compilation. + +**TODO**: The `~/.cargo/git/db` database is not yet persisted, support will be +added at a later point. diff --git a/action.yml b/action.yml index f4f667c..0abc7cc 100644 --- a/action.yml +++ b/action.yml @@ -1,9 +1,9 @@ name: "Rust Cache" -description: "A GitHub Action that implements smart caching for rust/cargo projects" +description: "A GitHub Action that implements smart caching for rust/cargo projects with sensible defaults." author: "Arpad Borsos " inputs: key: - description: "An explicit key for restoring and saving the target cache" + description: "An additional key for the cache" required: false runs: using: "node12" diff --git a/dist/restore/index.js b/dist/restore/index.js index 11c07c5..9330679 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -54612,6 +54612,8 @@ var __asyncValues = (undefined && undefined.__asyncValues) || function (o) { +const stateKey = "RUST_CACHE_KEY"; +const stateHash = "RUST_CACHE_HASH"; const home = external_os_default().homedir(); const paths = { index: external_path_default().join(home, ".cargo/registry/index"), @@ -54623,39 +54625,31 @@ const RefKey = "GITHUB_REF"; function isValidEvent() { return RefKey in process.env && Boolean(process.env[RefKey]); } -async function getCaches() { - const rustKey = await getRustKey(); - let lockHash = core.getState("lockHash"); +async function getCacheConfig() { + let lockHash = core.getState(stateHash); if (!lockHash) { lockHash = await getLockfileHash(); - core.saveState("lockHash", lockHash); + core.saveState(stateHash, lockHash); } - let targetKey = core.getInput("key"); - if (targetKey) { - targetKey = `${targetKey}-`; + let key = `v0-rust-`; + let inputKey = core.getInput("key"); + if (inputKey) { + key += `${inputKey}-`; } const job = process.env.GITHUB_JOB; if (job) { - targetKey = `${job}-${targetKey}`; + key += `${job}-`; } - const registry = `v0-registry`; - const target = `v0-target-${targetKey}${rustKey}`; + key += await getRustKey(); return { - registry: { - name: "Registry", - paths: [ - paths.index, - paths.cache, - ], - key: `${registry}-`, - restoreKeys: [registry], - }, - target: { - name: "Target", - paths: [paths.target], - key: `${target}-${lockHash}`, - restoreKeys: [target], - }, + paths: [ + paths.index, + paths.cache, + // TODO: paths.git, + paths.target, + ], + key: `${key}-${lockHash}`, + restoreKeys: [key], }; } async function getRustKey() { @@ -54680,15 +54674,6 @@ async function getCmdOutput(cmd, args = [], options = {}) { } }, options)); return stdout; } -async function getRegistryName() { - const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); - const files = await globber.glob(); - if (files.length > 1) { - core.warning(`got multiple registries: "${files.join('", "')}"`); - } - const first = files.shift(); - return external_path_default().basename(external_path_default().dirname(first)); -} async function getLockfileHash() { var e_1, _a; const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false }); @@ -54723,30 +54708,26 @@ async function run() { } try { core.exportVariable("CARGO_INCREMENTAL", 0); - const caches = await getCaches(); - for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) { - const start = Date.now(); - core.startGroup(`Restoring ${name}…`); - core.info(`Restoring paths:\n ${paths.join("\n ")}.`); - core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`); - try { - const restoreKey = await cache.restoreCache(paths, key, restoreKeys); - if (restoreKey) { - core.info(`Restored from cache key "${restoreKey}".`); - core.saveState(`CACHEKEY-${type}`, restoreKey); - } - else { - core.info("No cache found."); - } + const start = Date.now(); + const { paths, key, restoreKeys } = await getCacheConfig(); + core.info(`Restoring paths:\n ${paths.join("\n ")}.`); + core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`); + try { + const restoreKey = await cache.restoreCache(paths, key, restoreKeys); + if (restoreKey) { + core.info(`Restored from cache key "${restoreKey}".`); + core.saveState(stateKey, restoreKey); } - catch (e) { - core.info(`[warning] ${e.message}`); + else { + core.info("No cache found."); } - const duration = Math.round((Date.now() - start) / 1000); - if (duration) { - core.info(`Took ${duration}s.`); - } - core.endGroup(); + } + catch (e) { + core.info(`[warning] ${e.message}`); + } + const duration = Math.round((Date.now() - start) / 1000); + if (duration) { + core.info(`Took ${duration}s.`); } } catch (e) { diff --git a/dist/save/index.js b/dist/save/index.js index d314c71..70606b3 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -54585,13 +54585,13 @@ var glob = __webpack_require__(8090); var external_crypto_ = __webpack_require__(6417); var external_crypto_default = /*#__PURE__*/__webpack_require__.n(external_crypto_); -// EXTERNAL MODULE: ./node_modules/@actions/io/lib/io.js -var io = __webpack_require__(7436); - // EXTERNAL MODULE: external "fs" var external_fs_ = __webpack_require__(5747); var external_fs_default = /*#__PURE__*/__webpack_require__.n(external_fs_); +// EXTERNAL MODULE: ./node_modules/@actions/io/lib/io.js +var io = __webpack_require__(7436); + // EXTERNAL MODULE: external "os" var external_os_ = __webpack_require__(2087); var external_os_default = /*#__PURE__*/__webpack_require__.n(external_os_); @@ -54615,6 +54615,8 @@ var __asyncValues = (undefined && undefined.__asyncValues) || function (o) { +const stateKey = "RUST_CACHE_KEY"; +const stateHash = "RUST_CACHE_HASH"; const home = external_os_default().homedir(); const paths = { index: external_path_default().join(home, ".cargo/registry/index"), @@ -54626,39 +54628,31 @@ const RefKey = "GITHUB_REF"; function isValidEvent() { return RefKey in process.env && Boolean(process.env[RefKey]); } -async function getCaches() { - const rustKey = await getRustKey(); - let lockHash = core.getState("lockHash"); +async function getCacheConfig() { + let lockHash = core.getState(stateHash); if (!lockHash) { lockHash = await getLockfileHash(); - core.saveState("lockHash", lockHash); + core.saveState(stateHash, lockHash); } - let targetKey = core.getInput("key"); - if (targetKey) { - targetKey = `${targetKey}-`; + let key = `v0-rust-`; + let inputKey = core.getInput("key"); + if (inputKey) { + key += `${inputKey}-`; } const job = process.env.GITHUB_JOB; if (job) { - targetKey = `${job}-${targetKey}`; + key += `${job}-`; } - const registry = `v0-registry`; - const target = `v0-target-${targetKey}${rustKey}`; + key += await getRustKey(); return { - registry: { - name: "Registry", - paths: [ - paths.index, - paths.cache, - ], - key: `${registry}-`, - restoreKeys: [registry], - }, - target: { - name: "Target", - paths: [paths.target], - key: `${target}-${lockHash}`, - restoreKeys: [target], - }, + paths: [ + paths.index, + paths.cache, + // TODO: paths.git, + paths.target, + ], + key: `${key}-${lockHash}`, + restoreKeys: [key], }; } async function getRustKey() { @@ -54683,15 +54677,6 @@ async function getCmdOutput(cmd, args = [], options = {}) { } }, options)); return stdout; } -async function getRegistryName() { - const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); - const files = await globber.glob(); - if (files.length > 1) { - core.warning(`got multiple registries: "${files.join('", "')}"`); - } - const first = files.shift(); - return external_path_default().basename(external_path_default().dirname(first)); -} async function getLockfileHash() { var e_1, _a; const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false }); @@ -54731,50 +54716,35 @@ var save_asyncValues = (undefined && undefined.__asyncValues) || function (o) { + async function run() { if (!isValidEvent()) { return; } try { - const caches = await getCaches(); - let upToDate = true; - for (const [type, { key }] of Object.entries(caches)) { - if (core.getState(`CACHEKEY-${type}`) !== key) { - upToDate = false; - break; - } - } - if (upToDate) { - core.info(`All caches up-to-date`); + const start = Date.now(); + const { paths: savePaths, key } = await getCacheConfig(); + if (core.getState(stateKey) === key) { + core.info(`Cache up-to-date.`); return; } - const registryName = await getRegistryName(); - const packages = await getPackages(); // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands await macOsWorkaround(); - await io.rmRF(external_path_default().join(paths.index, registryName, ".cache")); - await pruneRegistryCache(registryName, packages); - await pruneTarget(packages); - for (const [type, { name, path: paths, key }] of Object.entries(caches)) { - if (core.getState(`CACHEKEY-${type}`) === key) { - core.info(`${name} up-to-date.`); - continue; - } - const start = Date.now(); - core.startGroup(`Saving ${name}…`); - core.info(`Saving paths:\n ${paths.join("\n ")}.`); - core.info(`Using key "${key}".`); - try { - await cache.saveCache(paths, key); - } - catch (e) { - core.info(`[warning] ${e.message}`); - } - const duration = Math.round((Date.now() - start) / 1000); - if (duration) { - core.info(`Took ${duration}s.`); - } - core.endGroup(); + const registryName = await getRegistryName(); + const packages = await getPackages(); + await cleanRegistry(registryName, packages); + await cleanTarget(packages); + core.info(`Saving paths:\n ${savePaths.join("\n ")}.`); + core.info(`Using key "${key}".`); + try { + await cache.saveCache(savePaths, key); + } + catch (e) { + core.info(`[warning] ${e.message}`); + } + const duration = Math.round((Date.now() - start) / 1000); + if (duration) { + core.info(`Took ${duration}s.`); } } catch (e) { @@ -54782,6 +54752,15 @@ async function run() { } } run(); +async function getRegistryName() { + const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); + const files = await globber.glob(); + if (files.length > 1) { + core.warning(`got multiple registries: "${files.join('", "')}"`); + } + const first = files.shift(); + return external_path_default().basename(external_path_default().dirname(first)); +} async function getPackages() { const cwd = process.cwd(); const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"])); @@ -54792,8 +54771,9 @@ async function getPackages() { return { name: p.name, version: p.version, targets }; }); } -async function pruneRegistryCache(registryName, packages) { +async function cleanRegistry(registryName, packages) { var e_1, _a; + await io.rmRF(external_path_default().join(paths.index, registryName, ".cache")); const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`)); const dir = await external_fs_default().promises.opendir(external_path_default().join(paths.cache, registryName)); try { @@ -54814,7 +54794,7 @@ async function pruneRegistryCache(registryName, packages) { finally { if (e_1) throw e_1.error; } } } -async function pruneTarget(packages) { +async function cleanTarget(packages) { var e_2, _a; await external_fs_default().promises.unlink("./target/.rustc_info.json"); await io.rmRF("./target/debug/examples"); @@ -54851,7 +54831,7 @@ async function pruneTarget(packages) { })); await rmExcept("./target/debug/deps", keepDeps); } -const twoWeeks = 14 * 24 * 3600 * 1000; +const oneWeek = 7 * 24 * 3600 * 1000; async function rmExcept(dirName, keepPrefix) { var e_3, _a; const dir = await external_fs_default().promises.opendir(dirName); @@ -54865,7 +54845,7 @@ async function rmExcept(dirName, keepPrefix) { } const fileName = external_path_default().join(dir.path, dirent.name); const { mtime } = await external_fs_default().promises.stat(fileName); - if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > twoWeeks) { + if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > oneWeek) { core.debug(`deleting "${fileName}"`); if (dirent.isFile()) { await external_fs_default().promises.unlink(fileName); diff --git a/package.json b/package.json index 9dbd5fd..ee086d7 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "private": true, "name": "rust-cache", "version": "1.0.2", - "description": "A GitHub Action that implements smart caching for rust/cargo projects", + "description": "A GitHub Action that implements smart caching for rust/cargo projects with sensible defaults.", "keywords": [ "actions", "rust", diff --git a/src/common.ts b/src/common.ts index f6bcce0..13eec16 100644 --- a/src/common.ts +++ b/src/common.ts @@ -6,6 +6,9 @@ import fs from "fs"; import os from "os"; import path from "path"; +export const stateKey = "RUST_CACHE_KEY"; +const stateHash = "RUST_CACHE_HASH"; + const home = os.homedir(); export const paths = { index: path.join(home, ".cargo/registry/index"), @@ -15,15 +18,9 @@ export const paths = { }; interface CacheConfig { - name: string; paths: Array; key: string; - restoreKeys?: Array; -} - -interface Caches { - registry: CacheConfig; - target: CacheConfig; + restoreKeys: Array; } const RefKey = "GITHUB_REF"; @@ -32,41 +29,36 @@ export function isValidEvent(): boolean { return RefKey in process.env && Boolean(process.env[RefKey]); } -export async function getCaches(): Promise { - const rustKey = await getRustKey(); - let lockHash = core.getState("lockHash"); +export async function getCacheConfig(): Promise { + let lockHash = core.getState(stateHash); if (!lockHash) { lockHash = await getLockfileHash(); - core.saveState("lockHash", lockHash); - } - let targetKey = core.getInput("key"); - if (targetKey) { - targetKey = `${targetKey}-`; - } - const job = process.env.GITHUB_JOB; - if (job) { - targetKey = `${job}-${targetKey}`; + core.saveState(stateHash, lockHash); } - const registry = `v0-registry`; - const target = `v0-target-${targetKey}${rustKey}`; + let key = `v0-rust-`; + + let inputKey = core.getInput("key"); + if (inputKey) { + key += `${inputKey}-`; + } + + const job = process.env.GITHUB_JOB; + if (job) { + key += `${job}-`; + } + + key += await getRustKey(); + return { - registry: { - name: "Registry", - paths: [ - paths.index, - paths.cache, - // TODO: paths.git, - ], - key: `${registry}-${lockHash}`, - restoreKeys: [registry], - }, - target: { - name: "Target", - paths: [paths.target], - key: `${target}-${lockHash}`, - restoreKeys: [target], - }, + paths: [ + paths.index, + paths.cache, + // TODO: paths.git, + paths.target, + ], + key: `${key}-${lockHash}`, + restoreKeys: [key], }; } @@ -109,17 +101,6 @@ export async function getCmdOutput( return stdout; } -export async function getRegistryName(): Promise { - const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); - const files = await globber.glob(); - if (files.length > 1) { - core.warning(`got multiple registries: "${files.join('", "')}"`); - } - - const first = files.shift()!; - return path.basename(path.dirname(first)); -} - async function getLockfileHash(): Promise { const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false }); const files = await globber.glob(); diff --git a/src/restore.ts b/src/restore.ts index 10d3c57..1113917 100644 --- a/src/restore.ts +++ b/src/restore.ts @@ -1,6 +1,6 @@ import * as cache from "@actions/cache"; import * as core from "@actions/core"; -import { getCaches, isValidEvent } from "./common"; +import { getCacheConfig, isValidEvent, stateKey } from "./common"; async function run() { if (!isValidEvent()) { @@ -10,28 +10,26 @@ async function run() { try { core.exportVariable("CARGO_INCREMENTAL", 0); - const caches = await getCaches(); - for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) { - const start = Date.now(); - core.startGroup(`Restoring ${name}…`); - core.info(`Restoring paths:\n ${paths.join("\n ")}.`); - core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`); - try { - const restoreKey = await cache.restoreCache(paths, key, restoreKeys); - if (restoreKey) { - core.info(`Restored from cache key "${restoreKey}".`); - core.saveState(`CACHEKEY-${type}`, restoreKey); - } else { - core.info("No cache found."); - } - } catch (e) { - core.info(`[warning] ${e.message}`); + const start = Date.now(); + const { paths, key, restoreKeys } = await getCacheConfig(); + + core.info(`Restoring paths:\n ${paths.join("\n ")}.`); + core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`); + try { + const restoreKey = await cache.restoreCache(paths, key, restoreKeys); + if (restoreKey) { + core.info(`Restored from cache key "${restoreKey}".`); + core.saveState(stateKey, restoreKey); + } else { + core.info("No cache found."); } - const duration = Math.round((Date.now() - start) / 1000); - if (duration) { - core.info(`Took ${duration}s.`); - } - core.endGroup(); + } catch (e) { + core.info(`[warning] ${e.message}`); + } + + const duration = Math.round((Date.now() - start) / 1000); + if (duration) { + core.info(`Took ${duration}s.`); } } catch (e) { core.info(`[warning] ${e.message}`); diff --git a/src/save.ts b/src/save.ts index 99d23a1..de0e280 100644 --- a/src/save.ts +++ b/src/save.ts @@ -1,10 +1,11 @@ import * as cache from "@actions/cache"; import * as core from "@actions/core"; import * as exec from "@actions/exec"; +import * as glob from "@actions/glob"; import * as io from "@actions/io"; import fs from "fs"; import path from "path"; -import { getCaches, getCmdOutput, getRegistryName, isValidEvent, paths } from "./common"; +import { getCacheConfig, getCmdOutput, isValidEvent, paths, stateKey } from "./common"; async function run() { if (!isValidEvent()) { @@ -12,49 +13,35 @@ async function run() { } try { - const caches = await getCaches(); - let upToDate = true; - for (const [type, { key }] of Object.entries(caches)) { - if (core.getState(`CACHEKEY-${type}`) !== key) { - upToDate = false; - break; - } - } - if (upToDate) { - core.info(`All caches up-to-date`); + const start = Date.now(); + const { paths: savePaths, key } = await getCacheConfig(); + + if (core.getState(stateKey) === key) { + core.info(`Cache up-to-date.`); return; } - const registryName = await getRegistryName(); - const packages = await getPackages(); - // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands await macOsWorkaround(); - await io.rmRF(path.join(paths.index, registryName, ".cache")); - await pruneRegistryCache(registryName, packages); + const registryName = await getRegistryName(); + const packages = await getPackages(); - await pruneTarget(packages); + await cleanRegistry(registryName, packages); - for (const [type, { name, path: paths, key }] of Object.entries(caches)) { - if (core.getState(`CACHEKEY-${type}`) === key) { - core.info(`${name} up-to-date.`); - continue; - } - const start = Date.now(); - core.startGroup(`Saving ${name}…`); - core.info(`Saving paths:\n ${paths.join("\n ")}.`); - core.info(`Using key "${key}".`); - try { - await cache.saveCache(paths, key); - } catch (e) { - core.info(`[warning] ${e.message}`); - } - const duration = Math.round((Date.now() - start) / 1000); - if (duration) { - core.info(`Took ${duration}s.`); - } - core.endGroup(); + await cleanTarget(packages); + + core.info(`Saving paths:\n ${savePaths.join("\n ")}.`); + core.info(`Using key "${key}".`); + try { + await cache.saveCache(savePaths, key); + } catch (e) { + core.info(`[warning] ${e.message}`); + } + + const duration = Math.round((Date.now() - start) / 1000); + if (duration) { + core.info(`Took ${duration}s.`); } } catch (e) { core.info(`[warning] ${e.message}`); @@ -80,6 +67,17 @@ interface Meta { }>; } +async function getRegistryName(): Promise { + const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); + const files = await globber.glob(); + if (files.length > 1) { + core.warning(`got multiple registries: "${files.join('", "')}"`); + } + + const first = files.shift()!; + return path.basename(path.dirname(first)); +} + async function getPackages(): Promise { const cwd = process.cwd(); const meta: Meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"])); @@ -92,7 +90,9 @@ async function getPackages(): Promise { }); } -async function pruneRegistryCache(registryName: string, packages: Packages) { +async function cleanRegistry(registryName: string, packages: Packages) { + await io.rmRF(path.join(paths.index, registryName, ".cache")); + const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`)); const dir = await fs.promises.opendir(path.join(paths.cache, registryName)); @@ -105,12 +105,12 @@ async function pruneRegistryCache(registryName: string, packages: Packages) { } } -async function pruneTarget(packages: Packages) { +async function cleanTarget(packages: Packages) { await fs.promises.unlink("./target/.rustc_info.json"); await io.rmRF("./target/debug/examples"); await io.rmRF("./target/debug/incremental"); - let dir: fs.Dir; + let dir: fs.Dir; // remove all *files* from debug dir = await fs.promises.opendir("./target/debug"); for await (const dirent of dir) { @@ -137,7 +137,7 @@ async function pruneTarget(packages: Packages) { await rmExcept("./target/debug/deps", keepDeps); } -const twoWeeks = 14 * 24 * 3600 * 1000; +const oneWeek = 7 * 24 * 3600 * 1000; async function rmExcept(dirName: string, keepPrefix: Set) { const dir = await fs.promises.opendir(dirName); @@ -149,7 +149,7 @@ async function rmExcept(dirName: string, keepPrefix: Set) { } const fileName = path.join(dir.path, dirent.name); const { mtime } = await fs.promises.stat(fileName); - if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > twoWeeks) { + if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > oneWeek) { core.debug(`deleting "${fileName}"`); if (dirent.isFile()) { await fs.promises.unlink(fileName);