mirrors/rust-cache
3
0
Fork 0
mirror of https://github.com/Swatinem/rust-cache synced 2025-04-05 21:24:07 +00:00
Code Activity

merge the registry caches together

Browse source
This commit is contained in:
Arpad Borsos 2020-10-03 18:10:54 +02:00
parent 2bcc375de8
commit f77cb1be47
6 changed files with 112 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
README.md
View file

@ -1,6 +1,7 @@
# Rust Cache Action # Rust Cache Action
A GitHub Action that implements smart caching for rust/cargo projects A GitHub Action that implements smart caching for rust/cargo projects with
sensible defaults.
## Example usage ## Example usage
@ -8,33 +9,29 @@ A GitHub Action that implements smart caching for rust/cargo projects
- uses: Swatinem/rust-cache@v1 - uses: Swatinem/rust-cache@v1
``` ```
## Specifics ### Registry Cache
This action tries to be better than just caching the following directories: - `~/.cargo/registry/index`
- `~/.cargo/registry/cache`
``` This cache is automatically keyed by hashing the `Cargo.lock` / `Cargo.toml`
~/.cargo/registry files. Before persisting, the cache is cleaned of intermediate artifacts and
~/.cargo/git unneeded dependencies.
target
```
It disables incremental compilation and only caches dependencies. The **TODO**: The `~/.cargo/git/db` database is not yet persisted, support will be
assumption is that we will likely recompile our own crate(s) anyway. added at a later point.
It also separates the cache into 3 groups, each treated differently: ### Target Cache
- Registry Index: `~/.cargo/registry/index/<registry>`: - `./target`
This is always restored from its latest snapshot, and persisted based on the This cache is automatically keyed by:
most recent revision.
- Registry Cache: `~/.cargo/registry/cache/<registry>`: - the github `job`,
- the rustc release / host / hash, and
- a hash of the `Cargo.lock` / `Cargo.toml` files.
Automatically keyed by the lockfile/toml hash, and is being pruned to only Before persisting, the cache is cleaned of anything that is not a needed
persist the dependencies that are being used. dependency. In particular, no caching of workspace crates will be done. For
this reason, this action will automatically set `CARGO_INCREMENTAL=0` to
- target: `./target` disable incremental compilation.
Automatically keyed by the lockfile, toml hash and job, and is being pruned
to only persist the dependencies that are being used. This is especially
throwing away any intermediate artifacts.

48
dist/restore/index.js vendored
View file

@ -54616,7 +54616,7 @@ const home = external_os_default().homedir();
const paths = { const paths = {
index: external_path_default().join(home, ".cargo/registry/index"), index: external_path_default().join(home, ".cargo/registry/index"),
cache: external_path_default().join(home, ".cargo/registry/cache"), cache: external_path_default().join(home, ".cargo/registry/cache"),
// git: path.join(home, ".cargo/git/db"), git: external_path_default().join(home, ".cargo/git/db"),
target: "target", target: "target",
}; };
const RefKey = "GITHUB_REF"; const RefKey = "GITHUB_REF";
@ -54638,30 +54638,21 @@ async function getCaches() {
if (job) { if (job) {
targetKey = `${job}-${targetKey}`; targetKey = `${job}-${targetKey}`;
} }
const registryIndex = `v0-registry-index`; const registry = `v0-registry`;
const registryCache = `v0-registry-cache`;
const target = `v0-target-${targetKey}${rustKey}`; const target = `v0-target-${targetKey}${rustKey}`;
return { return {
index: { registry: {
name: "Registry Index", name: "Registry",
path: paths.index, paths: [
key: `${registryIndex}-`, paths.index,
restoreKeys: [registryIndex], paths.cache,
],
key: `${registry}-`,
restoreKeys: [registry],
}, },
cache: {
name: "Registry Cache",
path: paths.cache,
key: `${registryCache}-${lockHash}`,
restoreKeys: [registryCache],
},
// git: {
// name: "Git Dependencies",
// path: paths.git,
// key: "git-db",
// },
target: { target: {
name: "Target", name: "Target",
path: paths.target, paths: [paths.target],
key: `${target}-${lockHash}`, key: `${target}-${lockHash}`,
restoreKeys: [target], restoreKeys: [target],
}, },
@ -54669,7 +54660,7 @@ async function getCaches() {
} }
async function getRustKey() { async function getRustKey() {
const rustc = await getRustVersion(); const rustc = await getRustVersion();
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`;
} }
async function getRustVersion() { async function getRustVersion() {
const stdout = await getCmdOutput("rustc", ["-vV"]); const stdout = await getCmdOutput("rustc", ["-vV"]);
@ -54693,12 +54684,9 @@ async function getRegistryName() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob(); const files = await globber.glob();
if (files.length > 1) { if (files.length > 1) {
core.debug(`got multiple registries: "${files.join('", "')}"`); core.warning(`got multiple registries: "${files.join('", "')}"`);
} }
const first = files.shift(); const first = files.shift();
if (!first) {
return;
}
return external_path_default().basename(external_path_default().dirname(first)); return external_path_default().basename(external_path_default().dirname(first));
} }
async function getLockfileHash() { async function getLockfileHash() {
@ -54722,7 +54710,7 @@ async function getLockfileHash() {
finally { if (e_1) throw e_1.error; } finally { if (e_1) throw e_1.error; }
} }
} }
return hasher.digest("hex"); return hasher.digest("hex").slice(0, 20);
} }
// CONCATENATED MODULE: ./src/restore.ts // CONCATENATED MODULE: ./src/restore.ts
@ -54736,16 +54724,16 @@ async function run() {
try { try {
core.exportVariable("CARGO_INCREMENTAL", 0); core.exportVariable("CARGO_INCREMENTAL", 0);
const caches = await getCaches(); const caches = await getCaches();
for (const [type, { name, path, key, restoreKeys }] of Object.entries(caches)) { for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) {
const start = Date.now(); const start = Date.now();
core.startGroup(`Restoring ${name}…`); core.startGroup(`Restoring ${name}…`);
core.info(`Restoring to path "${path}".`); core.info(`Restoring paths:\n ${paths.join("\n ")}.`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`); core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try { try {
const restoreKey = await cache.restoreCache([path], key, restoreKeys); const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
if (restoreKey) { if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`); core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(type, restoreKey); core.saveState(`CACHEKEY-${type}`, restoreKey);
} }
else { else {
core.info("No cache found."); core.info("No cache found.");

76
dist/save/index.js vendored
View file

@ -54619,7 +54619,7 @@ const home = external_os_default().homedir();
const paths = { const paths = {
index: external_path_default().join(home, ".cargo/registry/index"), index: external_path_default().join(home, ".cargo/registry/index"),
cache: external_path_default().join(home, ".cargo/registry/cache"), cache: external_path_default().join(home, ".cargo/registry/cache"),
// git: path.join(home, ".cargo/git/db"), git: external_path_default().join(home, ".cargo/git/db"),
target: "target", target: "target",
}; };
const RefKey = "GITHUB_REF"; const RefKey = "GITHUB_REF";
@ -54641,30 +54641,21 @@ async function getCaches() {
if (job) { if (job) {
targetKey = `${job}-${targetKey}`; targetKey = `${job}-${targetKey}`;
} }
const registryIndex = `v0-registry-index`; const registry = `v0-registry`;
const registryCache = `v0-registry-cache`;
const target = `v0-target-${targetKey}${rustKey}`; const target = `v0-target-${targetKey}${rustKey}`;
return { return {
index: { registry: {
name: "Registry Index", name: "Registry",
path: paths.index, paths: [
key: `${registryIndex}-`, paths.index,
restoreKeys: [registryIndex], paths.cache,
],
key: `${registry}-`,
restoreKeys: [registry],
}, },
cache: {
name: "Registry Cache",
path: paths.cache,
key: `${registryCache}-${lockHash}`,
restoreKeys: [registryCache],
},
// git: {
// name: "Git Dependencies",
// path: paths.git,
// key: "git-db",
// },
target: { target: {
name: "Target", name: "Target",
path: paths.target, paths: [paths.target],
key: `${target}-${lockHash}`, key: `${target}-${lockHash}`,
restoreKeys: [target], restoreKeys: [target],
}, },
@ -54672,7 +54663,7 @@ async function getCaches() {
} }
async function getRustKey() { async function getRustKey() {
const rustc = await getRustVersion(); const rustc = await getRustVersion();
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`;
} }
async function getRustVersion() { async function getRustVersion() {
const stdout = await getCmdOutput("rustc", ["-vV"]); const stdout = await getCmdOutput("rustc", ["-vV"]);
@ -54696,12 +54687,9 @@ async function getRegistryName() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob(); const files = await globber.glob();
if (files.length > 1) { if (files.length > 1) {
core.debug(`got multiple registries: "${files.join('", "')}"`); core.warning(`got multiple registries: "${files.join('", "')}"`);
} }
const first = files.shift(); const first = files.shift();
if (!first) {
return;
}
return external_path_default().basename(external_path_default().dirname(first)); return external_path_default().basename(external_path_default().dirname(first));
} }
async function getLockfileHash() { async function getLockfileHash() {
@ -54725,7 +54713,7 @@ async function getLockfileHash() {
finally { if (e_1) throw e_1.error; } finally { if (e_1) throw e_1.error; }
} }
} }
return hasher.digest("hex"); return hasher.digest("hex").slice(0, 20);
} }
// CONCATENATED MODULE: ./src/save.ts // CONCATENATED MODULE: ./src/save.ts
@ -54749,33 +54737,35 @@ async function run() {
} }
try { try {
const caches = await getCaches(); const caches = await getCaches();
let upToDate = true;
for (const [type, { key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) !== key) {
upToDate = false;
break;
}
}
if (upToDate) {
core.info(`All caches up-to-date`);
return;
}
const registryName = await getRegistryName(); const registryName = await getRegistryName();
const packages = await getPackages(); const packages = await getPackages();
// TODO: remove this once https://github.com/actions/toolkit/pull/553 lands // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
await macOsWorkaround(); await macOsWorkaround();
await io.rmRF(external_path_default().join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
await pruneTarget(packages); await pruneTarget(packages);
if (registryName) { for (const [type, { name, path: paths, key }] of Object.entries(caches)) {
// save the index based on its revision if (core.getState(`CACHEKEY-${type}`) === key) {
const indexRef = await getIndexRef(registryName);
caches.index.key += indexRef;
await io.rmRF(external_path_default().join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
}
else {
delete caches.index;
delete caches.cache;
}
for (const [type, { name, path, key }] of Object.entries(caches)) {
if (core.getState(type) === key) {
core.info(`${name} up-to-date.`); core.info(`${name} up-to-date.`);
continue; continue;
} }
const start = Date.now(); const start = Date.now();
core.startGroup(`Saving ${name}…`); core.startGroup(`Saving ${name}…`);
core.info(`Saving path "${path}".`); core.info(`Saving paths:\n ${paths.join("\n ")}.`);
core.info(`Using key "${key}".`); core.info(`Using key "${key}".`);
try { try {
await cache.saveCache([path], key); await cache.saveCache(paths, key);
} }
catch (e) { catch (e) {
core.info(`[warning] ${e.message}`); core.info(`[warning] ${e.message}`);
@ -54792,10 +54782,6 @@ async function run() {
} }
} }
run(); run();
async function getIndexRef(registryName) {
const cwd = external_path_default().join(paths.index, registryName);
return (await getCmdOutput("git", ["rev-parse", "--short", "origin/master"], { cwd })).trim();
}
async function getPackages() { async function getPackages() {
const cwd = process.cwd(); const cwd = process.cwd();
const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"])); const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"]));

53
src/common.ts
View file

@ -10,21 +10,19 @@ const home = os.homedir();
export const paths = { export const paths = {
index: path.join(home, ".cargo/registry/index"), index: path.join(home, ".cargo/registry/index"),
cache: path.join(home, ".cargo/registry/cache"), cache: path.join(home, ".cargo/registry/cache"),
// git: path.join(home, ".cargo/git/db"), git: path.join(home, ".cargo/git/db"),
target: "target", target: "target",
}; };
interface CacheConfig { interface CacheConfig {
name: string; name: string;
path: string; paths: Array<string>;
key: string; key: string;
restoreKeys?: Array<string>; restoreKeys?: Array<string>;
} }
interface Caches { interface Caches {
index: CacheConfig; registry: CacheConfig;
cache: CacheConfig;
// git: CacheConfig;
target: CacheConfig; target: CacheConfig;
} }
@ -50,30 +48,22 @@ export async function getCaches(): Promise<Caches> {
targetKey = `${job}-${targetKey}`; targetKey = `${job}-${targetKey}`;
} }
const registryIndex = `v0-registry-index`; const registry = `v0-registry`;
const registryCache = `v0-registry-cache`;
const target = `v0-target-${targetKey}${rustKey}`; const target = `v0-target-${targetKey}${rustKey}`;
return { return {
index: { registry: {
name: "Registry Index", name: "Registry",
path: paths.index, paths: [
key: `${registryIndex}-`, paths.index,
restoreKeys: [registryIndex], paths.cache,
// TODO: paths.git,
],
key: `${registry}-${lockHash}`,
restoreKeys: [registry],
}, },
cache: {
name: "Registry Cache",
path: paths.cache,
key: `${registryCache}-${lockHash}`,
restoreKeys: [registryCache],
},
// git: {
// name: "Git Dependencies",
// path: paths.git,
// key: "git-db",
// },
target: { target: {
name: "Target", name: "Target",
path: paths.target, paths: [paths.target],
key: `${target}-${lockHash}`, key: `${target}-${lockHash}`,
restoreKeys: [target], restoreKeys: [target],
}, },
@ -82,7 +72,7 @@ export async function getCaches(): Promise<Caches> {
async function getRustKey(): Promise<string> { async function getRustKey(): Promise<string> {
const rustc = await getRustVersion(); const rustc = await getRustVersion();
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`;
} }
interface RustVersion { interface RustVersion {
@ -119,21 +109,18 @@ export async function getCmdOutput(
return stdout; return stdout;
} }
export async function getRegistryName() { export async function getRegistryName(): Promise<string> {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob(); const files = await globber.glob();
if (files.length > 1) { if (files.length > 1) {
core.debug(`got multiple registries: "${files.join('", "')}"`); core.warning(`got multiple registries: "${files.join('", "')}"`);
} }
const first = files.shift(); const first = files.shift()!;
if (!first) {
return;
}
return path.basename(path.dirname(first)); return path.basename(path.dirname(first));
} }
async function getLockfileHash() { async function getLockfileHash(): Promise<string> {
const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false }); const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false });
const files = await globber.glob(); const files = await globber.glob();
files.sort((a, b) => a.localeCompare(b)); files.sort((a, b) => a.localeCompare(b));
@ -144,5 +131,5 @@ async function getLockfileHash() {
hasher.update(chunk); hasher.update(chunk);
} }
} }
return hasher.digest("hex"); return hasher.digest("hex").slice(0, 20);
} }

8
src/restore.ts
View file

@ -11,16 +11,16 @@ async function run() {
core.exportVariable("CARGO_INCREMENTAL", 0); core.exportVariable("CARGO_INCREMENTAL", 0);
const caches = await getCaches(); const caches = await getCaches();
for (const [type, { name, path, key, restoreKeys }] of Object.entries(caches)) { for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) {
const start = Date.now(); const start = Date.now();
core.startGroup(`Restoring ${name}…`); core.startGroup(`Restoring ${name}…`);
core.info(`Restoring to path "${path}".`); core.info(`Restoring paths:\n ${paths.join("\n ")}.`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`); core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try { try {
const restoreKey = await cache.restoreCache([path], key, restoreKeys); const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
if (restoreKey) { if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`); core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(type, restoreKey); core.saveState(`CACHEKEY-${type}`, restoreKey);
} else { } else {
core.info("No cache found."); core.info("No cache found.");
} }

40
src/save.ts
View file

@ -13,37 +13,40 @@ async function run() {
try { try {
const caches = await getCaches(); const caches = await getCaches();
let upToDate = true;
for (const [type, { key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) !== key) {
upToDate = false;
break;
}
}
if (upToDate) {
core.info(`All caches up-to-date`);
return;
}
const registryName = await getRegistryName(); const registryName = await getRegistryName();
const packages = await getPackages(); const packages = await getPackages();
// TODO: remove this once https://github.com/actions/toolkit/pull/553 lands // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
await macOsWorkaround(); await macOsWorkaround();
await io.rmRF(path.join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
await pruneTarget(packages); await pruneTarget(packages);
if (registryName) { for (const [type, { name, path: paths, key }] of Object.entries(caches)) {
// save the index based on its revision if (core.getState(`CACHEKEY-${type}`) === key) {
const indexRef = await getIndexRef(registryName);
caches.index.key += indexRef;
await io.rmRF(path.join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
} else {
delete (caches as any).index;
delete (caches as any).cache;
}
for (const [type, { name, path, key }] of Object.entries(caches)) {
if (core.getState(type) === key) {
core.info(`${name} up-to-date.`); core.info(`${name} up-to-date.`);
continue; continue;
} }
const start = Date.now(); const start = Date.now();
core.startGroup(`Saving ${name}…`); core.startGroup(`Saving ${name}…`);
core.info(`Saving path "${path}".`); core.info(`Saving paths:\n ${paths.join("\n ")}.`);
core.info(`Using key "${key}".`); core.info(`Using key "${key}".`);
try { try {
await cache.saveCache([path], key); await cache.saveCache(paths, key);
} catch (e) { } catch (e) {
core.info(`[warning] ${e.message}`); core.info(`[warning] ${e.message}`);
} }
@ -60,11 +63,6 @@ async function run() {
run(); run();
async function getIndexRef(registryName: string) {
const cwd = path.join(paths.index, registryName);
return (await getCmdOutput("git", ["rev-parse", "--short", "origin/master"], { cwd })).trim();
}
interface PackageDefinition { interface PackageDefinition {
name: string; name: string;
version: string; version: string;