mirrors/rust-cache
3
0
Fork 0
mirror of https://github.com/Swatinem/rust-cache synced 2025-10-31 12:52:28 +00:00
Code Activity

merge the registry caches together

Browse source
This commit is contained in:
Arpad Borsos 2020-10-03 18:10:54 +02:00
parent 2bcc375de8
commit f77cb1be47
6 changed files with 112 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

43
README.md
View file

@ -1,6 +1,7 @@
# Rust Cache Action
A GitHub Action that implements smart caching for rust/cargo projects
A GitHub Action that implements smart caching for rust/cargo projects with
sensible defaults.
## Example usage
@ -8,33 +9,29 @@ A GitHub Action that implements smart caching for rust/cargo projects
- uses: Swatinem/rust-cache@v1
```
## Specifics
### Registry Cache
This action tries to be better than just caching the following directories:
- `~/.cargo/registry/index`
- `~/.cargo/registry/cache`
```
~/.cargo/registry
~/.cargo/git
target
```
This cache is automatically keyed by hashing the `Cargo.lock` / `Cargo.toml`
files. Before persisting, the cache is cleaned of intermediate artifacts and
unneeded dependencies.
It disables incremental compilation and only caches dependencies. The
assumption is that we will likely recompile our own crate(s) anyway.
**TODO**: The `~/.cargo/git/db` database is not yet persisted, support will be
added at a later point.
It also separates the cache into 3 groups, each treated differently:
### Target Cache
- Registry Index: `~/.cargo/registry/index/<registry>`:
- `./target`
This is always restored from its latest snapshot, and persisted based on the
most recent revision.
This cache is automatically keyed by:
- Registry Cache: `~/.cargo/registry/cache/<registry>`:
- the github `job`,
- the rustc release / host / hash, and
- a hash of the `Cargo.lock` / `Cargo.toml` files.
Automatically keyed by the lockfile/toml hash, and is being pruned to only
persist the dependencies that are being used.
- target: `./target`
Automatically keyed by the lockfile, toml hash and job, and is being pruned
to only persist the dependencies that are being used. This is especially
throwing away any intermediate artifacts.
Before persisting, the cache is cleaned of anything that is not a needed
dependency. In particular, no caching of workspace crates will be done. For
this reason, this action will automatically set `CARGO_INCREMENTAL=0` to
disable incremental compilation.

48
dist/restore/index.js vendored
View file

@ -54616,7 +54616,7 @@ const home = external_os_default().homedir();
const paths = {
index: external_path_default().join(home, ".cargo/registry/index"),
cache: external_path_default().join(home, ".cargo/registry/cache"),
// git: path.join(home, ".cargo/git/db"),
git: external_path_default().join(home, ".cargo/git/db"),
target: "target",
};
const RefKey = "GITHUB_REF";
@ -54638,30 +54638,21 @@ async function getCaches() {
if (job) {
targetKey = `${job}-${targetKey}`;
}
const registryIndex = `v0-registry-index`;
const registryCache = `v0-registry-cache`;
const registry = `v0-registry`;
const target = `v0-target-${targetKey}${rustKey}`;
return {
index: {
name: "Registry Index",
path: paths.index,
key: `${registryIndex}-`,
restoreKeys: [registryIndex],
registry: {
name: "Registry",
paths: [
paths.index,
paths.cache,
],
key: `${registry}-`,
restoreKeys: [registry],
},
cache: {
name: "Registry Cache",
path: paths.cache,
key: `${registryCache}-${lockHash}`,
restoreKeys: [registryCache],
},
// git: {
// name: "Git Dependencies",
// path: paths.git,
// key: "git-db",
// },
target: {
name: "Target",
path: paths.target,
paths: [paths.target],
key: `${target}-${lockHash}`,
restoreKeys: [target],
},
@ -54669,7 +54660,7 @@ async function getCaches() {
}
async function getRustKey() {
const rustc = await getRustVersion();
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`;
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`;
}
async function getRustVersion() {
const stdout = await getCmdOutput("rustc", ["-vV"]);
@ -54693,12 +54684,9 @@ async function getRegistryName() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob();
if (files.length > 1) {
core.debug(`got multiple registries: "${files.join('", "')}"`);
core.warning(`got multiple registries: "${files.join('", "')}"`);
}
const first = files.shift();
if (!first) {
return;
}
return external_path_default().basename(external_path_default().dirname(first));
}
async function getLockfileHash() {
@ -54722,7 +54710,7 @@ async function getLockfileHash() {
finally { if (e_1) throw e_1.error; }
}
}
return hasher.digest("hex");
return hasher.digest("hex").slice(0, 20);
}
// CONCATENATED MODULE: ./src/restore.ts
@ -54736,16 +54724,16 @@ async function run() {
try {
core.exportVariable("CARGO_INCREMENTAL", 0);
const caches = await getCaches();
for (const [type, { name, path, key, restoreKeys }] of Object.entries(caches)) {
for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) {
const start = Date.now();
core.startGroup(`Restoring ${name}…`);
core.info(`Restoring to path "${path}".`);
core.info(`Restoring paths:\n ${paths.join("\n ")}.`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try {
const restoreKey = await cache.restoreCache([path], key, restoreKeys);
const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(type, restoreKey);
core.saveState(`CACHEKEY-${type}`, restoreKey);
}
else {
core.info("No cache found.");

76
dist/save/index.js vendored
View file

@ -54619,7 +54619,7 @@ const home = external_os_default().homedir();
const paths = {
index: external_path_default().join(home, ".cargo/registry/index"),
cache: external_path_default().join(home, ".cargo/registry/cache"),
// git: path.join(home, ".cargo/git/db"),
git: external_path_default().join(home, ".cargo/git/db"),
target: "target",
};
const RefKey = "GITHUB_REF";
@ -54641,30 +54641,21 @@ async function getCaches() {
if (job) {
targetKey = `${job}-${targetKey}`;
}
const registryIndex = `v0-registry-index`;
const registryCache = `v0-registry-cache`;
const registry = `v0-registry`;
const target = `v0-target-${targetKey}${rustKey}`;
return {
index: {
name: "Registry Index",
path: paths.index,
key: `${registryIndex}-`,
restoreKeys: [registryIndex],
registry: {
name: "Registry",
paths: [
paths.index,
paths.cache,
],
key: `${registry}-`,
restoreKeys: [registry],
},
cache: {
name: "Registry Cache",
path: paths.cache,
key: `${registryCache}-${lockHash}`,
restoreKeys: [registryCache],
},
// git: {
// name: "Git Dependencies",
// path: paths.git,
// key: "git-db",
// },
target: {
name: "Target",
path: paths.target,
paths: [paths.target],
key: `${target}-${lockHash}`,
restoreKeys: [target],
},
@ -54672,7 +54663,7 @@ async function getCaches() {
}
async function getRustKey() {
const rustc = await getRustVersion();
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`;
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`;
}
async function getRustVersion() {
const stdout = await getCmdOutput("rustc", ["-vV"]);
@ -54696,12 +54687,9 @@ async function getRegistryName() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob();
if (files.length > 1) {
core.debug(`got multiple registries: "${files.join('", "')}"`);
core.warning(`got multiple registries: "${files.join('", "')}"`);
}
const first = files.shift();
if (!first) {
return;
}
return external_path_default().basename(external_path_default().dirname(first));
}
async function getLockfileHash() {
@ -54725,7 +54713,7 @@ async function getLockfileHash() {
finally { if (e_1) throw e_1.error; }
}
}
return hasher.digest("hex");
return hasher.digest("hex").slice(0, 20);
}
// CONCATENATED MODULE: ./src/save.ts
@ -54749,33 +54737,35 @@ async function run() {
}
try {
const caches = await getCaches();
let upToDate = true;
for (const [type, { key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) !== key) {
upToDate = false;
break;
}
}
if (upToDate) {
core.info(`All caches up-to-date`);
return;
}
const registryName = await getRegistryName();
const packages = await getPackages();
// TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
await macOsWorkaround();
await io.rmRF(external_path_default().join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
await pruneTarget(packages);
if (registryName) {
// save the index based on its revision
const indexRef = await getIndexRef(registryName);
caches.index.key += indexRef;
await io.rmRF(external_path_default().join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
}
else {
delete caches.index;
delete caches.cache;
}
for (const [type, { name, path, key }] of Object.entries(caches)) {
if (core.getState(type) === key) {
for (const [type, { name, path: paths, key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) === key) {
core.info(`${name} up-to-date.`);
continue;
}
const start = Date.now();
core.startGroup(`Saving ${name}…`);
core.info(`Saving path "${path}".`);
core.info(`Saving paths:\n ${paths.join("\n ")}.`);
core.info(`Using key "${key}".`);
try {
await cache.saveCache([path], key);
await cache.saveCache(paths, key);
}
catch (e) {
core.info(`[warning] ${e.message}`);
@ -54792,10 +54782,6 @@ async function run() {
}
}
run();
async function getIndexRef(registryName) {
const cwd = external_path_default().join(paths.index, registryName);
return (await getCmdOutput("git", ["rev-parse", "--short", "origin/master"], { cwd })).trim();
}
async function getPackages() {
const cwd = process.cwd();
const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"]));

53
src/common.ts
View file

@ -10,21 +10,19 @@ const home = os.homedir();
export const paths = {
index: path.join(home, ".cargo/registry/index"),
cache: path.join(home, ".cargo/registry/cache"),
// git: path.join(home, ".cargo/git/db"),
git: path.join(home, ".cargo/git/db"),
target: "target",
};
interface CacheConfig {
name: string;
path: string;
paths: Array<string>;
key: string;
restoreKeys?: Array<string>;
}
interface Caches {
index: CacheConfig;
cache: CacheConfig;
// git: CacheConfig;
registry: CacheConfig;
target: CacheConfig;
}
@ -50,30 +48,22 @@ export async function getCaches(): Promise<Caches> {
targetKey = `${job}-${targetKey}`;
}
const registryIndex = `v0-registry-index`;
const registryCache = `v0-registry-cache`;
const registry = `v0-registry`;
const target = `v0-target-${targetKey}${rustKey}`;
return {
index: {
name: "Registry Index",
path: paths.index,
key: `${registryIndex}-`,
restoreKeys: [registryIndex],
registry: {
name: "Registry",
paths: [
paths.index,
paths.cache,
// TODO: paths.git,
],
key: `${registry}-${lockHash}`,
restoreKeys: [registry],
},
cache: {
name: "Registry Cache",
path: paths.cache,
key: `${registryCache}-${lockHash}`,
restoreKeys: [registryCache],
},
// git: {
// name: "Git Dependencies",
// path: paths.git,
// key: "git-db",
// },
target: {
name: "Target",
path: paths.target,
paths: [paths.target],
key: `${target}-${lockHash}`,
restoreKeys: [target],
},
@ -82,7 +72,7 @@ export async function getCaches(): Promise<Caches> {
async function getRustKey(): Promise<string> {
const rustc = await getRustVersion();
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`;
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"].slice(0, 12)}`;
}
interface RustVersion {
@ -119,21 +109,18 @@ export async function getCmdOutput(
return stdout;
}
export async function getRegistryName() {
export async function getRegistryName(): Promise<string> {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob();
if (files.length > 1) {
core.debug(`got multiple registries: "${files.join('", "')}"`);
core.warning(`got multiple registries: "${files.join('", "')}"`);
}
const first = files.shift();
if (!first) {
return;
}
const first = files.shift()!;
return path.basename(path.dirname(first));
}
async function getLockfileHash() {
async function getLockfileHash(): Promise<string> {
const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false });
const files = await globber.glob();
files.sort((a, b) => a.localeCompare(b));
@ -144,5 +131,5 @@ async function getLockfileHash() {
hasher.update(chunk);
}
}
return hasher.digest("hex");
return hasher.digest("hex").slice(0, 20);
}

8
src/restore.ts
View file

@ -11,16 +11,16 @@ async function run() {
core.exportVariable("CARGO_INCREMENTAL", 0);
const caches = await getCaches();
for (const [type, { name, path, key, restoreKeys }] of Object.entries(caches)) {
for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) {
const start = Date.now();
core.startGroup(`Restoring ${name}…`);
core.info(`Restoring to path "${path}".`);
core.info(`Restoring paths:\n ${paths.join("\n ")}.`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try {
const restoreKey = await cache.restoreCache([path], key, restoreKeys);
const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(type, restoreKey);
core.saveState(`CACHEKEY-${type}`, restoreKey);
} else {
core.info("No cache found.");
}

40
src/save.ts
View file

@ -13,37 +13,40 @@ async function run() {
try {
const caches = await getCaches();
let upToDate = true;
for (const [type, { key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) !== key) {
upToDate = false;
break;
}
}
if (upToDate) {
core.info(`All caches up-to-date`);
return;
}
const registryName = await getRegistryName();
const packages = await getPackages();
// TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
await macOsWorkaround();
await io.rmRF(path.join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
await pruneTarget(packages);
if (registryName) {
// save the index based on its revision
const indexRef = await getIndexRef(registryName);
caches.index.key += indexRef;
await io.rmRF(path.join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
} else {
delete (caches as any).index;
delete (caches as any).cache;
}
for (const [type, { name, path, key }] of Object.entries(caches)) {
if (core.getState(type) === key) {
for (const [type, { name, path: paths, key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) === key) {
core.info(`${name} up-to-date.`);
continue;
}
const start = Date.now();
core.startGroup(`Saving ${name}…`);
core.info(`Saving path "${path}".`);
core.info(`Saving paths:\n ${paths.join("\n ")}.`);
core.info(`Using key "${key}".`);
try {
await cache.saveCache([path], key);
await cache.saveCache(paths, key);
} catch (e) {
core.info(`[warning] ${e.message}`);
}
@ -60,11 +63,6 @@ async function run() {
run();
async function getIndexRef(registryName: string) {
const cwd = path.join(paths.index, registryName);
return (await getCmdOutput("git", ["rev-parse", "--short", "origin/master"], { cwd })).trim();
}
interface PackageDefinition {
name: string;
version: string;