mirrors/rust-cache
3
0
Fork 0
mirror of https://github.com/Swatinem/rust-cache synced 2025-04-29 06:35:53 +00:00
Code Activity

fix: cache key stability (#142)

Browse source 
Ensure consistency of main and post configuration by storing and
restoring it from state, which in turn ensures cache key stability.

Also:
* Fixed some typos.
* Use core.error for logging errors.
* Fix inverted condition on cache-all-crates.

Reverts: #138
Fixes #140
This commit is contained in:
Steven Hartland 2023-05-18 21:48:40 +01:00 • committed by GitHub
parent 060bda31e0
commit ad97570a01
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 260 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/cleanup.ts
View file

@ -3,7 +3,7 @@ import * as io from "@actions/io";
import fs from "fs";
import path from "path";
import { CARGO_HOME, STATE_BINS } from "./config";
import { CARGO_HOME } from "./config";
import { Packages } from "./workspace";
export async function cleanTargetDir(targetDir: string, packages: Packages, checkTimestamp = false) {
@ -69,9 +69,14 @@ export async function getCargoBins(): Promise<Set<string>> {
return bins;
}
export async function cleanBin() {
/**
* Clean the cargo bin directory, removing the binaries that existed
* when the action started, as they were not created by the build.
*
* @param oldBins The binaries that existed when the action started.
*/
export async function cleanBin(oldBins: Array<string>) {
const bins = await getCargoBins();
const oldBins = JSON.parse(core.getState(STATE_BINS));
for (const bin of oldBins) {
bins.delete(bin);
@ -186,10 +191,10 @@ const ONE_WEEK = 7 * 24 * 3600 * 1000;
/**
* Removes all files or directories in `dirName` matching some criteria.
*
*
* When the `checkTimestamp` flag is set, this will also remove anything older
* than one week.
*
*
* Otherwise, it will remove everything that does not match any string in the
* `keepPrefix` set.
* The matching strips and trailing `-$hash` suffix.

112
src/config.ts
View file

@ -7,14 +7,12 @@ import path from "path";
import { getCmdOutput } from "./utils";
import { Workspace } from "./workspace";
import { getCargoBins } from "./cleanup";
const HOME = os.homedir();
export const CARGO_HOME = process.env.CARGO_HOME || path.join(HOME, ".cargo");
const STATE_LOCKFILE_HASH = "RUST_CACHE_LOCKFILE_HASH";
const STATE_LOCKFILES = "RUST_CACHE_LOCKFILES";
export const STATE_BINS = "RUST_CACHE_BINS";
export const STATE_KEY = "RUST_CACHE_KEY";
const STATE_CONFIG = "RUST_CACHE_CONFIG";
export class CacheConfig {
/** All the paths we want to cache */
@ -27,6 +25,9 @@ export class CacheConfig {
/** The workspace configurations */
public workspaces: Array<Workspace> = [];
/** The cargo binaries present during main step */
public cargoBins: Array<string> = [];
/** The prefix portion of the cache key */
private keyPrefix = "";
/** The rust version considered for the cache key */
@ -104,10 +105,6 @@ export class CacheConfig {
self.keyEnvs = keyEnvs;
// Installed packages and their versions are also considered for the key.
const packages = await getPackages();
hasher.update(packages);
key += `-${hasher.digest("hex")}`;
self.restoreKey = key;
@ -115,13 +112,6 @@ export class CacheConfig {
// Construct the lockfiles portion of the key:
// This considers all the files found via globbing for various manifests
// and lockfiles.
// This part is computed in the "pre"/"restore" part of the job and persisted
// into the `state`. That state is loaded in the "post"/"save" part of the
// job so we have consistent values even though the "main" actions run
// might create/overwrite lockfiles.
let lockHash = core.getState(STATE_LOCKFILE_HASH);
let keyFiles: Array<string> = JSON.parse(core.getState(STATE_LOCKFILES) || "[]");
// Constructs the workspace config and paths to restore:
// The workspaces are given using a `$workspace -> $target` syntax.
@ -136,30 +126,25 @@ export class CacheConfig {
}
self.workspaces = workspaces;
if (!lockHash) {
keyFiles = keyFiles.concat(await globFiles("rust-toolchain\nrust-toolchain.toml"));
for (const workspace of workspaces) {
const root = workspace.root;
keyFiles.push(
...(await globFiles(
`${root}/**/Cargo.toml\n${root}/**/Cargo.lock\n${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`,
)),
);
}
keyFiles = keyFiles.filter(file => !fs.statSync(file).isDirectory());
keyFiles.sort((a, b) => a.localeCompare(b));
hasher = crypto.createHash("sha1");
for (const file of keyFiles) {
for await (const chunk of fs.createReadStream(file)) {
hasher.update(chunk);
}
}
lockHash = hasher.digest("hex");
core.saveState(STATE_LOCKFILE_HASH, lockHash);
core.saveState(STATE_LOCKFILES, JSON.stringify(keyFiles));
let keyFiles = await globFiles("rust-toolchain\nrust-toolchain.toml");
for (const workspace of workspaces) {
const root = workspace.root;
keyFiles.push(
...(await globFiles(
`${root}/**/Cargo.toml\n${root}/**/Cargo.lock\n${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`,
)),
);
}
keyFiles = keyFiles.filter(file => !fs.statSync(file).isDirectory());
keyFiles.sort((a, b) => a.localeCompare(b));
hasher = crypto.createHash("sha1");
for (const file of keyFiles) {
for await (const chunk of fs.createReadStream(file)) {
hasher.update(chunk);
}
}
let lockHash = hasher.digest("hex");
self.keyFiles = keyFiles;
@ -177,9 +162,37 @@ export class CacheConfig {
self.cachePaths.push(dir);
}
const bins = await getCargoBins();
self.cargoBins = Array.from(bins.values());
return self;
}
/**
* Reads and returns the cache config from the action `state`.
*
* @throws {Error} if the state is not present.
* @returns {CacheConfig} the configuration.
* @see {@link CacheConfig#saveState}
* @see {@link CacheConfig#new}
*/
static fromState(): CacheConfig {
const source = core.getState(STATE_CONFIG);
if (!source) {
throw new Error("Cache configuration not found in state");
}
const self = new CacheConfig();
Object.assign(self, JSON.parse(source));
self.workspaces = self.workspaces
.map((w: any) => new Workspace(w.root, w.target));
return self;
}
/**
* Prints the configuration to the action log.
*/
printInfo() {
core.startGroup("Cache Configuration");
core.info(`Workspaces:`);
@ -207,6 +220,23 @@ export class CacheConfig {
}
core.endGroup();
}
/**
* Saves the configuration to the state store.
* This is used to restore the configuration in the post action.
*/
saveState() {
core.saveState(STATE_CONFIG, this);
}
}
/**
* Checks if the cache is up to date.
*
* @returns `true` if the cache is up to date, `false` otherwise.
*/
export function isCacheUpToDate(): boolean {
return core.getState(STATE_CONFIG) === "";
}
interface RustVersion {
@ -225,12 +255,6 @@ async function getRustVersion(): Promise<RustVersion> {
return Object.fromEntries(splits);
}
async function getPackages(): Promise<string> {
let stdout = await getCmdOutput("cargo", ["install", "--list"]);
// Make OS independent.
return stdout.split(/[\n\r]+/).join("\n");
}
async function globFiles(pattern: string): Promise<string[]> {
const globber = await glob.create(pattern, {
followSymbolicLinks: false,

26
src/restore.ts
View file

@ -1,13 +1,13 @@
import * as cache from "@actions/cache";
import * as core from "@actions/core";
import { cleanTargetDir, getCargoBins } from "./cleanup";
import { CacheConfig, STATE_BINS, STATE_KEY } from "./config";
import { cleanTargetDir } from "./cleanup";
import { CacheConfig } from "./config";
process.on("uncaughtException", (e) => {
core.info(`[warning] ${e.message}`);
core.error(e.message);
if (e.stack) {
core.info(e.stack);
core.error(e.stack);
}
});
@ -29,9 +29,6 @@ async function run() {
config.printInfo();
core.info("");
const bins = await getCargoBins();
core.saveState(STATE_BINS, JSON.stringify([...bins]));
core.info(`... Restoring cache ...`);
const key = config.cacheKey;
// Pass a copy of cachePaths to avoid mutating the original array as reported by:
@ -39,28 +36,31 @@ async function run() {
// TODO: remove this once the underlying bug is fixed.
const restoreKey = await cache.restoreCache(config.cachePaths.slice(), key, [config.restoreKey]);
if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(STATE_KEY, restoreKey);
if (restoreKey !== key) {
const match = restoreKey === key;
core.info(`Restored from cache key "${restoreKey}" full match: ${match}.`);
if (!match) {
// pre-clean the target directory on cache mismatch
for (const workspace of config.workspaces) {
try {
await cleanTargetDir(workspace.target, [], true);
} catch {}
}
// We restored the cache but it is not a full match.
config.saveState();
}
setCacheHitOutput(restoreKey === key);
setCacheHitOutput(match);
} else {
core.info("No cache found.");
config.saveState();
setCacheHitOutput(false);
}
} catch (e) {
setCacheHitOutput(false);
core.info(`[warning] ${(e as any).stack}`);
core.error(`${(e as any).stack}`);
}
}

24
src/save.ts
View file

@ -3,12 +3,12 @@ import * as core from "@actions/core";
import * as exec from "@actions/exec";
import { cleanBin, cleanGit, cleanRegistry, cleanTargetDir } from "./cleanup";
import { CacheConfig, STATE_KEY } from "./config";
import { CacheConfig, isCacheUpToDate } from "./config";
process.on("uncaughtException", (e) => {
core.info(`[warning] ${e.message}`);
core.error(e.message);
if (e.stack) {
core.info(e.stack);
core.error(e.stack);
}
});
@ -20,15 +20,15 @@ async function run() {
}
try {
const config = await CacheConfig.new();
config.printInfo();
core.info("");
if (core.getState(STATE_KEY) === config.cacheKey) {
if (isCacheUpToDate()) {
core.info(`Cache up-to-date.`);
return;
}
const config = CacheConfig.fromState();
config.printInfo();
core.info("");
// TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
await macOsWorkaround();
@ -45,16 +45,16 @@ async function run() {
}
try {
const creates = core.getInput("cache-all-crates").toLowerCase() || "false";
core.info(`... Cleaning cargo registry cache-all-crates: ${creates} ...`);
await cleanRegistry(allPackages, creates === "true");
const crates = core.getInput("cache-all-crates").toLowerCase() || "false"
core.info(`... Cleaning cargo registry cache-all-crates: ${crates} ...`);
await cleanRegistry(allPackages, crates !== "true");
} catch (e) {
core.error(`${(e as any).stack}`);
}
try {
core.info(`... Cleaning cargo/bin ...`);
await cleanBin();
await cleanBin(config.cargoBins);
} catch (e) {
core.error(`${(e as any).stack}`);
}

4
src/utils.ts
View file

@ -22,8 +22,8 @@ export async function getCmdOutput(
...options,
});
} catch (e) {
core.info(`[warning] Command failed: ${cmd} ${args.join(" ")}`);
core.info(`[warning] ${stderr}`);
core.error(`Command failed: ${cmd} ${args.join(" ")}`);
core.error(stderr);
throw e;
}
return stdout;