From 3faf29c29b49073af6c77e1bf1e4951fda5a9e43 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Tue, 28 Jan 2025 18:13:52 -0800 Subject: [PATCH] wip: add incremental mtime restore --- action.yml | 4 ++++ src/cleanup.ts | 59 ++++++++++++++++++++++++++++++++++------------ src/config.ts | 5 +++- src/incremental.ts | 48 +++++++++++++++++++++++++++++++++++++ src/restore.ts | 17 ++++++++++--- src/save.ts | 4 ++-- 6 files changed, 116 insertions(+), 21 deletions(-) create mode 100644 src/incremental.ts diff --git a/action.yml b/action.yml index cb4c157..0f1b024 100644 --- a/action.yml +++ b/action.yml @@ -48,6 +48,10 @@ inputs: description: "Check if a cache entry exists without downloading the cache" required: false default: "false" + incremental: + description: "Determines whether to cache incremental builds - speeding up builds for more disk usage. Defaults to false." + required: false + default: "false" outputs: cache-hit: description: "A boolean value that indicates an exact match was found." diff --git a/src/cleanup.ts b/src/cleanup.ts index d84a9d5..19b0c74 100644 --- a/src/cleanup.ts +++ b/src/cleanup.ts @@ -7,7 +7,7 @@ import { CARGO_HOME } from "./config"; import { exists } from "./utils"; import { Packages } from "./workspace"; -export async function cleanTargetDir(targetDir: string, packages: Packages, checkTimestamp = false) { +export async function cleanTargetDir(targetDir: string, packages: Packages, checkTimestamp: boolean, incremental: boolean) { core.debug(`cleaning target directory "${targetDir}"`); // remove all *files* from the profile directory @@ -21,18 +21,18 @@ export async function cleanTargetDir(targetDir: string, packages: Packages, chec try { if (isNestedTarget) { - await cleanTargetDir(dirName, packages, checkTimestamp); + await cleanTargetDir(dirName, packages, checkTimestamp, incremental); } else { - await cleanProfileTarget(dirName, packages, checkTimestamp); + await cleanProfileTarget(dirName, packages, checkTimestamp, incremental); } - } catch {} + } catch { } } else if (dirent.name !== "CACHEDIR.TAG") { await rm(dir.path, dirent); } } } -async function cleanProfileTarget(profileDir: string, packages: Packages, checkTimestamp = false) { +async function cleanProfileTarget(profileDir: string, packages: Packages, checkTimestamp: boolean, incremental: boolean) { core.debug(`cleaning profile directory "${profileDir}"`); // Quite a few testing utility crates store compilation artifacts as nested @@ -42,15 +42,44 @@ async function cleanProfileTarget(profileDir: string, packages: Packages, checkT try { // https://github.com/vertexclique/kaos/blob/9876f6c890339741cc5be4b7cb9df72baa5a6d79/src/cargo.rs#L25 // https://github.com/eupn/macrotest/blob/c4151a5f9f545942f4971980b5d264ebcd0b1d11/src/cargo.rs#L27 - cleanTargetDir(path.join(profileDir, "target"), packages, checkTimestamp); - } catch {} + cleanTargetDir(path.join(profileDir, "target"), packages, checkTimestamp, incremental); + } catch { } + try { // https://github.com/dtolnay/trybuild/blob/eec8ca6cb9b8f53d0caf1aa499d99df52cae8b40/src/cargo.rs#L50 - cleanTargetDir(path.join(profileDir, "trybuild"), packages, checkTimestamp); - } catch {} + cleanTargetDir(path.join(profileDir, "trybuild"), packages, checkTimestamp, incremental); + } catch { } // Delete everything else. - await rmExcept(profileDir, new Set(["target", "trybuild"]), checkTimestamp); + let except = new Set(["target", "trybuild"]); + + // Keep the incremental folder if incremental builds are enabled + if (incremental) { + except.add("incremental"); + + // Traverse the incremental folder recursively and collect the modified times in a map + const incrementalDir = path.join(profileDir, "incremental"); + const modifiedTimes = new Map(); + const fillModifiedTimes = async (dir: string) => { + const dirEntries = await fs.promises.opendir(dir); + for await (const dirent of dirEntries) { + if (dirent.isDirectory()) { + await fillModifiedTimes(path.join(dir, dirent.name)); + } else { + const fileName = path.join(dir, dirent.name); + const { mtime } = await fs.promises.stat(fileName); + modifiedTimes.set(fileName, mtime.getTime()); + } + } + }; + await fillModifiedTimes(incrementalDir); + + // Write the modified times to the incremental folder + const contents = JSON.stringify({ modifiedTimes }); + await fs.promises.writeFile(path.join(incrementalDir, "incremental-restore.json"), contents); + } + + await rmExcept(profileDir, except, checkTimestamp); return; } @@ -86,7 +115,7 @@ export async function getCargoBins(): Promise> { bins.add(bin); } } - } catch {} + } catch { } return bins; } @@ -117,7 +146,7 @@ export async function cleanRegistry(packages: Packages, crates = true) { const credentials = path.join(CARGO_HOME, ".cargo", "credentials.toml"); core.debug(`deleting "${credentials}"`); await fs.promises.unlink(credentials); - } catch {} + } catch { } // `.cargo/registry/index` let pkgSet = new Set(packages.map((p) => p.name)); @@ -229,7 +258,7 @@ export async function cleanGit(packages: Packages) { await rm(dir.path, dirent); } } - } catch {} + } catch { } // clean the checkouts try { @@ -250,7 +279,7 @@ export async function cleanGit(packages: Packages) { } } } - } catch {} + } catch { } } const ONE_WEEK = 7 * 24 * 3600 * 1000; @@ -302,7 +331,7 @@ async function rm(parent: string, dirent: fs.Dirent) { } else if (dirent.isDirectory()) { await io.rmRF(fileName); } - } catch {} + } catch { } } async function rmRF(dirName: string) { diff --git a/src/config.ts b/src/config.ts index 5104f5c..08a6490 100644 --- a/src/config.ts +++ b/src/config.ts @@ -34,6 +34,9 @@ export class CacheConfig { /** The cargo binaries present during main step */ public cargoBins: Array = []; + /** Whether to cache incremental builds */ + public incremental: boolean = false; + /** The prefix portion of the cache key */ private keyPrefix = ""; /** The rust version considered for the cache key */ @@ -43,7 +46,7 @@ export class CacheConfig { /** The files considered for the cache key */ private keyFiles: Array = []; - private constructor() {} + private constructor() { } /** * Constructs a [`CacheConfig`] with all the paths and keys. diff --git a/src/incremental.ts b/src/incremental.ts new file mode 100644 index 0000000..3b054f9 --- /dev/null +++ b/src/incremental.ts @@ -0,0 +1,48 @@ +import * as core from "@actions/core"; +import * as io from "@actions/io"; +import fs from "fs"; +import path from "path"; + +import { CARGO_HOME } from "./config"; +import { exists } from "./utils"; +import { Packages } from "./workspace"; + + +export async function restoreIncremental(targetDir: string) { + core.debug(`restoring incremental directory "${targetDir}"`); + + + let dir = await fs.promises.opendir(targetDir); + for await (const dirent of dir) { + if (dirent.isDirectory()) { + let dirName = path.join(dir.path, dirent.name); + // is it a profile dir, or a nested target dir? + let isNestedTarget = + (await exists(path.join(dirName, "CACHEDIR.TAG"))) || (await exists(path.join(dirName, ".rustc_info.json"))); + + try { + if (isNestedTarget) { + await restoreIncremental(dirName); + } else { + await restoreIncrementalProfile(dirName); + } restoreIncrementalProfile + } catch { } + } + } +} + +async function restoreIncrementalProfile(dirName: string) { + core.debug(`restoring incremental profile directory "${dirName}"`); + const incrementalJson = path.join(dirName, "incremental-restore.json"); + if (await exists(incrementalJson)) { + const contents = await fs.promises.readFile(incrementalJson, "utf8"); + const { modifiedTimes } = JSON.parse(contents); + + // Write the mtimes to all the files in the profile directory + for (const fileName of Object.keys(modifiedTimes)) { + const mtime = modifiedTimes[fileName]; + const filePath = path.join(dirName, fileName); + await fs.promises.utimes(filePath, new Date(mtime), new Date(mtime)); + } + } +} diff --git a/src/restore.ts b/src/restore.ts index 21af56f..c02c2f0 100644 --- a/src/restore.ts +++ b/src/restore.ts @@ -3,6 +3,7 @@ import * as core from "@actions/core"; import { cleanTargetDir } from "./cleanup"; import { CacheConfig } from "./config"; import { getCacheProvider, reportError } from "./utils"; +import { restoreIncremental } from "./incremental"; process.on("uncaughtException", (e) => { core.error(e.message); @@ -27,12 +28,15 @@ async function run() { var lookupOnly = core.getInput("lookup-only").toLowerCase() === "true"; core.exportVariable("CACHE_ON_FAILURE", cacheOnFailure); - core.exportVariable("CARGO_INCREMENTAL", 0); const config = await CacheConfig.new(); config.printInfo(cacheProvider); core.info(""); + if (!config.incremental) { + core.exportVariable("CARGO_INCREMENTAL", 0); + } + core.info(`... ${lookupOnly ? "Checking" : "Restoring"} cache ...`); const key = config.cacheKey; // Pass a copy of cachePaths to avoid mutating the original array as reported by: @@ -44,12 +48,19 @@ async function run() { if (restoreKey) { const match = restoreKey === key; core.info(`${lookupOnly ? "Found" : "Restored from"} cache key "${restoreKey}" full match: ${match}.`); + + if (config.incremental) { + for (const workspace of config.workspaces) { + await restoreIncremental(workspace.target); + } + } + if (!match) { // pre-clean the target directory on cache mismatch for (const workspace of config.workspaces) { try { - await cleanTargetDir(workspace.target, [], true); - } catch {} + await cleanTargetDir(workspace.target, [], true, false); + } catch { } } // We restored the cache but it is not a full match. diff --git a/src/save.ts b/src/save.ts index a62019e..d199328 100644 --- a/src/save.ts +++ b/src/save.ts @@ -42,7 +42,7 @@ async function run() { allPackages.push(...packages); try { core.info(`... Cleaning ${workspace.target} ...`); - await cleanTargetDir(workspace.target, packages); + await cleanTargetDir(workspace.target, packages, false, config.incremental); } catch (e) { core.debug(`${(e as any).stack}`); } @@ -90,5 +90,5 @@ async function macOsWorkaround() { // Workaround for https://github.com/actions/cache/issues/403 // Also see https://github.com/rust-lang/cargo/issues/8603 await exec.exec("sudo", ["/usr/sbin/purge"], { silent: true }); - } catch {} + } catch { } }