Cache multiple paths and add glob pattern support (#212)

* Allow for multiple line-delimited paths to cache

* Add initial minimatch support

* Use @actions/glob for pattern matching

* Cache multiple entries using --files-from tar input

remove known failing test

Quote tar paths

Add salt to test cache

Try reading input files from manifest

bump salt

Run test on macos

more testing

Run caching tests on 3 platforms

Run tests on self-hosted

Apparently cant reference hosted runners by name

Bump salt

wait for some time after save

more timing out

smarter waiting

Cache in tmp dir that won't be deleted

Use child_process instead of actions/exec

Revert tempDir hack

bump salt

more logging

More console logging

Use filepath to with cacheHttpClient

Test cache restoration

Revert temp dir hack

debug logging

clean up cache.yml testing

Bump salt

change debug output

build actions

* unit test coverage for caching multiple dirs

* Ensure there's a locateable test folder at homedir

* Clean up code

* Version cache with all inputs

* Unit test getCacheVersion

* Include keys in getCacheEntry request

* Clean import orders

* Use fs promises in actionUtils tests

* Update import order for to fix linter errors

* Fix remaining linter error

* Remove platform-specific test code

* Add lerna example for caching multiple dirs

* Lerna example updated to v2

Co-Authored-By: Josh Gross <joshmgross@github.com>

Co-authored-by: Josh Gross <joshmgross@github.com>

src/cacheHttpClient.ts

@@ -6,8 +6,10 @@ import {
     IRequestOptions,
     ITypedResponse
 } from "@actions/http-client/interfaces";
+import * as crypto from "crypto";
 import * as fs from "fs";
 
+import { Inputs } from "./constants";
 import {
     ArtifactCacheEntry,
     CommitCacheRequest,
@@ -16,6 +18,8 @@ import {
 } from "./contracts";
 import * as utils from "./utils/actionUtils";
 
+const versionSalt = "1.0";
+
 function isSuccessStatusCode(statusCode?: number): boolean {
     if (!statusCode) {
         return false;
@@ -78,11 +82,27 @@ function createHttpClient(): HttpClient {
     );
 }
 
+export function getCacheVersion(): string {
+    // Add salt to cache version to support breaking changes in cache entry
+    const components = [
+        core.getInput(Inputs.Path, { required: true }),
+        versionSalt
+    ];
+
+    return crypto
+        .createHash("sha256")
+        .update(components.join("|"))
+        .digest("hex");
+}
+
 export async function getCacheEntry(
     keys: string[]
 ): Promise<ArtifactCacheEntry | null> {
     const httpClient = createHttpClient();
-    const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
+    const version = getCacheVersion();
+    const resource = `cache?keys=${encodeURIComponent(
+        keys.join(",")
+    )}&version=${version}`;
 
     const response = await httpClient.getJson<ArtifactCacheEntry>(
         getCacheApiUrl(resource)
@@ -130,9 +150,11 @@ export async function downloadCache(
 // Reserve Cache
 export async function reserveCache(key: string): Promise<number> {
     const httpClient = createHttpClient();
+    const version = getCacheVersion();
 
     const reserveCacheRequest: ReserveCacheRequest = {
-        key
+        key,
+        version
     };
     const response = await httpClient.postJson<ReserveCacheResponse>(
         getCacheApiUrl("caches"),

src/constants.ts

@@ -18,3 +18,5 @@ export enum Events {
     Push = "push",
     PullRequest = "pull_request"
 }
+
+export const CacheFilename = "cache.tgz";

src/restore.ts

@@ -20,11 +20,6 @@ async function run(): Promise<void> {
             return;
         }
 
-        const cachePath = utils.resolvePath(
-            core.getInput(Inputs.Path, { required: true })
-        );
-        core.debug(`Cache Path: ${cachePath}`);
-
         const primaryKey = core.getInput(Inputs.Key, { required: true });
         core.saveState(State.CacheKey, primaryKey);
 
@@ -89,7 +84,7 @@ async function run(): Promise<void> {
                     )} MB (${archiveFileSize} B)`
                 );
 
-                await extractTar(archivePath, cachePath);
+                await extractTar(archivePath);
             } finally {
                 // Try to delete the archive to save space
                 try {

src/save.ts

@@ -2,7 +2,7 @@ import * as core from "@actions/core";
 import * as path from "path";
 
 import * as cacheHttpClient from "./cacheHttpClient";
-import { Events, Inputs, State } from "./constants";
+import { CacheFilename, Events, Inputs, State } from "./constants";
 import { createTar } from "./tar";
 import * as utils from "./utils/actionUtils";
 
@@ -44,18 +44,21 @@ async function run(): Promise<void> {
             return;
         }
         core.debug(`Cache ID: ${cacheId}`);
-        const cachePath = utils.resolvePath(
-            core.getInput(Inputs.Path, { required: true })
+        const cachePaths = await utils.resolvePaths(
+            core
+                .getInput(Inputs.Path, { required: true })
+                .split("\n")
+                .filter(x => x !== "")
         );
-        core.debug(`Cache Path: ${cachePath}`);
 
-        const archivePath = path.join(
-            await utils.createTempDirectory(),
-            "cache.tgz"
-        );
+        core.debug("Cache Paths:");
+        core.debug(`${JSON.stringify(cachePaths)}`);
+
+        const archiveFolder = await utils.createTempDirectory();
+        const archivePath = path.join(archiveFolder, CacheFilename);
         core.debug(`Archive Path: ${archivePath}`);
 
-        await createTar(archivePath, cachePath);
+        await createTar(archiveFolder, cachePaths);
 
         const fileSizeLimit = 5 * 1024 * 1024 * 1024; // 5GB per repo limit
         const archiveFileSize = utils.getArchiveFileSize(archivePath);

src/tar.ts

@@ -1,6 +1,9 @@
 import { exec } from "@actions/exec";
 import * as io from "@actions/io";
-import { existsSync } from "fs";
+import { existsSync, writeFileSync } from "fs";
+import * as path from "path";
+
+import { CacheFilename } from "./constants";
 
 async function getTarPath(): Promise<string> {
     // Explicitly use BSD Tar on Windows
@@ -14,9 +17,9 @@ async function getTarPath(): Promise<string> {
     return await io.which("tar", true);
 }
 
-async function execTar(args: string[]): Promise<void> {
+async function execTar(args: string[], cwd?: string): Promise<void> {
     try {
-        await exec(`"${await getTarPath()}"`, args);
+        await exec(`"${await getTarPath()}"`, args, { cwd: cwd });
     } catch (error) {
         const IS_WINDOWS = process.platform === "win32";
         if (IS_WINDOWS) {
@@ -28,20 +31,38 @@ async function execTar(args: string[]): Promise<void> {
     }
 }
 
-export async function extractTar(
-    archivePath: string,
-    targetDirectory: string
-): Promise<void> {
+function getWorkingDirectory(): string {
+    return process.env["GITHUB_WORKSPACE"] ?? process.cwd();
+}
+
+export async function extractTar(archivePath: string): Promise<void> {
     // Create directory to extract tar into
-    await io.mkdirP(targetDirectory);
-    const args = ["-xz", "-f", archivePath, "-C", targetDirectory];
+    const workingDirectory = getWorkingDirectory();
+    await io.mkdirP(workingDirectory);
+    const args = ["-xz", "-f", archivePath, "-P", "-C", workingDirectory];
     await execTar(args);
 }
 
 export async function createTar(
-    archivePath: string,
-    sourceDirectory: string
+    archiveFolder: string,
+    sourceDirectories: string[]
 ): Promise<void> {
-    const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."];
-    await execTar(args);
+    // Write source directories to manifest.txt to avoid command length limits
+    const manifestFilename = "manifest.txt";
+    writeFileSync(
+        path.join(archiveFolder, manifestFilename),
+        sourceDirectories.join("\n")
+    );
+
+    const workingDirectory = getWorkingDirectory();
+    const args = [
+        "-cz",
+        "-f",
+        CacheFilename,
+        "-C",
+        workingDirectory,
+        "--files-from",
+        manifestFilename
+    ];
+    await execTar(args, archiveFolder);
 }

src/utils/actionUtils.ts

@@ -1,7 +1,7 @@
 import * as core from "@actions/core";
+import * as glob from "@actions/glob";
 import * as io from "@actions/io";
 import * as fs from "fs";
-import * as os from "os";
 import * as path from "path";
 import * as util from "util";
 import * as uuidV4 from "uuid/v4";
@@ -29,6 +29,7 @@ export async function createTempDirectory(): Promise<string> {
         }
         tempDirectory = path.join(baseLocation, "actions", "temp");
     }
+
     const dest = path.join(tempDirectory, uuidV4.default());
     await io.mkdirP(dest);
     return dest;
@@ -83,16 +84,21 @@ export function logWarning(message: string): void {
     core.info(`${warningPrefix}${message}`);
 }
 
-export function resolvePath(filePath: string): string {
-    if (filePath[0] === "~") {
-        const home = os.homedir();
-        if (!home) {
-            throw new Error("Unable to resolve `~` to HOME");
-        }
-        return path.join(home, filePath.slice(1));
+export async function resolvePaths(patterns: string[]): Promise<string[]> {
+    const paths: string[] = [];
+    const workspace = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
+    const globber = await glob.create(patterns.join("\n"), {
+        implicitDescendants: false
+    });
+
+    for await (const file of globber.globGenerator()) {
+        const relativeFile = path.relative(workspace, file);
+        core.debug(`Matched: ${relativeFile}`);
+        // Paths are made relative so the tar entries are all relative to the root of the workspace.
+        paths.push(`${relativeFile}`);
     }
 
-    return path.resolve(filePath);
+    return paths;
 }
 
 export function getSupportedEvents(): string[] {