Implement timeouts and retries logic

action.yml

@@ -14,6 +14,12 @@ inputs:
   workspaces:
     description: "Paths to multiple Cargo workspaces and their target directories, separated by newlines"
     required: false
+  maxRetryAttempts:
+    description: "The amount of attempts to retry the network operations after retriable errors"
+    required: false
+  timeout:
+    description: "The timeout for the networking operations"
+    required: false
   cache-on-failure:
     description: "Cache even if the build fails. Defaults to false"
     required: false

src/config.ts

@@ -27,6 +27,11 @@ export class CacheConfig {
   /** The workspace configurations */
   public workspaces: Array<Workspace> = [];
 
+  /** The max timeout for the networking operations */
+  public timeout: null | number = null;
+  /** The max retry attemtps for the networking operations */
+  public maxRetryAttempts: number = 0;
+
   /** The prefix portion of the cache key */
   private keyPrefix = "";
   /** The rust version considered for the cache key */
@@ -156,6 +161,12 @@ export class CacheConfig {
 
     self.cachePaths = [CARGO_HOME, ...workspaces.map((ws) => ws.target)];
 
+    const timeoutInput = core.getInput("timeout")
+    self.timeout = timeoutInput ? parseFloat(timeoutInput) : null;
+
+    const maxRetryAttemptsInput = core.getInput("maxRetryAttempts")
+    self.maxRetryAttempts =  maxRetryAttemptsInput ? parseFloat(maxRetryAttemptsInput) : 0;
+
     return self;
   }
 
@@ -184,6 +195,10 @@ export class CacheConfig {
     for (const file of this.keyFiles) {
       core.info(`  - ${file}`);
     }
+    core.info(`Network operations timeout:`);
+    core.info(`    ${this.timeout}`);
+    core.info(`Max retry attempts for the network operations:`);
+    core.info(`    ${this.maxRetryAttempts}`);
     core.endGroup();
   }
 }

src/restore.ts

@@ -3,6 +3,7 @@ import * as core from "@actions/core";
 
 import { cleanTargetDir, getCargoBins } from "./cleanup";
 import { CacheConfig, STATE_BINS, STATE_KEY } from "./config";
+import { withRetries, withTimeout } from "./utils";
 
 process.on("uncaughtException", (e) => {
   core.info(`[warning] ${e.message}`);
@@ -34,7 +35,16 @@ async function run() {
 
     core.info(`... Restoring cache ...`);
     const key = config.cacheKey;
-    const restoreKey = await cache.restoreCache(config.cachePaths, key, [config.restoreKey]);
+    const restoreKey = await withRetries(
+      () =>
+        withTimeout(
+          () => cache.restoreCache(config.cachePaths, key, [config.restoreKey]),
+          config.timeout
+        ),
+      config.maxRetryAttempts,
+      () => true
+    );
+
     if (restoreKey) {
       core.info(`Restored from cache key "${restoreKey}".`);
       core.saveState(STATE_KEY, restoreKey);

src/save.ts

@@ -4,6 +4,7 @@ import * as exec from "@actions/exec";
 
 import { cleanBin, cleanGit, cleanRegistry, cleanTargetDir } from "./cleanup";
 import { CacheConfig, STATE_KEY } from "./config";
+import { withRetries, withTimeout } from "./utils";
 
 process.on("uncaughtException", (e) => {
   core.info(`[warning] ${e.message}`);
@@ -64,7 +65,15 @@ async function run() {
     }
 
     core.info(`... Saving cache ...`);
-    await cache.saveCache(config.cachePaths, config.cacheKey);
+    await withRetries(
+      () =>
+        withTimeout(
+          () => cache.saveCache(config.cachePaths, config.cacheKey),
+          config.timeout
+        ),
+      config.maxRetryAttempts,
+      () => true
+    );
   } catch (e) {
     core.info(`[warning] ${(e as any).stack}`);
   }

src/utils.ts

@@ -28,3 +28,49 @@ export async function getCmdOutput(
   }
   return stdout;
 }
+
+export async function withRetries<T>(
+  operation: () => Promise<T>,
+  maxRetryAttempts: number,
+  isRetriable: (error: unknown) => boolean
+): Promise<T> {
+  let attemptsLeft = maxRetryAttempts;
+  while (true) {
+    try {
+      return await operation();
+    } catch (e: unknown) {
+      attemptsLeft -= 1;
+      if (attemptsLeft <= 0) {
+        throw e;
+      }
+      if (!isRetriable(e)) {
+        throw e;
+      }
+      core.info(
+        `[warning] Retrying after an error, ${attemptsLeft} attempts left, error: ${e}`
+      );
+    }
+  }
+}
+
+class TimeoutError extends Error {}
+
+export async function withTimeout<T>(
+  operation: (onTimeout: Promise<void>) => Promise<T>,
+  timeoutMs: null | number
+): Promise<T> {
+  const timeout = timeoutMs
+    ? new Promise<void>((resolve) => {
+        setTimeout(resolve, timeoutMs);
+      })
+    : new Promise<never>(() => {});
+
+  const timeoutSym = Symbol("timeout" as const);
+  const racingTimeout = timeout.then(() => timeoutSym);
+
+  const result = await Promise.race([racingTimeout, operation(timeout)]);
+  if (result === timeoutSym) {
+    throw new TimeoutError("operation timeout");
+  }
+  return result as Awaited<T>;
+}