mirror of
https://code.forgejo.org/actions/checkout.git
synced 2026-07-30 08:03:55 +00:00
migrate to post job uploads
This commit is contained in:
darshanime
parent
8b5fffd5f3
commit
41e5319596
6 changed files with 183 additions and 253 deletions
|
|
@ -1,10 +1,9 @@
|
|||
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||
import * as core from '@actions/core'
|
||||
|
||||
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
|
||||
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
|
||||
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
|
||||
// uploading this snapshot (skip upload); else = fall back.
|
||||
// Client for backend-core's git-mirror download-url endpoint, authed by the runner
|
||||
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
|
||||
// org; else = fall back. The upload half runs in the warpbuild-agent after the job.
|
||||
|
||||
const API_TIMEOUT_MS = 10_000
|
||||
|
||||
|
|
@ -20,12 +19,6 @@ export type SnapshotLookup =
|
|||
| {kind: 'disabled'}
|
||||
| {kind: 'error'}
|
||||
|
||||
export type UploadURLResult =
|
||||
| {kind: 'ok'; url: string}
|
||||
| {kind: 'disabled'}
|
||||
| {kind: 'locked'}
|
||||
| {kind: 'error'}
|
||||
|
||||
function baseUrl(): string {
|
||||
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '')
|
||||
}
|
||||
|
|
@ -65,40 +58,3 @@ export async function lookupSnapshot(
|
|||
return {kind: 'error'}
|
||||
}
|
||||
}
|
||||
|
||||
export async function requestUploadURL(
|
||||
repoKey: string,
|
||||
sha: string
|
||||
): Promise<UploadURLResult> {
|
||||
try {
|
||||
const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: authHeader(),
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({repo_key: repoKey, sha}),
|
||||
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||
})
|
||||
if (res.status === 200) {
|
||||
const body = (await res.json()) as {url?: string}
|
||||
if (body.url) {
|
||||
return {kind: 'ok', url: body.url}
|
||||
}
|
||||
return {kind: 'error'}
|
||||
}
|
||||
if (res.status === 403) {
|
||||
core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
|
||||
return {kind: 'disabled'}
|
||||
}
|
||||
if (res.status === 409) {
|
||||
core.debug(`[wb-cache] upload-url answered 409 (locked)`)
|
||||
return {kind: 'locked'}
|
||||
}
|
||||
core.debug(`[wb-cache] upload-url answered ${res.status}`)
|
||||
return {kind: 'error'}
|
||||
} catch (error) {
|
||||
core.debug(`[wb-cache] upload-url failed: ${error}`)
|
||||
return {kind: 'error'}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,10 +15,6 @@ import * as api from './backend-api.js'
|
|||
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
||||
|
||||
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
|
||||
const UPLOAD_TIMEOUT_MS = 15 * 60_000
|
||||
|
||||
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
|
||||
export const CACHE_STATE_FILE = 'wb-cache-state'
|
||||
|
||||
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
||||
export const SKIP_NOT_WARPBUILD =
|
||||
|
|
@ -97,15 +93,18 @@ export function computeDestinationRef(ref: string): string | null {
|
|||
}
|
||||
|
||||
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
||||
// Sets the `cache-hit` action output (true only on a restore).
|
||||
export async function setup(settings: IGitSourceSettings): Promise<boolean> {
|
||||
const restored = await setupImpl(settings)
|
||||
core.setOutput('cache-hit', restored ? 'true' : 'false')
|
||||
return restored
|
||||
}
|
||||
|
||||
async function setupImpl(settings: IGitSourceSettings): Promise<boolean> {
|
||||
decision = 'off'
|
||||
const skipReason = getMirrorCacheSkipReason(settings)
|
||||
if (skipReason) {
|
||||
if (skipReason === SKIP_NOT_WARPBUILD) {
|
||||
core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||
} else {
|
||||
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||
}
|
||||
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||
return false
|
||||
}
|
||||
core.startGroup('WarpBuild: checkout snapshot cache')
|
||||
|
|
@ -160,18 +159,16 @@ async function setupInner(settings: IGitSourceSettings): Promise<boolean> {
|
|||
return true
|
||||
}
|
||||
|
||||
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
|
||||
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
|
||||
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
|
||||
export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
||||
if (decision !== 'miss') {
|
||||
return
|
||||
}
|
||||
core.startGroup('WarpBuild: uploading checkout snapshot')
|
||||
try {
|
||||
await uploadSnapshot(settings)
|
||||
await writeSnapshotManifest(settings)
|
||||
} catch (error) {
|
||||
core.warning(`Snapshot upload skipped: ${error}`)
|
||||
} finally {
|
||||
core.endGroup()
|
||||
core.warning(`Snapshot manifest not written: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -242,80 +239,56 @@ async function restoreSnapshot(
|
|||
])
|
||||
}
|
||||
|
||||
await fs.promises.writeFile(
|
||||
path.join(gitDir, CACHE_STATE_FILE),
|
||||
`hit ${sha}\n`
|
||||
)
|
||||
return true
|
||||
} catch (error) {
|
||||
core.warning(`Snapshot restore failed: ${error}`)
|
||||
// Reset .git to its freshly-init state.
|
||||
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
||||
recursive: true,
|
||||
force: true
|
||||
})
|
||||
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
||||
recursive: true
|
||||
})
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||
recursive: true
|
||||
})
|
||||
await resetGitObjects(gitDir)
|
||||
return false
|
||||
} finally {
|
||||
await fs.promises.rm(tmpTar, {force: true})
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> {
|
||||
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
|
||||
const sha = settings.commit
|
||||
const gitDir = path.join(settings.repositoryPath, '.git')
|
||||
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`)
|
||||
try {
|
||||
const members = ['objects']
|
||||
if (fs.existsSync(path.join(gitDir, 'shallow'))) {
|
||||
members.push('shallow')
|
||||
}
|
||||
await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members])
|
||||
const size = (await fs.promises.stat(tmpTar)).size
|
||||
|
||||
const upload = await api.requestUploadURL(repoKey, sha)
|
||||
if (upload.kind !== 'ok') {
|
||||
core.info(
|
||||
upload.kind === 'locked'
|
||||
? 'Another job is already uploading this snapshot; skipping'
|
||||
: upload.kind === 'disabled'
|
||||
? 'Snapshot cache is disabled by the backend; not uploading'
|
||||
: 'Snapshot cache backend unavailable; not uploading'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
const init: RequestInit & {duplex: 'half'} = {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'content-length': String(size),
|
||||
'content-type': 'application/x-tar'
|
||||
},
|
||||
body: Readable.toWeb(
|
||||
fs.createReadStream(tmpTar)
|
||||
) as unknown as ReadableStream,
|
||||
duplex: 'half',
|
||||
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
||||
}
|
||||
const res = await fetch(upload.url, init)
|
||||
if (!res.ok) {
|
||||
throw new Error(`snapshot upload answered ${res.status}`)
|
||||
}
|
||||
await fs.promises.writeFile(
|
||||
path.join(gitDir, CACHE_STATE_FILE),
|
||||
`uploaded ${sha}\n`
|
||||
)
|
||||
core.info(
|
||||
`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`
|
||||
)
|
||||
} finally {
|
||||
await fs.promises.rm(tmpTar, {force: true})
|
||||
}
|
||||
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
|
||||
// creates, so a failed restore leaves no trace for the fallback fetch.
|
||||
export async function resetGitObjects(gitDir: string): Promise<void> {
|
||||
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
||||
recursive: true,
|
||||
force: true
|
||||
})
|
||||
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
||||
recursive: true
|
||||
})
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||
recursive: true
|
||||
})
|
||||
}
|
||||
|
||||
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
|
||||
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
|
||||
export function manifestDir(): string {
|
||||
const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir()
|
||||
return path.join(runnerTemp, 'wb-snapshots')
|
||||
}
|
||||
|
||||
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
|
||||
async function writeSnapshotManifest(
|
||||
settings: IGitSourceSettings
|
||||
): Promise<void> {
|
||||
const sha = settings.commit
|
||||
const manifest = {
|
||||
repo_key: process.env['GITHUB_REPOSITORY_ID'] as string,
|
||||
sha,
|
||||
git_dir: path.join(settings.repositoryPath, '.git')
|
||||
}
|
||||
const dir = manifestDir()
|
||||
await fs.promises.mkdir(dir, {recursive: true})
|
||||
await fs.promises.writeFile(
|
||||
path.join(dir, `${sha}.json`),
|
||||
JSON.stringify(manifest)
|
||||
)
|
||||
core.info(
|
||||
`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue