diff --git a/README.md b/README.md index 5509e7d1..e2e3c27e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,18 @@ +# WarpBuild Checkout + +This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in +replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's +bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git +alternates, so the fetch from GitHub downloads only the delta instead of the whole +repository. On a cache miss, the run hydrates the mirror once (full clone + upload); +mirrors expire on a server-configured TTL and re-hydrate. + +- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners. +- Fail-open — any cache error degrades to stock `actions/checkout` behavior. +- All fork code lives in `src/warpbuild/` + +--- + [![Build and Test](https://github.com/actions/checkout/actions/workflows/test.yml/badge.svg)](https://github.com/actions/checkout/actions/workflows/test.yml) # Checkout v7 diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts new file mode 100644 index 00000000..1f75b013 --- /dev/null +++ b/__test__/warpbuild-mirror.test.ts @@ -0,0 +1,199 @@ +import { + describe, + it, + expect, + beforeEach, + afterEach, + afterAll +} from '@jest/globals' +import { + SKIP_NOT_WARPBUILD, + computeDestinationRef, + getMirrorCacheSkipReason +} from '../src/warpbuild/mirror-cache.js' +import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js' +import {IGitSourceSettings} from '../src/git-source-settings.js' + +const WB_ENV = [ + 'WARPBUILD_RUNNER_VERIFICATION_TOKEN', + 'WARPBUILD_HOST_URL', + 'GITHUB_REPOSITORY_ID', + 'GITHUB_REPOSITORY' +] +const savedEnv: {[key: string]: string | undefined} = {} +for (const key of WB_ENV) { + savedEnv[key] = process.env[key] +} + +const SHA = 'cd5255d20e23e050238affc045ba9beee35eaaf7' + +function settingsFor( + overrides: Partial = {} +): IGitSourceSettings { + return { + repositoryOwner: 'octocat', + repositoryName: 'hello-world', + repositoryPath: '/tmp/does-not-matter', + ref: 'refs/heads/main', + commit: SHA, + fetchDepth: 1, + fetchTags: false, + filter: undefined, + sparseCheckout: undefined, + lfs: false, + ...overrides + } as unknown as IGitSourceSettings +} + +function setWarpBuildEnv(): void { + process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = 'test-token' + process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev' + process.env['GITHUB_REPOSITORY_ID'] = '123456789' + process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' +} + +describe('warpbuild snapshot cache', () => { + beforeEach(() => { + setWarpBuildEnv() + }) + + afterAll(() => { + for (const key of WB_ENV) { + if (savedEnv[key] === undefined) { + delete process.env[key] + } else { + process.env[key] = savedEnv[key] + } + } + }) + + describe('getMirrorCacheSkipReason', () => { + const winOnly = process.platform === 'win32' ? it.skip : it + + winOnly('returns null for the default checkout shape', () => { + expect(getMirrorCacheSkipReason(settingsFor())).toBeNull() + }) + + it('reports a non-WarpBuild runner', () => { + delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] + expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD) + }) + + it('reports repository: inputs that are not the workflow repo', () => { + expect( + getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'})) + ).toBe( + "repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'" + ) + }) + + it('requires an exact commit sha', () => { + expect(getMirrorCacheSkipReason(settingsFor({commit: ''}))).toBe( + 'no exact commit sha to key on' + ) + expect(getMirrorCacheSkipReason(settingsFor({commit: 'main'}))).toBe( + 'no exact commit sha to key on' + ) + }) + + it('only serves fetch-depth 1', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBe( + 'fetch-depth is 0, cache only serves fetch-depth 1' + ) + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBe( + 'fetch-depth is 50, cache only serves fetch-depth 1' + ) + }) + + it('skips fetch-tags, filters, sparse and lfs checkouts', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchTags: true}))).toBe( + 'fetch-tags is enabled' + ) + expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( + 'a fetch filter is configured' + ) + expect( + getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) + ).toBe('sparse checkout is configured') + expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe( + 'lfs is enabled (lfs objects are not in the snapshot)' + ) + }) + + winOnly('accepts sha-only checkouts (empty ref)', () => { + expect(getMirrorCacheSkipReason(settingsFor({ref: ''}))).toBeNull() + }) + + it('skips unqualified refs', () => { + expect(getMirrorCacheSkipReason(settingsFor({ref: 'main'}))).toBe( + "ref 'main' has no cacheable destination ref" + ) + }) + }) + + describe('computeDestinationRef', () => { + it('maps the refs the fetch would have created', () => { + expect(computeDestinationRef('refs/heads/main')).toBe( + 'refs/remotes/origin/main' + ) + expect(computeDestinationRef('refs/pull/42/merge')).toBe( + 'refs/remotes/pull/42/merge' + ) + expect(computeDestinationRef('refs/tags/v1.2.3')).toBe('refs/tags/v1.2.3') + expect(computeDestinationRef('')).toBe('') + expect(computeDestinationRef('main')).toBeNull() + }) + }) + + describe('backend api http contract', () => { + const realFetch = globalThis.fetch + let lastUrl = '' + + afterEach(() => { + globalThis.fetch = realFetch + }) + + function stubFetch(status: number, body: unknown): void { + globalThis.fetch = (async (input: unknown) => { + lastUrl = String(input) + return new Response(JSON.stringify(body), {status}) + }) as typeof fetch + } + + it('maps 200 to hit and keys by sha', async () => { + stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) + const result = await lookupSnapshot('123', SHA) + expect(result.kind).toBe('hit') + expect(lastUrl).toContain(`sha=${SHA}`) + }) + + it('maps 404 to miss (upload after the stock fetch)', async () => { + stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) + expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') + }) + + it('maps 403 to disabled (backend kill switch)', async () => { + stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) + expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') + expect((await requestUploadURL('123', SHA)).kind).toBe('disabled') + }) + + it('maps other statuses and network failures to error', async () => { + stubFetch(500, {}) + expect((await lookupSnapshot('123', SHA)).kind).toBe('error') + globalThis.fetch = (async () => { + throw new Error('boom') + }) as typeof fetch + expect((await lookupSnapshot('123', SHA)).kind).toBe('error') + expect((await requestUploadURL('123', SHA)).kind).toBe('error') + }) + + it('maps 200 upload responses to ok', async () => { + stubFetch(200, {url: 'https://s3/put'}) + expect(await requestUploadURL('123', SHA)).toEqual({ + kind: 'ok', + url: 'https://s3/put' + }) + }) + }) +}) diff --git a/dist/index.js b/dist/index.js index b381bd27..1cfe740e 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41692,6 +41692,315 @@ function ref_helper_select(obj, path) { return ref_helper_select(obj[key], path.substr(i + 1)); } +;// CONCATENATED MODULE: external "stream/promises" +const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises"); +;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts +/* eslint-disable i18n-text/no-en -- upstream convention */ + +// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner +// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the +// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. +const API_TIMEOUT_MS = 10_000; +function backend_api_baseUrl() { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); +} +function authHeader() { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; +} +async function lookupSnapshot(repoKey, sha) { + try { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}&sha=${encodeURIComponent(sha)}`, { + headers: { authorization: authHeader() }, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + return { kind: 'hit', info: (await res.json()) }; + } + if (res.status === 404) { + return { kind: 'miss' }; + } + if (res.status === 403) { + core_debug(`[wb-cache] download-url answered 403 (disabled)`); + return { kind: 'disabled' }; + } + core_debug(`[wb-cache] download-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-cache] download-url failed: ${error}`); + return { kind: 'error' }; + } +} +async function requestUploadURL(repoKey, sha) { + try { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify({ repo_key: repoKey, sha }), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + const body = (await res.json()); + if (body.url) { + return { kind: 'ok', url: body.url }; + } + return { kind: 'error' }; + } + if (res.status === 403) { + core_debug(`[wb-cache] upload-url answered 403 (disabled)`); + return { kind: 'disabled' }; + } + core_debug(`[wb-cache] upload-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-cache] upload-url failed: ${error}`); + return { kind: 'error' }; + } +} + +;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts +/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ + + + + + + + + +// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch +// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are +// immutable (no expiry). Fail-open: any error degrades to stock behavior. +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; +const UPLOAD_TIMEOUT_MS = 15 * 60_000; +// "hit " | "uploaded ", for e2e assertions. +const CACHE_STATE_FILE = 'wb-cache-state'; +// Logged at debug (normal state outside WarpBuild); other reasons log at info. +const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; +const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; +let decision = 'off'; +// Null = attempt the cache; else a reason to log. Only the default checkout shape +// is served. +function getMirrorCacheSkipReason(settings) { + if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL']) { + return SKIP_NOT_WARPBUILD; + } + if (process.platform === 'win32') { + return 'Windows is not supported by the snapshot cache yet'; + } + const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; + if (!repoKey) { + return 'GITHUB_REPOSITORY_ID is not set'; + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`; + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`; + } + const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, ''); + if (server !== 'https://github.com') { + return `server '${server}' is not github.com`; + } + if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { + return 'no exact commit sha to key on'; + } + if (settings.fetchDepth !== 1) { + return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`; + } + if (settings.fetchTags) { + return 'fetch-tags is enabled'; + } + if (settings.filter) { + return 'a fetch filter is configured'; + } + if (settings.sparseCheckout) { + return 'sparse checkout is configured'; + } + if (settings.lfs) { + return 'lfs is enabled (lfs objects are not in the snapshot)'; + } + if (settings.ref && computeDestinationRef(settings.ref) === null) { + return `ref '${settings.ref}' has no cacheable destination ref`; + } + return null; +} +// The local ref the fetch would have created ('' = none needed, null = uncacheable). +function computeDestinationRef(ref) { + if (!ref) { + return ''; + } + const upper = ref.toUpperCase(); + if (upper.startsWith('REFS/HEADS/')) { + return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`; + } + if (upper.startsWith('REFS/PULL/')) { + return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`; + } + if (upper.startsWith('REFS/TAGS/')) { + return ref; + } + return null; +} +// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. +async function setup(settings) { + decision = 'off'; + const skipReason = getMirrorCacheSkipReason(settings); + if (skipReason) { + if (skipReason === SKIP_NOT_WARPBUILD) { + core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`); + } + else { + info(`WarpBuild snapshot cache skipped: ${skipReason}`); + } + return false; + } + startGroup('WarpBuild: checkout snapshot cache'); + try { + return await setupInner(settings); + } + catch (error) { + warning(`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`); + return false; + } + finally { + endGroup(); + } +} +async function setupInner(settings) { + const repoKey = process.env['GITHUB_REPOSITORY_ID']; + const sha = settings.commit; + const lookup = await lookupSnapshot(repoKey, sha); + if (lookup.kind === 'disabled') { + info('Snapshot cache is disabled by the backend for this organization'); + return false; + } + if (lookup.kind === 'error') { + info('Snapshot cache backend unavailable; using standard checkout'); + return false; + } + if (lookup.kind === 'miss') { + info(`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`); + decision = 'miss'; + return false; + } + info(`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`); + if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { + return false; + } + info('Snapshot restored; skipping the GitHub fetch entirely'); + return true; +} +// Runs after checkout; uploads the fetch result on a miss. Failures only warn. +async function contribute(settings) { + if (decision !== 'miss') { + return; + } + startGroup('WarpBuild: uploading checkout snapshot'); + try { + await uploadSnapshot(settings); + } + catch (error) { + warning(`Snapshot upload skipped: ${error}`); + } + finally { + endGroup(); + } +} +async function restoreSnapshot(settings, url, sha) { + const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`); + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + throw new Error(`snapshot download answered ${res.status}`); + } + await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); + await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); + const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); + if (check !== 0) { + throw new Error(`restored snapshot does not contain ${sha}`); + } + // The ref the skipped fetch would have created; upstream's verification needs it. + const dstRef = computeDestinationRef(settings.ref); + if (dstRef) { + await exec_exec('git', [ + '-C', + settings.repositoryPath, + 'update-ref', + dstRef, + sha + ]); + } + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`); + return true; + } + catch (error) { + warning(`Snapshot restore failed: ${error}`); + // Reset .git to its freshly-init state. + await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), { + recursive: true, + force: true + }); + await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true }); + await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), { + recursive: true + }); + await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), { + recursive: true + }); + return false; + } + finally { + await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + } +} +async function uploadSnapshot(settings) { + const repoKey = process.env['GITHUB_REPOSITORY_ID']; + const sha = settings.commit; + const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`); + try { + const members = ['objects']; + if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) { + members.push('shallow'); + } + await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]); + const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; + const upload = await requestUploadURL(repoKey, sha); + if (upload.kind !== 'ok') { + info(upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading'); + return; + } + const init = { + method: 'PUT', + headers: { + 'content-length': String(size), + 'content-type': 'application/x-tar' + }, + body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)), + duplex: 'half', + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + }; + const res = await fetch(upload.url, init); + if (!res.ok) { + throw new Error(`snapshot upload answered ${res.status}`); + } + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`); + info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`); + } + finally { + await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + } +} + ;// CONCATENATED MODULE: ./src/git-source-provider.ts @@ -41705,6 +42014,7 @@ function ref_helper_select(obj, path) { + async function getSource(settings) { // Repository URL info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`); @@ -41724,6 +42034,7 @@ async function getSource(settings) { const git = await getGitCommandManager(settings); endGroup(); let authHelper = null; + let warpbuildRestored = false; try { if (git) { authHelper = createAuthHelper(git, settings); @@ -41774,6 +42085,8 @@ async function getSource(settings) { await git.init(objectFormat); await git.remoteAdd('origin', repositoryUrl); endGroup(); + // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped + warpbuildRestored = await setup(settings); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); @@ -41813,7 +42126,10 @@ async function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } - if (settings.fetchDepth <= 0) { + if (warpbuildRestored) { + info('Skipping fetch: checkout was restored from the snapshot cache'); + } + else if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit); await git.fetch(refSpec, fetchOptions); @@ -41879,6 +42195,8 @@ async function getSource(settings) { startGroup('Checking out the ref'); await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); endGroup(); + // WarpBuild snapshot cache: upload the fetch result on a miss + await contribute(settings); // Submodules if (settings.submodules) { // Temporarily override global config diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index b9c1d357..3b5adbc4 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -9,6 +9,7 @@ import * as path from 'path' import * as refHelper from './ref-helper.js' import * as stateHelper from './state-helper.js' import * as urlHelper from './url-helper.js' +import * as warpbuildMirror from './warpbuild/mirror-cache.js' import { MinimumGitSparseCheckoutVersion, IGitCommandManager @@ -40,6 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.endGroup() let authHelper: gitAuthHelper.IGitAuthHelper | null = null + let warpbuildRestored = false try { if (git) { authHelper = gitAuthHelper.createAuthHelper(git, settings) @@ -130,6 +132,9 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.init(objectFormat) await git.remoteAdd('origin', repositoryUrl) core.endGroup() + + // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped + warpbuildRestored = await warpbuildMirror.setup(settings) } // Disable automatic garbage collection @@ -185,7 +190,9 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } - if (settings.fetchDepth <= 0) { + if (warpbuildRestored) { + core.info('Skipping fetch: checkout was restored from the snapshot cache') + } else if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( settings.ref, @@ -271,6 +278,9 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) core.endGroup() + // WarpBuild snapshot cache: upload the fetch result on a miss + await warpbuildMirror.contribute(settings) + // Submodules if (settings.submodules) { // Temporarily override global config diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts new file mode 100644 index 00000000..d51cd064 --- /dev/null +++ b/src/warpbuild/backend-api.ts @@ -0,0 +1,98 @@ +/* eslint-disable i18n-text/no-en -- upstream convention */ +import * as core from '@actions/core' + +// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner +// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the +// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. + +const API_TIMEOUT_MS = 10_000 + +export interface SnapshotDownloadInfo { + url: string + size_bytes: number + created_at: string +} + +export type SnapshotLookup = + | {kind: 'hit'; info: SnapshotDownloadInfo} + | {kind: 'miss'} + | {kind: 'disabled'} + | {kind: 'error'} + +export type UploadURLResult = + | {kind: 'ok'; url: string} + | {kind: 'disabled'} + | {kind: 'error'} + +function baseUrl(): string { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '') +} + +function authHeader(): string { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` +} + +export async function lookupSnapshot( + repoKey: string, + sha: string +): Promise { + try { + const res = await fetch( + `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent( + repoKey + )}&sha=${encodeURIComponent(sha)}`, + { + headers: {authorization: authHeader()}, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + } + ) + if (res.status === 200) { + return {kind: 'hit', info: (await res.json()) as SnapshotDownloadInfo} + } + if (res.status === 404) { + return {kind: 'miss'} + } + if (res.status === 403) { + core.debug(`[wb-cache] download-url answered 403 (disabled)`) + return {kind: 'disabled'} + } + core.debug(`[wb-cache] download-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-cache] download-url failed: ${error}`) + return {kind: 'error'} + } +} + +export async function requestUploadURL( + repoKey: string, + sha: string +): Promise { + try { + const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify({repo_key: repoKey, sha}), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }) + if (res.status === 200) { + const body = (await res.json()) as {url?: string} + if (body.url) { + return {kind: 'ok', url: body.url} + } + return {kind: 'error'} + } + if (res.status === 403) { + core.debug(`[wb-cache] upload-url answered 403 (disabled)`) + return {kind: 'disabled'} + } + core.debug(`[wb-cache] upload-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-cache] upload-url failed: ${error}`) + return {kind: 'error'} + } +} diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts new file mode 100644 index 00000000..9e604e1b --- /dev/null +++ b/src/warpbuild/mirror-cache.ts @@ -0,0 +1,289 @@ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ +import * as core from '@actions/core' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {Readable} from 'stream' +import {pipeline} from 'stream/promises' +import {IGitSourceSettings} from '../git-source-settings.js' +import * as api from './backend-api.js' + +// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch +// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are +// immutable (no expiry). Fail-open: any error degrades to stock behavior. + +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 +const UPLOAD_TIMEOUT_MS = 15 * 60_000 + +// "hit " | "uploaded ", for e2e assertions. +export const CACHE_STATE_FILE = 'wb-cache-state' + +// Logged at debug (normal state outside WarpBuild); other reasons log at info. +export const SKIP_NOT_WARPBUILD = + 'not running on a WarpBuild runner (WARPBUILD_* env not present)' + +const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/ + +let decision: 'off' | 'miss' = 'off' + +// Null = attempt the cache; else a reason to log. Only the default checkout shape +// is served. +export function getMirrorCacheSkipReason( + settings: IGitSourceSettings +): string | null { + if ( + !process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL'] + ) { + return SKIP_NOT_WARPBUILD + } + if (process.platform === 'win32') { + return 'Windows is not supported by the snapshot cache yet' + } + const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' + if (!repoKey) { + return 'GITHUB_REPOSITORY_ID is not set' + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}` + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'` + } + const server = (settings.githubServerUrl || 'https://github.com').replace( + /\/+$/, + '' + ) + if (server !== 'https://github.com') { + return `server '${server}' is not github.com` + } + if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { + return 'no exact commit sha to key on' + } + if (settings.fetchDepth !== 1) { + return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1` + } + if (settings.fetchTags) { + return 'fetch-tags is enabled' + } + if (settings.filter) { + return 'a fetch filter is configured' + } + if (settings.sparseCheckout) { + return 'sparse checkout is configured' + } + if (settings.lfs) { + return 'lfs is enabled (lfs objects are not in the snapshot)' + } + if (settings.ref && computeDestinationRef(settings.ref) === null) { + return `ref '${settings.ref}' has no cacheable destination ref` + } + return null +} + +// The local ref the fetch would have created ('' = none needed, null = uncacheable). +export function computeDestinationRef(ref: string): string | null { + if (!ref) { + return '' + } + const upper = ref.toUpperCase() + if (upper.startsWith('REFS/HEADS/')) { + return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}` + } + if (upper.startsWith('REFS/PULL/')) { + return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}` + } + if (upper.startsWith('REFS/TAGS/')) { + return ref + } + return null +} + +// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. +export async function setup(settings: IGitSourceSettings): Promise { + decision = 'off' + const skipReason = getMirrorCacheSkipReason(settings) + if (skipReason) { + if (skipReason === SKIP_NOT_WARPBUILD) { + core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`) + } else { + core.info(`WarpBuild snapshot cache skipped: ${skipReason}`) + } + return false + } + core.startGroup('WarpBuild: checkout snapshot cache') + try { + return await setupInner(settings) + } catch (error) { + core.warning( + `WarpBuild snapshot cache unavailable, using standard checkout: ${error}` + ) + return false + } finally { + core.endGroup() + } +} + +async function setupInner(settings: IGitSourceSettings): Promise { + const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string + const sha = settings.commit + + const lookup = await api.lookupSnapshot(repoKey, sha) + + if (lookup.kind === 'disabled') { + core.info('Snapshot cache is disabled by the backend for this organization') + return false + } + if (lookup.kind === 'error') { + core.info('Snapshot cache backend unavailable; using standard checkout') + return false + } + + if (lookup.kind === 'miss') { + core.info( + `Cache miss for ${sha}: the standard fetch will run and its result will be uploaded` + ) + decision = 'miss' + return false + } + + core.info( + `Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)` + ) + if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { + return false + } + core.info('Snapshot restored; skipping the GitHub fetch entirely') + return true +} + +// Runs after checkout; uploads the fetch result on a miss. Failures only warn. +export async function contribute(settings: IGitSourceSettings): Promise { + if (decision !== 'miss') { + return + } + core.startGroup('WarpBuild: uploading checkout snapshot') + try { + await uploadSnapshot(settings) + } catch (error) { + core.warning(`Snapshot upload skipped: ${error}`) + } finally { + core.endGroup() + } +} + +async function restoreSnapshot( + settings: IGitSourceSettings, + url: string, + sha: string +): Promise { + const gitDir = path.join(settings.repositoryPath, '.git') + const tmpTar = path.join(os.tmpdir(), `wb-snapshot-${process.pid}.tar`) + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }) + if (!res.ok || !res.body) { + throw new Error(`snapshot download answered ${res.status}`) + } + await pipeline( + Readable.fromWeb(res.body as import('stream/web').ReadableStream), + fs.createWriteStream(tmpTar) + ) + await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir]) + + const check = await exec.exec( + 'git', + ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], + {ignoreReturnCode: true} + ) + if (check !== 0) { + throw new Error(`restored snapshot does not contain ${sha}`) + } + + // The ref the skipped fetch would have created; upstream's verification needs it. + const dstRef = computeDestinationRef(settings.ref) + if (dstRef) { + await exec.exec('git', [ + '-C', + settings.repositoryPath, + 'update-ref', + dstRef, + sha + ]) + } + + await fs.promises.writeFile( + path.join(gitDir, CACHE_STATE_FILE), + `hit ${sha}\n` + ) + return true + } catch (error) { + core.warning(`Snapshot restore failed: ${error}`) + // Reset .git to its freshly-init state. + await fs.promises.rm(path.join(gitDir, 'objects'), { + recursive: true, + force: true + }) + await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true}) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { + recursive: true + }) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { + recursive: true + }) + return false + } finally { + await fs.promises.rm(tmpTar, {force: true}) + } +} + +async function uploadSnapshot(settings: IGitSourceSettings): Promise { + const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string + const sha = settings.commit + const gitDir = path.join(settings.repositoryPath, '.git') + const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`) + try { + const members = ['objects'] + if (fs.existsSync(path.join(gitDir, 'shallow'))) { + members.push('shallow') + } + await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]) + const size = (await fs.promises.stat(tmpTar)).size + + const upload = await api.requestUploadURL(repoKey, sha) + if (upload.kind !== 'ok') { + core.info( + upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading' + ) + return + } + + const init: RequestInit & {duplex: 'half'} = { + method: 'PUT', + headers: { + 'content-length': String(size), + 'content-type': 'application/x-tar' + }, + body: Readable.toWeb( + fs.createReadStream(tmpTar) + ) as unknown as ReadableStream, + duplex: 'half', + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + } + const res = await fetch(upload.url, init) + if (!res.ok) { + throw new Error(`snapshot upload answered ${res.status}`) + } + await fs.promises.writeFile( + path.join(gitDir, CACHE_STATE_FILE), + `uploaded ${sha}\n` + ) + core.info( + `Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch` + ) + } finally { + await fs.promises.rm(tmpTar, {force: true}) + } +}