From 713b9094287731e8b4e51b5cbe2ecef7af51cb5d Mon Sep 17 00:00:00 2001 From: darshanime Date: Wed, 8 Jul 2026 09:40:20 +0530 Subject: [PATCH 1/2] init warpbuild caching --- README.md | 15 ++ __test__/warpbuild-mirror.test.ts | 197 +++++++++++++++++ dist/index.js | 345 ++++++++++++++++++++++++++++++ src/git-source-provider.ts | 5 + src/warpbuild/backend-api.ts | 101 +++++++++ src/warpbuild/mirror-cache.ts | 326 ++++++++++++++++++++++++++++ 6 files changed, 989 insertions(+) create mode 100644 __test__/warpbuild-mirror.test.ts create mode 100644 src/warpbuild/backend-api.ts create mode 100644 src/warpbuild/mirror-cache.ts diff --git a/README.md b/README.md index 5509e7d1..e2e3c27e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,18 @@ +# WarpBuild Checkout + +This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in +replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's +bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git +alternates, so the fetch from GitHub downloads only the delta instead of the whole +repository. On a cache miss, the run hydrates the mirror once (full clone + upload); +mirrors expire on a server-configured TTL and re-hydrate. + +- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners. +- Fail-open — any cache error degrades to stock `actions/checkout` behavior. +- All fork code lives in `src/warpbuild/` + +--- + [![Build and Test](https://github.com/actions/checkout/actions/workflows/test.yml/badge.svg)](https://github.com/actions/checkout/actions/workflows/test.yml) # Checkout v7 diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts new file mode 100644 index 00000000..023b257d --- /dev/null +++ b/__test__/warpbuild-mirror.test.ts @@ -0,0 +1,197 @@ +import { + describe, + it, + expect, + beforeEach, + afterEach, + afterAll +} from '@jest/globals' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { + ALTERNATES_CONTENT, + SKIP_NOT_WARPBUILD, + mirrorPath, + getMirrorCacheSkipReason, + writeAlternates +} from '../src/warpbuild/mirror-cache.js' +import {lookupMirror, requestUploadURL} from '../src/warpbuild/backend-api.js' +import {IGitSourceSettings} from '../src/git-source-settings.js' + +const WB_ENV = [ + 'WARPBUILD_RUNNER_VERIFICATION_TOKEN', + 'WARPBUILD_HOST_URL', + 'GITHUB_REPOSITORY_ID', + 'GITHUB_REPOSITORY' +] +const savedEnv: {[key: string]: string | undefined} = {} +for (const key of WB_ENV) { + savedEnv[key] = process.env[key] +} + +function settingsFor( + owner: string, + repo: string, + serverUrl?: string +): IGitSourceSettings { + return { + repositoryOwner: owner, + repositoryName: repo, + repositoryPath: '/tmp/does-not-matter', + githubServerUrl: serverUrl + } as unknown as IGitSourceSettings +} + +function setWarpBuildEnv(): void { + process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = 'test-token' + process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev' + process.env['GITHUB_REPOSITORY_ID'] = '123456789' + process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' +} + +describe('warpbuild mirror cache', () => { + beforeEach(() => { + setWarpBuildEnv() + }) + + afterAll(() => { + for (const key of WB_ENV) { + if (savedEnv[key] === undefined) { + delete process.env[key] + } else { + process.env[key] = savedEnv[key] + } + } + }) + + describe('getMirrorCacheSkipReason', () => { + const winOnly = process.platform === 'win32' ? it.skip : it + + winOnly('returns null for the workflow repo on a WarpBuild runner', () => { + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBeNull() + }) + + it('reports a non-WarpBuild runner', () => { + delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBe(SKIP_NOT_WARPBUILD) + }) + + it('reports repository: inputs that are not the workflow repo', () => { + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'other-repo')) + ).toBe( + "repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'" + ) + }) + + it('reports a missing GITHUB_REPOSITORY_ID', () => { + delete process.env['GITHUB_REPOSITORY_ID'] + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBe('GITHUB_REPOSITORY_ID is not set') + }) + + winOnly('accepts explicit github.com server urls', () => { + expect( + getMirrorCacheSkipReason( + settingsFor('octocat', 'hello-world', 'https://github.com') + ) + ).toBeNull() + }) + + it('reports GHES server urls', () => { + expect( + getMirrorCacheSkipReason( + settingsFor('octocat', 'hello-world', 'https://ghes.example.com') + ) + ).toBe("server 'https://ghes.example.com' is not github.com") + }) + }) + + describe('backend api http contract', () => { + const realFetch = globalThis.fetch + + afterEach(() => { + globalThis.fetch = realFetch + }) + + function stubFetch(status: number, body: unknown): void { + globalThis.fetch = (async () => + new Response(JSON.stringify(body), {status})) as typeof fetch + } + + it('maps 200 to hit', async () => { + stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) + const result = await lookupMirror('123') + expect(result.kind).toBe('hit') + if (result.kind === 'hit') { + expect(result.info.url).toBe('https://s3/x') + } + }) + + it('maps 404 to miss (hydrate)', async () => { + stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) + expect((await lookupMirror('123')).kind).toBe('miss') + }) + + it('maps 403 to disabled (backend kill switch — no hydration, no upload)', async () => { + stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) + expect((await lookupMirror('123')).kind).toBe('disabled') + expect((await requestUploadURL('123')).kind).toBe('disabled') + }) + + it('maps other statuses to error (fall back without hydrating)', async () => { + stubFetch(500, {}) + expect((await lookupMirror('123')).kind).toBe('error') + expect((await requestUploadURL('123')).kind).toBe('error') + }) + + it('maps network failures to error', async () => { + globalThis.fetch = (async () => { + throw new Error('boom') + }) as typeof fetch + expect((await lookupMirror('123')).kind).toBe('error') + expect((await requestUploadURL('123')).kind).toBe('error') + }) + + it('maps 200 upload responses to ok', async () => { + stubFetch(200, {url: 'https://s3/put'}) + const result = await requestUploadURL('123') + expect(result).toEqual({kind: 'ok', url: 'https://s3/put'}) + }) + }) + + describe('mirror layout', () => { + it('keeps the mirror inside .git', () => { + expect(mirrorPath('/work/repo')).toBe( + path.join('/work/repo', '.git', 'wb-mirror.git') + ) + }) + + it('uses a relative alternates path that never leaves .git', () => { + expect(ALTERNATES_CONTENT).toBe('../wb-mirror.git/objects\n') + expect(path.isAbsolute(ALTERNATES_CONTENT.trim())).toBe(false) + }) + + it('writeAlternates writes the relative path into objects/info', async () => { + const workspace = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'wb-mirror-test-') + ) + try { + await writeAlternates(workspace) + const content = await fs.promises.readFile( + path.join(workspace, '.git', 'objects', 'info', 'alternates'), + 'utf8' + ) + expect(content).toBe(ALTERNATES_CONTENT) + } finally { + await fs.promises.rm(workspace, {recursive: true, force: true}) + } + }) + }) +}) diff --git a/dist/index.js b/dist/index.js index b381bd27..0882dfb3 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41692,6 +41692,347 @@ function ref_helper_select(obj, path) { return ref_helper_select(obj[key], path.substr(i + 1)); } +;// CONCATENATED MODULE: external "stream/promises" +const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises"); +;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts +/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ + +// Thin client for backend-core's /api/v1/git-mirrors endpoints. +// +// Auth is the runner verification token every WarpBuild job carries in its env +// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the +// token alone. +// +// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): +// 200 -> use the presigned URL +// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) +// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror +// creation and upload entirely, behave exactly like stock actions/checkout +// else -> transient trouble: fall back WITHOUT the mirror download +const API_TIMEOUT_MS = 10_000; +function backend_api_baseUrl() { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); +} +function authHeader() { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; +} +async function lookupMirror(repoKey) { + try { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, { + headers: { authorization: authHeader() }, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + return { kind: 'hit', info: (await res.json()) }; + } + if (res.status === 404) { + return { kind: 'miss' }; + } + if (res.status === 403) { + core_debug(`[wb-mirror] download-url answered 403 (disabled)`); + return { kind: 'disabled' }; + } + core_debug(`[wb-mirror] download-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-mirror] download-url failed: ${error}`); + return { kind: 'error' }; + } +} +async function requestUploadURL(repoKey) { + try { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify({ repo_key: repoKey }), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + const body = (await res.json()); + if (body.url) { + return { kind: 'ok', url: body.url }; + } + return { kind: 'error' }; + } + if (res.status === 403) { + core_debug(`[wb-mirror] upload-url answered 403 (disabled)`); + return { kind: 'disabled' }; + } + core_debug(`[wb-mirror] upload-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-mirror] upload-url failed: ${error}`); + return { kind: 'error' }; + } +} + +;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts +/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and + .js-suffixed ESM imports both follow upstream's own conventions; the import plugin + has no TS resolver configured in this repo. */ + + + + + + + + +// WarpBuild git-mirror cache. +// +// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we +// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at +// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and +// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers +// 404), THIS run creates the mirror: one full download of all branches + tags from +// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is +// disabled for this org (backend-driven kill switch) and we skip everything — no +// mirror creation, no upload. +// +// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path +// is relative, so it survives every container mount scheme: `container:` jobs (/__w), +// Docker container actions (/github/workspace), and `docker build COPY .`. +// +// Everything here is fail-open: any error or timeout degrades to stock actions/checkout +// behavior with a warning, never a failed checkout. +const MIRROR_DIR = 'wb-mirror.git'; +const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n`; +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; +const UPLOAD_TIMEOUT_MS = 15 * 60_000; +// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state +// everywhere outside WarpBuild); every other reason is logged at info. +const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; +function mirrorPath(repositoryPath) { + return external_path_namespaceObject.join(repositoryPath, '.git', MIRROR_DIR); +} +// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache +// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and +// deliberately strict: anything unexpected means "behave exactly like upstream". +function getMirrorCacheSkipReason(settings) { + // Only on WarpBuild runners (these are injected into every job's env there). + if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL']) { + return SKIP_NOT_WARPBUILD; + } + // Linux + macOS in v1. + if (process.platform === 'win32') { + return 'Windows is not supported by the mirror cache yet'; + } + // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only + // cache when that is what is being checked out (`repository:` inputs fall back). + const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; + if (!repoKey) { + return 'GITHUB_REPOSITORY_ID is not set'; + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`; + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`; + } + // github.com only (repo ids and mirror keys assume it). + const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, ''); + if (server !== 'https://github.com') { + return `server '${server}' is not github.com`; + } + return null; +} +// setup is the single upstream splice point, called right after `git init` + +// `git remote add` for a fresh repository. It never throws. +async function setup(settings, repositoryUrl) { + const skipReason = getMirrorCacheSkipReason(settings); + if (skipReason) { + if (skipReason === SKIP_NOT_WARPBUILD) { + core_debug(`WarpBuild mirror cache skipped: ${skipReason}`); + } + else { + info(`WarpBuild mirror cache skipped: ${skipReason}`); + } + return; + } + startGroup('WarpBuild: setting up git mirror cache'); + try { + await setupInner(settings, repositoryUrl); + } + catch (error) { + warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + } + finally { + endGroup(); + } +} +async function setupInner(settings, repositoryUrl) { + const repoKey = process.env['GITHUB_REPOSITORY_ID']; + const mirror = mirrorPath(settings.repositoryPath); + // A second checkout of the same repo in one job finds the mirror already in place. + if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(mirror, 'objects'))) { + info('Mirror already present, reusing it'); + await writeAlternates(settings.repositoryPath); + return; + } + const lookup = await lookupMirror(repoKey); + if (lookup.kind === 'disabled') { + info('Mirror cache is disabled by the backend for this organization; using standard checkout'); + return; + } + if (lookup.kind === 'error') { + info('Mirror cache backend unavailable; using standard checkout'); + return; + } + if (lookup.kind === 'hit') { + info(`Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})`); + if (await restoreMirror(lookup.info.url, mirror)) { + await writeAlternates(settings.repositoryPath); + info('Mirror restored; the fetch below downloads only the delta'); + } + // Restore failure: fall through to plain checkout. The object exists, so the + // failure was transfer-shaped — re-downloading would just repeat the pain. + return; + } + // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or + // unreachable backend never costs a wasted full mirror clone. + const probe = await requestUploadURL(repoKey); + if (probe.kind !== 'ok') { + info(probe.kind === 'disabled' + ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' + : 'Mirror cache backend unavailable; skipping mirror creation'); + return; + } + info('Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)'); + await createMirrorFromGitHub(settings, repositoryUrl, mirror); + await writeAlternates(settings.repositoryPath); + // Upload failures only warn: the local mirror still accelerates THIS run. + await uploadMirror(repoKey, mirror); +} +// writeAlternates points the workspace repo's object lookups at the mirror. The path is +// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. +async function writeAlternates(repositoryPath) { + const infoDir = external_path_namespaceObject.join(repositoryPath, '.git', 'objects', 'info'); + await external_fs_namespaceObject.promises.mkdir(infoDir, { recursive: true }); + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(infoDir, 'alternates'), ALTERNATES_CONTENT); + info(`Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}`); +} +async function restoreMirror(url, mirror) { + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-restore-${process.pid}.tar`); + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + throw new Error(`mirror download answered ${res.status}`); + } + await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); + await external_fs_namespaceObject.promises.mkdir(mirror, { recursive: true }); + await exec_exec('tar', ['-xf', tmpTar, '-C', mirror]); + return true; + } + catch (error) { + warning(`Mirror restore failed: ${error}`); + // Never leave a partial mirror behind an alternates file — that is corruption. + await external_fs_namespaceObject.promises.rm(mirror, { recursive: true, force: true }); + return false; + } + finally { + await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + } +} +// createMirrorFromGitHub builds the bare mirror by downloading the repository from +// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. +// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also +// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large +// share of the download on PR-heavy repos). PR-triggered checkouts still work: their +// head SHA simply arrives as a small delta in the workspace fetch. +// +// The full history download itself cannot be safely avoided: a shallow or partial +// mirror behind an alternates file is an incomplete object store that git assumes is +// complete — the exact corruption Blacksmith hit and reverted. A mirror must be +// complete with respect to the refs it advertises. +async function createMirrorFromGitHub(settings, repositoryUrl, mirror) { + // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars + // (git >= 2.31) so the credential never appears in any process's argv. + const basicCredential = Buffer.from(`x-access-token:${settings.authToken}`, 'utf8').toString('base64'); + core_setSecret(basicCredential); + const env = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value; + } + } + env['GIT_CONFIG_COUNT'] = '1'; + env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader'; + env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}`; + await exec_exec('git', ['init', '--bare', '--quiet', mirror], { env }); + await exec_exec('git', ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], { + env + }); + // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. + await exec_exec('git', [ + '-c', + 'gc.auto=0', + '-C', + mirror, + 'fetch', + '--prune', + '--progress', + 'origin', + '+refs/heads/*:refs/heads/*', + '+refs/tags/*:refs/tags/*' + ], { env }); +} +async function uploadMirror(repoKey, mirror) { + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-upload-${process.pid}.tar`); + try { + // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic + // trims; the tar stays a valid bare repo either way. + await exec_exec('tar', [ + '-cf', + tmpTar, + '-C', + mirror, + '--exclude', + './hooks', + '--exclude', + './description', + '--exclude', + './FETCH_HEAD', + '.' + ]); + const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; + // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the + // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. + const fresh = await requestUploadURL(repoKey); + if (fresh.kind !== 'ok') { + throw new Error(fresh.kind === 'disabled' + ? 'mirror cache was disabled by the backend' + : 'upload-url unavailable'); + } + const init = { + method: 'PUT', + headers: { + 'content-length': String(size), + 'content-type': 'application/x-tar' + }, + body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)), + duplex: 'half', + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + }; + const res = await fetch(fresh.url, init); + if (!res.ok) { + throw new Error(`mirror upload answered ${res.status}`); + } + info(`Mirror uploaded (${size} bytes); future runs will restore it`); + } + catch (error) { + warning(`Mirror upload skipped: ${error}`); + } + finally { + await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + } +} + ;// CONCATENATED MODULE: ./src/git-source-provider.ts @@ -41705,6 +42046,7 @@ function ref_helper_select(obj, path) { + async function getSource(settings) { // Repository URL info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`); @@ -41774,6 +42116,9 @@ async function getSource(settings) { await git.init(objectFormat); await git.remoteAdd('origin', repositoryUrl); endGroup(); + // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and + // point alternates at it so the fetch below downloads only the delta from GitHub. + await setup(settings, repositoryUrl); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index b9c1d357..eedeb7f6 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -9,6 +9,7 @@ import * as path from 'path' import * as refHelper from './ref-helper.js' import * as stateHelper from './state-helper.js' import * as urlHelper from './url-helper.js' +import * as warpbuildMirror from './warpbuild/mirror-cache.js' import { MinimumGitSparseCheckoutVersion, IGitCommandManager @@ -130,6 +131,10 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.init(objectFormat) await git.remoteAdd('origin', repositoryUrl) core.endGroup() + + // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and + // point alternates at it so the fetch below downloads only the delta from GitHub. + await warpbuildMirror.setup(settings, repositoryUrl) } // Disable automatic garbage collection diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts new file mode 100644 index 00000000..23eb8a14 --- /dev/null +++ b/src/warpbuild/backend-api.ts @@ -0,0 +1,101 @@ +/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ +import * as core from '@actions/core' + +// Thin client for backend-core's /api/v1/git-mirrors endpoints. +// +// Auth is the runner verification token every WarpBuild job carries in its env +// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the +// token alone. +// +// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): +// 200 -> use the presigned URL +// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) +// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror +// creation and upload entirely, behave exactly like stock actions/checkout +// else -> transient trouble: fall back WITHOUT the mirror download + +const API_TIMEOUT_MS = 10_000 + +export interface MirrorDownloadInfo { + url: string + size_bytes: number + created_at: string +} + +export type MirrorLookup = + | {kind: 'hit'; info: MirrorDownloadInfo} + | {kind: 'miss'} + | {kind: 'disabled'} + | {kind: 'error'} + +export type UploadURLResult = + | {kind: 'ok'; url: string} + | {kind: 'disabled'} + | {kind: 'error'} + +function baseUrl(): string { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '') +} + +function authHeader(): string { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` +} + +export async function lookupMirror(repoKey: string): Promise { + try { + const res = await fetch( + `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, + { + headers: {authorization: authHeader()}, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + } + ) + if (res.status === 200) { + return {kind: 'hit', info: (await res.json()) as MirrorDownloadInfo} + } + if (res.status === 404) { + return {kind: 'miss'} + } + if (res.status === 403) { + core.debug(`[wb-mirror] download-url answered 403 (disabled)`) + return {kind: 'disabled'} + } + core.debug(`[wb-mirror] download-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-mirror] download-url failed: ${error}`) + return {kind: 'error'} + } +} + +export async function requestUploadURL( + repoKey: string +): Promise { + try { + const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify({repo_key: repoKey}), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }) + if (res.status === 200) { + const body = (await res.json()) as {url?: string} + if (body.url) { + return {kind: 'ok', url: body.url} + } + return {kind: 'error'} + } + if (res.status === 403) { + core.debug(`[wb-mirror] upload-url answered 403 (disabled)`) + return {kind: 'disabled'} + } + core.debug(`[wb-mirror] upload-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-mirror] upload-url failed: ${error}`) + return {kind: 'error'} + } +} diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts new file mode 100644 index 00000000..03f5f91b --- /dev/null +++ b/src/warpbuild/mirror-cache.ts @@ -0,0 +1,326 @@ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and + .js-suffixed ESM imports both follow upstream's own conventions; the import plugin + has no TS resolver configured in this repo. */ +import * as core from '@actions/core' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {Readable} from 'stream' +import {pipeline} from 'stream/promises' +import {IGitSourceSettings} from '../git-source-settings.js' +import * as api from './backend-api.js' + +// WarpBuild git-mirror cache. +// +// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we +// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at +// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and +// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers +// 404), THIS run creates the mirror: one full download of all branches + tags from +// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is +// disabled for this org (backend-driven kill switch) and we skip everything — no +// mirror creation, no upload. +// +// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path +// is relative, so it survives every container mount scheme: `container:` jobs (/__w), +// Docker container actions (/github/workspace), and `docker build COPY .`. +// +// Everything here is fail-open: any error or timeout degrades to stock actions/checkout +// behavior with a warning, never a failed checkout. + +const MIRROR_DIR = 'wb-mirror.git' +export const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n` + +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 +const UPLOAD_TIMEOUT_MS = 15 * 60_000 + +// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state +// everywhere outside WarpBuild); every other reason is logged at info. +export const SKIP_NOT_WARPBUILD = + 'not running on a WarpBuild runner (WARPBUILD_* env not present)' + +export function mirrorPath(repositoryPath: string): string { + return path.join(repositoryPath, '.git', MIRROR_DIR) +} + +// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache +// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and +// deliberately strict: anything unexpected means "behave exactly like upstream". +export function getMirrorCacheSkipReason( + settings: IGitSourceSettings +): string | null { + // Only on WarpBuild runners (these are injected into every job's env there). + if ( + !process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL'] + ) { + return SKIP_NOT_WARPBUILD + } + // Linux + macOS in v1. + if (process.platform === 'win32') { + return 'Windows is not supported by the mirror cache yet' + } + // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only + // cache when that is what is being checked out (`repository:` inputs fall back). + const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' + if (!repoKey) { + return 'GITHUB_REPOSITORY_ID is not set' + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}` + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'` + } + // github.com only (repo ids and mirror keys assume it). + const server = (settings.githubServerUrl || 'https://github.com').replace( + /\/+$/, + '' + ) + if (server !== 'https://github.com') { + return `server '${server}' is not github.com` + } + return null +} + +// setup is the single upstream splice point, called right after `git init` + +// `git remote add` for a fresh repository. It never throws. +export async function setup( + settings: IGitSourceSettings, + repositoryUrl: string +): Promise { + const skipReason = getMirrorCacheSkipReason(settings) + if (skipReason) { + if (skipReason === SKIP_NOT_WARPBUILD) { + core.debug(`WarpBuild mirror cache skipped: ${skipReason}`) + } else { + core.info(`WarpBuild mirror cache skipped: ${skipReason}`) + } + return + } + core.startGroup('WarpBuild: setting up git mirror cache') + try { + await setupInner(settings, repositoryUrl) + } catch (error) { + core.warning( + `WarpBuild mirror cache unavailable, using standard checkout: ${error}` + ) + } finally { + core.endGroup() + } +} + +async function setupInner( + settings: IGitSourceSettings, + repositoryUrl: string +): Promise { + const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string + const mirror = mirrorPath(settings.repositoryPath) + + // A second checkout of the same repo in one job finds the mirror already in place. + if (fs.existsSync(path.join(mirror, 'objects'))) { + core.info('Mirror already present, reusing it') + await writeAlternates(settings.repositoryPath) + return + } + + const lookup = await api.lookupMirror(repoKey) + + if (lookup.kind === 'disabled') { + core.info( + 'Mirror cache is disabled by the backend for this organization; using standard checkout' + ) + return + } + + if (lookup.kind === 'error') { + core.info('Mirror cache backend unavailable; using standard checkout') + return + } + + if (lookup.kind === 'hit') { + core.info( + `Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})` + ) + if (await restoreMirror(lookup.info.url, mirror)) { + await writeAlternates(settings.repositoryPath) + core.info('Mirror restored; the fetch below downloads only the delta') + } + return + } + + // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or + // unreachable backend never costs a wasted full mirror clone. + const probe = await api.requestUploadURL(repoKey) + if (probe.kind !== 'ok') { + core.info( + probe.kind === 'disabled' + ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' + : 'Mirror cache backend unavailable; skipping mirror creation' + ) + return + } + + core.info( + 'Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)' + ) + await createMirrorFromGitHub(settings, repositoryUrl, mirror) + await writeAlternates(settings.repositoryPath) + // Upload failures only warn: the local mirror still accelerates THIS run. + await uploadMirror(repoKey, mirror) +} + +// writeAlternates points the workspace repo's object lookups at the mirror. The path is +// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. +export async function writeAlternates(repositoryPath: string): Promise { + const infoDir = path.join(repositoryPath, '.git', 'objects', 'info') + await fs.promises.mkdir(infoDir, {recursive: true}) + await fs.promises.writeFile( + path.join(infoDir, 'alternates'), + ALTERNATES_CONTENT + ) + core.info( + `Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}` + ) +} + +async function restoreMirror(url: string, mirror: string): Promise { + const tmpTar = path.join(os.tmpdir(), `wb-mirror-restore-${process.pid}.tar`) + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }) + if (!res.ok || !res.body) { + throw new Error(`mirror download answered ${res.status}`) + } + await pipeline( + Readable.fromWeb(res.body as import('stream/web').ReadableStream), + fs.createWriteStream(tmpTar) + ) + await fs.promises.mkdir(mirror, {recursive: true}) + await exec.exec('tar', ['-xf', tmpTar, '-C', mirror]) + return true + } catch (error) { + core.warning(`Mirror restore failed: ${error}`) + // Never leave a partial mirror behind an alternates file — that is corruption. + await fs.promises.rm(mirror, {recursive: true, force: true}) + return false + } finally { + await fs.promises.rm(tmpTar, {force: true}) + } +} + +// createMirrorFromGitHub builds the bare mirror by downloading the repository from +// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. +// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also +// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large +// share of the download on PR-heavy repos). PR-triggered checkouts still work: their +// head SHA simply arrives as a small delta in the workspace fetch. +// +// The full history download itself cannot be safely avoided: a shallow or partial +// mirror behind an alternates file is an incomplete object store that git assumes is +// complete — the exact corruption Blacksmith hit and reverted. A mirror must be +// complete with respect to the refs it advertises. +async function createMirrorFromGitHub( + settings: IGitSourceSettings, + repositoryUrl: string, + mirror: string +): Promise { + // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars + // (git >= 2.31) so the credential never appears in any process's argv. + const basicCredential = Buffer.from( + `x-access-token:${settings.authToken}`, + 'utf8' + ).toString('base64') + core.setSecret(basicCredential) + + const env: {[key: string]: string} = {} + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value + } + } + env['GIT_CONFIG_COUNT'] = '1' + env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader' + env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}` + + await exec.exec('git', ['init', '--bare', '--quiet', mirror], {env}) + await exec.exec( + 'git', + ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], + { + env + } + ) + // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. + await exec.exec( + 'git', + [ + '-c', + 'gc.auto=0', + '-C', + mirror, + 'fetch', + '--prune', + '--progress', + 'origin', + '+refs/heads/*:refs/heads/*', + '+refs/tags/*:refs/tags/*' + ], + {env} + ) +} + +async function uploadMirror(repoKey: string, mirror: string): Promise { + const tmpTar = path.join(os.tmpdir(), `wb-mirror-upload-${process.pid}.tar`) + try { + // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic + // trims; the tar stays a valid bare repo either way. + await exec.exec('tar', [ + '-cf', + tmpTar, + '-C', + mirror, + '--exclude', + './hooks', + '--exclude', + './description', + '--exclude', + './FETCH_HEAD', + '.' + ]) + const size = (await fs.promises.stat(tmpTar)).size + + // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the + // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. + const fresh = await api.requestUploadURL(repoKey) + if (fresh.kind !== 'ok') { + throw new Error( + fresh.kind === 'disabled' + ? 'mirror cache was disabled by the backend' + : 'upload-url unavailable' + ) + } + + const init: RequestInit & {duplex: 'half'} = { + method: 'PUT', + headers: { + 'content-length': String(size), + 'content-type': 'application/x-tar' + }, + body: Readable.toWeb( + fs.createReadStream(tmpTar) + ) as unknown as ReadableStream, + duplex: 'half', + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + } + const res = await fetch(fresh.url, init) + if (!res.ok) { + throw new Error(`mirror upload answered ${res.status}`) + } + core.info(`Mirror uploaded (${size} bytes); future runs will restore it`) + } catch (error) { + core.warning(`Mirror upload skipped: ${error}`) + } finally { + await fs.promises.rm(tmpTar, {force: true}) + } +} From 4a9081b71c2365698c849f93032527a032475012 Mon Sep 17 00:00:00 2001 From: darshanime Date: Wed, 8 Jul 2026 13:29:51 +0530 Subject: [PATCH 2/2] move to using sha uploads --- __test__/warpbuild-mirror.test.ts | 182 +++++++-------- dist/index.js | 363 ++++++++++++++--------------- src/git-source-provider.ts | 13 +- src/warpbuild/backend-api.ts | 51 ++-- src/warpbuild/mirror-cache.ts | 371 ++++++++++++++---------------- 5 files changed, 460 insertions(+), 520 deletions(-) diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index 023b257d..1f75b013 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -6,17 +6,12 @@ import { afterEach, afterAll } from '@jest/globals' -import * as fs from 'fs' -import * as os from 'os' -import * as path from 'path' import { - ALTERNATES_CONTENT, SKIP_NOT_WARPBUILD, - mirrorPath, - getMirrorCacheSkipReason, - writeAlternates + computeDestinationRef, + getMirrorCacheSkipReason } from '../src/warpbuild/mirror-cache.js' -import {lookupMirror, requestUploadURL} from '../src/warpbuild/backend-api.js' +import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js' import {IGitSourceSettings} from '../src/git-source-settings.js' const WB_ENV = [ @@ -30,16 +25,23 @@ for (const key of WB_ENV) { savedEnv[key] = process.env[key] } +const SHA = 'cd5255d20e23e050238affc045ba9beee35eaaf7' + function settingsFor( - owner: string, - repo: string, - serverUrl?: string + overrides: Partial = {} ): IGitSourceSettings { return { - repositoryOwner: owner, - repositoryName: repo, + repositoryOwner: 'octocat', + repositoryName: 'hello-world', repositoryPath: '/tmp/does-not-matter', - githubServerUrl: serverUrl + ref: 'refs/heads/main', + commit: SHA, + fetchDepth: 1, + fetchTags: false, + filter: undefined, + sparseCheckout: undefined, + lfs: false, + ...overrides } as unknown as IGitSourceSettings } @@ -50,7 +52,7 @@ function setWarpBuildEnv(): void { process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' } -describe('warpbuild mirror cache', () => { +describe('warpbuild snapshot cache', () => { beforeEach(() => { setWarpBuildEnv() }) @@ -68,130 +70,130 @@ describe('warpbuild mirror cache', () => { describe('getMirrorCacheSkipReason', () => { const winOnly = process.platform === 'win32' ? it.skip : it - winOnly('returns null for the workflow repo on a WarpBuild runner', () => { - expect( - getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) - ).toBeNull() + winOnly('returns null for the default checkout shape', () => { + expect(getMirrorCacheSkipReason(settingsFor())).toBeNull() }) it('reports a non-WarpBuild runner', () => { delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] - expect( - getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) - ).toBe(SKIP_NOT_WARPBUILD) + expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD) }) it('reports repository: inputs that are not the workflow repo', () => { expect( - getMirrorCacheSkipReason(settingsFor('octocat', 'other-repo')) + getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'})) ).toBe( "repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'" ) }) - it('reports a missing GITHUB_REPOSITORY_ID', () => { - delete process.env['GITHUB_REPOSITORY_ID'] - expect( - getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) - ).toBe('GITHUB_REPOSITORY_ID is not set') + it('requires an exact commit sha', () => { + expect(getMirrorCacheSkipReason(settingsFor({commit: ''}))).toBe( + 'no exact commit sha to key on' + ) + expect(getMirrorCacheSkipReason(settingsFor({commit: 'main'}))).toBe( + 'no exact commit sha to key on' + ) }) - winOnly('accepts explicit github.com server urls', () => { - expect( - getMirrorCacheSkipReason( - settingsFor('octocat', 'hello-world', 'https://github.com') - ) - ).toBeNull() + it('only serves fetch-depth 1', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBe( + 'fetch-depth is 0, cache only serves fetch-depth 1' + ) + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBe( + 'fetch-depth is 50, cache only serves fetch-depth 1' + ) }) - it('reports GHES server urls', () => { + it('skips fetch-tags, filters, sparse and lfs checkouts', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchTags: true}))).toBe( + 'fetch-tags is enabled' + ) + expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( + 'a fetch filter is configured' + ) expect( - getMirrorCacheSkipReason( - settingsFor('octocat', 'hello-world', 'https://ghes.example.com') - ) - ).toBe("server 'https://ghes.example.com' is not github.com") + getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) + ).toBe('sparse checkout is configured') + expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe( + 'lfs is enabled (lfs objects are not in the snapshot)' + ) + }) + + winOnly('accepts sha-only checkouts (empty ref)', () => { + expect(getMirrorCacheSkipReason(settingsFor({ref: ''}))).toBeNull() + }) + + it('skips unqualified refs', () => { + expect(getMirrorCacheSkipReason(settingsFor({ref: 'main'}))).toBe( + "ref 'main' has no cacheable destination ref" + ) + }) + }) + + describe('computeDestinationRef', () => { + it('maps the refs the fetch would have created', () => { + expect(computeDestinationRef('refs/heads/main')).toBe( + 'refs/remotes/origin/main' + ) + expect(computeDestinationRef('refs/pull/42/merge')).toBe( + 'refs/remotes/pull/42/merge' + ) + expect(computeDestinationRef('refs/tags/v1.2.3')).toBe('refs/tags/v1.2.3') + expect(computeDestinationRef('')).toBe('') + expect(computeDestinationRef('main')).toBeNull() }) }) describe('backend api http contract', () => { const realFetch = globalThis.fetch + let lastUrl = '' afterEach(() => { globalThis.fetch = realFetch }) function stubFetch(status: number, body: unknown): void { - globalThis.fetch = (async () => - new Response(JSON.stringify(body), {status})) as typeof fetch + globalThis.fetch = (async (input: unknown) => { + lastUrl = String(input) + return new Response(JSON.stringify(body), {status}) + }) as typeof fetch } - it('maps 200 to hit', async () => { + it('maps 200 to hit and keys by sha', async () => { stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) - const result = await lookupMirror('123') + const result = await lookupSnapshot('123', SHA) expect(result.kind).toBe('hit') - if (result.kind === 'hit') { - expect(result.info.url).toBe('https://s3/x') - } + expect(lastUrl).toContain(`sha=${SHA}`) }) - it('maps 404 to miss (hydrate)', async () => { + it('maps 404 to miss (upload after the stock fetch)', async () => { stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) - expect((await lookupMirror('123')).kind).toBe('miss') + expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') }) - it('maps 403 to disabled (backend kill switch — no hydration, no upload)', async () => { + it('maps 403 to disabled (backend kill switch)', async () => { stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) - expect((await lookupMirror('123')).kind).toBe('disabled') - expect((await requestUploadURL('123')).kind).toBe('disabled') + expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') + expect((await requestUploadURL('123', SHA)).kind).toBe('disabled') }) - it('maps other statuses to error (fall back without hydrating)', async () => { + it('maps other statuses and network failures to error', async () => { stubFetch(500, {}) - expect((await lookupMirror('123')).kind).toBe('error') - expect((await requestUploadURL('123')).kind).toBe('error') - }) - - it('maps network failures to error', async () => { + expect((await lookupSnapshot('123', SHA)).kind).toBe('error') globalThis.fetch = (async () => { throw new Error('boom') }) as typeof fetch - expect((await lookupMirror('123')).kind).toBe('error') - expect((await requestUploadURL('123')).kind).toBe('error') + expect((await lookupSnapshot('123', SHA)).kind).toBe('error') + expect((await requestUploadURL('123', SHA)).kind).toBe('error') }) it('maps 200 upload responses to ok', async () => { stubFetch(200, {url: 'https://s3/put'}) - const result = await requestUploadURL('123') - expect(result).toEqual({kind: 'ok', url: 'https://s3/put'}) - }) - }) - - describe('mirror layout', () => { - it('keeps the mirror inside .git', () => { - expect(mirrorPath('/work/repo')).toBe( - path.join('/work/repo', '.git', 'wb-mirror.git') - ) - }) - - it('uses a relative alternates path that never leaves .git', () => { - expect(ALTERNATES_CONTENT).toBe('../wb-mirror.git/objects\n') - expect(path.isAbsolute(ALTERNATES_CONTENT.trim())).toBe(false) - }) - - it('writeAlternates writes the relative path into objects/info', async () => { - const workspace = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'wb-mirror-test-') - ) - try { - await writeAlternates(workspace) - const content = await fs.promises.readFile( - path.join(workspace, '.git', 'objects', 'info', 'alternates'), - 'utf8' - ) - expect(content).toBe(ALTERNATES_CONTENT) - } finally { - await fs.promises.rm(workspace, {recursive: true, force: true}) - } + expect(await requestUploadURL('123', SHA)).toEqual({ + kind: 'ok', + url: 'https://s3/put' + }) }) }) }) diff --git a/dist/index.js b/dist/index.js index 0882dfb3..1cfe740e 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41695,20 +41695,11 @@ function ref_helper_select(obj, path) { ;// CONCATENATED MODULE: external "stream/promises" const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises"); ;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts -/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ +/* eslint-disable i18n-text/no-en -- upstream convention */ -// Thin client for backend-core's /api/v1/git-mirrors endpoints. -// -// Auth is the runner verification token every WarpBuild job carries in its env -// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the -// token alone. -// -// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): -// 200 -> use the presigned URL -// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) -// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror -// creation and upload entirely, behave exactly like stock actions/checkout -// else -> transient trouble: fall back WITHOUT the mirror download +// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner +// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the +// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. const API_TIMEOUT_MS = 10_000; function backend_api_baseUrl() { return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); @@ -41716,9 +41707,9 @@ function backend_api_baseUrl() { function authHeader() { return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; } -async function lookupMirror(repoKey) { +async function lookupSnapshot(repoKey, sha) { try { - const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}&sha=${encodeURIComponent(sha)}`, { headers: { authorization: authHeader() }, signal: AbortSignal.timeout(API_TIMEOUT_MS) }); @@ -41729,18 +41720,18 @@ async function lookupMirror(repoKey) { return { kind: 'miss' }; } if (res.status === 403) { - core_debug(`[wb-mirror] download-url answered 403 (disabled)`); + core_debug(`[wb-cache] download-url answered 403 (disabled)`); return { kind: 'disabled' }; } - core_debug(`[wb-mirror] download-url answered ${res.status}`); + core_debug(`[wb-cache] download-url answered ${res.status}`); return { kind: 'error' }; } catch (error) { - core_debug(`[wb-mirror] download-url failed: ${error}`); + core_debug(`[wb-cache] download-url failed: ${error}`); return { kind: 'error' }; } } -async function requestUploadURL(repoKey) { +async function requestUploadURL(repoKey, sha) { try { const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, { method: 'POST', @@ -41748,7 +41739,7 @@ async function requestUploadURL(repoKey) { authorization: authHeader(), 'content-type': 'application/json' }, - body: JSON.stringify({ repo_key: repoKey }), + body: JSON.stringify({ repo_key: repoKey, sha }), signal: AbortSignal.timeout(API_TIMEOUT_MS) }); if (res.status === 200) { @@ -41759,22 +41750,20 @@ async function requestUploadURL(repoKey) { return { kind: 'error' }; } if (res.status === 403) { - core_debug(`[wb-mirror] upload-url answered 403 (disabled)`); + core_debug(`[wb-cache] upload-url answered 403 (disabled)`); return { kind: 'disabled' }; } - core_debug(`[wb-mirror] upload-url answered ${res.status}`); + core_debug(`[wb-cache] upload-url answered ${res.status}`); return { kind: 'error' }; } catch (error) { - core_debug(`[wb-mirror] upload-url failed: ${error}`); + core_debug(`[wb-cache] upload-url failed: ${error}`); return { kind: 'error' }; } } ;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts -/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and - .js-suffixed ESM imports both follow upstream's own conventions; the import plugin - has no TS resolver configured in this repo. */ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ @@ -41783,48 +41772,27 @@ async function requestUploadURL(repoKey) { -// WarpBuild git-mirror cache. -// -// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we -// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at -// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and -// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers -// 404), THIS run creates the mirror: one full download of all branches + tags from -// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is -// disabled for this org (backend-driven kill switch) and we skip everything — no -// mirror creation, no upload. -// -// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path -// is relative, so it survives every container mount scheme: `container:` jobs (/__w), -// Docker container actions (/github/workspace), and `docker build COPY .`. -// -// Everything here is fail-open: any error or timeout degrades to stock actions/checkout -// behavior with a warning, never a failed checkout. -const MIRROR_DIR = 'wb-mirror.git'; -const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n`; +// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch +// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are +// immutable (no expiry). Fail-open: any error degrades to stock behavior. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; const UPLOAD_TIMEOUT_MS = 15 * 60_000; -// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state -// everywhere outside WarpBuild); every other reason is logged at info. +// "hit " | "uploaded ", for e2e assertions. +const CACHE_STATE_FILE = 'wb-cache-state'; +// Logged at debug (normal state outside WarpBuild); other reasons log at info. const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; -function mirrorPath(repositoryPath) { - return external_path_namespaceObject.join(repositoryPath, '.git', MIRROR_DIR); -} -// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache -// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and -// deliberately strict: anything unexpected means "behave exactly like upstream". +const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; +let decision = 'off'; +// Null = attempt the cache; else a reason to log. Only the default checkout shape +// is served. function getMirrorCacheSkipReason(settings) { - // Only on WarpBuild runners (these are injected into every job's env there). if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_HOST_URL']) { return SKIP_NOT_WARPBUILD; } - // Linux + macOS in v1. if (process.platform === 'win32') { - return 'Windows is not supported by the mirror cache yet'; + return 'Windows is not supported by the snapshot cache yet'; } - // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only - // cache when that is what is being checked out (`repository:` inputs fall back). const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; if (!repoKey) { return 'GITHUB_REPOSITORY_ID is not set'; @@ -41833,181 +41801,183 @@ function getMirrorCacheSkipReason(settings) { if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`; } - // github.com only (repo ids and mirror keys assume it). const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, ''); if (server !== 'https://github.com') { return `server '${server}' is not github.com`; } + if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { + return 'no exact commit sha to key on'; + } + if (settings.fetchDepth !== 1) { + return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`; + } + if (settings.fetchTags) { + return 'fetch-tags is enabled'; + } + if (settings.filter) { + return 'a fetch filter is configured'; + } + if (settings.sparseCheckout) { + return 'sparse checkout is configured'; + } + if (settings.lfs) { + return 'lfs is enabled (lfs objects are not in the snapshot)'; + } + if (settings.ref && computeDestinationRef(settings.ref) === null) { + return `ref '${settings.ref}' has no cacheable destination ref`; + } return null; } -// setup is the single upstream splice point, called right after `git init` + -// `git remote add` for a fresh repository. It never throws. -async function setup(settings, repositoryUrl) { +// The local ref the fetch would have created ('' = none needed, null = uncacheable). +function computeDestinationRef(ref) { + if (!ref) { + return ''; + } + const upper = ref.toUpperCase(); + if (upper.startsWith('REFS/HEADS/')) { + return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`; + } + if (upper.startsWith('REFS/PULL/')) { + return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`; + } + if (upper.startsWith('REFS/TAGS/')) { + return ref; + } + return null; +} +// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. +async function setup(settings) { + decision = 'off'; const skipReason = getMirrorCacheSkipReason(settings); if (skipReason) { if (skipReason === SKIP_NOT_WARPBUILD) { - core_debug(`WarpBuild mirror cache skipped: ${skipReason}`); + core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`); } else { - info(`WarpBuild mirror cache skipped: ${skipReason}`); + info(`WarpBuild snapshot cache skipped: ${skipReason}`); } - return; + return false; } - startGroup('WarpBuild: setting up git mirror cache'); + startGroup('WarpBuild: checkout snapshot cache'); try { - await setupInner(settings, repositoryUrl); + return await setupInner(settings); } catch (error) { - warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + warning(`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`); + return false; } finally { endGroup(); } } -async function setupInner(settings, repositoryUrl) { +async function setupInner(settings) { const repoKey = process.env['GITHUB_REPOSITORY_ID']; - const mirror = mirrorPath(settings.repositoryPath); - // A second checkout of the same repo in one job finds the mirror already in place. - if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(mirror, 'objects'))) { - info('Mirror already present, reusing it'); - await writeAlternates(settings.repositoryPath); - return; - } - const lookup = await lookupMirror(repoKey); + const sha = settings.commit; + const lookup = await lookupSnapshot(repoKey, sha); if (lookup.kind === 'disabled') { - info('Mirror cache is disabled by the backend for this organization; using standard checkout'); - return; + info('Snapshot cache is disabled by the backend for this organization'); + return false; } if (lookup.kind === 'error') { - info('Mirror cache backend unavailable; using standard checkout'); - return; + info('Snapshot cache backend unavailable; using standard checkout'); + return false; } - if (lookup.kind === 'hit') { - info(`Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})`); - if (await restoreMirror(lookup.info.url, mirror)) { - await writeAlternates(settings.repositoryPath); - info('Mirror restored; the fetch below downloads only the delta'); - } - // Restore failure: fall through to plain checkout. The object exists, so the - // failure was transfer-shaped — re-downloading would just repeat the pain. - return; + if (lookup.kind === 'miss') { + info(`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`); + decision = 'miss'; + return false; } - // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or - // unreachable backend never costs a wasted full mirror clone. - const probe = await requestUploadURL(repoKey); - if (probe.kind !== 'ok') { - info(probe.kind === 'disabled' - ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' - : 'Mirror cache backend unavailable; skipping mirror creation'); - return; + info(`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`); + if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { + return false; } - info('Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)'); - await createMirrorFromGitHub(settings, repositoryUrl, mirror); - await writeAlternates(settings.repositoryPath); - // Upload failures only warn: the local mirror still accelerates THIS run. - await uploadMirror(repoKey, mirror); + info('Snapshot restored; skipping the GitHub fetch entirely'); + return true; } -// writeAlternates points the workspace repo's object lookups at the mirror. The path is -// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. -async function writeAlternates(repositoryPath) { - const infoDir = external_path_namespaceObject.join(repositoryPath, '.git', 'objects', 'info'); - await external_fs_namespaceObject.promises.mkdir(infoDir, { recursive: true }); - await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(infoDir, 'alternates'), ALTERNATES_CONTENT); - info(`Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}`); +// Runs after checkout; uploads the fetch result on a miss. Failures only warn. +async function contribute(settings) { + if (decision !== 'miss') { + return; + } + startGroup('WarpBuild: uploading checkout snapshot'); + try { + await uploadSnapshot(settings); + } + catch (error) { + warning(`Snapshot upload skipped: ${error}`); + } + finally { + endGroup(); + } } -async function restoreMirror(url, mirror) { - const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-restore-${process.pid}.tar`); +async function restoreSnapshot(settings, url, sha) { + const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`); try { const res = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) }); if (!res.ok || !res.body) { - throw new Error(`mirror download answered ${res.status}`); + throw new Error(`snapshot download answered ${res.status}`); } await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); - await external_fs_namespaceObject.promises.mkdir(mirror, { recursive: true }); - await exec_exec('tar', ['-xf', tmpTar, '-C', mirror]); + await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); + const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); + if (check !== 0) { + throw new Error(`restored snapshot does not contain ${sha}`); + } + // The ref the skipped fetch would have created; upstream's verification needs it. + const dstRef = computeDestinationRef(settings.ref); + if (dstRef) { + await exec_exec('git', [ + '-C', + settings.repositoryPath, + 'update-ref', + dstRef, + sha + ]); + } + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`); return true; } catch (error) { - warning(`Mirror restore failed: ${error}`); - // Never leave a partial mirror behind an alternates file — that is corruption. - await external_fs_namespaceObject.promises.rm(mirror, { recursive: true, force: true }); + warning(`Snapshot restore failed: ${error}`); + // Reset .git to its freshly-init state. + await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), { + recursive: true, + force: true + }); + await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true }); + await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), { + recursive: true + }); + await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), { + recursive: true + }); return false; } finally { await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); } } -// createMirrorFromGitHub builds the bare mirror by downloading the repository from -// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. -// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also -// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large -// share of the download on PR-heavy repos). PR-triggered checkouts still work: their -// head SHA simply arrives as a small delta in the workspace fetch. -// -// The full history download itself cannot be safely avoided: a shallow or partial -// mirror behind an alternates file is an incomplete object store that git assumes is -// complete — the exact corruption Blacksmith hit and reverted. A mirror must be -// complete with respect to the refs it advertises. -async function createMirrorFromGitHub(settings, repositoryUrl, mirror) { - // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars - // (git >= 2.31) so the credential never appears in any process's argv. - const basicCredential = Buffer.from(`x-access-token:${settings.authToken}`, 'utf8').toString('base64'); - core_setSecret(basicCredential); - const env = {}; - for (const [key, value] of Object.entries(process.env)) { - if (value !== undefined) { - env[key] = value; - } - } - env['GIT_CONFIG_COUNT'] = '1'; - env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader'; - env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}`; - await exec_exec('git', ['init', '--bare', '--quiet', mirror], { env }); - await exec_exec('git', ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], { - env - }); - // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. - await exec_exec('git', [ - '-c', - 'gc.auto=0', - '-C', - mirror, - 'fetch', - '--prune', - '--progress', - 'origin', - '+refs/heads/*:refs/heads/*', - '+refs/tags/*:refs/tags/*' - ], { env }); -} -async function uploadMirror(repoKey, mirror) { - const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-upload-${process.pid}.tar`); +async function uploadSnapshot(settings) { + const repoKey = process.env['GITHUB_REPOSITORY_ID']; + const sha = settings.commit; + const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`); try { - // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic - // trims; the tar stays a valid bare repo either way. - await exec_exec('tar', [ - '-cf', - tmpTar, - '-C', - mirror, - '--exclude', - './hooks', - '--exclude', - './description', - '--exclude', - './FETCH_HEAD', - '.' - ]); + const members = ['objects']; + if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) { + members.push('shallow'); + } + await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]); const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; - // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the - // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. - const fresh = await requestUploadURL(repoKey); - if (fresh.kind !== 'ok') { - throw new Error(fresh.kind === 'disabled' - ? 'mirror cache was disabled by the backend' - : 'upload-url unavailable'); + const upload = await requestUploadURL(repoKey, sha); + if (upload.kind !== 'ok') { + info(upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading'); + return; } const init = { method: 'PUT', @@ -42019,14 +41989,12 @@ async function uploadMirror(repoKey, mirror) { duplex: 'half', signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) }; - const res = await fetch(fresh.url, init); + const res = await fetch(upload.url, init); if (!res.ok) { - throw new Error(`mirror upload answered ${res.status}`); + throw new Error(`snapshot upload answered ${res.status}`); } - info(`Mirror uploaded (${size} bytes); future runs will restore it`); - } - catch (error) { - warning(`Mirror upload skipped: ${error}`); + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`); + info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`); } finally { await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); @@ -42066,6 +42034,7 @@ async function getSource(settings) { const git = await getGitCommandManager(settings); endGroup(); let authHelper = null; + let warpbuildRestored = false; try { if (git) { authHelper = createAuthHelper(git, settings); @@ -42116,9 +42085,8 @@ async function getSource(settings) { await git.init(objectFormat); await git.remoteAdd('origin', repositoryUrl); endGroup(); - // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and - // point alternates at it so the fetch below downloads only the delta from GitHub. - await setup(settings, repositoryUrl); + // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped + warpbuildRestored = await setup(settings); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); @@ -42158,7 +42126,10 @@ async function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } - if (settings.fetchDepth <= 0) { + if (warpbuildRestored) { + info('Skipping fetch: checkout was restored from the snapshot cache'); + } + else if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit); await git.fetch(refSpec, fetchOptions); @@ -42224,6 +42195,8 @@ async function getSource(settings) { startGroup('Checking out the ref'); await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); endGroup(); + // WarpBuild snapshot cache: upload the fetch result on a miss + await contribute(settings); // Submodules if (settings.submodules) { // Temporarily override global config diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index eedeb7f6..3b5adbc4 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -41,6 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.endGroup() let authHelper: gitAuthHelper.IGitAuthHelper | null = null + let warpbuildRestored = false try { if (git) { authHelper = gitAuthHelper.createAuthHelper(git, settings) @@ -132,9 +133,8 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.remoteAdd('origin', repositoryUrl) core.endGroup() - // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and - // point alternates at it so the fetch below downloads only the delta from GitHub. - await warpbuildMirror.setup(settings, repositoryUrl) + // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped + warpbuildRestored = await warpbuildMirror.setup(settings) } // Disable automatic garbage collection @@ -190,7 +190,9 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } - if (settings.fetchDepth <= 0) { + if (warpbuildRestored) { + core.info('Skipping fetch: checkout was restored from the snapshot cache') + } else if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( settings.ref, @@ -276,6 +278,9 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) core.endGroup() + // WarpBuild snapshot cache: upload the fetch result on a miss + await warpbuildMirror.contribute(settings) + // Submodules if (settings.submodules) { // Temporarily override global config diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts index 23eb8a14..d51cd064 100644 --- a/src/warpbuild/backend-api.ts +++ b/src/warpbuild/backend-api.ts @@ -1,29 +1,20 @@ -/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ +/* eslint-disable i18n-text/no-en -- upstream convention */ import * as core from '@actions/core' -// Thin client for backend-core's /api/v1/git-mirrors endpoints. -// -// Auth is the runner verification token every WarpBuild job carries in its env -// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the -// token alone. -// -// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): -// 200 -> use the presigned URL -// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) -// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror -// creation and upload entirely, behave exactly like stock actions/checkout -// else -> transient trouble: fall back WITHOUT the mirror download +// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner +// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the +// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. const API_TIMEOUT_MS = 10_000 -export interface MirrorDownloadInfo { +export interface SnapshotDownloadInfo { url: string size_bytes: number created_at: string } -export type MirrorLookup = - | {kind: 'hit'; info: MirrorDownloadInfo} +export type SnapshotLookup = + | {kind: 'hit'; info: SnapshotDownloadInfo} | {kind: 'miss'} | {kind: 'disabled'} | {kind: 'error'} @@ -41,35 +32,41 @@ function authHeader(): string { return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` } -export async function lookupMirror(repoKey: string): Promise { +export async function lookupSnapshot( + repoKey: string, + sha: string +): Promise { try { const res = await fetch( - `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, + `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent( + repoKey + )}&sha=${encodeURIComponent(sha)}`, { headers: {authorization: authHeader()}, signal: AbortSignal.timeout(API_TIMEOUT_MS) } ) if (res.status === 200) { - return {kind: 'hit', info: (await res.json()) as MirrorDownloadInfo} + return {kind: 'hit', info: (await res.json()) as SnapshotDownloadInfo} } if (res.status === 404) { return {kind: 'miss'} } if (res.status === 403) { - core.debug(`[wb-mirror] download-url answered 403 (disabled)`) + core.debug(`[wb-cache] download-url answered 403 (disabled)`) return {kind: 'disabled'} } - core.debug(`[wb-mirror] download-url answered ${res.status}`) + core.debug(`[wb-cache] download-url answered ${res.status}`) return {kind: 'error'} } catch (error) { - core.debug(`[wb-mirror] download-url failed: ${error}`) + core.debug(`[wb-cache] download-url failed: ${error}`) return {kind: 'error'} } } export async function requestUploadURL( - repoKey: string + repoKey: string, + sha: string ): Promise { try { const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, { @@ -78,7 +75,7 @@ export async function requestUploadURL( authorization: authHeader(), 'content-type': 'application/json' }, - body: JSON.stringify({repo_key: repoKey}), + body: JSON.stringify({repo_key: repoKey, sha}), signal: AbortSignal.timeout(API_TIMEOUT_MS) }) if (res.status === 200) { @@ -89,13 +86,13 @@ export async function requestUploadURL( return {kind: 'error'} } if (res.status === 403) { - core.debug(`[wb-mirror] upload-url answered 403 (disabled)`) + core.debug(`[wb-cache] upload-url answered 403 (disabled)`) return {kind: 'disabled'} } - core.debug(`[wb-mirror] upload-url answered ${res.status}`) + core.debug(`[wb-cache] upload-url answered ${res.status}`) return {kind: 'error'} } catch (error) { - core.debug(`[wb-mirror] upload-url failed: ${error}`) + core.debug(`[wb-cache] upload-url failed: ${error}`) return {kind: 'error'} } } diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 03f5f91b..9e604e1b 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -1,6 +1,4 @@ -/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and - .js-suffixed ESM imports both follow upstream's own conventions; the import plugin - has no TS resolver configured in this repo. */ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ import * as core from '@actions/core' import * as exec from '@actions/exec' import * as fs from 'fs' @@ -11,58 +9,38 @@ import {pipeline} from 'stream/promises' import {IGitSourceSettings} from '../git-source-settings.js' import * as api from './backend-api.js' -// WarpBuild git-mirror cache. -// -// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we -// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at -// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and -// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers -// 404), THIS run creates the mirror: one full download of all branches + tags from -// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is -// disabled for this org (backend-driven kill switch) and we skip everything — no -// mirror creation, no upload. -// -// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path -// is relative, so it survives every container mount scheme: `container:` jobs (/__w), -// Docker container actions (/github/workspace), and `docker build COPY .`. -// -// Everything here is fail-open: any error or timeout degrades to stock actions/checkout -// behavior with a warning, never a failed checkout. - -const MIRROR_DIR = 'wb-mirror.git' -export const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n` +// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch +// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are +// immutable (no expiry). Fail-open: any error degrades to stock behavior. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 const UPLOAD_TIMEOUT_MS = 15 * 60_000 -// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state -// everywhere outside WarpBuild); every other reason is logged at info. +// "hit " | "uploaded ", for e2e assertions. +export const CACHE_STATE_FILE = 'wb-cache-state' + +// Logged at debug (normal state outside WarpBuild); other reasons log at info. export const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)' -export function mirrorPath(repositoryPath: string): string { - return path.join(repositoryPath, '.git', MIRROR_DIR) -} +const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/ -// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache -// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and -// deliberately strict: anything unexpected means "behave exactly like upstream". +let decision: 'off' | 'miss' = 'off' + +// Null = attempt the cache; else a reason to log. Only the default checkout shape +// is served. export function getMirrorCacheSkipReason( settings: IGitSourceSettings ): string | null { - // Only on WarpBuild runners (these are injected into every job's env there). if ( !process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_HOST_URL'] ) { return SKIP_NOT_WARPBUILD } - // Linux + macOS in v1. if (process.platform === 'win32') { - return 'Windows is not supported by the mirror cache yet' + return 'Windows is not supported by the snapshot cache yet' } - // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only - // cache when that is what is being checked out (`repository:` inputs fall back). const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' if (!repoKey) { return 'GITHUB_REPOSITORY_ID is not set' @@ -71,7 +49,6 @@ export function getMirrorCacheSkipReason( if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'` } - // github.com only (repo ids and mirror keys assume it). const server = (settings.githubServerUrl || 'https://github.com').replace( /\/+$/, '' @@ -79,226 +56,208 @@ export function getMirrorCacheSkipReason( if (server !== 'https://github.com') { return `server '${server}' is not github.com` } + if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { + return 'no exact commit sha to key on' + } + if (settings.fetchDepth !== 1) { + return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1` + } + if (settings.fetchTags) { + return 'fetch-tags is enabled' + } + if (settings.filter) { + return 'a fetch filter is configured' + } + if (settings.sparseCheckout) { + return 'sparse checkout is configured' + } + if (settings.lfs) { + return 'lfs is enabled (lfs objects are not in the snapshot)' + } + if (settings.ref && computeDestinationRef(settings.ref) === null) { + return `ref '${settings.ref}' has no cacheable destination ref` + } return null } -// setup is the single upstream splice point, called right after `git init` + -// `git remote add` for a fresh repository. It never throws. -export async function setup( - settings: IGitSourceSettings, - repositoryUrl: string -): Promise { +// The local ref the fetch would have created ('' = none needed, null = uncacheable). +export function computeDestinationRef(ref: string): string | null { + if (!ref) { + return '' + } + const upper = ref.toUpperCase() + if (upper.startsWith('REFS/HEADS/')) { + return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}` + } + if (upper.startsWith('REFS/PULL/')) { + return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}` + } + if (upper.startsWith('REFS/TAGS/')) { + return ref + } + return null +} + +// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. +export async function setup(settings: IGitSourceSettings): Promise { + decision = 'off' const skipReason = getMirrorCacheSkipReason(settings) if (skipReason) { if (skipReason === SKIP_NOT_WARPBUILD) { - core.debug(`WarpBuild mirror cache skipped: ${skipReason}`) + core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`) } else { - core.info(`WarpBuild mirror cache skipped: ${skipReason}`) + core.info(`WarpBuild snapshot cache skipped: ${skipReason}`) } - return + return false } - core.startGroup('WarpBuild: setting up git mirror cache') + core.startGroup('WarpBuild: checkout snapshot cache') try { - await setupInner(settings, repositoryUrl) + return await setupInner(settings) } catch (error) { core.warning( - `WarpBuild mirror cache unavailable, using standard checkout: ${error}` + `WarpBuild snapshot cache unavailable, using standard checkout: ${error}` ) + return false } finally { core.endGroup() } } -async function setupInner( - settings: IGitSourceSettings, - repositoryUrl: string -): Promise { +async function setupInner(settings: IGitSourceSettings): Promise { const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string - const mirror = mirrorPath(settings.repositoryPath) + const sha = settings.commit - // A second checkout of the same repo in one job finds the mirror already in place. - if (fs.existsSync(path.join(mirror, 'objects'))) { - core.info('Mirror already present, reusing it') - await writeAlternates(settings.repositoryPath) - return - } - - const lookup = await api.lookupMirror(repoKey) + const lookup = await api.lookupSnapshot(repoKey, sha) if (lookup.kind === 'disabled') { - core.info( - 'Mirror cache is disabled by the backend for this organization; using standard checkout' - ) - return + core.info('Snapshot cache is disabled by the backend for this organization') + return false } - if (lookup.kind === 'error') { - core.info('Mirror cache backend unavailable; using standard checkout') - return + core.info('Snapshot cache backend unavailable; using standard checkout') + return false } - if (lookup.kind === 'hit') { + if (lookup.kind === 'miss') { core.info( - `Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})` + `Cache miss for ${sha}: the standard fetch will run and its result will be uploaded` ) - if (await restoreMirror(lookup.info.url, mirror)) { - await writeAlternates(settings.repositoryPath) - core.info('Mirror restored; the fetch below downloads only the delta') - } - return - } - - // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or - // unreachable backend never costs a wasted full mirror clone. - const probe = await api.requestUploadURL(repoKey) - if (probe.kind !== 'ok') { - core.info( - probe.kind === 'disabled' - ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' - : 'Mirror cache backend unavailable; skipping mirror creation' - ) - return + decision = 'miss' + return false } core.info( - 'Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)' + `Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)` ) - await createMirrorFromGitHub(settings, repositoryUrl, mirror) - await writeAlternates(settings.repositoryPath) - // Upload failures only warn: the local mirror still accelerates THIS run. - await uploadMirror(repoKey, mirror) + if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { + return false + } + core.info('Snapshot restored; skipping the GitHub fetch entirely') + return true } -// writeAlternates points the workspace repo's object lookups at the mirror. The path is -// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. -export async function writeAlternates(repositoryPath: string): Promise { - const infoDir = path.join(repositoryPath, '.git', 'objects', 'info') - await fs.promises.mkdir(infoDir, {recursive: true}) - await fs.promises.writeFile( - path.join(infoDir, 'alternates'), - ALTERNATES_CONTENT - ) - core.info( - `Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}` - ) +// Runs after checkout; uploads the fetch result on a miss. Failures only warn. +export async function contribute(settings: IGitSourceSettings): Promise { + if (decision !== 'miss') { + return + } + core.startGroup('WarpBuild: uploading checkout snapshot') + try { + await uploadSnapshot(settings) + } catch (error) { + core.warning(`Snapshot upload skipped: ${error}`) + } finally { + core.endGroup() + } } -async function restoreMirror(url: string, mirror: string): Promise { - const tmpTar = path.join(os.tmpdir(), `wb-mirror-restore-${process.pid}.tar`) +async function restoreSnapshot( + settings: IGitSourceSettings, + url: string, + sha: string +): Promise { + const gitDir = path.join(settings.repositoryPath, '.git') + const tmpTar = path.join(os.tmpdir(), `wb-snapshot-${process.pid}.tar`) try { const res = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) }) if (!res.ok || !res.body) { - throw new Error(`mirror download answered ${res.status}`) + throw new Error(`snapshot download answered ${res.status}`) } await pipeline( Readable.fromWeb(res.body as import('stream/web').ReadableStream), fs.createWriteStream(tmpTar) ) - await fs.promises.mkdir(mirror, {recursive: true}) - await exec.exec('tar', ['-xf', tmpTar, '-C', mirror]) + await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir]) + + const check = await exec.exec( + 'git', + ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], + {ignoreReturnCode: true} + ) + if (check !== 0) { + throw new Error(`restored snapshot does not contain ${sha}`) + } + + // The ref the skipped fetch would have created; upstream's verification needs it. + const dstRef = computeDestinationRef(settings.ref) + if (dstRef) { + await exec.exec('git', [ + '-C', + settings.repositoryPath, + 'update-ref', + dstRef, + sha + ]) + } + + await fs.promises.writeFile( + path.join(gitDir, CACHE_STATE_FILE), + `hit ${sha}\n` + ) return true } catch (error) { - core.warning(`Mirror restore failed: ${error}`) - // Never leave a partial mirror behind an alternates file — that is corruption. - await fs.promises.rm(mirror, {recursive: true, force: true}) + core.warning(`Snapshot restore failed: ${error}`) + // Reset .git to its freshly-init state. + await fs.promises.rm(path.join(gitDir, 'objects'), { + recursive: true, + force: true + }) + await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true}) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { + recursive: true + }) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { + recursive: true + }) return false } finally { await fs.promises.rm(tmpTar, {force: true}) } } -// createMirrorFromGitHub builds the bare mirror by downloading the repository from -// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. -// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also -// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large -// share of the download on PR-heavy repos). PR-triggered checkouts still work: their -// head SHA simply arrives as a small delta in the workspace fetch. -// -// The full history download itself cannot be safely avoided: a shallow or partial -// mirror behind an alternates file is an incomplete object store that git assumes is -// complete — the exact corruption Blacksmith hit and reverted. A mirror must be -// complete with respect to the refs it advertises. -async function createMirrorFromGitHub( - settings: IGitSourceSettings, - repositoryUrl: string, - mirror: string -): Promise { - // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars - // (git >= 2.31) so the credential never appears in any process's argv. - const basicCredential = Buffer.from( - `x-access-token:${settings.authToken}`, - 'utf8' - ).toString('base64') - core.setSecret(basicCredential) - - const env: {[key: string]: string} = {} - for (const [key, value] of Object.entries(process.env)) { - if (value !== undefined) { - env[key] = value - } - } - env['GIT_CONFIG_COUNT'] = '1' - env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader' - env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}` - - await exec.exec('git', ['init', '--bare', '--quiet', mirror], {env}) - await exec.exec( - 'git', - ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], - { - env - } - ) - // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. - await exec.exec( - 'git', - [ - '-c', - 'gc.auto=0', - '-C', - mirror, - 'fetch', - '--prune', - '--progress', - 'origin', - '+refs/heads/*:refs/heads/*', - '+refs/tags/*:refs/tags/*' - ], - {env} - ) -} - -async function uploadMirror(repoKey: string, mirror: string): Promise { - const tmpTar = path.join(os.tmpdir(), `wb-mirror-upload-${process.pid}.tar`) +async function uploadSnapshot(settings: IGitSourceSettings): Promise { + const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string + const sha = settings.commit + const gitDir = path.join(settings.repositoryPath, '.git') + const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`) try { - // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic - // trims; the tar stays a valid bare repo either way. - await exec.exec('tar', [ - '-cf', - tmpTar, - '-C', - mirror, - '--exclude', - './hooks', - '--exclude', - './description', - '--exclude', - './FETCH_HEAD', - '.' - ]) + const members = ['objects'] + if (fs.existsSync(path.join(gitDir, 'shallow'))) { + members.push('shallow') + } + await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]) const size = (await fs.promises.stat(tmpTar)).size - // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the - // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. - const fresh = await api.requestUploadURL(repoKey) - if (fresh.kind !== 'ok') { - throw new Error( - fresh.kind === 'disabled' - ? 'mirror cache was disabled by the backend' - : 'upload-url unavailable' + const upload = await api.requestUploadURL(repoKey, sha) + if (upload.kind !== 'ok') { + core.info( + upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading' ) + return } const init: RequestInit & {duplex: 'half'} = { @@ -313,13 +272,17 @@ async function uploadMirror(repoKey: string, mirror: string): Promise { duplex: 'half', signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) } - const res = await fetch(fresh.url, init) + const res = await fetch(upload.url, init) if (!res.ok) { - throw new Error(`mirror upload answered ${res.status}`) + throw new Error(`snapshot upload answered ${res.status}`) } - core.info(`Mirror uploaded (${size} bytes); future runs will restore it`) - } catch (error) { - core.warning(`Mirror upload skipped: ${error}`) + await fs.promises.writeFile( + path.join(gitDir, CACHE_STATE_FILE), + `uploaded ${sha}\n` + ) + core.info( + `Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch` + ) } finally { await fs.promises.rm(tmpTar, {force: true}) }