From 713b9094287731e8b4e51b5cbe2ecef7af51cb5d Mon Sep 17 00:00:00 2001 From: darshanime Date: Wed, 8 Jul 2026 09:40:20 +0530 Subject: [PATCH] init warpbuild caching --- README.md | 15 ++ __test__/warpbuild-mirror.test.ts | 197 +++++++++++++++++ dist/index.js | 345 ++++++++++++++++++++++++++++++ src/git-source-provider.ts | 5 + src/warpbuild/backend-api.ts | 101 +++++++++ src/warpbuild/mirror-cache.ts | 326 ++++++++++++++++++++++++++++ 6 files changed, 989 insertions(+) create mode 100644 __test__/warpbuild-mirror.test.ts create mode 100644 src/warpbuild/backend-api.ts create mode 100644 src/warpbuild/mirror-cache.ts diff --git a/README.md b/README.md index 5509e7d1..e2e3c27e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,18 @@ +# WarpBuild Checkout + +This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in +replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's +bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git +alternates, so the fetch from GitHub downloads only the delta instead of the whole +repository. On a cache miss, the run hydrates the mirror once (full clone + upload); +mirrors expire on a server-configured TTL and re-hydrate. + +- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners. +- Fail-open — any cache error degrades to stock `actions/checkout` behavior. +- All fork code lives in `src/warpbuild/` + +--- + [![Build and Test](https://github.com/actions/checkout/actions/workflows/test.yml/badge.svg)](https://github.com/actions/checkout/actions/workflows/test.yml) # Checkout v7 diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts new file mode 100644 index 00000000..023b257d --- /dev/null +++ b/__test__/warpbuild-mirror.test.ts @@ -0,0 +1,197 @@ +import { + describe, + it, + expect, + beforeEach, + afterEach, + afterAll +} from '@jest/globals' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { + ALTERNATES_CONTENT, + SKIP_NOT_WARPBUILD, + mirrorPath, + getMirrorCacheSkipReason, + writeAlternates +} from '../src/warpbuild/mirror-cache.js' +import {lookupMirror, requestUploadURL} from '../src/warpbuild/backend-api.js' +import {IGitSourceSettings} from '../src/git-source-settings.js' + +const WB_ENV = [ + 'WARPBUILD_RUNNER_VERIFICATION_TOKEN', + 'WARPBUILD_HOST_URL', + 'GITHUB_REPOSITORY_ID', + 'GITHUB_REPOSITORY' +] +const savedEnv: {[key: string]: string | undefined} = {} +for (const key of WB_ENV) { + savedEnv[key] = process.env[key] +} + +function settingsFor( + owner: string, + repo: string, + serverUrl?: string +): IGitSourceSettings { + return { + repositoryOwner: owner, + repositoryName: repo, + repositoryPath: '/tmp/does-not-matter', + githubServerUrl: serverUrl + } as unknown as IGitSourceSettings +} + +function setWarpBuildEnv(): void { + process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = 'test-token' + process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev' + process.env['GITHUB_REPOSITORY_ID'] = '123456789' + process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' +} + +describe('warpbuild mirror cache', () => { + beforeEach(() => { + setWarpBuildEnv() + }) + + afterAll(() => { + for (const key of WB_ENV) { + if (savedEnv[key] === undefined) { + delete process.env[key] + } else { + process.env[key] = savedEnv[key] + } + } + }) + + describe('getMirrorCacheSkipReason', () => { + const winOnly = process.platform === 'win32' ? it.skip : it + + winOnly('returns null for the workflow repo on a WarpBuild runner', () => { + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBeNull() + }) + + it('reports a non-WarpBuild runner', () => { + delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBe(SKIP_NOT_WARPBUILD) + }) + + it('reports repository: inputs that are not the workflow repo', () => { + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'other-repo')) + ).toBe( + "repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'" + ) + }) + + it('reports a missing GITHUB_REPOSITORY_ID', () => { + delete process.env['GITHUB_REPOSITORY_ID'] + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBe('GITHUB_REPOSITORY_ID is not set') + }) + + winOnly('accepts explicit github.com server urls', () => { + expect( + getMirrorCacheSkipReason( + settingsFor('octocat', 'hello-world', 'https://github.com') + ) + ).toBeNull() + }) + + it('reports GHES server urls', () => { + expect( + getMirrorCacheSkipReason( + settingsFor('octocat', 'hello-world', 'https://ghes.example.com') + ) + ).toBe("server 'https://ghes.example.com' is not github.com") + }) + }) + + describe('backend api http contract', () => { + const realFetch = globalThis.fetch + + afterEach(() => { + globalThis.fetch = realFetch + }) + + function stubFetch(status: number, body: unknown): void { + globalThis.fetch = (async () => + new Response(JSON.stringify(body), {status})) as typeof fetch + } + + it('maps 200 to hit', async () => { + stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) + const result = await lookupMirror('123') + expect(result.kind).toBe('hit') + if (result.kind === 'hit') { + expect(result.info.url).toBe('https://s3/x') + } + }) + + it('maps 404 to miss (hydrate)', async () => { + stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) + expect((await lookupMirror('123')).kind).toBe('miss') + }) + + it('maps 403 to disabled (backend kill switch — no hydration, no upload)', async () => { + stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) + expect((await lookupMirror('123')).kind).toBe('disabled') + expect((await requestUploadURL('123')).kind).toBe('disabled') + }) + + it('maps other statuses to error (fall back without hydrating)', async () => { + stubFetch(500, {}) + expect((await lookupMirror('123')).kind).toBe('error') + expect((await requestUploadURL('123')).kind).toBe('error') + }) + + it('maps network failures to error', async () => { + globalThis.fetch = (async () => { + throw new Error('boom') + }) as typeof fetch + expect((await lookupMirror('123')).kind).toBe('error') + expect((await requestUploadURL('123')).kind).toBe('error') + }) + + it('maps 200 upload responses to ok', async () => { + stubFetch(200, {url: 'https://s3/put'}) + const result = await requestUploadURL('123') + expect(result).toEqual({kind: 'ok', url: 'https://s3/put'}) + }) + }) + + describe('mirror layout', () => { + it('keeps the mirror inside .git', () => { + expect(mirrorPath('/work/repo')).toBe( + path.join('/work/repo', '.git', 'wb-mirror.git') + ) + }) + + it('uses a relative alternates path that never leaves .git', () => { + expect(ALTERNATES_CONTENT).toBe('../wb-mirror.git/objects\n') + expect(path.isAbsolute(ALTERNATES_CONTENT.trim())).toBe(false) + }) + + it('writeAlternates writes the relative path into objects/info', async () => { + const workspace = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'wb-mirror-test-') + ) + try { + await writeAlternates(workspace) + const content = await fs.promises.readFile( + path.join(workspace, '.git', 'objects', 'info', 'alternates'), + 'utf8' + ) + expect(content).toBe(ALTERNATES_CONTENT) + } finally { + await fs.promises.rm(workspace, {recursive: true, force: true}) + } + }) + }) +}) diff --git a/dist/index.js b/dist/index.js index b381bd27..0882dfb3 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41692,6 +41692,347 @@ function ref_helper_select(obj, path) { return ref_helper_select(obj[key], path.substr(i + 1)); } +;// CONCATENATED MODULE: external "stream/promises" +const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises"); +;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts +/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ + +// Thin client for backend-core's /api/v1/git-mirrors endpoints. +// +// Auth is the runner verification token every WarpBuild job carries in its env +// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the +// token alone. +// +// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): +// 200 -> use the presigned URL +// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) +// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror +// creation and upload entirely, behave exactly like stock actions/checkout +// else -> transient trouble: fall back WITHOUT the mirror download +const API_TIMEOUT_MS = 10_000; +function backend_api_baseUrl() { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); +} +function authHeader() { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; +} +async function lookupMirror(repoKey) { + try { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, { + headers: { authorization: authHeader() }, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + return { kind: 'hit', info: (await res.json()) }; + } + if (res.status === 404) { + return { kind: 'miss' }; + } + if (res.status === 403) { + core_debug(`[wb-mirror] download-url answered 403 (disabled)`); + return { kind: 'disabled' }; + } + core_debug(`[wb-mirror] download-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-mirror] download-url failed: ${error}`); + return { kind: 'error' }; + } +} +async function requestUploadURL(repoKey) { + try { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify({ repo_key: repoKey }), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + const body = (await res.json()); + if (body.url) { + return { kind: 'ok', url: body.url }; + } + return { kind: 'error' }; + } + if (res.status === 403) { + core_debug(`[wb-mirror] upload-url answered 403 (disabled)`); + return { kind: 'disabled' }; + } + core_debug(`[wb-mirror] upload-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-mirror] upload-url failed: ${error}`); + return { kind: 'error' }; + } +} + +;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts +/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and + .js-suffixed ESM imports both follow upstream's own conventions; the import plugin + has no TS resolver configured in this repo. */ + + + + + + + + +// WarpBuild git-mirror cache. +// +// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we +// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at +// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and +// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers +// 404), THIS run creates the mirror: one full download of all branches + tags from +// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is +// disabled for this org (backend-driven kill switch) and we skip everything — no +// mirror creation, no upload. +// +// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path +// is relative, so it survives every container mount scheme: `container:` jobs (/__w), +// Docker container actions (/github/workspace), and `docker build COPY .`. +// +// Everything here is fail-open: any error or timeout degrades to stock actions/checkout +// behavior with a warning, never a failed checkout. +const MIRROR_DIR = 'wb-mirror.git'; +const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n`; +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; +const UPLOAD_TIMEOUT_MS = 15 * 60_000; +// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state +// everywhere outside WarpBuild); every other reason is logged at info. +const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; +function mirrorPath(repositoryPath) { + return external_path_namespaceObject.join(repositoryPath, '.git', MIRROR_DIR); +} +// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache +// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and +// deliberately strict: anything unexpected means "behave exactly like upstream". +function getMirrorCacheSkipReason(settings) { + // Only on WarpBuild runners (these are injected into every job's env there). + if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL']) { + return SKIP_NOT_WARPBUILD; + } + // Linux + macOS in v1. + if (process.platform === 'win32') { + return 'Windows is not supported by the mirror cache yet'; + } + // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only + // cache when that is what is being checked out (`repository:` inputs fall back). + const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; + if (!repoKey) { + return 'GITHUB_REPOSITORY_ID is not set'; + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`; + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`; + } + // github.com only (repo ids and mirror keys assume it). + const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, ''); + if (server !== 'https://github.com') { + return `server '${server}' is not github.com`; + } + return null; +} +// setup is the single upstream splice point, called right after `git init` + +// `git remote add` for a fresh repository. It never throws. +async function setup(settings, repositoryUrl) { + const skipReason = getMirrorCacheSkipReason(settings); + if (skipReason) { + if (skipReason === SKIP_NOT_WARPBUILD) { + core_debug(`WarpBuild mirror cache skipped: ${skipReason}`); + } + else { + info(`WarpBuild mirror cache skipped: ${skipReason}`); + } + return; + } + startGroup('WarpBuild: setting up git mirror cache'); + try { + await setupInner(settings, repositoryUrl); + } + catch (error) { + warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + } + finally { + endGroup(); + } +} +async function setupInner(settings, repositoryUrl) { + const repoKey = process.env['GITHUB_REPOSITORY_ID']; + const mirror = mirrorPath(settings.repositoryPath); + // A second checkout of the same repo in one job finds the mirror already in place. + if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(mirror, 'objects'))) { + info('Mirror already present, reusing it'); + await writeAlternates(settings.repositoryPath); + return; + } + const lookup = await lookupMirror(repoKey); + if (lookup.kind === 'disabled') { + info('Mirror cache is disabled by the backend for this organization; using standard checkout'); + return; + } + if (lookup.kind === 'error') { + info('Mirror cache backend unavailable; using standard checkout'); + return; + } + if (lookup.kind === 'hit') { + info(`Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})`); + if (await restoreMirror(lookup.info.url, mirror)) { + await writeAlternates(settings.repositoryPath); + info('Mirror restored; the fetch below downloads only the delta'); + } + // Restore failure: fall through to plain checkout. The object exists, so the + // failure was transfer-shaped — re-downloading would just repeat the pain. + return; + } + // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or + // unreachable backend never costs a wasted full mirror clone. + const probe = await requestUploadURL(repoKey); + if (probe.kind !== 'ok') { + info(probe.kind === 'disabled' + ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' + : 'Mirror cache backend unavailable; skipping mirror creation'); + return; + } + info('Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)'); + await createMirrorFromGitHub(settings, repositoryUrl, mirror); + await writeAlternates(settings.repositoryPath); + // Upload failures only warn: the local mirror still accelerates THIS run. + await uploadMirror(repoKey, mirror); +} +// writeAlternates points the workspace repo's object lookups at the mirror. The path is +// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. +async function writeAlternates(repositoryPath) { + const infoDir = external_path_namespaceObject.join(repositoryPath, '.git', 'objects', 'info'); + await external_fs_namespaceObject.promises.mkdir(infoDir, { recursive: true }); + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(infoDir, 'alternates'), ALTERNATES_CONTENT); + info(`Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}`); +} +async function restoreMirror(url, mirror) { + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-restore-${process.pid}.tar`); + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + throw new Error(`mirror download answered ${res.status}`); + } + await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); + await external_fs_namespaceObject.promises.mkdir(mirror, { recursive: true }); + await exec_exec('tar', ['-xf', tmpTar, '-C', mirror]); + return true; + } + catch (error) { + warning(`Mirror restore failed: ${error}`); + // Never leave a partial mirror behind an alternates file — that is corruption. + await external_fs_namespaceObject.promises.rm(mirror, { recursive: true, force: true }); + return false; + } + finally { + await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + } +} +// createMirrorFromGitHub builds the bare mirror by downloading the repository from +// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. +// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also +// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large +// share of the download on PR-heavy repos). PR-triggered checkouts still work: their +// head SHA simply arrives as a small delta in the workspace fetch. +// +// The full history download itself cannot be safely avoided: a shallow or partial +// mirror behind an alternates file is an incomplete object store that git assumes is +// complete — the exact corruption Blacksmith hit and reverted. A mirror must be +// complete with respect to the refs it advertises. +async function createMirrorFromGitHub(settings, repositoryUrl, mirror) { + // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars + // (git >= 2.31) so the credential never appears in any process's argv. + const basicCredential = Buffer.from(`x-access-token:${settings.authToken}`, 'utf8').toString('base64'); + core_setSecret(basicCredential); + const env = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value; + } + } + env['GIT_CONFIG_COUNT'] = '1'; + env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader'; + env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}`; + await exec_exec('git', ['init', '--bare', '--quiet', mirror], { env }); + await exec_exec('git', ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], { + env + }); + // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. + await exec_exec('git', [ + '-c', + 'gc.auto=0', + '-C', + mirror, + 'fetch', + '--prune', + '--progress', + 'origin', + '+refs/heads/*:refs/heads/*', + '+refs/tags/*:refs/tags/*' + ], { env }); +} +async function uploadMirror(repoKey, mirror) { + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-upload-${process.pid}.tar`); + try { + // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic + // trims; the tar stays a valid bare repo either way. + await exec_exec('tar', [ + '-cf', + tmpTar, + '-C', + mirror, + '--exclude', + './hooks', + '--exclude', + './description', + '--exclude', + './FETCH_HEAD', + '.' + ]); + const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; + // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the + // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. + const fresh = await requestUploadURL(repoKey); + if (fresh.kind !== 'ok') { + throw new Error(fresh.kind === 'disabled' + ? 'mirror cache was disabled by the backend' + : 'upload-url unavailable'); + } + const init = { + method: 'PUT', + headers: { + 'content-length': String(size), + 'content-type': 'application/x-tar' + }, + body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)), + duplex: 'half', + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + }; + const res = await fetch(fresh.url, init); + if (!res.ok) { + throw new Error(`mirror upload answered ${res.status}`); + } + info(`Mirror uploaded (${size} bytes); future runs will restore it`); + } + catch (error) { + warning(`Mirror upload skipped: ${error}`); + } + finally { + await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + } +} + ;// CONCATENATED MODULE: ./src/git-source-provider.ts @@ -41705,6 +42046,7 @@ function ref_helper_select(obj, path) { + async function getSource(settings) { // Repository URL info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`); @@ -41774,6 +42116,9 @@ async function getSource(settings) { await git.init(objectFormat); await git.remoteAdd('origin', repositoryUrl); endGroup(); + // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and + // point alternates at it so the fetch below downloads only the delta from GitHub. + await setup(settings, repositoryUrl); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index b9c1d357..eedeb7f6 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -9,6 +9,7 @@ import * as path from 'path' import * as refHelper from './ref-helper.js' import * as stateHelper from './state-helper.js' import * as urlHelper from './url-helper.js' +import * as warpbuildMirror from './warpbuild/mirror-cache.js' import { MinimumGitSparseCheckoutVersion, IGitCommandManager @@ -130,6 +131,10 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.init(objectFormat) await git.remoteAdd('origin', repositoryUrl) core.endGroup() + + // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and + // point alternates at it so the fetch below downloads only the delta from GitHub. + await warpbuildMirror.setup(settings, repositoryUrl) } // Disable automatic garbage collection diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts new file mode 100644 index 00000000..23eb8a14 --- /dev/null +++ b/src/warpbuild/backend-api.ts @@ -0,0 +1,101 @@ +/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ +import * as core from '@actions/core' + +// Thin client for backend-core's /api/v1/git-mirrors endpoints. +// +// Auth is the runner verification token every WarpBuild job carries in its env +// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the +// token alone. +// +// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): +// 200 -> use the presigned URL +// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) +// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror +// creation and upload entirely, behave exactly like stock actions/checkout +// else -> transient trouble: fall back WITHOUT the mirror download + +const API_TIMEOUT_MS = 10_000 + +export interface MirrorDownloadInfo { + url: string + size_bytes: number + created_at: string +} + +export type MirrorLookup = + | {kind: 'hit'; info: MirrorDownloadInfo} + | {kind: 'miss'} + | {kind: 'disabled'} + | {kind: 'error'} + +export type UploadURLResult = + | {kind: 'ok'; url: string} + | {kind: 'disabled'} + | {kind: 'error'} + +function baseUrl(): string { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '') +} + +function authHeader(): string { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` +} + +export async function lookupMirror(repoKey: string): Promise { + try { + const res = await fetch( + `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, + { + headers: {authorization: authHeader()}, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + } + ) + if (res.status === 200) { + return {kind: 'hit', info: (await res.json()) as MirrorDownloadInfo} + } + if (res.status === 404) { + return {kind: 'miss'} + } + if (res.status === 403) { + core.debug(`[wb-mirror] download-url answered 403 (disabled)`) + return {kind: 'disabled'} + } + core.debug(`[wb-mirror] download-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-mirror] download-url failed: ${error}`) + return {kind: 'error'} + } +} + +export async function requestUploadURL( + repoKey: string +): Promise { + try { + const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify({repo_key: repoKey}), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }) + if (res.status === 200) { + const body = (await res.json()) as {url?: string} + if (body.url) { + return {kind: 'ok', url: body.url} + } + return {kind: 'error'} + } + if (res.status === 403) { + core.debug(`[wb-mirror] upload-url answered 403 (disabled)`) + return {kind: 'disabled'} + } + core.debug(`[wb-mirror] upload-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-mirror] upload-url failed: ${error}`) + return {kind: 'error'} + } +} diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts new file mode 100644 index 00000000..03f5f91b --- /dev/null +++ b/src/warpbuild/mirror-cache.ts @@ -0,0 +1,326 @@ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and + .js-suffixed ESM imports both follow upstream's own conventions; the import plugin + has no TS resolver configured in this repo. */ +import * as core from '@actions/core' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {Readable} from 'stream' +import {pipeline} from 'stream/promises' +import {IGitSourceSettings} from '../git-source-settings.js' +import * as api from './backend-api.js' + +// WarpBuild git-mirror cache. +// +// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we +// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at +// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and +// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers +// 404), THIS run creates the mirror: one full download of all branches + tags from +// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is +// disabled for this org (backend-driven kill switch) and we skip everything — no +// mirror creation, no upload. +// +// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path +// is relative, so it survives every container mount scheme: `container:` jobs (/__w), +// Docker container actions (/github/workspace), and `docker build COPY .`. +// +// Everything here is fail-open: any error or timeout degrades to stock actions/checkout +// behavior with a warning, never a failed checkout. + +const MIRROR_DIR = 'wb-mirror.git' +export const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n` + +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 +const UPLOAD_TIMEOUT_MS = 15 * 60_000 + +// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state +// everywhere outside WarpBuild); every other reason is logged at info. +export const SKIP_NOT_WARPBUILD = + 'not running on a WarpBuild runner (WARPBUILD_* env not present)' + +export function mirrorPath(repositoryPath: string): string { + return path.join(repositoryPath, '.git', MIRROR_DIR) +} + +// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache +// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and +// deliberately strict: anything unexpected means "behave exactly like upstream". +export function getMirrorCacheSkipReason( + settings: IGitSourceSettings +): string | null { + // Only on WarpBuild runners (these are injected into every job's env there). + if ( + !process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL'] + ) { + return SKIP_NOT_WARPBUILD + } + // Linux + macOS in v1. + if (process.platform === 'win32') { + return 'Windows is not supported by the mirror cache yet' + } + // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only + // cache when that is what is being checked out (`repository:` inputs fall back). + const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' + if (!repoKey) { + return 'GITHUB_REPOSITORY_ID is not set' + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}` + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'` + } + // github.com only (repo ids and mirror keys assume it). + const server = (settings.githubServerUrl || 'https://github.com').replace( + /\/+$/, + '' + ) + if (server !== 'https://github.com') { + return `server '${server}' is not github.com` + } + return null +} + +// setup is the single upstream splice point, called right after `git init` + +// `git remote add` for a fresh repository. It never throws. +export async function setup( + settings: IGitSourceSettings, + repositoryUrl: string +): Promise { + const skipReason = getMirrorCacheSkipReason(settings) + if (skipReason) { + if (skipReason === SKIP_NOT_WARPBUILD) { + core.debug(`WarpBuild mirror cache skipped: ${skipReason}`) + } else { + core.info(`WarpBuild mirror cache skipped: ${skipReason}`) + } + return + } + core.startGroup('WarpBuild: setting up git mirror cache') + try { + await setupInner(settings, repositoryUrl) + } catch (error) { + core.warning( + `WarpBuild mirror cache unavailable, using standard checkout: ${error}` + ) + } finally { + core.endGroup() + } +} + +async function setupInner( + settings: IGitSourceSettings, + repositoryUrl: string +): Promise { + const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string + const mirror = mirrorPath(settings.repositoryPath) + + // A second checkout of the same repo in one job finds the mirror already in place. + if (fs.existsSync(path.join(mirror, 'objects'))) { + core.info('Mirror already present, reusing it') + await writeAlternates(settings.repositoryPath) + return + } + + const lookup = await api.lookupMirror(repoKey) + + if (lookup.kind === 'disabled') { + core.info( + 'Mirror cache is disabled by the backend for this organization; using standard checkout' + ) + return + } + + if (lookup.kind === 'error') { + core.info('Mirror cache backend unavailable; using standard checkout') + return + } + + if (lookup.kind === 'hit') { + core.info( + `Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})` + ) + if (await restoreMirror(lookup.info.url, mirror)) { + await writeAlternates(settings.repositoryPath) + core.info('Mirror restored; the fetch below downloads only the delta') + } + return + } + + // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or + // unreachable backend never costs a wasted full mirror clone. + const probe = await api.requestUploadURL(repoKey) + if (probe.kind !== 'ok') { + core.info( + probe.kind === 'disabled' + ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' + : 'Mirror cache backend unavailable; skipping mirror creation' + ) + return + } + + core.info( + 'Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)' + ) + await createMirrorFromGitHub(settings, repositoryUrl, mirror) + await writeAlternates(settings.repositoryPath) + // Upload failures only warn: the local mirror still accelerates THIS run. + await uploadMirror(repoKey, mirror) +} + +// writeAlternates points the workspace repo's object lookups at the mirror. The path is +// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. +export async function writeAlternates(repositoryPath: string): Promise { + const infoDir = path.join(repositoryPath, '.git', 'objects', 'info') + await fs.promises.mkdir(infoDir, {recursive: true}) + await fs.promises.writeFile( + path.join(infoDir, 'alternates'), + ALTERNATES_CONTENT + ) + core.info( + `Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}` + ) +} + +async function restoreMirror(url: string, mirror: string): Promise { + const tmpTar = path.join(os.tmpdir(), `wb-mirror-restore-${process.pid}.tar`) + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }) + if (!res.ok || !res.body) { + throw new Error(`mirror download answered ${res.status}`) + } + await pipeline( + Readable.fromWeb(res.body as import('stream/web').ReadableStream), + fs.createWriteStream(tmpTar) + ) + await fs.promises.mkdir(mirror, {recursive: true}) + await exec.exec('tar', ['-xf', tmpTar, '-C', mirror]) + return true + } catch (error) { + core.warning(`Mirror restore failed: ${error}`) + // Never leave a partial mirror behind an alternates file — that is corruption. + await fs.promises.rm(mirror, {recursive: true, force: true}) + return false + } finally { + await fs.promises.rm(tmpTar, {force: true}) + } +} + +// createMirrorFromGitHub builds the bare mirror by downloading the repository from +// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. +// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also +// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large +// share of the download on PR-heavy repos). PR-triggered checkouts still work: their +// head SHA simply arrives as a small delta in the workspace fetch. +// +// The full history download itself cannot be safely avoided: a shallow or partial +// mirror behind an alternates file is an incomplete object store that git assumes is +// complete — the exact corruption Blacksmith hit and reverted. A mirror must be +// complete with respect to the refs it advertises. +async function createMirrorFromGitHub( + settings: IGitSourceSettings, + repositoryUrl: string, + mirror: string +): Promise { + // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars + // (git >= 2.31) so the credential never appears in any process's argv. + const basicCredential = Buffer.from( + `x-access-token:${settings.authToken}`, + 'utf8' + ).toString('base64') + core.setSecret(basicCredential) + + const env: {[key: string]: string} = {} + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value + } + } + env['GIT_CONFIG_COUNT'] = '1' + env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader' + env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}` + + await exec.exec('git', ['init', '--bare', '--quiet', mirror], {env}) + await exec.exec( + 'git', + ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], + { + env + } + ) + // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. + await exec.exec( + 'git', + [ + '-c', + 'gc.auto=0', + '-C', + mirror, + 'fetch', + '--prune', + '--progress', + 'origin', + '+refs/heads/*:refs/heads/*', + '+refs/tags/*:refs/tags/*' + ], + {env} + ) +} + +async function uploadMirror(repoKey: string, mirror: string): Promise { + const tmpTar = path.join(os.tmpdir(), `wb-mirror-upload-${process.pid}.tar`) + try { + // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic + // trims; the tar stays a valid bare repo either way. + await exec.exec('tar', [ + '-cf', + tmpTar, + '-C', + mirror, + '--exclude', + './hooks', + '--exclude', + './description', + '--exclude', + './FETCH_HEAD', + '.' + ]) + const size = (await fs.promises.stat(tmpTar)).size + + // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the + // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. + const fresh = await api.requestUploadURL(repoKey) + if (fresh.kind !== 'ok') { + throw new Error( + fresh.kind === 'disabled' + ? 'mirror cache was disabled by the backend' + : 'upload-url unavailable' + ) + } + + const init: RequestInit & {duplex: 'half'} = { + method: 'PUT', + headers: { + 'content-length': String(size), + 'content-type': 'application/x-tar' + }, + body: Readable.toWeb( + fs.createReadStream(tmpTar) + ) as unknown as ReadableStream, + duplex: 'half', + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + } + const res = await fetch(fresh.url, init) + if (!res.ok) { + throw new Error(`mirror upload answered ${res.status}`) + } + core.info(`Mirror uploaded (${size} bytes); future runs will restore it`) + } catch (error) { + core.warning(`Mirror upload skipped: ${error}`) + } finally { + await fs.promises.rm(tmpTar, {force: true}) + } +}