mirror of
https://github.com/actions/checkout.git
synced 2026-07-09 09:21:49 +00:00
Merge 4a9081b71c into e8d4307400
This commit is contained in:
commit
efd4f60fcc
15
README.md
15
README.md
@ -1,3 +1,18 @@
|
|||||||
|
# WarpBuild Checkout
|
||||||
|
|
||||||
|
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
|
||||||
|
replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's
|
||||||
|
bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git
|
||||||
|
alternates, so the fetch from GitHub downloads only the delta instead of the whole
|
||||||
|
repository. On a cache miss, the run hydrates the mirror once (full clone + upload);
|
||||||
|
mirrors expire on a server-configured TTL and re-hydrate.
|
||||||
|
|
||||||
|
- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners.
|
||||||
|
- Fail-open — any cache error degrades to stock `actions/checkout` behavior.
|
||||||
|
- All fork code lives in `src/warpbuild/`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
[](https://github.com/actions/checkout/actions/workflows/test.yml)
|
[](https://github.com/actions/checkout/actions/workflows/test.yml)
|
||||||
|
|
||||||
# Checkout v7
|
# Checkout v7
|
||||||
|
|||||||
199
__test__/warpbuild-mirror.test.ts
Normal file
199
__test__/warpbuild-mirror.test.ts
Normal file
@ -0,0 +1,199 @@
|
|||||||
|
import {
|
||||||
|
describe,
|
||||||
|
it,
|
||||||
|
expect,
|
||||||
|
beforeEach,
|
||||||
|
afterEach,
|
||||||
|
afterAll
|
||||||
|
} from '@jest/globals'
|
||||||
|
import {
|
||||||
|
SKIP_NOT_WARPBUILD,
|
||||||
|
computeDestinationRef,
|
||||||
|
getMirrorCacheSkipReason
|
||||||
|
} from '../src/warpbuild/mirror-cache.js'
|
||||||
|
import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js'
|
||||||
|
import {IGitSourceSettings} from '../src/git-source-settings.js'
|
||||||
|
|
||||||
|
const WB_ENV = [
|
||||||
|
'WARPBUILD_RUNNER_VERIFICATION_TOKEN',
|
||||||
|
'WARPBUILD_HOST_URL',
|
||||||
|
'GITHUB_REPOSITORY_ID',
|
||||||
|
'GITHUB_REPOSITORY'
|
||||||
|
]
|
||||||
|
const savedEnv: {[key: string]: string | undefined} = {}
|
||||||
|
for (const key of WB_ENV) {
|
||||||
|
savedEnv[key] = process.env[key]
|
||||||
|
}
|
||||||
|
|
||||||
|
const SHA = 'cd5255d20e23e050238affc045ba9beee35eaaf7'
|
||||||
|
|
||||||
|
function settingsFor(
|
||||||
|
overrides: Partial<IGitSourceSettings> = {}
|
||||||
|
): IGitSourceSettings {
|
||||||
|
return {
|
||||||
|
repositoryOwner: 'octocat',
|
||||||
|
repositoryName: 'hello-world',
|
||||||
|
repositoryPath: '/tmp/does-not-matter',
|
||||||
|
ref: 'refs/heads/main',
|
||||||
|
commit: SHA,
|
||||||
|
fetchDepth: 1,
|
||||||
|
fetchTags: false,
|
||||||
|
filter: undefined,
|
||||||
|
sparseCheckout: undefined,
|
||||||
|
lfs: false,
|
||||||
|
...overrides
|
||||||
|
} as unknown as IGitSourceSettings
|
||||||
|
}
|
||||||
|
|
||||||
|
function setWarpBuildEnv(): void {
|
||||||
|
process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = 'test-token'
|
||||||
|
process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev'
|
||||||
|
process.env['GITHUB_REPOSITORY_ID'] = '123456789'
|
||||||
|
process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world'
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('warpbuild snapshot cache', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
setWarpBuildEnv()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
for (const key of WB_ENV) {
|
||||||
|
if (savedEnv[key] === undefined) {
|
||||||
|
delete process.env[key]
|
||||||
|
} else {
|
||||||
|
process.env[key] = savedEnv[key]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('getMirrorCacheSkipReason', () => {
|
||||||
|
const winOnly = process.platform === 'win32' ? it.skip : it
|
||||||
|
|
||||||
|
winOnly('returns null for the default checkout shape', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor())).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports a non-WarpBuild runner', () => {
|
||||||
|
delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN']
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports repository: inputs that are not the workflow repo', () => {
|
||||||
|
expect(
|
||||||
|
getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'}))
|
||||||
|
).toBe(
|
||||||
|
"repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('requires an exact commit sha', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({commit: ''}))).toBe(
|
||||||
|
'no exact commit sha to key on'
|
||||||
|
)
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({commit: 'main'}))).toBe(
|
||||||
|
'no exact commit sha to key on'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('only serves fetch-depth 1', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBe(
|
||||||
|
'fetch-depth is 0, cache only serves fetch-depth 1'
|
||||||
|
)
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBe(
|
||||||
|
'fetch-depth is 50, cache only serves fetch-depth 1'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('skips fetch-tags, filters, sparse and lfs checkouts', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchTags: true}))).toBe(
|
||||||
|
'fetch-tags is enabled'
|
||||||
|
)
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe(
|
||||||
|
'a fetch filter is configured'
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']}))
|
||||||
|
).toBe('sparse checkout is configured')
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe(
|
||||||
|
'lfs is enabled (lfs objects are not in the snapshot)'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
winOnly('accepts sha-only checkouts (empty ref)', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({ref: ''}))).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('skips unqualified refs', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({ref: 'main'}))).toBe(
|
||||||
|
"ref 'main' has no cacheable destination ref"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('computeDestinationRef', () => {
|
||||||
|
it('maps the refs the fetch would have created', () => {
|
||||||
|
expect(computeDestinationRef('refs/heads/main')).toBe(
|
||||||
|
'refs/remotes/origin/main'
|
||||||
|
)
|
||||||
|
expect(computeDestinationRef('refs/pull/42/merge')).toBe(
|
||||||
|
'refs/remotes/pull/42/merge'
|
||||||
|
)
|
||||||
|
expect(computeDestinationRef('refs/tags/v1.2.3')).toBe('refs/tags/v1.2.3')
|
||||||
|
expect(computeDestinationRef('')).toBe('')
|
||||||
|
expect(computeDestinationRef('main')).toBeNull()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('backend api http contract', () => {
|
||||||
|
const realFetch = globalThis.fetch
|
||||||
|
let lastUrl = ''
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
globalThis.fetch = realFetch
|
||||||
|
})
|
||||||
|
|
||||||
|
function stubFetch(status: number, body: unknown): void {
|
||||||
|
globalThis.fetch = (async (input: unknown) => {
|
||||||
|
lastUrl = String(input)
|
||||||
|
return new Response(JSON.stringify(body), {status})
|
||||||
|
}) as typeof fetch
|
||||||
|
}
|
||||||
|
|
||||||
|
it('maps 200 to hit and keys by sha', async () => {
|
||||||
|
stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'})
|
||||||
|
const result = await lookupSnapshot('123', SHA)
|
||||||
|
expect(result.kind).toBe('hit')
|
||||||
|
expect(lastUrl).toContain(`sha=${SHA}`)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('maps 404 to miss (upload after the stock fetch)', async () => {
|
||||||
|
stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
|
||||||
|
expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('maps 403 to disabled (backend kill switch)', async () => {
|
||||||
|
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
|
||||||
|
expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')
|
||||||
|
expect((await requestUploadURL('123', SHA)).kind).toBe('disabled')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('maps other statuses and network failures to error', async () => {
|
||||||
|
stubFetch(500, {})
|
||||||
|
expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
|
||||||
|
globalThis.fetch = (async () => {
|
||||||
|
throw new Error('boom')
|
||||||
|
}) as typeof fetch
|
||||||
|
expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
|
||||||
|
expect((await requestUploadURL('123', SHA)).kind).toBe('error')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('maps 200 upload responses to ok', async () => {
|
||||||
|
stubFetch(200, {url: 'https://s3/put'})
|
||||||
|
expect(await requestUploadURL('123', SHA)).toEqual({
|
||||||
|
kind: 'ok',
|
||||||
|
url: 'https://s3/put'
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
320
dist/index.js
vendored
320
dist/index.js
vendored
@ -41692,6 +41692,315 @@ function ref_helper_select(obj, path) {
|
|||||||
return ref_helper_select(obj[key], path.substr(i + 1));
|
return ref_helper_select(obj[key], path.substr(i + 1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
;// CONCATENATED MODULE: external "stream/promises"
|
||||||
|
const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises");
|
||||||
|
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
|
||||||
|
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||||
|
|
||||||
|
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
|
||||||
|
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
|
||||||
|
// stock fetch); 403 = unservable org (skip cache + upload); else = fall back.
|
||||||
|
const API_TIMEOUT_MS = 10_000;
|
||||||
|
function backend_api_baseUrl() {
|
||||||
|
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
|
||||||
|
}
|
||||||
|
function authHeader() {
|
||||||
|
return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`;
|
||||||
|
}
|
||||||
|
async function lookupSnapshot(repoKey, sha) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}&sha=${encodeURIComponent(sha)}`, {
|
||||||
|
headers: { authorization: authHeader() },
|
||||||
|
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||||
|
});
|
||||||
|
if (res.status === 200) {
|
||||||
|
return { kind: 'hit', info: (await res.json()) };
|
||||||
|
}
|
||||||
|
if (res.status === 404) {
|
||||||
|
return { kind: 'miss' };
|
||||||
|
}
|
||||||
|
if (res.status === 403) {
|
||||||
|
core_debug(`[wb-cache] download-url answered 403 (disabled)`);
|
||||||
|
return { kind: 'disabled' };
|
||||||
|
}
|
||||||
|
core_debug(`[wb-cache] download-url answered ${res.status}`);
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
core_debug(`[wb-cache] download-url failed: ${error}`);
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function requestUploadURL(repoKey, sha) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
authorization: authHeader(),
|
||||||
|
'content-type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ repo_key: repoKey, sha }),
|
||||||
|
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||||
|
});
|
||||||
|
if (res.status === 200) {
|
||||||
|
const body = (await res.json());
|
||||||
|
if (body.url) {
|
||||||
|
return { kind: 'ok', url: body.url };
|
||||||
|
}
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
if (res.status === 403) {
|
||||||
|
core_debug(`[wb-cache] upload-url answered 403 (disabled)`);
|
||||||
|
return { kind: 'disabled' };
|
||||||
|
}
|
||||||
|
core_debug(`[wb-cache] upload-url answered ${res.status}`);
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
core_debug(`[wb-cache] upload-url failed: ${error}`);
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
|
||||||
|
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch
|
||||||
|
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
|
||||||
|
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
||||||
|
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000;
|
||||||
|
const UPLOAD_TIMEOUT_MS = 15 * 60_000;
|
||||||
|
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
|
||||||
|
const CACHE_STATE_FILE = 'wb-cache-state';
|
||||||
|
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
||||||
|
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
|
||||||
|
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
|
||||||
|
let decision = 'off';
|
||||||
|
// Null = attempt the cache; else a reason to log. Only the default checkout shape
|
||||||
|
// is served.
|
||||||
|
function getMirrorCacheSkipReason(settings) {
|
||||||
|
if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
|
||||||
|
!process.env['WARPBUILD_HOST_URL']) {
|
||||||
|
return SKIP_NOT_WARPBUILD;
|
||||||
|
}
|
||||||
|
if (process.platform === 'win32') {
|
||||||
|
return 'Windows is not supported by the snapshot cache yet';
|
||||||
|
}
|
||||||
|
const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '';
|
||||||
|
if (!repoKey) {
|
||||||
|
return 'GITHUB_REPOSITORY_ID is not set';
|
||||||
|
}
|
||||||
|
const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`;
|
||||||
|
if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) {
|
||||||
|
return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`;
|
||||||
|
}
|
||||||
|
const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, '');
|
||||||
|
if (server !== 'https://github.com') {
|
||||||
|
return `server '${server}' is not github.com`;
|
||||||
|
}
|
||||||
|
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
|
||||||
|
return 'no exact commit sha to key on';
|
||||||
|
}
|
||||||
|
if (settings.fetchDepth !== 1) {
|
||||||
|
return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`;
|
||||||
|
}
|
||||||
|
if (settings.fetchTags) {
|
||||||
|
return 'fetch-tags is enabled';
|
||||||
|
}
|
||||||
|
if (settings.filter) {
|
||||||
|
return 'a fetch filter is configured';
|
||||||
|
}
|
||||||
|
if (settings.sparseCheckout) {
|
||||||
|
return 'sparse checkout is configured';
|
||||||
|
}
|
||||||
|
if (settings.lfs) {
|
||||||
|
return 'lfs is enabled (lfs objects are not in the snapshot)';
|
||||||
|
}
|
||||||
|
if (settings.ref && computeDestinationRef(settings.ref) === null) {
|
||||||
|
return `ref '${settings.ref}' has no cacheable destination ref`;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// The local ref the fetch would have created ('' = none needed, null = uncacheable).
|
||||||
|
function computeDestinationRef(ref) {
|
||||||
|
if (!ref) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
const upper = ref.toUpperCase();
|
||||||
|
if (upper.startsWith('REFS/HEADS/')) {
|
||||||
|
return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`;
|
||||||
|
}
|
||||||
|
if (upper.startsWith('REFS/PULL/')) {
|
||||||
|
return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`;
|
||||||
|
}
|
||||||
|
if (upper.startsWith('REFS/TAGS/')) {
|
||||||
|
return ref;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
||||||
|
async function setup(settings) {
|
||||||
|
decision = 'off';
|
||||||
|
const skipReason = getMirrorCacheSkipReason(settings);
|
||||||
|
if (skipReason) {
|
||||||
|
if (skipReason === SKIP_NOT_WARPBUILD) {
|
||||||
|
core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
info(`WarpBuild snapshot cache skipped: ${skipReason}`);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
startGroup('WarpBuild: checkout snapshot cache');
|
||||||
|
try {
|
||||||
|
return await setupInner(settings);
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
endGroup();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function setupInner(settings) {
|
||||||
|
const repoKey = process.env['GITHUB_REPOSITORY_ID'];
|
||||||
|
const sha = settings.commit;
|
||||||
|
const lookup = await lookupSnapshot(repoKey, sha);
|
||||||
|
if (lookup.kind === 'disabled') {
|
||||||
|
info('Snapshot cache is disabled by the backend for this organization');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (lookup.kind === 'error') {
|
||||||
|
info('Snapshot cache backend unavailable; using standard checkout');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (lookup.kind === 'miss') {
|
||||||
|
info(`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`);
|
||||||
|
decision = 'miss';
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
info(`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`);
|
||||||
|
if (!(await restoreSnapshot(settings, lookup.info.url, sha))) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
info('Snapshot restored; skipping the GitHub fetch entirely');
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
|
||||||
|
async function contribute(settings) {
|
||||||
|
if (decision !== 'miss') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
startGroup('WarpBuild: uploading checkout snapshot');
|
||||||
|
try {
|
||||||
|
await uploadSnapshot(settings);
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`Snapshot upload skipped: ${error}`);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
endGroup();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function restoreSnapshot(settings, url, sha) {
|
||||||
|
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
|
||||||
|
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`);
|
||||||
|
try {
|
||||||
|
const res = await fetch(url, {
|
||||||
|
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
|
||||||
|
});
|
||||||
|
if (!res.ok || !res.body) {
|
||||||
|
throw new Error(`snapshot download answered ${res.status}`);
|
||||||
|
}
|
||||||
|
await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar));
|
||||||
|
await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]);
|
||||||
|
const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true });
|
||||||
|
if (check !== 0) {
|
||||||
|
throw new Error(`restored snapshot does not contain ${sha}`);
|
||||||
|
}
|
||||||
|
// The ref the skipped fetch would have created; upstream's verification needs it.
|
||||||
|
const dstRef = computeDestinationRef(settings.ref);
|
||||||
|
if (dstRef) {
|
||||||
|
await exec_exec('git', [
|
||||||
|
'-C',
|
||||||
|
settings.repositoryPath,
|
||||||
|
'update-ref',
|
||||||
|
dstRef,
|
||||||
|
sha
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`Snapshot restore failed: ${error}`);
|
||||||
|
// Reset .git to its freshly-init state.
|
||||||
|
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
|
||||||
|
recursive: true,
|
||||||
|
force: true
|
||||||
|
});
|
||||||
|
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
|
||||||
|
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
|
||||||
|
recursive: true
|
||||||
|
});
|
||||||
|
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
|
||||||
|
recursive: true
|
||||||
|
});
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function uploadSnapshot(settings) {
|
||||||
|
const repoKey = process.env['GITHUB_REPOSITORY_ID'];
|
||||||
|
const sha = settings.commit;
|
||||||
|
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
|
||||||
|
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`);
|
||||||
|
try {
|
||||||
|
const members = ['objects'];
|
||||||
|
if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) {
|
||||||
|
members.push('shallow');
|
||||||
|
}
|
||||||
|
await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]);
|
||||||
|
const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size;
|
||||||
|
const upload = await requestUploadURL(repoKey, sha);
|
||||||
|
if (upload.kind !== 'ok') {
|
||||||
|
info(upload.kind === 'disabled'
|
||||||
|
? 'Snapshot cache is disabled by the backend; not uploading'
|
||||||
|
: 'Snapshot cache backend unavailable; not uploading');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const init = {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: {
|
||||||
|
'content-length': String(size),
|
||||||
|
'content-type': 'application/x-tar'
|
||||||
|
},
|
||||||
|
body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)),
|
||||||
|
duplex: 'half',
|
||||||
|
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
||||||
|
};
|
||||||
|
const res = await fetch(upload.url, init);
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`snapshot upload answered ${res.status}`);
|
||||||
|
}
|
||||||
|
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`);
|
||||||
|
info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
;// CONCATENATED MODULE: ./src/git-source-provider.ts
|
;// CONCATENATED MODULE: ./src/git-source-provider.ts
|
||||||
|
|
||||||
|
|
||||||
@ -41705,6 +42014,7 @@ function ref_helper_select(obj, path) {
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async function getSource(settings) {
|
async function getSource(settings) {
|
||||||
// Repository URL
|
// Repository URL
|
||||||
info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`);
|
info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`);
|
||||||
@ -41724,6 +42034,7 @@ async function getSource(settings) {
|
|||||||
const git = await getGitCommandManager(settings);
|
const git = await getGitCommandManager(settings);
|
||||||
endGroup();
|
endGroup();
|
||||||
let authHelper = null;
|
let authHelper = null;
|
||||||
|
let warpbuildRestored = false;
|
||||||
try {
|
try {
|
||||||
if (git) {
|
if (git) {
|
||||||
authHelper = createAuthHelper(git, settings);
|
authHelper = createAuthHelper(git, settings);
|
||||||
@ -41774,6 +42085,8 @@ async function getSource(settings) {
|
|||||||
await git.init(objectFormat);
|
await git.init(objectFormat);
|
||||||
await git.remoteAdd('origin', repositoryUrl);
|
await git.remoteAdd('origin', repositoryUrl);
|
||||||
endGroup();
|
endGroup();
|
||||||
|
// WarpBuild snapshot cache: hit = objects restored, fetch below is skipped
|
||||||
|
warpbuildRestored = await setup(settings);
|
||||||
}
|
}
|
||||||
// Disable automatic garbage collection
|
// Disable automatic garbage collection
|
||||||
startGroup('Disabling automatic garbage collection');
|
startGroup('Disabling automatic garbage collection');
|
||||||
@ -41813,7 +42126,10 @@ async function getSource(settings) {
|
|||||||
else if (settings.sparseCheckout) {
|
else if (settings.sparseCheckout) {
|
||||||
fetchOptions.filter = 'blob:none';
|
fetchOptions.filter = 'blob:none';
|
||||||
}
|
}
|
||||||
if (settings.fetchDepth <= 0) {
|
if (warpbuildRestored) {
|
||||||
|
info('Skipping fetch: checkout was restored from the snapshot cache');
|
||||||
|
}
|
||||||
|
else if (settings.fetchDepth <= 0) {
|
||||||
// Fetch all branches and tags
|
// Fetch all branches and tags
|
||||||
let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit);
|
let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit);
|
||||||
await git.fetch(refSpec, fetchOptions);
|
await git.fetch(refSpec, fetchOptions);
|
||||||
@ -41879,6 +42195,8 @@ async function getSource(settings) {
|
|||||||
startGroup('Checking out the ref');
|
startGroup('Checking out the ref');
|
||||||
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint);
|
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint);
|
||||||
endGroup();
|
endGroup();
|
||||||
|
// WarpBuild snapshot cache: upload the fetch result on a miss
|
||||||
|
await contribute(settings);
|
||||||
// Submodules
|
// Submodules
|
||||||
if (settings.submodules) {
|
if (settings.submodules) {
|
||||||
// Temporarily override global config
|
// Temporarily override global config
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import * as path from 'path'
|
|||||||
import * as refHelper from './ref-helper.js'
|
import * as refHelper from './ref-helper.js'
|
||||||
import * as stateHelper from './state-helper.js'
|
import * as stateHelper from './state-helper.js'
|
||||||
import * as urlHelper from './url-helper.js'
|
import * as urlHelper from './url-helper.js'
|
||||||
|
import * as warpbuildMirror from './warpbuild/mirror-cache.js'
|
||||||
import {
|
import {
|
||||||
MinimumGitSparseCheckoutVersion,
|
MinimumGitSparseCheckoutVersion,
|
||||||
IGitCommandManager
|
IGitCommandManager
|
||||||
@ -40,6 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
|||||||
core.endGroup()
|
core.endGroup()
|
||||||
|
|
||||||
let authHelper: gitAuthHelper.IGitAuthHelper | null = null
|
let authHelper: gitAuthHelper.IGitAuthHelper | null = null
|
||||||
|
let warpbuildRestored = false
|
||||||
try {
|
try {
|
||||||
if (git) {
|
if (git) {
|
||||||
authHelper = gitAuthHelper.createAuthHelper(git, settings)
|
authHelper = gitAuthHelper.createAuthHelper(git, settings)
|
||||||
@ -130,6 +132,9 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
|||||||
await git.init(objectFormat)
|
await git.init(objectFormat)
|
||||||
await git.remoteAdd('origin', repositoryUrl)
|
await git.remoteAdd('origin', repositoryUrl)
|
||||||
core.endGroup()
|
core.endGroup()
|
||||||
|
|
||||||
|
// WarpBuild snapshot cache: hit = objects restored, fetch below is skipped
|
||||||
|
warpbuildRestored = await warpbuildMirror.setup(settings)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Disable automatic garbage collection
|
// Disable automatic garbage collection
|
||||||
@ -185,7 +190,9 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
|||||||
fetchOptions.filter = 'blob:none'
|
fetchOptions.filter = 'blob:none'
|
||||||
}
|
}
|
||||||
|
|
||||||
if (settings.fetchDepth <= 0) {
|
if (warpbuildRestored) {
|
||||||
|
core.info('Skipping fetch: checkout was restored from the snapshot cache')
|
||||||
|
} else if (settings.fetchDepth <= 0) {
|
||||||
// Fetch all branches and tags
|
// Fetch all branches and tags
|
||||||
let refSpec = refHelper.getRefSpecForAllHistory(
|
let refSpec = refHelper.getRefSpecForAllHistory(
|
||||||
settings.ref,
|
settings.ref,
|
||||||
@ -271,6 +278,9 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
|||||||
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
|
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
|
||||||
core.endGroup()
|
core.endGroup()
|
||||||
|
|
||||||
|
// WarpBuild snapshot cache: upload the fetch result on a miss
|
||||||
|
await warpbuildMirror.contribute(settings)
|
||||||
|
|
||||||
// Submodules
|
// Submodules
|
||||||
if (settings.submodules) {
|
if (settings.submodules) {
|
||||||
// Temporarily override global config
|
// Temporarily override global config
|
||||||
|
|||||||
98
src/warpbuild/backend-api.ts
Normal file
98
src/warpbuild/backend-api.ts
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||||
|
import * as core from '@actions/core'
|
||||||
|
|
||||||
|
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
|
||||||
|
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
|
||||||
|
// stock fetch); 403 = unservable org (skip cache + upload); else = fall back.
|
||||||
|
|
||||||
|
const API_TIMEOUT_MS = 10_000
|
||||||
|
|
||||||
|
export interface SnapshotDownloadInfo {
|
||||||
|
url: string
|
||||||
|
size_bytes: number
|
||||||
|
created_at: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SnapshotLookup =
|
||||||
|
| {kind: 'hit'; info: SnapshotDownloadInfo}
|
||||||
|
| {kind: 'miss'}
|
||||||
|
| {kind: 'disabled'}
|
||||||
|
| {kind: 'error'}
|
||||||
|
|
||||||
|
export type UploadURLResult =
|
||||||
|
| {kind: 'ok'; url: string}
|
||||||
|
| {kind: 'disabled'}
|
||||||
|
| {kind: 'error'}
|
||||||
|
|
||||||
|
function baseUrl(): string {
|
||||||
|
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '')
|
||||||
|
}
|
||||||
|
|
||||||
|
function authHeader(): string {
|
||||||
|
return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function lookupSnapshot(
|
||||||
|
repoKey: string,
|
||||||
|
sha: string
|
||||||
|
): Promise<SnapshotLookup> {
|
||||||
|
try {
|
||||||
|
const res = await fetch(
|
||||||
|
`${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(
|
||||||
|
repoKey
|
||||||
|
)}&sha=${encodeURIComponent(sha)}`,
|
||||||
|
{
|
||||||
|
headers: {authorization: authHeader()},
|
||||||
|
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if (res.status === 200) {
|
||||||
|
return {kind: 'hit', info: (await res.json()) as SnapshotDownloadInfo}
|
||||||
|
}
|
||||||
|
if (res.status === 404) {
|
||||||
|
return {kind: 'miss'}
|
||||||
|
}
|
||||||
|
if (res.status === 403) {
|
||||||
|
core.debug(`[wb-cache] download-url answered 403 (disabled)`)
|
||||||
|
return {kind: 'disabled'}
|
||||||
|
}
|
||||||
|
core.debug(`[wb-cache] download-url answered ${res.status}`)
|
||||||
|
return {kind: 'error'}
|
||||||
|
} catch (error) {
|
||||||
|
core.debug(`[wb-cache] download-url failed: ${error}`)
|
||||||
|
return {kind: 'error'}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function requestUploadURL(
|
||||||
|
repoKey: string,
|
||||||
|
sha: string
|
||||||
|
): Promise<UploadURLResult> {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
authorization: authHeader(),
|
||||||
|
'content-type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify({repo_key: repoKey, sha}),
|
||||||
|
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||||
|
})
|
||||||
|
if (res.status === 200) {
|
||||||
|
const body = (await res.json()) as {url?: string}
|
||||||
|
if (body.url) {
|
||||||
|
return {kind: 'ok', url: body.url}
|
||||||
|
}
|
||||||
|
return {kind: 'error'}
|
||||||
|
}
|
||||||
|
if (res.status === 403) {
|
||||||
|
core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
|
||||||
|
return {kind: 'disabled'}
|
||||||
|
}
|
||||||
|
core.debug(`[wb-cache] upload-url answered ${res.status}`)
|
||||||
|
return {kind: 'error'}
|
||||||
|
} catch (error) {
|
||||||
|
core.debug(`[wb-cache] upload-url failed: ${error}`)
|
||||||
|
return {kind: 'error'}
|
||||||
|
}
|
||||||
|
}
|
||||||
289
src/warpbuild/mirror-cache.ts
Normal file
289
src/warpbuild/mirror-cache.ts
Normal file
@ -0,0 +1,289 @@
|
|||||||
|
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
|
||||||
|
import * as core from '@actions/core'
|
||||||
|
import * as exec from '@actions/exec'
|
||||||
|
import * as fs from 'fs'
|
||||||
|
import * as os from 'os'
|
||||||
|
import * as path from 'path'
|
||||||
|
import {Readable} from 'stream'
|
||||||
|
import {pipeline} from 'stream/promises'
|
||||||
|
import {IGitSourceSettings} from '../git-source-settings.js'
|
||||||
|
import * as api from './backend-api.js'
|
||||||
|
|
||||||
|
// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch
|
||||||
|
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
|
||||||
|
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
||||||
|
|
||||||
|
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
|
||||||
|
const UPLOAD_TIMEOUT_MS = 15 * 60_000
|
||||||
|
|
||||||
|
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
|
||||||
|
export const CACHE_STATE_FILE = 'wb-cache-state'
|
||||||
|
|
||||||
|
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
||||||
|
export const SKIP_NOT_WARPBUILD =
|
||||||
|
'not running on a WarpBuild runner (WARPBUILD_* env not present)'
|
||||||
|
|
||||||
|
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/
|
||||||
|
|
||||||
|
let decision: 'off' | 'miss' = 'off'
|
||||||
|
|
||||||
|
// Null = attempt the cache; else a reason to log. Only the default checkout shape
|
||||||
|
// is served.
|
||||||
|
export function getMirrorCacheSkipReason(
|
||||||
|
settings: IGitSourceSettings
|
||||||
|
): string | null {
|
||||||
|
if (
|
||||||
|
!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
|
||||||
|
!process.env['WARPBUILD_HOST_URL']
|
||||||
|
) {
|
||||||
|
return SKIP_NOT_WARPBUILD
|
||||||
|
}
|
||||||
|
if (process.platform === 'win32') {
|
||||||
|
return 'Windows is not supported by the snapshot cache yet'
|
||||||
|
}
|
||||||
|
const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''
|
||||||
|
if (!repoKey) {
|
||||||
|
return 'GITHUB_REPOSITORY_ID is not set'
|
||||||
|
}
|
||||||
|
const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`
|
||||||
|
if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) {
|
||||||
|
return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`
|
||||||
|
}
|
||||||
|
const server = (settings.githubServerUrl || 'https://github.com').replace(
|
||||||
|
/\/+$/,
|
||||||
|
''
|
||||||
|
)
|
||||||
|
if (server !== 'https://github.com') {
|
||||||
|
return `server '${server}' is not github.com`
|
||||||
|
}
|
||||||
|
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
|
||||||
|
return 'no exact commit sha to key on'
|
||||||
|
}
|
||||||
|
if (settings.fetchDepth !== 1) {
|
||||||
|
return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`
|
||||||
|
}
|
||||||
|
if (settings.fetchTags) {
|
||||||
|
return 'fetch-tags is enabled'
|
||||||
|
}
|
||||||
|
if (settings.filter) {
|
||||||
|
return 'a fetch filter is configured'
|
||||||
|
}
|
||||||
|
if (settings.sparseCheckout) {
|
||||||
|
return 'sparse checkout is configured'
|
||||||
|
}
|
||||||
|
if (settings.lfs) {
|
||||||
|
return 'lfs is enabled (lfs objects are not in the snapshot)'
|
||||||
|
}
|
||||||
|
if (settings.ref && computeDestinationRef(settings.ref) === null) {
|
||||||
|
return `ref '${settings.ref}' has no cacheable destination ref`
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// The local ref the fetch would have created ('' = none needed, null = uncacheable).
|
||||||
|
export function computeDestinationRef(ref: string): string | null {
|
||||||
|
if (!ref) {
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
const upper = ref.toUpperCase()
|
||||||
|
if (upper.startsWith('REFS/HEADS/')) {
|
||||||
|
return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`
|
||||||
|
}
|
||||||
|
if (upper.startsWith('REFS/PULL/')) {
|
||||||
|
return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`
|
||||||
|
}
|
||||||
|
if (upper.startsWith('REFS/TAGS/')) {
|
||||||
|
return ref
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
||||||
|
export async function setup(settings: IGitSourceSettings): Promise<boolean> {
|
||||||
|
decision = 'off'
|
||||||
|
const skipReason = getMirrorCacheSkipReason(settings)
|
||||||
|
if (skipReason) {
|
||||||
|
if (skipReason === SKIP_NOT_WARPBUILD) {
|
||||||
|
core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||||
|
} else {
|
||||||
|
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
core.startGroup('WarpBuild: checkout snapshot cache')
|
||||||
|
try {
|
||||||
|
return await setupInner(settings)
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(
|
||||||
|
`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`
|
||||||
|
)
|
||||||
|
return false
|
||||||
|
} finally {
|
||||||
|
core.endGroup()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setupInner(settings: IGitSourceSettings): Promise<boolean> {
|
||||||
|
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
|
||||||
|
const sha = settings.commit
|
||||||
|
|
||||||
|
const lookup = await api.lookupSnapshot(repoKey, sha)
|
||||||
|
|
||||||
|
if (lookup.kind === 'disabled') {
|
||||||
|
core.info('Snapshot cache is disabled by the backend for this organization')
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if (lookup.kind === 'error') {
|
||||||
|
core.info('Snapshot cache backend unavailable; using standard checkout')
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lookup.kind === 'miss') {
|
||||||
|
core.info(
|
||||||
|
`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`
|
||||||
|
)
|
||||||
|
decision = 'miss'
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
core.info(
|
||||||
|
`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`
|
||||||
|
)
|
||||||
|
if (!(await restoreSnapshot(settings, lookup.info.url, sha))) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
core.info('Snapshot restored; skipping the GitHub fetch entirely')
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
|
||||||
|
export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
||||||
|
if (decision !== 'miss') {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
core.startGroup('WarpBuild: uploading checkout snapshot')
|
||||||
|
try {
|
||||||
|
await uploadSnapshot(settings)
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(`Snapshot upload skipped: ${error}`)
|
||||||
|
} finally {
|
||||||
|
core.endGroup()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function restoreSnapshot(
|
||||||
|
settings: IGitSourceSettings,
|
||||||
|
url: string,
|
||||||
|
sha: string
|
||||||
|
): Promise<boolean> {
|
||||||
|
const gitDir = path.join(settings.repositoryPath, '.git')
|
||||||
|
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-${process.pid}.tar`)
|
||||||
|
try {
|
||||||
|
const res = await fetch(url, {
|
||||||
|
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
|
||||||
|
})
|
||||||
|
if (!res.ok || !res.body) {
|
||||||
|
throw new Error(`snapshot download answered ${res.status}`)
|
||||||
|
}
|
||||||
|
await pipeline(
|
||||||
|
Readable.fromWeb(res.body as import('stream/web').ReadableStream),
|
||||||
|
fs.createWriteStream(tmpTar)
|
||||||
|
)
|
||||||
|
await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir])
|
||||||
|
|
||||||
|
const check = await exec.exec(
|
||||||
|
'git',
|
||||||
|
['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`],
|
||||||
|
{ignoreReturnCode: true}
|
||||||
|
)
|
||||||
|
if (check !== 0) {
|
||||||
|
throw new Error(`restored snapshot does not contain ${sha}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The ref the skipped fetch would have created; upstream's verification needs it.
|
||||||
|
const dstRef = computeDestinationRef(settings.ref)
|
||||||
|
if (dstRef) {
|
||||||
|
await exec.exec('git', [
|
||||||
|
'-C',
|
||||||
|
settings.repositoryPath,
|
||||||
|
'update-ref',
|
||||||
|
dstRef,
|
||||||
|
sha
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
await fs.promises.writeFile(
|
||||||
|
path.join(gitDir, CACHE_STATE_FILE),
|
||||||
|
`hit ${sha}\n`
|
||||||
|
)
|
||||||
|
return true
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(`Snapshot restore failed: ${error}`)
|
||||||
|
// Reset .git to its freshly-init state.
|
||||||
|
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
||||||
|
recursive: true,
|
||||||
|
force: true
|
||||||
|
})
|
||||||
|
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
|
||||||
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
||||||
|
recursive: true
|
||||||
|
})
|
||||||
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||||
|
recursive: true
|
||||||
|
})
|
||||||
|
return false
|
||||||
|
} finally {
|
||||||
|
await fs.promises.rm(tmpTar, {force: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> {
|
||||||
|
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
|
||||||
|
const sha = settings.commit
|
||||||
|
const gitDir = path.join(settings.repositoryPath, '.git')
|
||||||
|
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`)
|
||||||
|
try {
|
||||||
|
const members = ['objects']
|
||||||
|
if (fs.existsSync(path.join(gitDir, 'shallow'))) {
|
||||||
|
members.push('shallow')
|
||||||
|
}
|
||||||
|
await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members])
|
||||||
|
const size = (await fs.promises.stat(tmpTar)).size
|
||||||
|
|
||||||
|
const upload = await api.requestUploadURL(repoKey, sha)
|
||||||
|
if (upload.kind !== 'ok') {
|
||||||
|
core.info(
|
||||||
|
upload.kind === 'disabled'
|
||||||
|
? 'Snapshot cache is disabled by the backend; not uploading'
|
||||||
|
: 'Snapshot cache backend unavailable; not uploading'
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const init: RequestInit & {duplex: 'half'} = {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: {
|
||||||
|
'content-length': String(size),
|
||||||
|
'content-type': 'application/x-tar'
|
||||||
|
},
|
||||||
|
body: Readable.toWeb(
|
||||||
|
fs.createReadStream(tmpTar)
|
||||||
|
) as unknown as ReadableStream,
|
||||||
|
duplex: 'half',
|
||||||
|
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
||||||
|
}
|
||||||
|
const res = await fetch(upload.url, init)
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`snapshot upload answered ${res.status}`)
|
||||||
|
}
|
||||||
|
await fs.promises.writeFile(
|
||||||
|
path.join(gitDir, CACHE_STATE_FILE),
|
||||||
|
`uploaded ${sha}\n`
|
||||||
|
)
|
||||||
|
core.info(
|
||||||
|
`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`
|
||||||
|
)
|
||||||
|
} finally {
|
||||||
|
await fs.promises.rm(tmpTar, {force: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user