diff --git a/.github/scripts/cleanup-cloudsmith.mjs b/.github/scripts/cleanup-cloudsmith.mjs index c1018736..4efa5871 100644 --- a/.github/scripts/cleanup-cloudsmith.mjs +++ b/.github/scripts/cleanup-cloudsmith.mjs @@ -1,4 +1,5 @@ import {forEachSequential} from './async-iteration.mjs'; +import {CleanupStopped, createCleanupRuntime} from './cleanup-runtime.mjs'; const CLOUDSMITH_API = 'https://api.cloudsmith.io/v1'; const DEFAULT_GRACE_MS = 24 * 60 * 60 * 1000; @@ -134,7 +135,7 @@ async function responseError(response) { return new Error(`${response.status} ${response.statusText}: ${body.slice(0, 500)}`); } -async function listCloudsmithPackages({owner, repository, token}) { +async function listCloudsmithPackages({owner, repository, token}, request) { const packages = []; const pageSize = 100; @@ -142,7 +143,7 @@ async function listCloudsmithPackages({owner, repository, token}) { const url = new URL(`${CLOUDSMITH_API}/packages/${owner}/${repository}/`); url.searchParams.set('page', page); url.searchParams.set('page_size', pageSize); - const response = await fetch(url, { + const response = await request(url, { headers: { Authorization: `Bearer ${token}`, 'User-Agent': 'LizardByte-artifact-cleanup', @@ -163,8 +164,8 @@ async function listCloudsmithPackages({owner, repository, token}) { return packages; } -async function deleteCloudsmithPackage({owner, repository, token, identifier}) { - const response = await fetch( +async function deleteCloudsmithPackage({owner, repository, token, identifier}, request) { + const response = await request( `${CLOUDSMITH_API}/packages/${owner}/${repository}/${encodeURIComponent(identifier)}/`, { method: 'DELETE', @@ -211,6 +212,24 @@ function packageLabel(packageData) { return `${packageData.name} ${packageData.version} (${target})`; } +async function writeCloudsmithSummary({core, owner, repository, dryRun, counts, progress, status, inventoryComplete}) { + await core.summary + .addHeading(`Cloudsmith ${owner}/${repository} cleanup`, 2) + .addTable([ + [{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}], + [{data: 'Status', header: true}, {data: status}], + [{data: 'Inventory scan', header: true}, {data: inventoryComplete ? 'Complete' : 'Partial'}], + [{data: 'Packages scanned', header: true}, {data: String(counts.scanned)}], + [{data: 'Packages retained', header: true}, {data: String(counts.retained)}], + [{data: 'Packages selected', header: true}, {data: String(counts.selected)}], + [{data: 'Delete operations processed', header: true}, {data: String(progress.operations)}], + [{data: 'Packages deleted', header: true}, {data: String(progress.deleted)}], + [{data: 'Known packages deferred', header: true}, {data: String(counts.selected - progress.operations)}], + [{data: 'Unknown package names', header: true}, {data: String(progress.unknownNames)}], + ]) + .write(); +} + /** * Clean a Cloudsmith repository against the organization's published releases. * @@ -225,61 +244,61 @@ export async function cleanupCloudsmith({ organization = 'LizardByte', owner = 'lizardbyte', repository = 'beta', + runtime = createCleanupRuntime(), }) { - const [packages, repositories] = await Promise.all([ - listCloudsmithPackages({owner, repository, token}), - loadRepositories(github, organization), - ]); - const knownNames = new Set(repositories.map(({name}) => name.toLowerCase())); - const matchedRepositories = new Set( - packages - .map(({name}) => repositories.find( + const counts = {scanned: 0, retained: 0, selected: 0}; + const progress = {operations: 0, deleted: 0, unknownNames: 0}; + let inventoryComplete = false; + let status = 'Completed'; + try { + await runtime.run(async () => { + const api = runtime.github(github); + const [packages, repositories] = await Promise.all([ + listCloudsmithPackages({owner, repository, token}, runtime.fetch), + loadRepositories(api, organization), + ]); + counts.scanned = packages.length; + const knownNames = new Set(repositories.map(({name}) => name.toLowerCase())); + progress.unknownNames = new Set(packages.map(({name}) => String(name).toLowerCase()) + .filter((name) => !knownNames.has(name))).size; + const matchedRepositories = new Set(packages.map(({name}) => repositories.find( (repositoryData) => repositoryData.name.toLowerCase() === String(name).toLowerCase(), - )?.name) - .filter(Boolean), - ); - const releaseTagsByRepository = await loadReleaseTags( - github, - organization, - [...matchedRepositories], - ); - const plan = planCloudsmithCleanup({packages, repositories, releaseTagsByRepository}); - - for (const {package: packageData, reason} of plan.keep) { - if (reason === 'unknown GitHub repository') { - core.warning(`Keeping ${packageLabel(packageData)}: ${reason}.`); - } - } - - await forEachSequential(plan.remove, async ({package: packageData}) => { - const identifier = packageData.identifier_perm ?? packageData.slug_perm; - if (dryRun) { - core.info(`[dry-run] Delete ${packageLabel(packageData)} (${identifier}).`); + )?.name).filter(Boolean)); + const releaseTagsByRepository = await loadReleaseTags(api, organization, [...matchedRepositories]); + runtime.check(); + const plan = planCloudsmithCleanup({packages, repositories, releaseTagsByRepository}); + counts.retained = plan.keep.length; + counts.selected = plan.remove.length; + inventoryComplete = true; + for (const {package: packageData, reason} of plan.keep) { + if (reason === 'unknown GitHub repository') { + core.warning(`Keeping ${packageLabel(packageData)}: ${reason}.`); + } + } + await forEachSequential(plan.remove, async ({package: packageData}) => { + runtime.check(); + const identifier = packageData.identifier_perm ?? packageData.slug_perm; + if (dryRun) { + core.info(`[dry-run] Delete ${packageLabel(packageData)} (${identifier}).`); + } else { + core.info(`Deleting ${packageLabel(packageData)} (${identifier}).`); + await deleteCloudsmithPackage({owner, repository, token, identifier}, runtime.fetch); + } + progress.operations += 1; + progress.deleted += dryRun ? 0 : 1; + }); + }); + } catch (error) { + if (error instanceof CleanupStopped) { + status = error.message; + core.warning(`Stopping Cloudsmith cleanup: ${status}`); } else { - core.info(`Deleting ${packageLabel(packageData)} (${identifier}).`); - await deleteCloudsmithPackage({owner, repository, token, identifier}); + status = `Failed: ${error.message}`; + throw error; } - }); - - const unknownNames = new Set( - packages - .map(({name}) => String(name).toLowerCase()) - .filter((name) => !knownNames.has(name)), - ); - await core.summary - .addHeading(`Cloudsmith ${owner}/${repository} cleanup`, 2) - .addTable([ - [{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}], - [{data: 'Packages scanned', header: true}, {data: String(packages.length)}], - [{data: 'Packages retained', header: true}, {data: String(plan.keep.length)}], - [{data: 'Packages selected', header: true}, {data: String(plan.remove.length)}], - [{data: 'Unknown package names', header: true}, {data: String(unknownNames.size)}], - ]) - .write(); - - return { - scanned: packages.length, - retained: plan.keep.length, - selected: plan.remove.length, - }; + } finally { + runtime.dispose(); + await writeCloudsmithSummary({core, owner, repository, dryRun, counts, progress, status, inventoryComplete}); + } + return counts; } diff --git a/.github/scripts/cleanup-containers.mjs b/.github/scripts/cleanup-containers.mjs index 38d55fef..ca133842 100644 --- a/.github/scripts/cleanup-containers.mjs +++ b/.github/scripts/cleanup-containers.mjs @@ -1,8 +1,10 @@ import {collectPages, forEachSequential} from './async-iteration.mjs'; +import {CleanupStopped, createCleanupRuntime} from './cleanup-runtime.mjs'; const DOCKER_HUB_API = 'https://hub.docker.com/v2'; const DOCKER_REGISTRY = 'https://registry-1.docker.io'; const GHCR_REGISTRY = 'https://ghcr.io'; +const GITHUB_API_RESERVE = 500; const DEFAULT_GRACE_MS = 24 * 60 * 60 * 1000; const MANIFEST_ACCEPT = [ 'application/vnd.oci.image.index.v1+json', @@ -207,12 +209,12 @@ async function fetchJson(url, options = {}, request = fetch) { return response.json(); } -async function dockerHubToken(username, accessToken) { +async function dockerHubToken(username, accessToken, request) { const response = await fetchJson(`${DOCKER_HUB_API}/auth/token`, { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({identifier: username, secret: accessToken}), - }); + }, request); const token = response.access_token ?? response.token; if (!token) { throw new Error('Docker Hub authentication did not return an access token.'); @@ -227,10 +229,10 @@ async function dockerHubToken(username, accessToken) { * @param {() => Promise} loadToken Docker Hub or scoped registry token loader. * @returns {Promise} Authenticated request function. */ -async function refreshingDockerRequest(loadToken) { +async function refreshingDockerRequest(loadToken, fetcher) { let token = await loadToken(); return async (url, options = {}) => { - const request = () => fetch(url, { + const request = () => fetcher(url, { ...options, headers: {...options.headers, Authorization: `Bearer ${token}`}, }); @@ -251,14 +253,14 @@ async function paginatedHubResults(url, request) { }); } -async function dockerRegistryToken({namespace, repository, username, accessToken}) { +async function dockerRegistryToken({namespace, repository, username, accessToken}, request) { const url = new URL('https://auth.docker.io/token'); url.searchParams.set('service', 'registry.docker.io'); url.searchParams.set('scope', `repository:${namespace}/${repository}:pull,push,delete`); const credentials = Buffer.from(`${username}:${accessToken}`).toString('base64'); const response = await fetchJson(url, { headers: {Authorization: `Basic ${credentials}`}, - }); + }, request); if (!response.token) { throw new Error(`Docker Registry authentication failed for ${namespace}/${repository}.`); } @@ -328,118 +330,135 @@ function deletionBudget(maxDeletions) { return maxDeletions; } -/** - * Clean every Docker Hub repository in a namespace which maps to a GitHub repo. - * - * @param {object} options Runtime dependencies and settings. - * @returns {Promise} Cleanup counts. - */ -export async function cleanupDockerHub({ - github, - core, - username, - accessToken, - dryRun, - maxDeletions, - organization = 'LizardByte', - namespace = 'lizardbyte', +async function cleanupHubRepository({ + github, core, namespace, organization, username, accessToken, dryRun, runtime, + hubRequest, hubRepository, githubRepositories, counts, progress, budget, }) { - let remaining = deletionBudget(maxDeletions); - const [hubRequest, repositories] = await Promise.all([ - refreshingDockerRequest(() => dockerHubToken(username, accessToken)), - loadRepositories(github, organization), + runtime.check(); + const githubRepository = githubRepositories.get(hubRepository.name.toLowerCase()); + if (!githubRepository) { + core.warning(`Skipping Docker Hub ${namespace}/${hubRepository.name}: no matching GitHub repository.`); + return; + } + const [tags, releaseTags] = await Promise.all([ + paginatedHubResults( + `${DOCKER_HUB_API}/namespaces/${namespace}/repositories/` + + `${encodeURIComponent(hubRepository.name)}/tags?page_size=100`, hubRequest, + ), + loadReleaseTags(github, organization, githubRepository), ]); - const githubRepositories = repositoryLookup(repositories); - const hubRepositories = await paginatedHubResults( - `${DOCKER_HUB_API}/namespaces/${namespace}/repositories?page_size=100`, - hubRequest, - ); - let scanned = 0; - let selectedTags = 0; - let deletedActions = 0; - let deferredActions = 0; - - // Keep repository budgets and registry mutations in deterministic order. - const sortedRepositories = hubRepositories.toSorted((a, b) => a.name.localeCompare(b.name)); - await forEachSequential(sortedRepositories, async (hubRepository) => { - const githubRepository = githubRepositories.get(hubRepository.name.toLowerCase()); - if (!githubRepository) { - core.warning(`Skipping Docker Hub ${namespace}/${hubRepository.name}: no matching GitHub repository.`); - return; + runtime.check(); + counts.scanned += tags.length; + const plan = planDockerHubCleanup(tags, releaseTags); + counts.selectedTags += plan.selectedTags; + counts.deferredActions += plan.actions.length; + const selectedActions = plan.actions.slice(0, Math.max(budget.remaining, 0)); + let registryRequest; + await forEachSequential(selectedActions, async (action) => { + runtime.check(); + const description = action.kind === 'manifest' + ? `${namespace}/${hubRepository.name}@${action.digest} (${action.tags.length} tags)` + : `${namespace}/${hubRepository.name}:${action.tag}`; + if (dryRun) { + core.info(`[dry-run] Delete ${description}.`); + } else if (action.kind === 'manifest') { + registryRequest ??= await refreshingDockerRequest(() => dockerRegistryToken({ + namespace, repository: hubRepository.name, username, accessToken, + }, runtime.fetch), runtime.fetch); + core.info(`Deleting ${description}.`); + await deleteDockerManifest({ + namespace, repository: hubRepository.name, digest: action.digest, request: registryRequest, + }); + } else { + core.info(`Deleting ${description}.`); + await deleteDockerHubTag({namespace, repository: hubRepository.name, tag: action.tag, request: hubRequest}); } - - const [tags, releaseTags] = await Promise.all([ - paginatedHubResults( - `${DOCKER_HUB_API}/namespaces/${namespace}/repositories/` - + `${encodeURIComponent(hubRepository.name)}/tags?page_size=100`, - hubRequest, - ), - loadReleaseTags(github, organization, githubRepository), - ]); - scanned += tags.length; - const plan = planDockerHubCleanup(tags, releaseTags); - selectedTags += plan.selectedTags; - const selectedActions = plan.actions.slice(0, Math.max(remaining, 0)); - deferredActions += plan.actions.length - selectedActions.length; - let registryRequest; - - await forEachSequential(selectedActions, async (action) => { - const description = action.kind === 'manifest' - ? `${namespace}/${hubRepository.name}@${action.digest} (${action.tags.length} tags)` - : `${namespace}/${hubRepository.name}:${action.tag}`; - if (dryRun) { - core.info(`[dry-run] Delete ${description}.`); - } else if (action.kind === 'manifest') { - registryRequest ??= await refreshingDockerRequest(() => dockerRegistryToken({ - namespace, - repository: hubRepository.name, - username, - accessToken, - })); - core.info(`Deleting ${description}.`); - await deleteDockerManifest({ - namespace, - repository: hubRepository.name, - digest: action.digest, - request: registryRequest, - }); - } else { - core.info(`Deleting ${description}.`); - await deleteDockerHubTag({ - namespace, - repository: hubRepository.name, - tag: action.tag, - request: hubRequest, - }); - } - deletedActions += 1; - remaining -= 1; - }); + counts.deletedActions += 1; + counts.deferredActions -= 1; + progress.deleted += dryRun ? 0 : 1; + progress.tagsDeleted += dryRun ? 0 : action.tags.length; + budget.remaining -= 1; }); +} +async function writeHubSummary({core, namespace, dryRun, counts, progress, status, inventoryComplete}) { await core.summary .addHeading(`Docker Hub ${namespace} cleanup`, 2) .addTable([ [{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}], - [{data: 'Repositories scanned', header: true}, {data: String(hubRepositories.length)}], - [{data: 'Tags scanned', header: true}, {data: String(scanned)}], - [{data: 'Tags selected', header: true}, {data: String(selectedTags)}], - [{data: 'Delete operations selected', header: true}, {data: String(deletedActions)}], - [{data: 'Operations deferred', header: true}, {data: String(deferredActions)}], + [{data: 'Status', header: true}, {data: status}], + [{data: 'Inventory scan', header: true}, {data: inventoryComplete ? 'Complete' : 'Partial'}], + [{data: 'Repositories listed', header: true}, {data: String(counts.repositories)}], + [{data: 'Tags scanned', header: true}, {data: String(counts.scanned)}], + [{data: 'Tags selected', header: true}, {data: String(counts.selectedTags)}], + [{data: 'Delete operations processed', header: true}, {data: String(counts.deletedActions)}], + [{data: 'Delete operations completed', header: true}, {data: String(progress.deleted)}], + [{data: 'Tags cleaned', header: true}, {data: String(progress.tagsDeleted)}], + [{data: 'Known operations deferred', header: true}, {data: String(counts.deferredActions)}], ]) .write(); +} - return {repositories: hubRepositories.length, scanned, selectedTags, deletedActions, deferredActions}; +/** + * Clean every Docker Hub repository in a namespace which maps to a GitHub repo. + * + * @param {object} options Runtime dependencies and settings. + * @returns {Promise} Cleanup counts. + */ +export async function cleanupDockerHub({ + github, core, username, accessToken, dryRun, maxDeletions, + organization = 'LizardByte', namespace = 'lizardbyte', runtime, +}) { + const budget = {remaining: deletionBudget(maxDeletions)}; + runtime ??= createCleanupRuntime(); + const counts = {repositories: 0, scanned: 0, selectedTags: 0, deletedActions: 0, deferredActions: 0}; + const progress = {deleted: 0, tagsDeleted: 0}; + let inventoryComplete = false; + let status = 'Completed'; + try { + await runtime.run(async () => { + const api = runtime.github(github); + const [hubRequest, repositories] = await Promise.all([ + refreshingDockerRequest(() => dockerHubToken(username, accessToken, runtime.fetch), runtime.fetch), + loadRepositories(api, organization), + ]); + const githubRepositories = repositoryLookup(repositories); + const hubRepositories = await paginatedHubResults( + `${DOCKER_HUB_API}/namespaces/${namespace}/repositories?page_size=100`, hubRequest, + ); + counts.repositories = hubRepositories.length; + // Keep repository budgets and registry mutations in deterministic order. + await forEachSequential(hubRepositories.toSorted((a, b) => a.name.localeCompare(b.name)), (hubRepository) => ( + cleanupHubRepository({ + github: api, core, namespace, organization, username, accessToken, dryRun, runtime, + hubRequest, hubRepository, githubRepositories, counts, progress, budget, + }) + )); + inventoryComplete = true; + }); + } catch (error) { + if (error instanceof CleanupStopped) { + status = error.message; + core.warning(`Stopping Docker Hub cleanup: ${status}`); + } else { + status = `Failed: ${error.message}`; + throw error; + } + } finally { + runtime.dispose(); + await writeHubSummary({core, namespace, dryRun, counts, progress, status, inventoryComplete}); + } + return counts; } -async function ghcrToken({namespace, packageName, username, accessToken}) { +async function ghcrToken({namespace, packageName, username, accessToken}, request) { const url = new URL(`${GHCR_REGISTRY}/token`); url.searchParams.set('service', 'ghcr.io'); url.searchParams.set('scope', `repository:${namespace}/${packageName}:pull`); const credentials = Buffer.from(`${username}:${accessToken}`).toString('base64'); const response = await fetchJson(url, { headers: {Authorization: `Basic ${credentials}`}, - }); + }, request); const token = response.token ?? response.access_token; if (!token) { throw new Error(`GHCR authentication failed for ${namespace}/${packageName}.`); @@ -447,14 +466,95 @@ async function ghcrToken({namespace, packageName, username, accessToken}) { return token; } -async function getGhcrManifest({namespace, packageName, digest, token}) { +async function getGhcrManifest({namespace, packageName, digest, token}, request) { const packagePath = packageName.split('/').map(encodeURIComponent).join('/'); return fetchJson(`${GHCR_REGISTRY}/v2/${namespace}/${packagePath}/manifests/${digest}`, { headers: { Accept: MANIFEST_ACCEPT, Authorization: `Bearer ${token}`, }, - }); + }, request); +} + +class GitHubCleanupStopped extends CleanupStopped {} + +function githubCallsRemaining(headers) { + const value = headers?.['x-ratelimit-remaining']; + const remaining = Number(value); + return value !== undefined && value !== null && String(value).trim() !== '' + && Number.isInteger(remaining) && remaining >= 0 + ? remaining : undefined; +} + +function isGitHubRateLimitError(error) { + const status = error.status ?? error.response?.status; + const headers = error.response?.headers; + const message = `${error.message ?? ''} ${error.response?.data?.message ?? ''}`; + return (status === 403 || status === 429) && ( + status === 429 + || githubCallsRemaining(headers) === 0 + || headers?.['retry-after'] !== undefined + || /rate limit|abuse detection/i.test(message) + ); +} + +/** + * Reserve GitHub API calls, including every page of inventory and release reads. + * Stop pagination rather than returning an incomplete inventory for deletion planning. + * + * @param {object} github GitHub Actions Octokit client. + * @returns {Promise} Guarded client with the last known remaining call count. + */ +async function cleanupGitHubApi(github) { + // This endpoint does not consume the primary rate limit. + const response = await github.request('GET /rate_limit'); + let remaining = githubCallsRemaining(response.headers) ?? response.data?.resources?.core?.remaining; + if (!Number.isInteger(remaining) || remaining < 0) { + throw new GitHubCleanupStopped('Cannot determine the remaining GitHub API budget.'); + } + + const checkReserve = () => { + if (remaining <= GITHUB_API_RESERVE) { + throw new GitHubCleanupStopped( + `GitHub API budget is at or below the ${GITHUB_API_RESERVE}-call reserve (${remaining} remaining).`, + ); + } + }; + const observe = (headers) => { + remaining = githubCallsRemaining(headers) ?? Math.max(remaining - 1, 0); + }; + const run = async (operation) => { + checkReserve(); + try { + return await operation(); + } catch (error) { + if (isGitHubRateLimitError(error)) { + observe(error.response?.headers); + throw new GitHubCleanupStopped( + 'GitHub API rate limit reached; remaining cleanup is deferred.', {cause: error}, + ); + } + throw error; + } + }; + + return { + rest: github.rest, + get remaining() { return remaining; }, + paginate: (route, options) => run(() => github.paginate(route, options, (page) => { + observe(page.headers); + // Check between pages so a long inventory scan cannot consume the reserve. + if (page.headers?.link?.includes('rel="next"')) { + checkReserve(); + } + return page.data; + })), + request: (route, options) => run(async () => { + const result = await github.request(route, options); + observe(result.headers); + return result; + }), + }; } async function deleteGhcrVersion({github, organization, packageName, versionId}) { @@ -477,9 +577,12 @@ async function deleteSelectedGhcrVersions({ packageName, selectedVersions, dryRun, + onProcessed, + runtime, }) { // Tagged parent versions must be deleted before their orphaned children. await forEachSequential(selectedVersions, async (version) => { + runtime.check(); const tags = versionTags(version); const description = tags.length > 0 ? `${namespace}/${packageName}:${tags.join(',')}` @@ -495,9 +598,86 @@ async function deleteSelectedGhcrVersions({ versionId: version.id, }); } + // Record only confirmed deletions (or dry-run selections), even if the next request stops cleanup. + onProcessed(); }); } +async function cleanupGhcrPackage({ + github, core, organization, namespace, packageData, githubRepositories, username, accessToken, + dryRun, counts, budget, runtime, +}) { + runtime.check(); + const githubRepository = githubRepositories.get(packageData.name.toLowerCase()); + if (!githubRepository) { + core.warning(`Skipping GHCR ${namespace}/${packageData.name}: no matching GitHub repository.`); + return; + } + + // Serialize GitHub calls so each request checks the budget from the preceding response. + const versions = await github.paginate( + 'GET /orgs/{org}/packages/{package_type}/{package_name}/versions', + {org: organization, package_type: 'container', package_name: packageData.name, per_page: 100}, + ); + const releaseTags = await loadReleaseTags(github, organization, githubRepository); + counts.scanned += versions.length; + const classification = classifyGhcrVersions(versions, releaseTags); + let reachable; + try { + const registryToken = await ghcrToken({ + namespace, packageName: packageData.name, username, accessToken, + }, runtime.fetch); + reachable = await resolveReachableDigests( + classification.roots.map(({name}) => name), + (digest) => getGhcrManifest({ + namespace, packageName: packageData.name, digest, token: registryToken, + }, runtime.fetch), + ); + } catch (error) { + runtime.check(); + if (error instanceof CleanupStopped) { + throw error; + } + core.warning( + `Skipping GHCR ${namespace}/${packageData.name}: retained manifest graph could not be read: ` + error.message, + ); + return; + } + + const candidates = ghcrCleanupCandidates(classification, reachable); + counts.selected += candidates.length; + counts.deferred += candidates.length; + const selectedVersions = candidates.slice(0, Math.max(budget.remaining, 0)); + await deleteSelectedGhcrVersions({ + github, core, organization, namespace, packageName: packageData.name, selectedVersions, dryRun, runtime, + onProcessed() { + counts.operations += 1; + counts.deleted += dryRun ? 0 : 1; + counts.deferred -= 1; + budget.remaining -= 1; + }, + }); +} + +async function writeGhcrSummary({core, namespace, dryRun, counts, status, inventoryComplete, apiRemaining}) { + await core.summary + .addHeading(`GHCR ${namespace} cleanup`, 2) + .addTable([ + [{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}], + [{data: 'Status', header: true}, {data: status}], + [{data: 'Inventory scan', header: true}, {data: inventoryComplete ? 'Complete' : 'Partial'}], + [{data: 'Packages listed', header: true}, {data: String(counts.packages)}], + [{data: 'Versions scanned', header: true}, {data: String(counts.scanned)}], + [{data: 'Versions selected', header: true}, {data: String(counts.selected)}], + [{data: 'Delete operations processed', header: true}, {data: String(counts.operations)}], + [{data: 'Versions deleted', header: true}, {data: String(counts.deleted)}], + [{data: 'Known versions deferred', header: true}, {data: String(counts.deferred)}], + [{data: 'GitHub API calls remaining', header: true}, {data: String(apiRemaining ?? 'Unknown')}], + [{data: 'GitHub API reserve target', header: true}, {data: String(GITHUB_API_RESERVE)}], + ]) + .write(); +} + /** * Clean GHCR package versions while preserving retained multi-arch graphs. * @@ -513,98 +693,46 @@ export async function cleanupGhcr({ maxDeletions, organization = 'LizardByte', namespace = 'lizardbyte', + runtime, }) { - let remaining = deletionBudget(maxDeletions); - const [packages, repositories] = await Promise.all([ - github.paginate('GET /orgs/{org}/packages', { - org: organization, - package_type: 'container', - per_page: 100, - }), - loadRepositories(github, organization), - ]); - const githubRepositories = repositoryLookup(repositories); - let scanned = 0; - let selected = 0; - let operations = 0; - let deferred = 0; - - // Finish each package before consuming the shared deletion budget for the next. - await forEachSequential(packages.sort((a, b) => a.name.localeCompare(b.name)), async (packageData) => { - const githubRepository = githubRepositories.get(packageData.name.toLowerCase()); - if (!githubRepository) { - core.warning(`Skipping GHCR ${namespace}/${packageData.name}: no matching GitHub repository.`); - return; - } - - const [versions, releaseTags] = await Promise.all([ - github.paginate( - 'GET /orgs/{org}/packages/{package_type}/{package_name}/versions', - { - org: organization, - package_type: 'container', - package_name: packageData.name, - per_page: 100, - }, - ), - loadReleaseTags(github, organization, githubRepository), - ]); - scanned += versions.length; - const classification = classifyGhcrVersions(versions, releaseTags); - let reachable; - try { - const registryToken = await ghcrToken({ - namespace, - packageName: packageData.name, - username, - accessToken, + const budget = {remaining: deletionBudget(maxDeletions)}; + runtime ??= createCleanupRuntime(); + const counts = {packages: 0, scanned: 0, selected: 0, operations: 0, deleted: 0, deferred: 0}; + let api; + let apiRemaining; + let inventoryComplete = false; + let status = 'Completed'; + try { + await runtime.run(async () => { + api = await cleanupGitHubApi(runtime.github(github)); + const packages = await api.paginate('GET /orgs/{org}/packages', { + org: organization, package_type: 'container', per_page: 100, }); - reachable = await resolveReachableDigests( - classification.roots.map(({name}) => name), - (digest) => getGhcrManifest({ - namespace, - packageName: packageData.name, - digest, - token: registryToken, - }), - ); - } catch (error) { - core.warning( - `Skipping GHCR ${namespace}/${packageData.name}: retained manifest graph could not be read: ` - + error.message, - ); - return; + counts.packages = packages.length; + const githubRepositories = repositoryLookup(await loadRepositories(api, organization)); + // Finish each package before consuming the shared deletion and API budgets for the next. + await forEachSequential(packages.toSorted((a, b) => a.name.localeCompare(b.name)), (packageData) => ( + cleanupGhcrPackage({ + github: api, core, organization, namespace, packageData, githubRepositories, username, accessToken, + dryRun, counts, budget, runtime, + }) + )); + inventoryComplete = true; + }); + } catch (error) { + if (error instanceof CleanupStopped || isGitHubRateLimitError(error)) { + status = error instanceof CleanupStopped ? error.message : 'GitHub API rate limit reached.'; + apiRemaining = githubCallsRemaining(error.response?.headers); + core.warning(`Stopping GHCR cleanup: ${status}`); + } else { + status = `Failed: ${error.message}`; + throw error; } - - const candidates = ghcrCleanupCandidates(classification, reachable); - selected += candidates.length; - const selectedVersions = candidates.slice(0, Math.max(remaining, 0)); - deferred += candidates.length - selectedVersions.length; - - await deleteSelectedGhcrVersions({ - github, - core, - organization, - namespace, - packageName: packageData.name, - selectedVersions, - dryRun, + } finally { + runtime.dispose(); + await writeGhcrSummary({ + core, namespace, dryRun, counts, status, inventoryComplete, apiRemaining: apiRemaining ?? api?.remaining, }); - operations += selectedVersions.length; - remaining -= selectedVersions.length; - }); - - await core.summary - .addHeading(`GHCR ${namespace} cleanup`, 2) - .addTable([ - [{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}], - [{data: 'Packages scanned', header: true}, {data: String(packages.length)}], - [{data: 'Versions scanned', header: true}, {data: String(scanned)}], - [{data: 'Versions selected', header: true}, {data: String(selected)}], - [{data: 'Delete operations selected', header: true}, {data: String(operations)}], - [{data: 'Versions deferred', header: true}, {data: String(deferred)}], - ]) - .write(); - - return {packages: packages.length, scanned, selected, operations, deferred}; + } + return counts; } diff --git a/.github/scripts/cleanup-runtime.mjs b/.github/scripts/cleanup-runtime.mjs new file mode 100644 index 00000000..674fc812 --- /dev/null +++ b/.github/scripts/cleanup-runtime.mjs @@ -0,0 +1,65 @@ +const DEFAULT_MAX_RUNTIME_MS = 5.5 * 60 * 60 * 1000; + +/** A cleanup budget was reached; unfinished work can wait until the next run. */ +export class CleanupStopped extends Error {} + +/** + * Stop cleanup after 5.5 hours, leaving 30 minutes for the job summary and shutdown. + * Bound pending requests as well as the time between sequential operations. + * + * @param {object} [options] Time budget and clock dependencies. + * @returns {object} Deadline checks and bounded HTTP/GitHub clients. + */ +export function createCleanupRuntime({maxRuntimeMs = DEFAULT_MAX_RUNTIME_MS, now = () => performance.now()} = {}) { + const deadline = now() + maxRuntimeMs; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), maxRuntimeMs); + timer.unref(); + const stopped = () => new CleanupStopped('Cleanup time budget reached; remaining cleanup is deferred.'); + const check = () => { + if (now() >= deadline) { + controller.abort(); + } + if (controller.signal.aborted) { + throw stopped(); + } + }; + const run = async (operation) => { + check(); + let onAbort; + const aborted = new Promise((_, reject) => { + onAbort = () => reject(stopped()); + controller.signal.addEventListener('abort', onAbort, {once: true}); + }); + try { + // Do not check after success: a confirmed deletion must still be counted at the deadline. + return await Promise.race([operation(), aborted]); + } finally { + controller.signal.removeEventListener('abort', onAbort); + } + }; + const requestOptions = (options = {}) => ({ + ...options, + request: {...options.request, signal: controller.signal}, + }); + return { + check, + run, + dispose() { + clearTimeout(timer); + controller.abort(); + }, + fetch: (url, options = {}) => run(() => fetch(url, { + ...options, + signal: options.signal ? AbortSignal.any([controller.signal, options.signal]) : controller.signal, + })), + github: (github) => ({ + rest: github.rest, + request: (route, options) => run(() => github.request(route, requestOptions(options))), + paginate: (route, options, mapPage) => run(() => github.paginate(route, requestOptions(options), (page, done) => { + check(); + return mapPage ? mapPage(page, done) : page.data; + })), + }), + }; +} diff --git a/.github/workflows/__cleanup-cloudsmith.yml b/.github/workflows/__cleanup-cloudsmith.yml index 503a3711..c973c606 100644 --- a/.github/workflows/__cleanup-cloudsmith.yml +++ b/.github/workflows/__cleanup-cloudsmith.yml @@ -27,6 +27,7 @@ jobs: permissions: contents: read runs-on: ubuntu-latest + timeout-minutes: 360 # Cleanup stops after 330 minutes to leave time for its summary. steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/__cleanup-containers.yml b/.github/workflows/__cleanup-containers.yml index 71fbc00e..44fc4b26 100644 --- a/.github/workflows/__cleanup-containers.yml +++ b/.github/workflows/__cleanup-containers.yml @@ -40,6 +40,7 @@ jobs: contents: read packages: write runs-on: ubuntu-latest + timeout-minutes: 360 # Cleanup stops after 330 minutes to leave time for its summary. strategy: fail-fast: false matrix: diff --git a/.github/workflows/__cleanup.yml b/.github/workflows/__cleanup.yml index 0a30dce0..70b27cf0 100644 --- a/.github/workflows/__cleanup.yml +++ b/.github/workflows/__cleanup.yml @@ -14,6 +14,11 @@ on: required: false type: boolean default: true + max_deletions: + description: Maximum container delete operations per registry; 0 means no limit + required: false + type: number + default: 0 concurrency: group: cleanup-artifacts @@ -43,6 +48,7 @@ jobs: docker_hub_username: ${{ vars.DOCKER_HUB_USERNAME }} dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} gh_bot_name: ${{ vars.GH_BOT_NAME }} + max_deletions: ${{ inputs.max_deletions || 0 }} secrets: DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} GH_BOT_TOKEN: ${{ secrets.GH_BOT_TOKEN }} diff --git a/tests/cleanup-cloudsmith.test.mjs b/tests/cleanup-cloudsmith.test.mjs index 51926450..da4700af 100644 --- a/tests/cleanup-cloudsmith.test.mjs +++ b/tests/cleanup-cloudsmith.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import {setImmediate} from 'node:timers/promises'; import test from 'node:test'; +import {createCleanupRuntime} from '../.github/scripts/cleanup-runtime.mjs'; import { cleanupCloudsmith, @@ -121,6 +122,16 @@ test('Cloudsmith cleanup collects all pages and processes releases and deletions assert.deepEqual(result, {scanned: 102, retained: 100, selected: 2}); }); +function summaryCore() { + const summary = { + rows: [], writes: 0, + addHeading() { return this; }, + addTable(rows) { this.rows = rows; return this; }, + async write() { this.writes += 1; }, + }; + return {info() {}, warning() {}, summary}; +} + test('Cloudsmith cleanup stops before deletion if loading release tags fails', async (t) => { t.mock.method(globalThis, 'fetch', async (input, options) => { assert.notEqual(options.method, 'DELETE'); @@ -141,6 +152,116 @@ test('Cloudsmith cleanup stops before deletion if loading release tags fails', a throw failure; }, }; - await assert.rejects(cleanupCloudsmith({github, core: {}, token: 'token', dryRun: false}), failure); + const core = summaryCore(); + await assert.rejects(cleanupCloudsmith({github, core, token: 'token', dryRun: false}), failure); assert.deepEqual(releases, ['Alpha']); + assert.equal(core.summary.writes, 1); + assert.equal(Object.fromEntries(core.summary.rows.map(([label, value]) => [label.data, value.data])) + ['Packages deleted'], '0'); +}); + +function cleanupGithub() { + return { + rest: {repos: {listForOrg: 'repos', listReleases: 'releases'}}, + async paginate(route) { return route === 'repos' ? [{name: 'Alpha'}] : []; }, + }; +} + +function summaryRows(core) { + assert.equal(core.summary.writes, 1); + return Object.fromEntries(core.summary.rows.map(([key, value]) => [key.data, value.data])); +} + +for (const dryRun of [false, true]) { + test(`Cloudsmith writes progress when its deadline stops deletions (dryRun=${dryRun})`, async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + const core = summaryCore(); + if (dryRun) { + core.info = () => { now = 1000; }; + } + const deleted = []; + t.mock.method(globalThis, 'fetch', async (input, options) => { + if (options.method === 'DELETE') { + deleted.push(new URL(input).pathname.split('/').at(-2)); + now = 1000; + return new Response(null, {status: 204}); + } + return Response.json([1, 2, 3].map((id) => ({name: 'alpha', version: 'old', identifier_perm: `stale-${id}`}))); + }); + const result = await cleanupCloudsmith({github: cleanupGithub(), core, token: 'token', dryRun, runtime}); + assert.deepEqual(result, {scanned: 3, retained: 0, selected: 3}); + assert.deepEqual(deleted, dryRun ? [] : ['stale-1']); + const summary = summaryRows(core); + assert.match(summary.Status, /time budget reached/); + assert.equal(summary['Delete operations processed'], '1'); + assert.equal(summary['Packages deleted'], dryRun ? '0' : '1'); + assert.equal(summary['Known packages deferred'], '2'); + }); +} + +test('Cloudsmith never deletes from an inventory interrupted by its deadline', async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + const core = summaryCore(); + let pages = 0; + t.mock.method(globalThis, 'fetch', async (input, options) => { + assert.notEqual(options.method, 'DELETE'); + pages += 1; + now = 1000; + return Response.json(Array.from({length: 100}, (_, id) => ({ + name: 'alpha', version: 'old', identifier_perm: `stale-${id}`, + })), {headers: {'x-pagination-pagetotal': '2'}}); + }); + const result = await cleanupCloudsmith({github: cleanupGithub(), core, token: 'token', dryRun: false, runtime}); + assert.equal(result.selected, 0); + assert.equal(pages, 1); + const summary = summaryRows(core); + assert.match(summary.Status, /time budget reached/); + assert.equal(summary['Inventory scan'], 'Partial'); + assert.equal(summary['Packages deleted'], '0'); +}); + +test('Cloudsmith reports prior deletions when a pending delete stalls until the deadline', async (t) => { + const runtime = createCleanupRuntime({maxRuntimeMs: 100}); + const keepAlive = setTimeout(() => {}, 1000); + t.after(() => clearTimeout(keepAlive)); + const core = summaryCore(); + let attempts = 0; + let stalledSignal; + t.mock.method(globalThis, 'fetch', async (input, options) => { + if (options.method === 'DELETE') { + attempts += 1; + if (attempts === 2) { + stalledSignal = options.signal; + return new Promise(() => {}); + } + return new Response(null, {status: 204}); + } + return Response.json([1, 2, 3].map((id) => ({name: 'alpha', version: 'old', identifier_perm: `stale-${id}`}))); + }); + await cleanupCloudsmith({github: cleanupGithub(), core, token: 'token', dryRun: false, runtime}); + assert.equal(attempts, 2); + assert.equal(stalledSignal.aborted, true); + const summary = summaryRows(core); + assert.equal(summary['Packages deleted'], '1'); + assert.equal(summary['Known packages deferred'], '2'); +}); + +test('Cloudsmith writes successful deletion counts before propagating an unrelated failure', async (t) => { + const core = summaryCore(); + let attempts = 0; + t.mock.method(globalThis, 'fetch', async (input, options) => { + if (options.method === 'DELETE') { + attempts += 1; + return attempts === 1 ? new Response(null, {status: 204}) : new Response('server failure', {status: 500}); + } + return Response.json([1, 2, 3].map((id) => ({name: 'alpha', version: 'old', identifier_perm: `stale-${id}`}))); + }); + await assert.rejects(cleanupCloudsmith({github: cleanupGithub(), core, token: 'token', dryRun: false}), /500/); + assert.equal(attempts, 2); + const summary = summaryRows(core); + assert.match(summary.Status, /^Failed:/); + assert.equal(summary['Packages deleted'], '1'); + assert.equal(summary['Known packages deferred'], '2'); }); diff --git a/tests/cleanup-containers.test.mjs b/tests/cleanup-containers.test.mjs index 03b8ab00..85a4453f 100644 --- a/tests/cleanup-containers.test.mjs +++ b/tests/cleanup-containers.test.mjs @@ -2,6 +2,8 @@ import assert from 'node:assert/strict'; import {setImmediate} from 'node:timers/promises'; import test from 'node:test'; +import {forEachSequential} from '../.github/scripts/async-iteration.mjs'; +import {createCleanupRuntime} from '../.github/scripts/cleanup-runtime.mjs'; import { classifyGhcrVersions, cleanupDockerHub, @@ -126,13 +128,35 @@ test('resolveReachableDigests reads cyclic and duplicate digests only once', asy function summaryCore() { const summary = { + rows: [], + writes: 0, addHeading() { return this; }, - addTable() { return this; }, - async write() {}, + addTable(rows) { this.rows = rows; return this; }, + async write() { this.writes += 1; }, }; return {info() {}, warning() {}, summary}; } +function githubResponses(github) { + let remaining = 5000; + return { + ...github, + async paginate(route, options, mapPage) { + const data = await github.paginate(route, options); + remaining -= 1; + return mapPage({data, headers: {'x-ratelimit-remaining': String(remaining)}}); + }, + async request(route, options) { + if (route === 'GET /rate_limit') { + return {data: {resources: {core: {remaining}}}}; + } + await github.request(route, options); + remaining -= 1; + return {status: 204, headers: {'x-ratelimit-remaining': String(remaining)}}; + }, + }; +} + function cleanupGithub(repositories = ['Alpha'], releases = []) { return { rest: {repos: {listForOrg: 'repos', listReleases: 'releases'}}, @@ -431,10 +455,10 @@ test('GHCR cleans legacy themerr-plex versions against Themerr releases and pres }; const result = await cleanupGhcr({ - github, core: summaryCore(), username: 'user', accessToken: 'token', dryRun: false, + github: githubResponses(github), core: summaryCore(), username: 'user', accessToken: 'token', dryRun: false, }); assert.deepEqual(deleted, ['stale', 'orphan']); - assert.deepEqual(result, {packages: 1, scanned: 4, selected: 2, operations: 2, deferred: 0}); + assert.deepEqual(result, {packages: 1, scanned: 4, selected: 2, operations: 2, deleted: 2, deferred: 0}); }); for (const dryRun of [false, true]) { @@ -521,11 +545,13 @@ for (const maxDeletions of [3, undefined, 0]) { }, }; const result = await cleanupGhcr({ - github, core: summaryCore(), username: 'user', accessToken: 'token', dryRun, maxDeletions, + github: githubResponses(github), core: summaryCore(), username: 'user', accessToken: 'token', + dryRun, maxDeletions, }); const limited = maxDeletions === 3; assert.deepEqual(result, { - packages: 2, scanned: 4, selected: 4, operations: limited ? 3 : 4, deferred: limited ? 1 : 0, + packages: 2, scanned: 4, selected: 4, operations: limited ? 3 : 4, + deleted: dryRun ? 0 : (limited ? 3 : 4), deferred: limited ? 1 : 0, }); const expectedEvents = ['start alpha/parent', 'end alpha/parent', 'start alpha/child', 'end alpha/child', 'start beta/parent', 'end beta/parent']; @@ -561,8 +587,440 @@ test('GHCR makes no deletions when a retained manifest cannot be read', async (t async request() { assert.fail('Unsafe deletion'); }, }; const result = await cleanupGhcr({ - github, core: summaryCore(), username: 'user', accessToken: 'token', dryRun: false, maxDeletions: 3, + github: githubResponses(github), core: summaryCore(), username: 'user', accessToken: 'token', + dryRun: false, maxDeletions: 3, }); assert.equal(result.operations, 0); assert.equal(result.selected, 0); }); + +function ghcrRateFixture(t, {remaining = 1000, versionPageSize = 100, onResponse, onRegistryResponse} = {}) { + const calls = []; + const deleted = []; + const manifests = []; + const core = summaryCore(); + const warnings = []; + core.warning = (message) => warnings.push(message); + t.mock.method(globalThis, 'fetch', async (input) => { + const url = new URL(input); + assert.equal(url.hostname, 'ghcr.io'); + if (url.pathname === '/token') { + return Response.json({token: 'registry-token'}); + } + manifests.push(url.pathname); + const response = Response.json(url.pathname.endsWith('/sha256:root') + ? {manifests: [{digest: 'sha256:child'}]} : {}); + return onRegistryResponse?.(url, response) ?? response; + }); + + const versions = [ + {id: 'root', name: 'sha256:root', metadata: {container: {tags: ['v1.0.0']}}}, + {id: 'child', name: 'sha256:child', metadata: {container: {tags: []}}}, + {id: 'parent-1', name: 'sha256:parent-1', metadata: {container: {tags: ['old-1']}}}, + {id: 'parent-2', name: 'sha256:parent-2', metadata: {container: {tags: ['old-2']}}}, + {id: 'orphan', name: 'sha256:orphan', metadata: {container: {tags: []}}}, + ]; + const respond = (call, data, headers = {}) => { + calls.push(call); + if (call.route !== 'GET /rate_limit') { + remaining -= 1; + } + const response = { + status: call.route.startsWith('DELETE') ? 204 : 200, data, + headers: {'x-ratelimit-remaining': String(remaining), ...headers}, + }; + return onResponse?.(call, response) ?? response; + }; + const github = { + rest: {repos: {listForOrg: 'repos', listReleases: 'releases'}}, + async paginate(route, options, mapPage) { + const data = route === 'repos' ? [{name: 'Alpha'}, {name: 'Beta'}] + : route === 'releases' ? [{tag_name: 'v1.0.0', draft: false}] + : options.package_name ? versions : [{name: 'alpha'}, {name: 'beta'}]; + const pageSize = options.package_name ? versionPageSize : 100; + const pages = Array.from({length: Math.ceil(data.length / pageSize)}, (_, index) => index + 1); + const results = []; + // A page callback failure must prevent the next HTTP request, as it does in Octokit. + await forEachSequential(pages, async (page) => { + await setImmediate(); + const headers = page < pages.length ? {link: '; rel="next"'} : {}; + const response = respond({route, options, page}, data.slice((page - 1) * pageSize, page * pageSize), headers); + results.push(...mapPage(response)); + }); + return results; + }, + async request(route, options = {}) { + const data = route === 'GET /rate_limit' ? {resources: {core: {remaining}}} : undefined; + const response = await respond({route, options}, data); + if (route.startsWith('DELETE')) { + deleted.push(`${options.package_name}/${options.package_version_id}`); + } + return response; + }, + }; + return { + calls, deleted, manifests, core, warnings, + run: (options = {}) => cleanupGhcr({ + github, core, username: 'user', accessToken: 'token', dryRun: false, ...options, + }), + }; +} + +function ghcrSummary(fixture) { + assert.equal(fixture.core.summary.writes, 1); + return Object.fromEntries(fixture.core.summary.rows.map(([key, value]) => [key.data, value.data])); +} + +function githubFailure(status, headers, message) { + return Object.assign(new Error(message), {status, response: {status, headers, data: {message}}}); +} + +for (const remaining of [0, 499, 500]) { + test(`GHCR makes no charged API calls when ${remaining} remain`, async (t) => { + const fixture = ghcrRateFixture(t, {remaining}); + const result = await fixture.run(); + assert.deepEqual(result, {packages: 0, scanned: 0, selected: 0, operations: 0, deleted: 0, deferred: 0}); + assert.deepEqual(fixture.calls.map(({route}) => route), ['GET /rate_limit']); + assert.deepEqual(fixture.deleted, []); + assert.deepEqual(fixture.manifests, []); + const summary = ghcrSummary(fixture); + assert.match(summary.Status, /500-call reserve/); + assert.equal(summary['GitHub API calls remaining'], String(remaining)); + assert.equal(summary['Versions deleted'], '0'); + assert.equal(summary['Inventory scan'], 'Partial'); + }); +} + +test('GHCR reserves exactly 500 calls and summarizes the deletion completed before stopping', async (t) => { + const fixture = ghcrRateFixture(t, {remaining: 505}); + const result = await fixture.run(); + assert.deepEqual(result, {packages: 2, scanned: 5, selected: 3, operations: 1, deleted: 1, deferred: 2}); + assert.deepEqual(fixture.deleted, ['alpha/parent-1']); + assert.equal(fixture.calls.length, 6); // One free budget query, four metadata reads, and one deletion. + assert.ok(!fixture.calls.some(({options}) => options.package_name === 'beta')); + const summary = ghcrSummary(fixture); + assert.equal(summary['Versions deleted'], '1'); + assert.equal(summary['Known versions deferred'], '2'); + assert.equal(summary['GitHub API calls remaining'], '500'); + assert.equal(summary['GitHub API reserve target'], '500'); + assert.equal(summary['Inventory scan'], 'Partial'); + assert.equal(fixture.warnings.length, 1); +}); + +test('GHCR stops between inventory pages and never plans deletion from a partial inventory', async (t) => { + const fixture = ghcrRateFixture(t, {remaining: 504, versionPageSize: 2}); + const result = await fixture.run(); + assert.deepEqual(result, {packages: 2, scanned: 0, selected: 0, operations: 0, deleted: 0, deferred: 0}); + assert.deepEqual(fixture.calls.filter(({options}) => options.package_name).map(({page}) => page), [1, 2]); + assert.ok(!fixture.calls.some(({route}) => route === 'releases')); + assert.deepEqual(fixture.deleted, []); + assert.deepEqual(fixture.manifests, []); + assert.equal(ghcrSummary(fixture)['GitHub API calls remaining'], '500'); +}); + +const rateLimitFailures = [ + [403, {'x-ratelimit-remaining': '0'}, 'API rate limit exceeded'], + [403, {'x-ratelimit-remaining': '995', 'retry-after': '60'}, 'Request throttled'], + [403, {'x-ratelimit-remaining': '995'}, 'You have exceeded a secondary rate limit'], + [403, {'x-ratelimit-remaining': '995'}, 'Forbidden', 'You have exceeded a secondary rate limit'], + [429, {'x-ratelimit-remaining': '995'}, 'Too many requests'], +]; +for (const [status, headers, message, apiMessage = message] of rateLimitFailures) { + test(`GHCR stops cleanly on ${status} ${message} and counts only successful deletions`, async (t) => { + const fixture = ghcrRateFixture(t, { + onResponse(call, response) { + if (call.options.package_version_id === 'parent-2') { + const failure = githubFailure(status, headers, message); + failure.response.data.message = apiMessage; + throw failure; + } + return response; + }, + }); + const result = await fixture.run(); + assert.deepEqual(result, {packages: 2, scanned: 5, selected: 3, operations: 1, deleted: 1, deferred: 2}); + assert.deepEqual(fixture.deleted, ['alpha/parent-1']); + assert.equal(fixture.calls.filter(({route}) => route.startsWith('DELETE')).length, 2); + assert.ok(!fixture.calls.some(({options}) => options.package_name === 'beta')); + const summary = ghcrSummary(fixture); + assert.match(summary.Status, /rate limit reached/); + assert.equal(summary['Versions deleted'], '1'); + assert.equal(summary['Known versions deferred'], '2'); + assert.equal(summary['GitHub API calls remaining'], headers['x-ratelimit-remaining']); + assert.equal(fixture.warnings.length, 1); + }); +} + +test('GHCR preserves prior deletion counts if a later package inventory is rate limited', async (t) => { + const fixture = ghcrRateFixture(t, { + versionPageSize: 2, + onResponse(call, response) { + if (call.options.package_name === 'beta' && call.page === 2) { + throw githubFailure(403, {'x-ratelimit-remaining': '0'}, 'API rate limit exceeded'); + } + return response; + }, + }); + const result = await fixture.run(); + assert.deepEqual(result, {packages: 2, scanned: 5, selected: 3, operations: 3, deleted: 3, deferred: 0}); + assert.deepEqual(fixture.deleted, ['alpha/parent-1', 'alpha/parent-2', 'alpha/orphan']); + const summary = ghcrSummary(fixture); + assert.equal(summary['Versions deleted'], '3'); + assert.equal(summary['Inventory scan'], 'Partial'); + assert.equal(summary['GitHub API calls remaining'], '0'); +}); + +test('GHCR dry runs use the reserve guard and report zero actual deletions', async (t) => { + const fixture = ghcrRateFixture(t, {remaining: 505}); + const result = await fixture.run({dryRun: true}); + assert.deepEqual(result, {packages: 2, scanned: 5, selected: 3, operations: 3, deleted: 0, deferred: 0}); + assert.deepEqual(fixture.deleted, []); + const summary = ghcrSummary(fixture); + assert.equal(summary.Mode, 'Dry run'); + assert.equal(summary['Versions deleted'], '0'); + assert.equal(summary['Delete operations processed'], '3'); + assert.equal(summary['GitHub API calls remaining'], '500'); +}); + +for (const status of [403, 500]) { + test(`GHCR preserves its summary but still fails for an unrelated ${status} error`, async (t) => { + const failure = githubFailure(status, {'x-ratelimit-remaining': '995'}, 'Resource not accessible'); + const fixture = ghcrRateFixture(t, { + onResponse(call, response) { + if (call.options.package_version_id === 'parent-2') { + throw failure; + } + return response; + }, + }); + await assert.rejects(fixture.run(), (error) => error === failure); + assert.deepEqual(fixture.deleted, ['alpha/parent-1']); + const summary = ghcrSummary(fixture); + assert.match(summary.Status, /^Failed: Resource not accessible/); + assert.equal(summary['Versions deleted'], '1'); + assert.equal(summary['Known versions deferred'], '2'); + assert.deepEqual(fixture.warnings, []); + }); +} + +test('GHCR writes a zero-deletion summary if the initial rate limit query is throttled', async (t) => { + const fixture = ghcrRateFixture(t, { + onResponse() { throw githubFailure(429, {'x-ratelimit-remaining': '0'}, 'Too many requests'); }, + }); + const result = await fixture.run(); + assert.equal(result.deleted, 0); + assert.equal(fixture.calls.length, 1); + const summary = ghcrSummary(fixture); + assert.equal(summary['Versions deleted'], '0'); + assert.equal(summary['GitHub API calls remaining'], '0'); +}); + +test('GHCR stops safely if it cannot determine the initial API budget', async (t) => { + const fixture = ghcrRateFixture(t, {onResponse() { return {headers: {}, data: {}}; }}); + const result = await fixture.run(); + assert.equal(result.deleted, 0); + assert.equal(fixture.calls.length, 1); + const summary = ghcrSummary(fixture); + assert.match(summary.Status, /Cannot determine/); + assert.equal(summary['GitHub API calls remaining'], 'Unknown'); +}); + +test('GHCR conservatively counts requests when response rate limit headers are unavailable', async (t) => { + const fixture = ghcrRateFixture(t, { + remaining: 505, + onResponse(call, response) { return {...response, headers: {}}; }, + }); + const result = await fixture.run(); + assert.equal(result.deleted, 1); + assert.deepEqual(fixture.deleted, ['alpha/parent-1']); + assert.equal(ghcrSummary(fixture)['GitHub API calls remaining'], '500'); +}); + +for (const kind of ['manifest', 'tag']) { + for (const dryRun of [false, true]) { + test(`Docker Hub writes progress when its time budget stops ${kind} cleanup (dryRun=${dryRun})`, async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + const core = summaryCore(); + const deleted = []; + if (dryRun) { + core.info = () => { now = 1000; }; + } + t.mock.method(globalThis, 'fetch', async (input, options) => { + const url = new URL(input); + if (url.pathname === '/v2/auth/token') { + return Response.json({token: 'hub-token'}); + } + if (url.hostname === 'auth.docker.io') { + return Response.json({token: 'registry-token'}); + } + if (options.method === 'DELETE') { + deleted.push(url.pathname); + now = 1000; + return new Response(null, {status: 204}); + } + if (url.pathname.endsWith('/repositories')) { + return Response.json({results: [{name: 'alpha'}, {name: 'beta'}]}); + } + assert.ok(url.pathname.endsWith('/alpha/tags')); + const tags = kind === 'manifest' ? [ + {name: 'old-1a', digest: 'sha256:1'}, {name: 'old-1b', digest: 'sha256:1'}, + {name: 'old-2', digest: 'sha256:2'}, + ] : [{name: 'old-1'}, {name: 'old-2'}]; + return Response.json({results: tags.map((tag) => ({...tag, last_updated: '2020-01-01'}))}); + }); + const result = await cleanupDockerHub({ + github: cleanupGithub(['Alpha', 'Beta']), core, username: 'user', accessToken: 'token', dryRun, runtime, + }); + assert.equal(result.deletedActions, 1); + assert.equal(result.deferredActions, 1); + assert.equal(deleted.length, dryRun ? 0 : 1); + assert.equal(core.summary.writes, 1); + const summary = Object.fromEntries(core.summary.rows.map(([key, value]) => [key.data, value.data])); + assert.match(summary.Status, /time budget reached/); + assert.equal(summary['Inventory scan'], 'Partial'); + assert.equal(summary['Delete operations completed'], dryRun ? '0' : '1'); + assert.equal(summary['Tags cleaned'], dryRun ? '0' : String(kind === 'manifest' ? 2 : 1)); + assert.equal(summary['Known operations deferred'], '1'); + }); + } +} + +test('Docker Hub never deletes from a tag inventory cut short by the time budget', async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + const core = summaryCore(); + let tagPages = 0; + t.mock.method(globalThis, 'fetch', async (input, options) => { + assert.notEqual(options.method, 'DELETE'); + const url = new URL(input); + if (url.pathname === '/v2/auth/token') { + return Response.json({token: 'hub-token'}); + } + if (url.pathname.endsWith('/repositories')) { + return Response.json({results: [{name: 'alpha'}]}); + } + tagPages += 1; + now = 1000; + return Response.json({ + results: [{name: 'stale', digest: 'sha256:1', last_updated: '2020-01-01'}], + next: `${url.origin}${url.pathname}?page=2`, + }); + }); + const result = await cleanupDockerHub({ + github: cleanupGithub(), core, username: 'user', accessToken: 'token', dryRun: false, runtime, + }); + assert.equal(tagPages, 1); + assert.equal(result.deletedActions, 0); + assert.equal(core.summary.writes, 1); +}); + +for (const stall of [false, true]) { + test(`Docker Hub reports prior deletions when a later delete ${stall ? 'stalls' : 'fails'}`, async (t) => { + const core = summaryCore(); + const runtime = createCleanupRuntime({maxRuntimeMs: 100}); + const keepAlive = setTimeout(() => {}, 1000); + t.after(() => clearTimeout(keepAlive)); + let attempts = 0; + let stalledSignal; + t.mock.method(globalThis, 'fetch', async (input, options) => { + const url = new URL(input); + if (url.pathname === '/v2/auth/token') { + return Response.json({token: 'hub-token'}); + } + if (options.method === 'DELETE') { + attempts += 1; + if (attempts === 2) { + stalledSignal = options.signal; + return stall ? new Promise(() => {}) : new Response('server failure', {status: 500}); + } + return new Response(null, {status: 204}); + } + return Response.json({results: url.pathname.endsWith('/repositories') ? [{name: 'alpha'}] + : [1, 2, 3].map((id) => ({name: `old-${id}`, last_updated: '2020-01-01'}))}); + }); + const cleanup = cleanupDockerHub({ + github: cleanupGithub(), core, username: 'user', accessToken: 'token', dryRun: false, runtime, + }); + if (stall) { + assert.equal((await cleanup).deletedActions, 1); + assert.equal(stalledSignal.aborted, true); + } else { + await assert.rejects(cleanup, /500/); + } + assert.equal(attempts, 2); + assert.equal(core.summary.writes, 1); + const summary = Object.fromEntries(core.summary.rows.map(([key, value]) => [key.data, value.data])); + assert.equal(summary['Delete operations completed'], '1'); + assert.equal(summary['Known operations deferred'], '2'); + assert.match(summary.Status, stall ? /time budget reached/ : /^Failed:/); + }); +} + +test('GHCR summarizes confirmed deletions before its time budget stops further packages', async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + const fixture = ghcrRateFixture(t, { + onResponse(call, response) { + if (call.options.package_version_id === 'parent-1') { + now = 1000; + } + return response; + }, + }); + const result = await fixture.run({runtime}); + assert.equal(result.deleted, 1); + assert.equal(result.deferred, 2); + assert.deepEqual(fixture.deleted, ['alpha/parent-1']); + const summary = ghcrSummary(fixture); + assert.match(summary.Status, /time budget reached/); + assert.equal(summary['Versions deleted'], '1'); + assert.equal(summary['Known versions deferred'], '2'); +}); + +test('GHCR propagates a manifest graph deadline to the cleanup summary', async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + const fixture = ghcrRateFixture(t, { + onRegistryResponse(url, response) { now = 1000; return response; }, + }); + const result = await fixture.run({runtime}); + assert.equal(result.deleted, 0); + assert.equal(fixture.manifests.length, 1); + assert.equal(fixture.warnings.length, 1); + assert.match(ghcrSummary(fixture).Status, /time budget reached/); +}); + +test('GHCR stops inventory pagination at its deadline without making a deletion plan', async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + const fixture = ghcrRateFixture(t, { + versionPageSize: 2, + onResponse(call, response) { if (call.options.package_name) { now = 1000; } return response; }, + }); + const result = await fixture.run({runtime}); + assert.equal(result.selected, 0); + assert.equal(fixture.calls.filter(({options}) => options.package_name).length, 1); + assert.match(ghcrSummary(fixture).Status, /time budget reached/); +}); + +test('GHCR writes its summary when an in-flight deletion stalls until the deadline', async (t) => { + const runtime = createCleanupRuntime({maxRuntimeMs: 100}); + const keepAlive = setTimeout(() => {}, 1000); + t.after(() => clearTimeout(keepAlive)); + let stalledSignal; + const fixture = ghcrRateFixture(t, { + onResponse(call, response) { + if (call.options.package_version_id === 'parent-2') { + stalledSignal = call.options.request.signal; + return new Promise(() => {}); + } + return response; + }, + }); + const result = await fixture.run({runtime}); + assert.equal(stalledSignal.aborted, true); + assert.equal(result.deleted, 1); + assert.deepEqual(fixture.deleted, ['alpha/parent-1']); + assert.equal(ghcrSummary(fixture)['Versions deleted'], '1'); +}); diff --git a/tests/cleanup-runtime.test.mjs b/tests/cleanup-runtime.test.mjs new file mode 100644 index 00000000..c5bbc865 --- /dev/null +++ b/tests/cleanup-runtime.test.mjs @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import {CleanupStopped, createCleanupRuntime} from '../.github/scripts/cleanup-runtime.mjs'; + +test('cleanup defaults to 5.5 hours and refuses to start further work at the deadline', async (t) => { + let now = 0; + const runtime = createCleanupRuntime({now: () => now}); + t.after(() => runtime.dispose()); + now = 5.5 * 60 * 60 * 1000 - 1; + assert.equal(await runtime.run(async () => 'before deadline'), 'before deadline'); + now += 1; + await assert.rejects(runtime.run(() => assert.fail('Work must not start')), CleanupStopped); +}); + +test('cleanup counts a confirmed operation even if it finishes at the deadline', async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + t.after(() => runtime.dispose()); + assert.equal(await runtime.run(async () => { now = 1000; return 'confirmed'; }), 'confirmed'); + assert.throws(() => runtime.check(), CleanupStopped); +}); + +test('cleanup aborts a pending HTTP request when its wall-clock budget expires', async (t) => { + const runtime = createCleanupRuntime({maxRuntimeMs: 20}); + const keepAlive = setTimeout(() => {}, 1000); + t.after(() => { runtime.dispose(); clearTimeout(keepAlive); }); + let requestSignal; + t.mock.method(globalThis, 'fetch', (url, {signal}) => { + requestSignal = signal; + return new Promise((_, reject) => signal.addEventListener('abort', () => reject(signal.reason), {once: true})); + }); + await assert.rejects(runtime.fetch('https://registry.example/stalled'), CleanupStopped); + assert.equal(requestSignal.aborted, true); +}); + +test('cleanup bounds a stalled Octokit request and passes its abort signal', async (t) => { + const runtime = createCleanupRuntime({maxRuntimeMs: 20}); + const keepAlive = setTimeout(() => {}, 1000); + t.after(() => { runtime.dispose(); clearTimeout(keepAlive); }); + let requestSignal; + const api = runtime.github({ + request(route, options) { + assert.equal(options.request.retries, 0); + requestSignal = options.request.signal; + return new Promise(() => {}); + }, + }); + await assert.rejects(api.request('DELETE /stalled', {request: {retries: 0}}), CleanupStopped); + assert.equal(requestSignal.aborted, true); +}); + +test('cleanup stops pagination at the deadline without returning a partial inventory', async (t) => { + let now = 0; + const runtime = createCleanupRuntime({maxRuntimeMs: 1000, now: () => now}); + t.after(() => runtime.dispose()); + let pages = 0; + const api = runtime.github({ + async paginate(route, options, mapPage) { + assert.ok(options.request.signal instanceof AbortSignal); + pages += 1; + now = 1000; + mapPage({data: ['partial']}); + pages += 1; + return ['partial', 'more']; + }, + }); + await assert.rejects(api.paginate('GET /pages'), CleanupStopped); + assert.equal(pages, 1); +});