From 4c04936db0a0234679b731edf66dc598582f8a22 Mon Sep 17 00:00:00 2001 From: v-robaiken Date: Fri, 31 Jul 2026 11:14:38 +0000 Subject: [PATCH 1/4] Implement split fetch/update phases with separate containers and proxies --- __tests__/container-service.test.ts | 31 ++++ __tests__/proxy-integration.test.ts | 48 ++++++ __tests__/updater-builder.test.ts | 46 ++++++ __tests__/updater-integration.test.ts | 29 ++++ __tests__/updater.test.ts | 211 ++++++++++++++++++++++++++ src/container-service.ts | 154 ++++++++++++------- src/proxy.ts | 14 +- src/updater-builder.ts | 12 +- src/updater.ts | 192 +++++++++++++++++++---- 9 files changed, 647 insertions(+), 90 deletions(-) diff --git a/__tests__/container-service.test.ts b/__tests__/container-service.test.ts index a300405a6..489b4b7df 100644 --- a/__tests__/container-service.test.ts +++ b/__tests__/container-service.test.ts @@ -52,4 +52,35 @@ describe('ContainerService', () => { ) }) }) + + describe('updaterCommands', () => { + test('the fetch phase only runs the file fetcher', () => { + expect(ContainerService.updaterCommands('fetch')).toEqual([ + 'mkdir -p /home/dependabot/dependabot-updater/output', + '$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files' + ]) + }) + + test('the update phase only runs the file updater', () => { + expect(ContainerService.updaterCommands('update')).toEqual([ + 'mkdir -p /home/dependabot/dependabot-updater/output', + '$DEPENDABOT_HOME/dependabot-updater/bin/run update_files' + ]) + }) + + test('the update phase honours the graph command', () => { + expect(ContainerService.updaterCommands('update', 'graph')).toEqual([ + 'mkdir -p /home/dependabot/dependabot-updater/output', + '$DEPENDABOT_HOME/dependabot-updater/bin/run update_graph' + ]) + }) + + test('the all phase runs both halves in one container', () => { + expect(ContainerService.updaterCommands('all')).toEqual([ + 'mkdir -p /home/dependabot/dependabot-updater/output', + '$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files', + '$DEPENDABOT_HOME/dependabot-updater/bin/run update_files' + ]) + }) + }) }) diff --git a/__tests__/proxy-integration.test.ts b/__tests__/proxy-integration.test.ts index a607c4b9a..637af6a4d 100644 --- a/__tests__/proxy-integration.test.ts +++ b/__tests__/proxy-integration.test.ts @@ -82,6 +82,54 @@ integration('ProxyBuilder', () => { await proxy.shutdown() }) + jest.setTimeout(20000) + it('namespaces the container and networks per phase', async () => { + const fetchProxy = await builder.run( + jobId, + jobToken, + dependabotApiUrl, + credentials, + 'fetch' + ) + await fetchProxy.container.start() + + const updateProxy = await builder.run( + jobId, + jobToken, + dependabotApiUrl, + [], + 'update' + ) + await updateProxy.container.start() + + const fetchInfo = await fetchProxy.container.inspect() + const updateInfo = await updateProxy.container.inspect() + + expect(fetchInfo.Name).toBe('/dependabot-job-1-fetch-proxy') + expect(updateInfo.Name).toBe('/dependabot-job-1-update-proxy') + + expect(fetchProxy.networkName).toBe( + 'dependabot-job-1-fetch-internal-network' + ) + expect(updateProxy.networkName).toBe( + 'dependabot-job-1-update-internal-network' + ) + + // Each phase gets its own networks so that the two proxies are reachable + // only from their own phase's container. + expect(Object.keys(fetchInfo.NetworkSettings.Networks)).toEqual([ + 'dependabot-job-1-fetch-external-network', + 'dependabot-job-1-fetch-internal-network' + ]) + expect(Object.keys(updateInfo.NetworkSettings.Networks)).toEqual([ + 'dependabot-job-1-update-external-network', + 'dependabot-job-1-update-internal-network' + ]) + + await updateProxy.shutdown() + await fetchProxy.shutdown() + }) + jest.setTimeout(20000) it('copies in a custom root CA if configured', async () => { // make a tmp dir at the repo root unless it already exists diff --git a/__tests__/updater-builder.test.ts b/__tests__/updater-builder.test.ts index 8d00e4720..eba114236 100644 --- a/__tests__/updater-builder.test.ts +++ b/__tests__/updater-builder.test.ts @@ -110,4 +110,50 @@ describe('UpdaterBuilder', () => { }) ) }) + + it('sets UPDATER_ONE_CONTAINER when both phases share a container', async () => { + mockExtractUpdaterSha.mockReturnValue(null) + + const updaterBuilder = new UpdaterBuilder( + mockDocker, + jobParams, + input, + mockProxy, + 'test-image' + ) + + await updaterBuilder.run('test-container') + + expect(mockCreateContainer).toHaveBeenCalledWith( + expect.objectContaining({ + Env: expect.arrayContaining(['UPDATER_ONE_CONTAINER=1']) + }) + ) + }) + + it.each(['fetch', 'update'] as const)( + 'does not set UPDATER_ONE_CONTAINER for the %s phase', + async phase => { + mockExtractUpdaterSha.mockReturnValue(null) + + const updaterBuilder = new UpdaterBuilder( + mockDocker, + jobParams, + input, + mockProxy, + 'test-image', + phase + ) + + await updaterBuilder.run('test-container') + + expect(mockCreateContainer).toHaveBeenCalledWith( + expect.objectContaining({ + Env: expect.not.arrayContaining([ + expect.stringMatching(/UPDATER_ONE_CONTAINER/) + ]) + }) + ) + } + ) }) diff --git a/__tests__/updater-integration.test.ts b/__tests__/updater-integration.test.ts index e20cd0a78..9b7b8fad3 100644 --- a/__tests__/updater-integration.test.ts +++ b/__tests__/updater-integration.test.ts @@ -79,4 +79,33 @@ integration('Updater', () => { 'Bump fetch-factory from 0.0.1 to 0.2.1' ) }) + + jest.setTimeout(120000) + it('should create the same pull request when the phases are split', async () => { + const details = await apiClient.getJobDetails() + const credentials = await apiClient.getCredentials() + + const updater = new Updater( + updaterImageName('npm_and_yarn'), + PROXY_IMAGE_NAME, + apiClient, + { + ...details, + experiments: { + ...details.experiments, + 'split-fetch-update-containers': true + } + }, + credentials + ) + + await updater.runUpdater() + + const res = await client.getJson(`${dependabotApiUrl}/pull_requests/1`) + + expect(res.statusCode).toEqual(200) + expect(res.result['pr-title']).toEqual( + 'Bump fetch-factory from 0.0.1 to 0.2.1' + ) + }) }) diff --git a/__tests__/updater.test.ts b/__tests__/updater.test.ts index c401aa99e..4dce57a06 100644 --- a/__tests__/updater.test.ts +++ b/__tests__/updater.test.ts @@ -490,4 +490,215 @@ describe('Updater', () => { ]) }) }) + + describe('when the split fetch/update experiment is enabled', () => { + const splitJobDetails: any = { + ...mockJobDetails, + experiments: {'split-fetch-update-containers': true}, + source: {repo: 'dependabot/example'} + } + + const credentials = [ + { + type: 'git_source', + host: 'github.com', + username: 'x-access-token', + password: 'target-repo-token' + }, + { + type: 'git_source', + host: 'github.com', + repo: 'dependabot/other', + username: 'x-access-token', + password: 'other-repo-token' + }, + { + type: 'npm_registry', + host: 'registry.npmjs.org', + token: 'npm_token' + } + ] + + const fetcherOutput = JSON.stringify({ + base_commit_sha: 'sha', + base64_dependency_files: [ + { + name: 'package.json', + content: Buffer.from('{}').toString('base64'), + directory: '/' + } + ] + }) + + let proxyRun: jest.SpyInstance + let runFileFetcher: jest.SpyInstance + let runFileUpdater: jest.SpyInstance + let runSingleContainer: jest.SpyInstance + let storeInput: jest.SpyInstance + let createContainer: jest.SpyInstance + + const updater = new Updater( + 'MOCK_UPDATER_IMAGE_NAME', + 'MOCK_PROXY_IMAGE_NAME', + mockApiClient, + splitJobDetails, + credentials + ) + + beforeEach(async () => { + createContainer = jest + .spyOn(Docker.prototype, 'createContainer') + .mockResolvedValue(mockContainer) + + proxyRun = jest + .spyOn(ProxyBuilder.prototype, 'run') + .mockResolvedValue(mockProxy) + + runFileFetcher = jest + .spyOn(ContainerService, 'runFileFetcher') + .mockResolvedValue(fetcherOutput) + runFileUpdater = jest + .spyOn(ContainerService, 'runFileUpdater') + .mockResolvedValue() + runSingleContainer = jest + .spyOn(ContainerService, 'run') + .mockResolvedValue(true) + storeInput = jest.spyOn(ContainerService, 'storeInput') + }) + + it('runs the fetch and update phases in separate containers', async () => { + expect(await updater.runUpdater()).toBe(true) + + expect(runFileFetcher).toHaveBeenCalledTimes(1) + expect(runFileUpdater).toHaveBeenCalledTimes(1) + expect(runSingleContainer).not.toHaveBeenCalled() + }) + + it('starts a separate proxy per phase', async () => { + await updater.runUpdater() + + expect(proxyRun).toHaveBeenCalledTimes(2) + expect(proxyRun.mock.calls[0][4]).toBe('fetch') + expect(proxyRun.mock.calls[1][4]).toBe('update') + }) + + it('excludes target-repo credentials from the update phase proxy', async () => { + await updater.runUpdater() + + expect(proxyRun.mock.calls[0][3]).toEqual(credentials) + expect(proxyRun.mock.calls[1][3]).toEqual([ + credentials[1], + credentials[2] + ]) + }) + + it('shuts the fetch proxy down before the update phase starts', async () => { + const order: string[] = [] + mockProxy.shutdown.mockImplementation(async () => { + order.push('shutdown') + }) + runFileUpdater.mockImplementation(async () => { + order.push('update') + }) + + await updater.runUpdater() + + expect(order).toEqual(['shutdown', 'update', 'shutdown']) + }) + + it('raises an error when the fetcher produces no output', async () => { + runFileFetcher.mockResolvedValue(undefined) + + await expect(updater.runUpdater()).rejects.toThrow( + 'No output.json created by the fetcher container' + ) + }) + + it('gives each phase its own container name', async () => { + await updater.runUpdater() + + expect(createContainer.mock.calls[0][0].name).toBe( + 'dependabot-job-1-file-fetcher' + ) + expect(createContainer.mock.calls[1][0].name).toBe( + 'dependabot-job-1-updater' + ) + }) + + it('hands the decoded file subset off to the update container', async () => { + await updater.runUpdater() + + expect(storeInput.mock.calls[0][3]).toEqual({job: splitJobDetails}) + expect(storeInput.mock.calls[1][3]).toEqual({ + job: splitJobDetails, + base_commit_sha: 'sha', + base64_dependency_files: [ + { + name: 'package.json', + content: Buffer.from('{}').toString('base64'), + directory: '/' + } + ], + dependency_files: [ + {name: 'package.json', content: '{}', directory: '/'} + ] + }) + }) + + it('does not clean up the update phase proxy until the update fails', async () => { + runFileUpdater.mockRejectedValue(new Error('update failed')) + + await expect(updater.runUpdater()).rejects.toThrow('update failed') + + // One shutdown for the fetch proxy, one for the update proxy. + expect(mockProxy.shutdown).toHaveBeenCalledTimes(2) + }) + + it('does not start the update phase when the fetch phase fails', async () => { + runFileFetcher.mockRejectedValue(new Error('fetch failed')) + + await expect(updater.runUpdater()).rejects.toThrow('fetch failed') + + expect(runFileUpdater).not.toHaveBeenCalled() + expect(proxyRun).toHaveBeenCalledTimes(1) + expect(mockProxy.shutdown).toHaveBeenCalledTimes(1) + }) + + it('forwards the graph command to the update phase', async () => { + const graphUpdater = new Updater( + 'MOCK_UPDATER_IMAGE_NAME', + 'MOCK_PROXY_IMAGE_NAME', + mockApiClient, + {...splitJobDetails, command: 'graph'}, + credentials + ) + + await graphUpdater.runUpdater() + + expect(runFileUpdater).toHaveBeenCalledWith(mockContainer, 'graph') + }) + + it.each([ + ['the experiment is absent', {}], + ['the experiment is disabled', {'split-fetch-update-containers': false}] + ])('uses the single container path when %s', async (_name, experiments) => { + const legacyUpdater = new Updater( + 'MOCK_UPDATER_IMAGE_NAME', + 'MOCK_PROXY_IMAGE_NAME', + mockApiClient, + {...splitJobDetails, experiments}, + credentials + ) + + expect(await legacyUpdater.runUpdater()).toBe(true) + + expect(runSingleContainer).toHaveBeenCalledTimes(1) + expect(runFileFetcher).not.toHaveBeenCalled() + expect(runFileUpdater).not.toHaveBeenCalled() + // The legacy path starts one unphased proxy with the full credential set. + expect(proxyRun).toHaveBeenCalledTimes(1) + expect(proxyRun.mock.calls[0][3]).toEqual(credentials) + expect(proxyRun.mock.calls[0][4]).toBeUndefined() + }) + }) }) diff --git a/src/container-service.ts b/src/container-service.ts index 51b358697..05a50d410 100644 --- a/src/container-service.ts +++ b/src/container-service.ts @@ -9,6 +9,15 @@ export class ContainerRuntimeError extends Error {} const RWX_ALL = 0o777 +const OUTPUT_PATH = '/home/dependabot/dependabot-updater/output' +const OUTPUT_FILE_PATH = `${OUTPUT_PATH}/output.json` +const SUMMARY_FILE_PATH = `${OUTPUT_PATH}/summary.md` + +// 'fetch' clones the repo and writes the handoff artifact, 'update' runs the +// package manager against that artifact. 'all' runs both in one container, +// which is the legacy UPDATER_ONE_CONTAINER behaviour. +export type UpdaterPhase = 'fetch' | 'update' | 'all' + export const ContainerService = { async storeInput( name: string, @@ -35,6 +44,45 @@ export const ContainerService = { }, async run(container: Container, command?: string): Promise { + await this.runPhase(container, 'all', command) + return true + }, + + /** + * Run the fetch phase and return the raw contents of the handoff artifact + * written by the fetcher, or undefined if it produced no output. + */ + async runFileFetcher(container: Container): Promise { + return await this.runPhase(container, 'fetch') + }, + + async runFileUpdater(container: Container, command?: string): Promise { + await this.runPhase(container, 'update', command) + }, + + updaterCommands(phase: UpdaterPhase, command?: string): string[] { + const commands = [`mkdir -p ${OUTPUT_PATH}`] + + if (phase === 'all' || phase === 'fetch') { + commands.push('$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files') + } + + if (phase === 'all' || phase === 'update') { + commands.push( + command === 'graph' + ? '$DEPENDABOT_HOME/dependabot-updater/bin/run update_graph' + : '$DEPENDABOT_HOME/dependabot-updater/bin/run update_files' + ) + } + + return commands + }, + + async runPhase( + container: Container, + phase: UpdaterPhase, + command?: string + ): Promise { try { // Start the container await container.start() @@ -46,52 +94,40 @@ export const ContainerService = { env.startsWith('DEPENDABOT_JOB_ID=') ) - if (isDependabotContainer) { - // For dependabot containers, run CA certificates update as root first - await this.execCommand( - container, - ['/usr/sbin/update-ca-certificates'], - 'root' - ) - - // Then run the dependabot commands as dependabot user - const dependabotCommands = [ - 'mkdir -p /home/dependabot/dependabot-updater/output', - '$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files' - ] - - if (command === 'graph') { - dependabotCommands.push( - '$DEPENDABOT_HOME/dependabot-updater/bin/run update_graph' - ) - } else { - dependabotCommands.push( - '$DEPENDABOT_HOME/dependabot-updater/bin/run update_files' - ) - } - - for (const cmd of dependabotCommands) { - await this.execCommand( - container, - ['/bin/sh', '-c', cmd], - 'dependabot' - ) - } - - // Extract job summary only after all commands have succeeded. - // This prevents malicious code executed during fetch_files from - // injecting content — our updater overwrites the file at the end - // of a successful run. - await this.extractJobSummary(container) - } else { + if (!isDependabotContainer) { // For test containers and other containers, just wait for completion const outcome = await container.wait() if (outcome.StatusCode !== 0) { throw new Error(`Container exited with code ${outcome.StatusCode}`) } + return undefined + } + + // For dependabot containers, run CA certificates update as root first + await this.execCommand( + container, + ['/usr/sbin/update-ca-certificates'], + 'root' + ) + + // Then run the dependabot commands as dependabot user + for (const cmd of this.updaterCommands(phase, command)) { + await this.execCommand(container, ['/bin/sh', '-c', cmd], 'dependabot') } - return true + if (phase === 'fetch') { + // The fetch phase hands its file subset off to the update container, so + // read it out before this container is torn down. + return await this.readFile(container, OUTPUT_FILE_PATH) + } + + // Extract job summary only after all commands have succeeded. + // This prevents malicious code executed during fetch_files from + // injecting content — our updater overwrites the file at the end + // of a successful run. + await this.extractJobSummary(container) + + return undefined } catch (error) { core.info(`Failure running container ${container.id}: ${error}`) throw new ContainerRuntimeError( @@ -149,18 +185,14 @@ export const ContainerService = { } }, - async extractJobSummary(container: Container): Promise { - const summaryPath = '/home/dependabot/dependabot-updater/output/summary.md' - const stepSummaryPath = process.env.GITHUB_STEP_SUMMARY - - if (!stepSummaryPath) { - return - } - + async readFile( + container: Container, + path: string + ): Promise { try { - const archiveStream = await container.getArchive({path: summaryPath}) + const archiveStream = await container.getArchive({path}) - const content = await new Promise((resolve, reject) => { + return await new Promise((resolve, reject) => { const extractor = extract() let data = '' @@ -177,14 +209,24 @@ export const ContainerService = { archiveStream.pipe(extractor) }) - - if (content.length > 0) { - fs.appendFileSync(stepSummaryPath, content) - core.info('Job summary written to GITHUB_STEP_SUMMARY') - } } catch { - // File doesn't exist in container (older updater image) — skip gracefully - core.debug('No job summary file found in container') + core.debug(`No file found in container at ${path}`) + return undefined + } + }, + + async extractJobSummary(container: Container): Promise { + const stepSummaryPath = process.env.GITHUB_STEP_SUMMARY + + if (!stepSummaryPath) { + return + } + + const content = await this.readFile(container, SUMMARY_FILE_PATH) + + if (content && content.length > 0) { + fs.appendFileSync(stepSummaryPath, content) + core.info('Job summary written to GITHUB_STEP_SUMMARY') } } } diff --git a/src/proxy.ts b/src/proxy.ts index af8f8cda6..9c2cb3bbf 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -62,16 +62,22 @@ export class ProxyBuilder { jobId: number, jobToken: string, dependabotApiUrl: string, - credentials: Credential[] + credentials: Credential[], + phase?: string ): Promise { - const name = `dependabot-job-${jobId}-proxy` + // When a phase is given, every Docker resource is namespaced by it so the + // fetch and update phases get their own proxy on their own networks. + const prefix = phase + ? `dependabot-job-${jobId}-${phase}` + : `dependabot-job-${jobId}` + const name = `${prefix}-proxy` const config = this.buildProxyConfig(credentials) const cert = config.ca.cert - const externalNetworkName = `dependabot-job-${jobId}-external-network` + const externalNetworkName = `${prefix}-external-network` const externalNetwork = await this.ensureNetwork(externalNetworkName, false) - const internalNetworkName = `dependabot-job-${jobId}-internal-network` + const internalNetworkName = `${prefix}-internal-network` const internalNetwork = await this.ensureNetwork(internalNetworkName, true) const container = await this.createContainer( diff --git a/src/updater-builder.ts b/src/updater-builder.ts index 3740ed15b..cd7f84480 100644 --- a/src/updater-builder.ts +++ b/src/updater-builder.ts @@ -1,6 +1,6 @@ import * as core from '@actions/core' import Docker, {Container} from 'dockerode' -import {ContainerService} from './container-service' +import {ContainerService, UpdaterPhase} from './container-service' import {FileFetcherInput, FileUpdaterInput} from './config-types' import {JobParameters} from './inputs' import {Proxy} from './proxy' @@ -22,7 +22,8 @@ export class UpdaterBuilder { private readonly input: FileFetcherInput | FileUpdaterInput, private readonly proxy: Proxy, - private readonly updaterImage: string + private readonly updaterImage: string, + private readonly phase: UpdaterPhase = 'all' ) {} async run(containerName: string): Promise { @@ -43,7 +44,6 @@ export class UpdaterBuilder { `HTTP_PROXY=${proxyUrl}`, `https_proxy=${proxyUrl}`, `HTTPS_PROXY=${proxyUrl}`, - `UPDATER_ONE_CONTAINER=1`, `ENABLE_CONNECTIVITY_CHECK=${ process.env.DEPENDABOT_ENABLE_CONNECTIVITY_CHECK || '1' }`, @@ -57,6 +57,12 @@ export class UpdaterBuilder { `NODE_OPTIONS=--max-old-space-size=4096` ] + // When the fetch and update phases run in separate containers each one only + // performs its own half of the job, so the single-container hint is omitted. + if (this.phase === 'all') { + envVars.push(`UPDATER_ONE_CONTAINER=1`) + } + // Add DEPENDABOT_UPDATER_SHA if we successfully extracted a SHA if (updaterSha !== null) { envVars.push(`DEPENDABOT_UPDATER_SHA=${updaterSha}`) diff --git a/src/updater.ts b/src/updater.ts index 71f3764f0..28b3dc235 100644 --- a/src/updater.ts +++ b/src/updater.ts @@ -1,10 +1,27 @@ import * as core from '@actions/core' import Docker, {Container} from 'dockerode' import {JobDetails, ApiClient, Credential} from './api-client' -import {ContainerService} from './container-service' -import {FileUpdaterInput, FileFetcherInput} from './config-types' +import {ContainerService, UpdaterPhase} from './container-service' +import { + DependencyFile, + FetchedFiles, + FileUpdaterInput, + FileFetcherInput +} from './config-types' import {ProxyBuilder, Proxy} from './proxy' import {UpdaterBuilder} from './updater-builder' +import {base64DecodeDependencyFile} from './utils' + +// Experiment which opts a job into running the fetch and update phases in +// separate containers, each with its own proxy and credential set. +const FEATURE_SPLIT_FETCH_UPDATE = 'split-fetch-update-containers' + +export class UpdaterFetchError extends Error { + constructor(msg: string) { + super(msg) + Object.setPrototypeOf(this, UpdaterFetchError.prototype) + } +} export class Updater { docker: Docker @@ -24,6 +41,144 @@ export class Updater { * Execute an update job and report the result to Dependabot API. */ async runUpdater(): Promise { + if (this.splitPhasesEnabled()) { + return await this.runSplitPhaseUpdate() + } + + return await this.runSingleContainerUpdate() + } + + private splitPhasesEnabled(): boolean { + const experiments = (this.details.experiments || {}) as { + [key: string]: unknown + } + return experiments[FEATURE_SPLIT_FETCH_UPDATE] === true + } + + private async runSingleContainerUpdate(): Promise { + const proxy = await this.startProxy(this.credentials) + + try { + await this.runUpdate(proxy) + return true + } finally { + await this.cleanup(proxy) + } + } + + /** + * Run the job as two sequential containers. The fetch container clones the + * repository behind a proxy holding the target-repo credential and hands off + * the file subset it selected. The update container then runs the package + * manager against only that subset, behind its own proxy. + */ + private async runSplitPhaseUpdate(): Promise { + const files = await this.runFetchPhase() + await this.runUpdatePhase(files) + return true + } + + private async runFetchPhase(): Promise { + core.info(`Fetching files for job ${this.apiClient.params.jobId}`) + + const proxy = await this.startProxy(this.credentials, 'fetch') + + try { + const name = `dependabot-job-${this.apiClient.params.jobId}-file-fetcher` + const container = await this.createContainer( + proxy, + name, + {job: this.details}, + 'fetch' + ) + + const output = await ContainerService.runFileFetcher(container) + if (!output) { + throw new UpdaterFetchError( + 'No output.json created by the fetcher container' + ) + } + + const fileFetcherOutput = JSON.parse(output) + + return { + base_commit_sha: fileFetcherOutput.base_commit_sha, + base64_dependency_files: fileFetcherOutput.base64_dependency_files, + dependency_files: fileFetcherOutput.base64_dependency_files.map( + (file: DependencyFile) => base64DecodeDependencyFile(file) + ) + } + } finally { + // Tear the fetch proxy and its networks down before the update phase + // starts, so the two phases never share a proxy or a network. + await this.cleanup(proxy) + } + } + + private async runUpdatePhase(files: FetchedFiles): Promise { + core.info(`Running update job ${this.apiClient.params.jobId}`) + + const proxy = await this.startProxy(this.updatePhaseCredentials(), 'update') + + try { + const name = `dependabot-job-${this.apiClient.params.jobId}-updater` + const input: FileUpdaterInput = { + base_commit_sha: files.base_commit_sha, + base64_dependency_files: files.base64_dependency_files, + dependency_files: files.dependency_files, + job: this.details + } + const container = await this.createContainer(proxy, name, input, 'update') + + await ContainerService.runFileUpdater(container, this.details.command) + } finally { + await this.cleanup(proxy) + } + } + + /** + * The credential set handed to the update phase's proxy. Credentials which + * resolve to the target repository are dropped, since the update phase works + * from the file subset handed over by the fetch phase rather than the repo. + */ + private updatePhaseCredentials(): Credential[] { + const targetRepo = this.details.source?.repo + + const credentials = this.credentials.filter( + credential => !this.canReadTargetRepo(credential, targetRepo) + ) + + const dropped = this.credentials.length - credentials.length + if (dropped > 0) { + core.info( + `Excluding ${dropped} target-repo credential(s) from the update phase proxy` + ) + } + + return credentials + } + + private canReadTargetRepo( + credential: Credential, + targetRepo: string | undefined + ): boolean { + if (credential.type !== 'git_source') { + return false + } + + // A git_source credential scoped to a different repository does not resolve + // to the target repo, so it is kept for git-sourced dependencies. + if (credential.repo && targetRepo && credential.repo !== targetRepo) { + return false + } + + return true + } + + private async startProxy( + credentials: Credential[], + phase?: string + ): Promise { const cachedMode = Object.hasOwn( this.details.experiments ?? {}, 'proxy-cached' @@ -39,32 +194,13 @@ export class Updater { this.apiClient.params.jobId, this.apiClient.getJobToken(), this.apiClient.params.dependabotApiUrl, - this.credentials + credentials, + phase ) await proxy.container.start() + await proxy.waitUntilReady() - try { - await proxy.waitUntilReady() - await this.runUpdate(proxy) - } catch (error) { - try { - await this.cleanup(proxy) - } catch (cleanupError) { - const cleanupErrors = - cleanupError instanceof AggregateError - ? cleanupError.errors - : [cleanupError] - for (const cleanupFailure of cleanupErrors) { - core.info( - `Failed to clean up proxy after update failure: ${cleanupFailure}` - ) - } - } - throw error - } - - await this.cleanup(proxy) - return true + return proxy } private generateCredentialsMetadata(): Credential[] { @@ -171,14 +307,16 @@ export class Updater { private async createContainer( proxy: Proxy, containerName: string, - input: FileFetcherInput | FileUpdaterInput + input: FileFetcherInput | FileUpdaterInput, + phase: UpdaterPhase = 'all' ): Promise { return new UpdaterBuilder( this.docker, this.apiClient.params, input, proxy, - this.updaterImage + this.updaterImage, + phase ).run(containerName) } From 1bac259f4e6370ff8e5a466ae8216575a0da1635 Mon Sep 17 00:00:00 2001 From: v-robaiken Date: Fri, 31 Jul 2026 12:51:11 +0000 Subject: [PATCH 2/4] Refactor updater integration tests to initialize server within test cases --- __tests__/updater-integration.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/__tests__/updater-integration.test.ts b/__tests__/updater-integration.test.ts index 9b7b8fad3..64c0e5e0d 100644 --- a/__tests__/updater-integration.test.ts +++ b/__tests__/updater-integration.test.ts @@ -45,9 +45,6 @@ integration('Updater', () => { beforeAll(async () => { await ImageService.pull(updaterImageName('npm_and_yarn')) await ImageService.pull(PROXY_IMAGE_NAME) - - const testRetry = true - server = await runFakeDependabotApi(FAKE_SERVER_PORT, testRetry) }) afterEach(async () => { @@ -57,6 +54,9 @@ integration('Updater', () => { jest.setTimeout(120000) it('should run the updater, retry on apiClient failure, and create a pull request', async () => { + const testRetry = true + server = await runFakeDependabotApi(FAKE_SERVER_PORT, testRetry) + const details = await apiClient.getJobDetails() const credentials = await apiClient.getCredentials() @@ -81,7 +81,15 @@ integration('Updater', () => { }) jest.setTimeout(120000) - it('should create the same pull request when the phases are split', async () => { + // Skipped until the updater image supports running the phases separately. + // `bin/run fetch_files` is currently a no-op kept for backward compatibility + // ("fetch_files command is no longer used directly"), and `bin/update_files.rb` + // runs the file fetcher in-process and hands the files straight to + // UpdateFilesCommand, so no output.json is produced for the handoff. + it.skip('should create the same pull request when the phases are split', async () => { + // Each test gets its own server, as afterEach tears the previous one down. + server = await runFakeDependabotApi(FAKE_SERVER_PORT) + const details = await apiClient.getJobDetails() const credentials = await apiClient.getCredentials() From b802a967fb15345bf8c3027eedc9371bb01ef9a4 Mon Sep 17 00:00:00 2001 From: v-robaiken Date: Fri, 31 Jul 2026 13:25:16 +0000 Subject: [PATCH 3/4] Add support for opt-in split fetch/update phases with environment variable --- __tests__/updater-integration.test.ts | 6 +++++- __tests__/updater.test.ts | 16 ++++++++++++++++ src/updater.ts | 22 +++++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/__tests__/updater-integration.test.ts b/__tests__/updater-integration.test.ts index 64c0e5e0d..3774a89fb 100644 --- a/__tests__/updater-integration.test.ts +++ b/__tests__/updater-integration.test.ts @@ -85,10 +85,14 @@ integration('Updater', () => { // `bin/run fetch_files` is currently a no-op kept for backward compatibility // ("fetch_files command is no longer used directly"), and `bin/update_files.rb` // runs the file fetcher in-process and hands the files straight to - // UpdateFilesCommand, so no output.json is produced for the handoff. + // UpdateFilesCommand, so no output.json is produced for the handoff. The same + // gap is why DEPENDABOT_SPLIT_FETCH_UPDATE is required alongside the + // experiment; both this test and that opt-in can go once an image ships with + // standalone phase entrypoints. it.skip('should create the same pull request when the phases are split', async () => { // Each test gets its own server, as afterEach tears the previous one down. server = await runFakeDependabotApi(FAKE_SERVER_PORT) + process.env.DEPENDABOT_SPLIT_FETCH_UPDATE = '1' const details = await apiClient.getJobDetails() const credentials = await apiClient.getCredentials() diff --git a/__tests__/updater.test.ts b/__tests__/updater.test.ts index 4dce57a06..f67907584 100644 --- a/__tests__/updater.test.ts +++ b/__tests__/updater.test.ts @@ -546,6 +546,8 @@ describe('Updater', () => { ) beforeEach(async () => { + process.env.DEPENDABOT_SPLIT_FETCH_UPDATE = '1' + createContainer = jest .spyOn(Docker.prototype, 'createContainer') .mockResolvedValue(mockContainer) @@ -566,6 +568,10 @@ describe('Updater', () => { storeInput = jest.spyOn(ContainerService, 'storeInput') }) + afterEach(() => { + delete process.env.DEPENDABOT_SPLIT_FETCH_UPDATE + }) + it('runs the fetch and update phases in separate containers', async () => { expect(await updater.runUpdater()).toBe(true) @@ -700,5 +706,15 @@ describe('Updater', () => { expect(proxyRun.mock.calls[0][3]).toEqual(credentials) expect(proxyRun.mock.calls[0][4]).toBeUndefined() }) + + it('stays on the single container path without the opt-in', async () => { + delete process.env.DEPENDABOT_SPLIT_FETCH_UPDATE + + expect(await updater.runUpdater()).toBe(true) + + expect(runSingleContainer).toHaveBeenCalledTimes(1) + expect(runFileFetcher).not.toHaveBeenCalled() + expect(runFileUpdater).not.toHaveBeenCalled() + }) }) }) diff --git a/src/updater.ts b/src/updater.ts index 28b3dc235..de97958e1 100644 --- a/src/updater.ts +++ b/src/updater.ts @@ -16,6 +16,14 @@ import {base64DecodeDependencyFile} from './utils' // separate containers, each with its own proxy and credential set. const FEATURE_SPLIT_FETCH_UPDATE = 'split-fetch-update-containers' +// Opt-in required alongside the experiment. The updater image does not yet +// expose standalone phase entrypoints — `bin/run fetch_files` is a no-op kept +// for backward compatibility, and `update_files` runs the file fetcher +// in-process — so there is no handoff artifact for the update container to +// consume. This keeps the split path unreachable for real jobs until an image +// providing those entrypoints ships. +const SPLIT_FETCH_UPDATE_ENV = 'DEPENDABOT_SPLIT_FETCH_UPDATE' + export class UpdaterFetchError extends Error { constructor(msg: string) { super(msg) @@ -52,7 +60,19 @@ export class Updater { const experiments = (this.details.experiments || {}) as { [key: string]: unknown } - return experiments[FEATURE_SPLIT_FETCH_UPDATE] === true + + if (experiments[FEATURE_SPLIT_FETCH_UPDATE] !== true) { + return false + } + + if (process.env[SPLIT_FETCH_UPDATE_ENV] !== '1') { + core.info( + `The ${FEATURE_SPLIT_FETCH_UPDATE} experiment is enabled but the updater image does not yet support split phases, running both phases in one container` + ) + return false + } + + return true } private async runSingleContainerUpdate(): Promise { From 713394e139312f8cae26ee40b6e8a4bc294286b5 Mon Sep 17 00:00:00 2001 From: v-robaiken Date: Tue, 18 Aug 2026 17:55:31 +0000 Subject: [PATCH 4/4] Isolate fetch and update containers --- __tests__/container-service.test.ts | 67 ++- __tests__/updater-builder.test.ts | 85 ++++ __tests__/updater-integration.test.ts | 83 ++-- __tests__/updater.test.ts | 123 +++--- dist/main.js | 598 +++++++++++++++++--------- src/config-types.ts | 10 - src/container-service.ts | 53 ++- src/updater-builder.ts | 35 +- src/updater.ts | 250 ++++++----- 9 files changed, 829 insertions(+), 475 deletions(-) diff --git a/__tests__/container-service.test.ts b/__tests__/container-service.test.ts index 489b4b7df..8b5313132 100644 --- a/__tests__/container-service.test.ts +++ b/__tests__/container-service.test.ts @@ -54,10 +54,11 @@ describe('ContainerService', () => { }) describe('updaterCommands', () => { - test('the fetch phase only runs the file fetcher', () => { + test('the fetch phase clones and copies the checkout to the handoff volume', () => { expect(ContainerService.updaterCommands('fetch')).toEqual([ 'mkdir -p /home/dependabot/dependabot-updater/output', - '$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files' + '$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files', + 'cp -a /home/dependabot/dependabot-updater/repo/. /home/dependabot/dependabot-updater/repo-handoff/' ]) }) @@ -75,12 +76,70 @@ describe('ContainerService', () => { ]) }) - test('the all phase runs both halves in one container', () => { + test('the all phase lets update_files fetch in-process', () => { expect(ContainerService.updaterCommands('all')).toEqual([ 'mkdir -p /home/dependabot/dependabot-updater/output', - '$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files', '$DEPENDABOT_HOME/dependabot-updater/bin/run update_files' ]) }) }) + + describe('the fetch phase', () => { + test('prepares the repository volume for the dependabot user', async () => { + const dependabotContainer: any = { + id: 'fetch-container', + start: jest.fn().mockResolvedValue(undefined), + inspect: jest.fn().mockResolvedValue({ + Config: {Env: ['DEPENDABOT_JOB_ID=1']} + }), + remove: jest.fn().mockResolvedValue(undefined) + } + const execCommand = jest + .spyOn(ContainerService, 'execCommand') + .mockResolvedValue(undefined) + + await ContainerService.runFileFetcher(dependabotContainer) + + expect(execCommand.mock.calls).toEqual([ + [dependabotContainer, ['/usr/sbin/update-ca-certificates'], 'root'], + [ + dependabotContainer, + [ + 'chown', + 'dependabot', + '/home/dependabot/dependabot-updater/repo', + '/home/dependabot/dependabot-updater/repo-handoff' + ], + 'root' + ], + [ + dependabotContainer, + [ + '/bin/sh', + '-c', + 'mkdir -p /home/dependabot/dependabot-updater/output' + ], + 'dependabot' + ], + [ + dependabotContainer, + [ + '/bin/sh', + '-c', + '$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files' + ], + 'dependabot' + ], + [ + dependabotContainer, + [ + '/bin/sh', + '-c', + 'cp -a /home/dependabot/dependabot-updater/repo/. /home/dependabot/dependabot-updater/repo-handoff/' + ], + 'dependabot' + ] + ]) + }) + }) }) diff --git a/__tests__/updater-builder.test.ts b/__tests__/updater-builder.test.ts index eba114236..5ac0920fe 100644 --- a/__tests__/updater-builder.test.ts +++ b/__tests__/updater-builder.test.ts @@ -156,4 +156,89 @@ describe('UpdaterBuilder', () => { ) } ) + + it('mounts clone and handoff volumes in the fetch phase', async () => { + mockExtractUpdaterSha.mockReturnValue(null) + + const updaterBuilder = new UpdaterBuilder( + mockDocker, + jobParams, + input, + mockProxy, + 'test-image', + 'fetch', + 'clone-volume', + 'handoff-volume' + ) + + await updaterBuilder.run('test-container') + + expect(mockCreateContainer).toHaveBeenCalledWith( + expect.objectContaining({ + HostConfig: expect.objectContaining({ + Mounts: [ + { + Type: 'volume', + Source: 'clone-volume', + Target: '/home/dependabot/dependabot-updater/repo' + }, + { + Type: 'volume', + Source: 'handoff-volume', + Target: '/home/dependabot/dependabot-updater/repo-handoff' + } + ] + }) + }) + ) + }) + + it('requires a local checkout only in the update phase', async () => { + mockExtractUpdaterSha.mockReturnValue(null) + + const updaterBuilder = new UpdaterBuilder( + mockDocker, + jobParams, + input, + mockProxy, + 'test-image', + 'update', + 'repo-volume' + ) + + await updaterBuilder.run('test-container') + + expect(mockCreateContainer).toHaveBeenCalledWith( + expect.objectContaining({ + Env: expect.arrayContaining(['DEPENDABOT_LOCAL_CHECKOUT_ONLY=true']) + }) + ) + }) + + it.each(['all', 'fetch'] as const)( + 'does not require a local checkout in the %s phase', + async phase => { + mockExtractUpdaterSha.mockReturnValue(null) + + const updaterBuilder = new UpdaterBuilder( + mockDocker, + jobParams, + input, + mockProxy, + 'test-image', + phase, + phase === 'fetch' ? 'repo-volume' : undefined + ) + + await updaterBuilder.run('test-container') + + expect(mockCreateContainer).toHaveBeenCalledWith( + expect.objectContaining({ + Env: expect.not.arrayContaining([ + 'DEPENDABOT_LOCAL_CHECKOUT_ONLY=true' + ]) + }) + ) + } + ) }) diff --git a/__tests__/updater-integration.test.ts b/__tests__/updater-integration.test.ts index 3774a89fb..e6b51a286 100644 --- a/__tests__/updater-integration.test.ts +++ b/__tests__/updater-integration.test.ts @@ -81,43 +81,48 @@ integration('Updater', () => { }) jest.setTimeout(120000) - // Skipped until the updater image supports running the phases separately. - // `bin/run fetch_files` is currently a no-op kept for backward compatibility - // ("fetch_files command is no longer used directly"), and `bin/update_files.rb` - // runs the file fetcher in-process and hands the files straight to - // UpdateFilesCommand, so no output.json is produced for the handoff. The same - // gap is why DEPENDABOT_SPLIT_FETCH_UPDATE is required alongside the - // experiment; both this test and that opt-in can go once an image ships with - // standalone phase entrypoints. - it.skip('should create the same pull request when the phases are split', async () => { - // Each test gets its own server, as afterEach tears the previous one down. - server = await runFakeDependabotApi(FAKE_SERVER_PORT) - process.env.DEPENDABOT_SPLIT_FETCH_UPDATE = '1' - - const details = await apiClient.getJobDetails() - const credentials = await apiClient.getCredentials() - - const updater = new Updater( - updaterImageName('npm_and_yarn'), - PROXY_IMAGE_NAME, - apiClient, - { - ...details, - experiments: { - ...details.experiments, - 'split-fetch-update-containers': true - } - }, - credentials - ) - - await updater.runUpdater() - - const res = await client.getJson(`${dependabotApiUrl}/pull_requests/1`) - - expect(res.statusCode).toEqual(200) - expect(res.result['pr-title']).toEqual( - 'Bump fetch-factory from 0.0.1 to 0.2.1' - ) - }) + const splitUpdaterImage = process.env.DEPENDABOT_SPLIT_TEST_UPDATER_IMAGE + const splitIntegration = splitUpdaterImage ? it : it.skip + + // Use a locally patched image until the pinned updater image contains the + // Core fetch_files entrypoint and local-checkout-only behavior. + splitIntegration( + 'should create the same pull request when the phases are split', + async () => { + if (!splitUpdaterImage) { + throw new Error('DEPENDABOT_SPLIT_TEST_UPDATER_IMAGE is required') + } + + // Each test gets its own server, as afterEach tears the previous one down. + server = await runFakeDependabotApi(FAKE_SERVER_PORT) + + const details = await apiClient.getJobDetails() + const credentials = await apiClient.getCredentials() + + const updater = new Updater( + splitUpdaterImage, + PROXY_IMAGE_NAME, + apiClient, + { + ...details, + experiments: { + ...details.experiments, + isolate_fetch_update: true + } + }, + credentials + ) + + await updater.runUpdater() + + const res = await client.getJson( + `${dependabotApiUrl}/pull_requests/1` + ) + + expect(res.statusCode).toEqual(200) + expect(res.result['pr-title']).toEqual( + 'Bump fetch-factory from 0.0.1 to 0.2.1' + ) + } + ) }) diff --git a/__tests__/updater.test.ts b/__tests__/updater.test.ts index f67907584..19f8615d0 100644 --- a/__tests__/updater.test.ts +++ b/__tests__/updater.test.ts @@ -494,7 +494,7 @@ describe('Updater', () => { describe('when the split fetch/update experiment is enabled', () => { const splitJobDetails: any = { ...mockJobDetails, - experiments: {'split-fetch-update-containers': true}, + experiments: {isolate_fetch_update: true}, source: {repo: 'dependabot/example'} } @@ -519,23 +519,14 @@ describe('Updater', () => { } ] - const fetcherOutput = JSON.stringify({ - base_commit_sha: 'sha', - base64_dependency_files: [ - { - name: 'package.json', - content: Buffer.from('{}').toString('base64'), - directory: '/' - } - ] - }) - let proxyRun: jest.SpyInstance let runFileFetcher: jest.SpyInstance let runFileUpdater: jest.SpyInstance let runSingleContainer: jest.SpyInstance let storeInput: jest.SpyInstance let createContainer: jest.SpyInstance + let removeCloneVolume: jest.Mock + let removeHandoffVolume: jest.Mock const updater = new Updater( 'MOCK_UPDATER_IMAGE_NAME', @@ -546,7 +537,18 @@ describe('Updater', () => { ) beforeEach(async () => { - process.env.DEPENDABOT_SPLIT_FETCH_UPDATE = '1' + removeCloneVolume = jest.fn().mockResolvedValue(undefined) + removeHandoffVolume = jest.fn().mockResolvedValue(undefined) + jest + .spyOn(Docker.prototype, 'createVolume') + .mockResolvedValueOnce({ + name: 'dependabot-job-1-clone', + remove: removeCloneVolume + } as any) + .mockResolvedValueOnce({ + name: 'dependabot-job-1-handoff', + remove: removeHandoffVolume + } as any) createContainer = jest .spyOn(Docker.prototype, 'createContainer') @@ -558,7 +560,7 @@ describe('Updater', () => { runFileFetcher = jest .spyOn(ContainerService, 'runFileFetcher') - .mockResolvedValue(fetcherOutput) + .mockResolvedValue() runFileUpdater = jest .spyOn(ContainerService, 'runFileUpdater') .mockResolvedValue() @@ -568,10 +570,6 @@ describe('Updater', () => { storeInput = jest.spyOn(ContainerService, 'storeInput') }) - afterEach(() => { - delete process.env.DEPENDABOT_SPLIT_FETCH_UPDATE - }) - it('runs the fetch and update phases in separate containers', async () => { expect(await updater.runUpdater()).toBe(true) @@ -588,14 +586,11 @@ describe('Updater', () => { expect(proxyRun.mock.calls[1][4]).toBe('update') }) - it('excludes target-repo credentials from the update phase proxy', async () => { + it('passes the full credential set to both phase proxies', async () => { await updater.runUpdater() expect(proxyRun.mock.calls[0][3]).toEqual(credentials) - expect(proxyRun.mock.calls[1][3]).toEqual([ - credentials[1], - credentials[2] - ]) + expect(proxyRun.mock.calls[1][3]).toEqual(credentials) }) it('shuts the fetch proxy down before the update phase starts', async () => { @@ -612,14 +607,6 @@ describe('Updater', () => { expect(order).toEqual(['shutdown', 'update', 'shutdown']) }) - it('raises an error when the fetcher produces no output', async () => { - runFileFetcher.mockResolvedValue(undefined) - - await expect(updater.runUpdater()).rejects.toThrow( - 'No output.json created by the fetcher container' - ) - }) - it('gives each phase its own container name', async () => { await updater.runUpdater() @@ -631,24 +618,52 @@ describe('Updater', () => { ) }) - it('hands the decoded file subset off to the update container', async () => { + it('gives both phases the normal job input', async () => { await updater.runUpdater() expect(storeInput.mock.calls[0][3]).toEqual({job: splitJobDetails}) - expect(storeInput.mock.calls[1][3]).toEqual({ - job: splitJobDetails, - base_commit_sha: 'sha', - base64_dependency_files: [ - { - name: 'package.json', - content: Buffer.from('{}').toString('base64'), - directory: '/' - } - ], - dependency_files: [ - {name: 'package.json', content: '{}', directory: '/'} - ] - }) + expect(storeInput.mock.calls[1][3]).toEqual({job: splitJobDetails}) + }) + + it('hands the fetched checkout to the update phase in a separate volume', async () => { + await updater.runUpdater() + + expect(createContainer.mock.calls[0][0].HostConfig.Mounts).toEqual([ + { + Type: 'volume', + Source: 'dependabot-job-1-clone', + Target: '/home/dependabot/dependabot-updater/repo' + }, + { + Type: 'volume', + Source: 'dependabot-job-1-handoff', + Target: '/home/dependabot/dependabot-updater/repo-handoff' + } + ]) + expect(createContainer.mock.calls[1][0].HostConfig.Mounts).toEqual([ + { + Type: 'volume', + Source: 'dependabot-job-1-handoff', + Target: '/home/dependabot/dependabot-updater/repo' + } + ]) + expect(removeCloneVolume).toHaveBeenCalledTimes(1) + expect(removeHandoffVolume).toHaveBeenCalledTimes(1) + }) + + it('reports a repository volume cleanup failure after a successful run', async () => { + removeCloneVolume.mockRejectedValue(new Error('volume cleanup failed')) + + await expect(updater.runUpdater()).rejects.toThrow( + 'volume cleanup failed' + ) + }) + + it('does not mask a fetch failure with a volume cleanup failure', async () => { + runFileFetcher.mockRejectedValue(new Error('fetch failed')) + removeCloneVolume.mockRejectedValue(new Error('volume cleanup failed')) + + await expect(updater.runUpdater()).rejects.toThrow('fetch failed') }) it('does not clean up the update phase proxy until the update fails', async () => { @@ -658,6 +673,8 @@ describe('Updater', () => { // One shutdown for the fetch proxy, one for the update proxy. expect(mockProxy.shutdown).toHaveBeenCalledTimes(2) + expect(removeCloneVolume).toHaveBeenCalledTimes(1) + expect(removeHandoffVolume).toHaveBeenCalledTimes(1) }) it('does not start the update phase when the fetch phase fails', async () => { @@ -668,6 +685,8 @@ describe('Updater', () => { expect(runFileUpdater).not.toHaveBeenCalled() expect(proxyRun).toHaveBeenCalledTimes(1) expect(mockProxy.shutdown).toHaveBeenCalledTimes(1) + expect(removeCloneVolume).toHaveBeenCalledTimes(1) + expect(removeHandoffVolume).toHaveBeenCalledTimes(1) }) it('forwards the graph command to the update phase', async () => { @@ -686,7 +705,7 @@ describe('Updater', () => { it.each([ ['the experiment is absent', {}], - ['the experiment is disabled', {'split-fetch-update-containers': false}] + ['the experiment is disabled', {isolate_fetch_update: false}] ])('uses the single container path when %s', async (_name, experiments) => { const legacyUpdater = new Updater( 'MOCK_UPDATER_IMAGE_NAME', @@ -706,15 +725,5 @@ describe('Updater', () => { expect(proxyRun.mock.calls[0][3]).toEqual(credentials) expect(proxyRun.mock.calls[0][4]).toBeUndefined() }) - - it('stays on the single container path without the opt-in', async () => { - delete process.env.DEPENDABOT_SPLIT_FETCH_UPDATE - - expect(await updater.runUpdater()).toBe(true) - - expect(runSingleContainer).toHaveBeenCalledTimes(1) - expect(runFileFetcher).not.toHaveBeenCalled() - expect(runFileUpdater).not.toHaveBeenCalled() - }) }) }) diff --git a/dist/main.js b/dist/main.js index 01363344d..d0cd12e6e 100644 --- a/dist/main.js +++ b/dist/main.js @@ -418,7 +418,7 @@ var require_tunnel = __commonJS({ connectOptions.headers = connectOptions.headers || {}; connectOptions.headers["Proxy-Authorization"] = "Basic " + new Buffer(connectOptions.proxyAuth).toString("base64"); } - debug2("making CONNECT request"); + debug3("making CONNECT request"); var connectReq = self2.request(connectOptions); connectReq.useChunkedEncodingByDefault = false; connectReq.once("response", onResponse); @@ -438,7 +438,7 @@ var require_tunnel = __commonJS({ connectReq.removeAllListeners(); socket.removeAllListeners(); if (res.statusCode !== 200) { - debug2( + debug3( "tunneling socket could not be established, statusCode=%d", res.statusCode ); @@ -450,7 +450,7 @@ var require_tunnel = __commonJS({ return; } if (head.length > 0) { - debug2("got illegal response body from proxy"); + debug3("got illegal response body from proxy"); socket.destroy(); var error3 = new Error("got illegal response body from proxy"); error3.code = "ECONNRESET"; @@ -458,13 +458,13 @@ var require_tunnel = __commonJS({ self2.removeSocket(placeholder); return; } - debug2("tunneling connection has established"); + debug3("tunneling connection has established"); self2.sockets[self2.sockets.indexOf(placeholder)] = socket; return cb(socket); } function onError(cause) { connectReq.removeAllListeners(); - debug2( + debug3( "tunneling socket could not be established, cause=%s\n", cause.message, cause.stack @@ -526,9 +526,9 @@ var require_tunnel = __commonJS({ } return target; } - var debug2; + var debug3; if (process.env.NODE_DEBUG && /\btunnel\b/.test(process.env.NODE_DEBUG)) { - debug2 = function() { + debug3 = function() { var args = Array.prototype.slice.call(arguments); if (typeof args[0] === "string") { args[0] = "TUNNEL: " + args[0]; @@ -538,10 +538,10 @@ var require_tunnel = __commonJS({ console.error.apply(console, args); }; } else { - debug2 = function() { + debug3 = function() { }; } - exports2.debug = debug2; + exports2.debug = debug3; } }); @@ -19273,7 +19273,7 @@ var require_core = __commonJS({ exports2.setCommandEcho = setCommandEcho; exports2.setFailed = setFailed3; exports2.isDebug = isDebug; - exports2.debug = debug2; + exports2.debug = debug3; exports2.error = error3; exports2.warning = warning5; exports2.notice = notice; @@ -19362,7 +19362,7 @@ Support boolean input list: \`true | True | TRUE | false | False | FALSE\``); function isDebug() { return process.env["RUNNER_DEBUG"] === "1"; } - function debug2(message) { + function debug3(message) { (0, command_1.issueCommand)("debug", {}, message); } function error3(message, properties = {}) { @@ -34184,10 +34184,10 @@ var require_kex = __commonJS({ let clientList; let serverList; let i; - const debug2 = self2._debug; - debug2 && debug2("Inbound: Handshake in progress"); - debug2 && debug2(`Handshake: (local) KEX method: ${localKex}`); - debug2 && debug2(`Handshake: (remote) KEX method: ${remote.kex}`); + const debug3 = self2._debug; + debug3 && debug3("Inbound: Handshake in progress"); + debug3 && debug3(`Handshake: (local) KEX method: ${localKex}`); + debug3 && debug3(`Handshake: (remote) KEX method: ${remote.kex}`); let remoteExtInfoEnabled; if (self2._server) { serverList = localKex; @@ -34205,10 +34205,10 @@ var require_kex = __commonJS({ self2._strictMode = serverList.indexOf("kex-strict-s-v00@openssh.com") !== -1; } if (self2._strictMode) { - debug2 && debug2("Handshake: strict KEX mode enabled"); + debug3 && debug3("Handshake: strict KEX mode enabled"); if (self2._decipher.inSeqno !== 1) { - if (debug2) - debug2("Handshake: KEXINIT not first packet in strict KEX mode"); + if (debug3) + debug3("Handshake: KEXINIT not first packet in strict KEX mode"); return doFatalError( self2, "Handshake failed: KEXINIT not first packet in strict KEX mode", @@ -34220,7 +34220,7 @@ var require_kex = __commonJS({ } for (i = 0; i < clientList.length && serverList.indexOf(clientList[i]) === -1; ++i) ; if (i === clientList.length) { - debug2 && debug2("Handshake: no matching key exchange algorithm"); + debug3 && debug3("Handshake: no matching key exchange algorithm"); return doFatalError( self2, "Handshake failed: no matching key exchange algorithm", @@ -34229,13 +34229,13 @@ var require_kex = __commonJS({ ); } init.kex = clientList[i]; - debug2 && debug2(`Handshake: KEX algorithm: ${clientList[i]}`); + debug3 && debug3(`Handshake: KEX algorithm: ${clientList[i]}`); if (firstFollows && (!remote.kex.length || clientList[i] !== remote.kex[0])) { self2._skipNextInboundPacket = true; } const localSrvHostKey = local.lists.serverHostKey.array; - debug2 && debug2(`Handshake: (local) Host key format: ${localSrvHostKey}`); - debug2 && debug2( + debug3 && debug3(`Handshake: (local) Host key format: ${localSrvHostKey}`); + debug3 && debug3( `Handshake: (remote) Host key format: ${remote.serverHostKey}` ); if (self2._server) { @@ -34247,7 +34247,7 @@ var require_kex = __commonJS({ } for (i = 0; i < clientList.length && serverList.indexOf(clientList[i]) === -1; ++i) ; if (i === clientList.length) { - debug2 && debug2("Handshake: No matching host key format"); + debug3 && debug3("Handshake: No matching host key format"); return doFatalError( self2, "Handshake failed: no matching host key format", @@ -34256,10 +34256,10 @@ var require_kex = __commonJS({ ); } init.serverHostKey = clientList[i]; - debug2 && debug2(`Handshake: Host key format: ${clientList[i]}`); + debug3 && debug3(`Handshake: Host key format: ${clientList[i]}`); const localCSCipher = local.lists.cs.cipher.array; - debug2 && debug2(`Handshake: (local) C->S cipher: ${localCSCipher}`); - debug2 && debug2(`Handshake: (remote) C->S cipher: ${remote.cs.cipher}`); + debug3 && debug3(`Handshake: (local) C->S cipher: ${localCSCipher}`); + debug3 && debug3(`Handshake: (remote) C->S cipher: ${remote.cs.cipher}`); if (self2._server) { serverList = localCSCipher; clientList = remote.cs.cipher; @@ -34269,7 +34269,7 @@ var require_kex = __commonJS({ } for (i = 0; i < clientList.length && serverList.indexOf(clientList[i]) === -1; ++i) ; if (i === clientList.length) { - debug2 && debug2("Handshake: No matching C->S cipher"); + debug3 && debug3("Handshake: No matching C->S cipher"); return doFatalError( self2, "Handshake failed: no matching C->S cipher", @@ -34278,10 +34278,10 @@ var require_kex = __commonJS({ ); } init.cs.cipher = clientList[i]; - debug2 && debug2(`Handshake: C->S Cipher: ${clientList[i]}`); + debug3 && debug3(`Handshake: C->S Cipher: ${clientList[i]}`); const localSCCipher = local.lists.sc.cipher.array; - debug2 && debug2(`Handshake: (local) S->C cipher: ${localSCCipher}`); - debug2 && debug2(`Handshake: (remote) S->C cipher: ${remote.sc.cipher}`); + debug3 && debug3(`Handshake: (local) S->C cipher: ${localSCCipher}`); + debug3 && debug3(`Handshake: (remote) S->C cipher: ${remote.sc.cipher}`); if (self2._server) { serverList = localSCCipher; clientList = remote.sc.cipher; @@ -34291,7 +34291,7 @@ var require_kex = __commonJS({ } for (i = 0; i < clientList.length && serverList.indexOf(clientList[i]) === -1; ++i) ; if (i === clientList.length) { - debug2 && debug2("Handshake: No matching S->C cipher"); + debug3 && debug3("Handshake: No matching S->C cipher"); return doFatalError( self2, "Handshake failed: no matching S->C cipher", @@ -34300,13 +34300,13 @@ var require_kex = __commonJS({ ); } init.sc.cipher = clientList[i]; - debug2 && debug2(`Handshake: S->C cipher: ${clientList[i]}`); + debug3 && debug3(`Handshake: S->C cipher: ${clientList[i]}`); const localCSMAC = local.lists.cs.mac.array; - debug2 && debug2(`Handshake: (local) C->S MAC: ${localCSMAC}`); - debug2 && debug2(`Handshake: (remote) C->S MAC: ${remote.cs.mac}`); + debug3 && debug3(`Handshake: (local) C->S MAC: ${localCSMAC}`); + debug3 && debug3(`Handshake: (remote) C->S MAC: ${remote.cs.mac}`); if (CIPHER_INFO[init.cs.cipher].authLen > 0) { init.cs.mac = ""; - debug2 && debug2("Handshake: C->S MAC: "); + debug3 && debug3("Handshake: C->S MAC: "); } else { if (self2._server) { serverList = localCSMAC; @@ -34317,7 +34317,7 @@ var require_kex = __commonJS({ } for (i = 0; i < clientList.length && serverList.indexOf(clientList[i]) === -1; ++i) ; if (i === clientList.length) { - debug2 && debug2("Handshake: No matching C->S MAC"); + debug3 && debug3("Handshake: No matching C->S MAC"); return doFatalError( self2, "Handshake failed: no matching C->S MAC", @@ -34326,14 +34326,14 @@ var require_kex = __commonJS({ ); } init.cs.mac = clientList[i]; - debug2 && debug2(`Handshake: C->S MAC: ${clientList[i]}`); + debug3 && debug3(`Handshake: C->S MAC: ${clientList[i]}`); } const localSCMAC = local.lists.sc.mac.array; - debug2 && debug2(`Handshake: (local) S->C MAC: ${localSCMAC}`); - debug2 && debug2(`Handshake: (remote) S->C MAC: ${remote.sc.mac}`); + debug3 && debug3(`Handshake: (local) S->C MAC: ${localSCMAC}`); + debug3 && debug3(`Handshake: (remote) S->C MAC: ${remote.sc.mac}`); if (CIPHER_INFO[init.sc.cipher].authLen > 0) { init.sc.mac = ""; - debug2 && debug2("Handshake: S->C MAC: "); + debug3 && debug3("Handshake: S->C MAC: "); } else { if (self2._server) { serverList = localSCMAC; @@ -34344,7 +34344,7 @@ var require_kex = __commonJS({ } for (i = 0; i < clientList.length && serverList.indexOf(clientList[i]) === -1; ++i) ; if (i === clientList.length) { - debug2 && debug2("Handshake: No matching S->C MAC"); + debug3 && debug3("Handshake: No matching S->C MAC"); return doFatalError( self2, "Handshake failed: no matching S->C MAC", @@ -34353,11 +34353,11 @@ var require_kex = __commonJS({ ); } init.sc.mac = clientList[i]; - debug2 && debug2(`Handshake: S->C MAC: ${clientList[i]}`); + debug3 && debug3(`Handshake: S->C MAC: ${clientList[i]}`); } const localCSCompress = local.lists.cs.compress.array; - debug2 && debug2(`Handshake: (local) C->S compression: ${localCSCompress}`); - debug2 && debug2(`Handshake: (remote) C->S compression: ${remote.cs.compress}`); + debug3 && debug3(`Handshake: (local) C->S compression: ${localCSCompress}`); + debug3 && debug3(`Handshake: (remote) C->S compression: ${remote.cs.compress}`); if (self2._server) { serverList = localCSCompress; clientList = remote.cs.compress; @@ -34367,7 +34367,7 @@ var require_kex = __commonJS({ } for (i = 0; i < clientList.length && serverList.indexOf(clientList[i]) === -1; ++i) ; if (i === clientList.length) { - debug2 && debug2("Handshake: No matching C->S compression"); + debug3 && debug3("Handshake: No matching C->S compression"); return doFatalError( self2, "Handshake failed: no matching C->S compression", @@ -34376,10 +34376,10 @@ var require_kex = __commonJS({ ); } init.cs.compress = clientList[i]; - debug2 && debug2(`Handshake: C->S compression: ${clientList[i]}`); + debug3 && debug3(`Handshake: C->S compression: ${clientList[i]}`); const localSCCompress = local.lists.sc.compress.array; - debug2 && debug2(`Handshake: (local) S->C compression: ${localSCCompress}`); - debug2 && debug2(`Handshake: (remote) S->C compression: ${remote.sc.compress}`); + debug3 && debug3(`Handshake: (local) S->C compression: ${localSCCompress}`); + debug3 && debug3(`Handshake: (remote) S->C compression: ${remote.sc.compress}`); if (self2._server) { serverList = localSCCompress; clientList = remote.sc.compress; @@ -34389,7 +34389,7 @@ var require_kex = __commonJS({ } for (i = 0; i < clientList.length && serverList.indexOf(clientList[i]) === -1; ++i) ; if (i === clientList.length) { - debug2 && debug2("Handshake: No matching S->C compression"); + debug3 && debug3("Handshake: No matching S->C compression"); return doFatalError( self2, "Handshake failed: no matching S->C compression", @@ -34398,7 +34398,7 @@ var require_kex = __commonJS({ ); } init.sc.compress = clientList[i]; - debug2 && debug2(`Handshake: S->C compression: ${clientList[i]}`); + debug3 && debug3(`Handshake: S->C compression: ${clientList[i]}`); init.cs.lang = ""; init.sc.lang = ""; if (self2._kex) { @@ -35774,9 +35774,9 @@ var require_Protocol = __commonJS({ this._onError = (err) => { onError(err); }; - const debug2 = config.debug; - this._debug = typeof debug2 === "function" ? (msg) => { - debug2(msg); + const debug3 = config.debug; + this._debug = typeof debug3 === "function" ? (msg) => { + debug3(msg); } : void 0; const onHeader = config.onHeader; this._onHeader = typeof onHeader === "function" ? (...args) => { @@ -41100,7 +41100,7 @@ var require_client2 = __commonJS({ this.config.allowAgentFwd = cfg.agentForward === true && this.config.agent !== void 0; let authHandler = this.config.authHandler = typeof cfg.authHandler === "function" || Array.isArray(cfg.authHandler) ? cfg.authHandler : void 0; this.config.strictVendor = typeof cfg.strictVendor === "boolean" ? cfg.strictVendor : true; - const debug2 = this.config.debug = typeof cfg.debug === "function" ? cfg.debug : void 0; + const debug3 = this.config.debug = typeof cfg.debug === "function" ? cfg.debug : void 0; if (cfg.agentForward === true && !this.config.allowAgentFwd) { throw new Error( "You must set a valid agent path to allow agent forwarding" @@ -41148,8 +41148,8 @@ var require_client2 = __commonJS({ let sawHeader = false; if (this._protocol) this._protocol.cleanup(); - const DEBUG_HANDLER = !debug2 ? void 0 : (p, display, msg) => { - debug2(`Debug output from server: ${JSON.stringify(msg)}`); + const DEBUG_HANDLER = !debug3 ? void 0 : (p, display, msg) => { + debug3(`Debug output from server: ${JSON.stringify(msg)}`); }; let serverSigAlgs; const proto = this._protocol = new Protocol({ @@ -41183,7 +41183,7 @@ var require_client2 = __commonJS({ proto.service("ssh-userauth"); } }, - debug: debug2, + debug: debug3, hostVerifier, messageHandlers: { DEBUG: DEBUG_HANDLER, @@ -41226,8 +41226,8 @@ var require_client2 = __commonJS({ USERAUTH_FAILURE: (p, authMethods, partialSuccess) => { if (curAuth.keyAlgos) { const oldKeyAlgo = curAuth.keyAlgos[0][0]; - if (debug2) - debug2(`Client: ${curAuth.type} (${oldKeyAlgo}) auth failed`); + if (debug3) + debug3(`Client: ${curAuth.type} (${oldKeyAlgo}) auth failed`); curAuth.keyAlgos.shift(); if (curAuth.keyAlgos.length) { const [keyAlgo, hashAlgo] = curAuth.keyAlgos[0]; @@ -41268,10 +41268,10 @@ var require_client2 = __commonJS({ } if (curAuth.type === "agent") { const pos = curAuth.agentCtx.pos(); - debug2 && debug2(`Client: Agent key #${pos + 1} failed`); + debug3 && debug3(`Client: Agent key #${pos + 1} failed`); return tryNextAgentKey(); } - debug2 && debug2(`Client: ${curAuth.type} auth failed`); + debug3 && debug3(`Client: ${curAuth.type} auth failed`); curPartial = partialSuccess; curAuthsLeft = authMethods; tryNextAuth(); @@ -41323,7 +41323,7 @@ var require_client2 = __commonJS({ if (curAuth.type === "keyboard-interactive") { const nprompts = Array.isArray(prompts) ? prompts.length : 0; if (nprompts === 0) { - debug2 && debug2( + debug3 && debug3( "Client: Sending automatic USERAUTH_INFO_RESPONSE" ); proto.authInfoRes(); @@ -41388,7 +41388,7 @@ var require_client2 = __commonJS({ state: "open" } }; - const instance = isSFTP ? new SFTP(this, chanInfo, { debug: debug2 }) : new Channel(this, chanInfo); + const instance = isSFTP ? new SFTP(this, chanInfo, { debug: debug3 }) : new Channel(this, chanInfo); this._chanMgr.update(info8.recipient, instance); channel(void 0, instance); }, @@ -41559,7 +41559,7 @@ var require_client2 = __commonJS({ return; called = true; wasConnected = true; - debug2 && debug2("Socket connected"); + debug3 && debug3("Socket connected"); this.emit("connect"); cryptoInit.then(() => { proto.start(); @@ -41592,19 +41592,19 @@ var require_client2 = __commonJS({ sock.on("connect", onConnect).on("timeout", () => { this.emit("timeout"); }).on("error", (err) => { - debug2 && debug2(`Socket error: ${err.message}`); + debug3 && debug3(`Socket error: ${err.message}`); clearTimeout(this._readyTimeout); err.level = "client-socket"; this.emit("error", err); }).on("end", () => { - debug2 && debug2("Socket ended"); + debug3 && debug3("Socket ended"); onDone(); proto.cleanup(); clearTimeout(this._readyTimeout); clearInterval(katimer); this.emit("end"); }).on("close", () => { - debug2 && debug2("Socket closed"); + debug3 && debug3("Socket closed"); onDone(); proto.cleanup(); clearTimeout(this._readyTimeout); @@ -41838,7 +41838,7 @@ var require_client2 = __commonJS({ } }; function skipAuth(msg) { - debug2 && debug2(msg); + debug3 && debug3(msg); process.nextTick(tryNextAuth); } function tryNextAuth() { @@ -41852,8 +41852,8 @@ var require_client2 = __commonJS({ if (curAuth.type === "agent") { const key = curAuth.agentCtx.nextKey(); if (key === false) { - debug2 && debug2("Agent: No more keys left to try"); - debug2 && debug2("Client: agent auth failed"); + debug3 && debug3("Agent: No more keys left to try"); + debug3 && debug3("Client: agent auth failed"); tryNextAuth(); } else { const pos = curAuth.agentCtx.pos(); @@ -41863,14 +41863,14 @@ var require_client2 = __commonJS({ if (curAuth.keyAlgos.length) { keyAlgo = curAuth.keyAlgos[0][0]; } else { - debug2 && debug2( + debug3 && debug3( `Agent: Skipping key #${pos + 1} (no mutual hash algorithm)` ); tryNextAgentKey(); return; } } - debug2 && debug2(`Agent: Trying key #${pos + 1}`); + debug3 && debug3(`Agent: Trying key #${pos + 1}`); proto.authPK(curAuth.username, key, keyAlgo); } } @@ -41889,7 +41889,7 @@ var require_client2 = __commonJS({ let host = this.config.host; const forceIPv4 = this.config.forceIPv4; const forceIPv6 = this.config.forceIPv6; - debug2 && debug2(`Client: Trying ${host} on port ${this.config.port} ...`); + debug3 && debug3(`Client: Trying ${host} on port ${this.config.port} ...`); const doConnect = () => { startTimeout(); sock.connect({ @@ -43075,14 +43075,14 @@ var require_server = __commonJS({ socket.once("close", () => { --this._connections; }); - let debug2; + let debug3; if (origDebug) { const debugPrefix = `[${process.hrtime().join(".")}] `; - debug2 = (msg) => { + debug3 = (msg) => { origDebug(`${debugPrefix}${msg}`); }; } - new Client(socket, hostKeys, ident, offer, debug2, this, cfg); + new Client(socket, hostKeys, ident, offer, debug3, this, cfg); }).on("error", (err) => { this.emit("error", err); }).on("listening", () => { @@ -43123,7 +43123,7 @@ var require_server = __commonJS({ Server.KEEPALIVE_CLIENT_INTERVAL = 15e3; Server.KEEPALIVE_CLIENT_COUNT_MAX = 3; var Client = class extends EventEmitter { - constructor(socket, hostKeys, ident, offer, debug2, server, srvCfg) { + constructor(socket, hostKeys, ident, offer, debug3, server, srvCfg) { super(); let exchanges = 0; let acceptedAuthSvc = false; @@ -43134,14 +43134,14 @@ var require_server = __commonJS({ const unsentGlobalRequestsReplies = []; this._sock = socket; this._chanMgr = new ChannelManager(this); - this._debug = debug2; + this._debug = debug3; this.noMoreSessions = false; this.authenticated = false; function onClientPreHeaderError(err) { } this.on("error", onClientPreHeaderError); - const DEBUG_HANDLER = !debug2 ? void 0 : (p, display, msg) => { - debug2(`Debug output from client: ${JSON.stringify(msg)}`); + const DEBUG_HANDLER = !debug3 ? void 0 : (p, display, msg) => { + debug3(`Debug output from client: ${JSON.stringify(msg)}`); }; const kaIntvl = typeof srvCfg.keepaliveInterval === "number" && isFinite(srvCfg.keepaliveInterval) && srvCfg.keepaliveInterval > 0 ? srvCfg.keepaliveInterval : typeof Server.KEEPALIVE_CLIENT_INTERVAL === "number" && isFinite(Server.KEEPALIVE_CLIENT_INTERVAL) && Server.KEEPALIVE_CLIENT_INTERVAL > 0 ? Server.KEEPALIVE_CLIENT_INTERVAL : -1; const kaCountMax = typeof srvCfg.keepaliveCountMax === "number" && isFinite(srvCfg.keepaliveCountMax) && srvCfg.keepaliveCountMax >= 0 ? srvCfg.keepaliveCountMax : typeof Server.KEEPALIVE_CLIENT_COUNT_MAX === "number" && isFinite(Server.KEEPALIVE_CLIENT_COUNT_MAX) && Server.KEEPALIVE_CLIENT_COUNT_MAX >= 0 ? Server.KEEPALIVE_CLIENT_COUNT_MAX : -1; @@ -43211,7 +43211,7 @@ var require_server = __commonJS({ this.emit("rekey"); this.emit("handshake", negotiated); }, - debug: debug2, + debug: debug3, messageHandlers: { DEBUG: DEBUG_HANDLER, DISCONNECT: (p, reason, desc) => { @@ -43254,8 +43254,8 @@ var require_server = __commonJS({ localChan = this._chanMgr.add(); if (localChan === -1) { reason = CHANNEL_OPEN_FAILURE.RESOURCE_SHORTAGE; - if (debug2) { - debug2("Automatic rejection of incoming channel open: no channels available"); + if (debug3) { + debug3("Automatic rejection of incoming channel open: no channels available"); } } return localChan !== -1; @@ -43354,14 +43354,14 @@ var require_server = __commonJS({ break; default: reason = CHANNEL_OPEN_FAILURE.UNKNOWN_CHANNEL_TYPE; - if (debug2) { - debug2(`Automatic rejection of unsupported incoming channel open type: ${info8.type}`); + if (debug3) { + debug3(`Automatic rejection of unsupported incoming channel open type: ${info8.type}`); } } if (reason === void 0) { reason = CHANNEL_OPEN_FAILURE.ADMINISTRATIVELY_PROHIBITED; - if (debug2) { - debug2(`Automatic rejection of unexpected incoming channel open for: ${info8.type}`); + if (debug3) { + debug3(`Automatic rejection of unexpected incoming channel open for: ${info8.type}`); } } reject(); @@ -43603,7 +43603,7 @@ var require_server = __commonJS({ if (useSFTP) { instance = new SFTP(this, session._chanInfo, { server: true, - debug: debug2 + debug: debug3 }); } else { instance = new Channel( @@ -43632,7 +43632,7 @@ var require_server = __commonJS({ break; } } - debug2 && debug2( + debug3 && debug3( `Automatic rejection of incoming channel request: ${type}` ); reject && reject(); @@ -43842,11 +43842,11 @@ var require_server = __commonJS({ err.level = "socket"; this.emit("error", err); }).once("end", () => { - debug2 && debug2("Socket ended"); + debug3 && debug3("Socket ended"); proto.cleanup(); this.emit("end"); }).once("close", () => { - debug2 && debug2("Socket closed"); + debug3 && debug3("Socket closed"); proto.cleanup(); this.emit("close"); const err = new Error("No response from server"); @@ -46246,11 +46246,11 @@ var require_stream_readable = __commonJS({ return Buffer2.isBuffer(obj) || obj instanceof OurUint8Array; } var debugUtil = require("util"); - var debug2; + var debug3; if (debugUtil && debugUtil.debuglog) { - debug2 = debugUtil.debuglog("stream"); + debug3 = debugUtil.debuglog("stream"); } else { - debug2 = function debug3() { + debug3 = function debug4() { }; } var BufferList = require_buffer_list(); @@ -46365,7 +46365,7 @@ var require_stream_readable = __commonJS({ return readableAddChunk(this, chunk, null, true, false); }; function readableAddChunk(stream2, chunk, encoding, addToFront, skipChunkCheck) { - debug2("readableAddChunk", chunk); + debug3("readableAddChunk", chunk); var state = stream2._readableState; if (chunk === null) { state.reading = false; @@ -46472,13 +46472,13 @@ var require_stream_readable = __commonJS({ return state.length; } Readable2.prototype.read = function(n) { - debug2("read", n); + debug3("read", n); n = parseInt(n, 10); var state = this._readableState; var nOrig = n; if (n !== 0) state.emittedReadable = false; if (n === 0 && state.needReadable && ((state.highWaterMark !== 0 ? state.length >= state.highWaterMark : state.length > 0) || state.ended)) { - debug2("read: emitReadable", state.length, state.ended); + debug3("read: emitReadable", state.length, state.ended); if (state.length === 0 && state.ended) endReadable(this); else emitReadable(this); return null; @@ -46489,16 +46489,16 @@ var require_stream_readable = __commonJS({ return null; } var doRead = state.needReadable; - debug2("need readable", doRead); + debug3("need readable", doRead); if (state.length === 0 || state.length - n < state.highWaterMark) { doRead = true; - debug2("length less than watermark", doRead); + debug3("length less than watermark", doRead); } if (state.ended || state.reading) { doRead = false; - debug2("reading or ended", doRead); + debug3("reading or ended", doRead); } else if (doRead) { - debug2("do read"); + debug3("do read"); state.reading = true; state.sync = true; if (state.length === 0) state.needReadable = true; @@ -46524,7 +46524,7 @@ var require_stream_readable = __commonJS({ return ret; }; function onEofChunk(stream2, state) { - debug2("onEofChunk"); + debug3("onEofChunk"); if (state.ended) return; if (state.decoder) { var chunk = state.decoder.end(); @@ -46546,17 +46546,17 @@ var require_stream_readable = __commonJS({ } function emitReadable(stream2) { var state = stream2._readableState; - debug2("emitReadable", state.needReadable, state.emittedReadable); + debug3("emitReadable", state.needReadable, state.emittedReadable); state.needReadable = false; if (!state.emittedReadable) { - debug2("emitReadable", state.flowing); + debug3("emitReadable", state.flowing); state.emittedReadable = true; process.nextTick(emitReadable_, stream2); } } function emitReadable_(stream2) { var state = stream2._readableState; - debug2("emitReadable_", state.destroyed, state.length, state.ended); + debug3("emitReadable_", state.destroyed, state.length, state.ended); if (!state.destroyed && (state.length || state.ended)) { stream2.emit("readable"); state.emittedReadable = false; @@ -46573,7 +46573,7 @@ var require_stream_readable = __commonJS({ function maybeReadMore_(stream2, state) { while (!state.reading && !state.ended && (state.length < state.highWaterMark || state.flowing && state.length === 0)) { var len = state.length; - debug2("maybeReadMore read 0"); + debug3("maybeReadMore read 0"); stream2.read(0); if (len === state.length) break; @@ -46598,14 +46598,14 @@ var require_stream_readable = __commonJS({ break; } state.pipesCount += 1; - debug2("pipe count=%d opts=%j", state.pipesCount, pipeOpts); + debug3("pipe count=%d opts=%j", state.pipesCount, pipeOpts); var doEnd = (!pipeOpts || pipeOpts.end !== false) && dest !== process.stdout && dest !== process.stderr; var endFn = doEnd ? onend : unpipe; if (state.endEmitted) process.nextTick(endFn); else src.once("end", endFn); dest.on("unpipe", onunpipe); function onunpipe(readable, unpipeInfo) { - debug2("onunpipe"); + debug3("onunpipe"); if (readable === src) { if (unpipeInfo && unpipeInfo.hasUnpiped === false) { unpipeInfo.hasUnpiped = true; @@ -46614,14 +46614,14 @@ var require_stream_readable = __commonJS({ } } function onend() { - debug2("onend"); + debug3("onend"); dest.end(); } var ondrain = pipeOnDrain(src); dest.on("drain", ondrain); var cleanedUp = false; function cleanup() { - debug2("cleanup"); + debug3("cleanup"); dest.removeListener("close", onclose); dest.removeListener("finish", onfinish); dest.removeListener("drain", ondrain); @@ -46635,19 +46635,19 @@ var require_stream_readable = __commonJS({ } src.on("data", ondata); function ondata(chunk) { - debug2("ondata"); + debug3("ondata"); var ret = dest.write(chunk); - debug2("dest.write", ret); + debug3("dest.write", ret); if (ret === false) { if ((state.pipesCount === 1 && state.pipes === dest || state.pipesCount > 1 && indexOf(state.pipes, dest) !== -1) && !cleanedUp) { - debug2("false write response, pause", state.awaitDrain); + debug3("false write response, pause", state.awaitDrain); state.awaitDrain++; } src.pause(); } } function onerror(er) { - debug2("onerror", er); + debug3("onerror", er); unpipe(); dest.removeListener("error", onerror); if (EElistenerCount(dest, "error") === 0) errorOrDestroy(dest, er); @@ -46659,18 +46659,18 @@ var require_stream_readable = __commonJS({ } dest.once("close", onclose); function onfinish() { - debug2("onfinish"); + debug3("onfinish"); dest.removeListener("close", onclose); unpipe(); } dest.once("finish", onfinish); function unpipe() { - debug2("unpipe"); + debug3("unpipe"); src.unpipe(dest); } dest.emit("pipe", src); if (!state.flowing) { - debug2("pipe resume"); + debug3("pipe resume"); src.resume(); } return dest; @@ -46678,7 +46678,7 @@ var require_stream_readable = __commonJS({ function pipeOnDrain(src) { return function pipeOnDrainFunctionResult() { var state = src._readableState; - debug2("pipeOnDrain", state.awaitDrain); + debug3("pipeOnDrain", state.awaitDrain); if (state.awaitDrain) state.awaitDrain--; if (state.awaitDrain === 0 && EElistenerCount(src, "data")) { state.flowing = true; @@ -46733,7 +46733,7 @@ var require_stream_readable = __commonJS({ state.readableListening = state.needReadable = true; state.flowing = false; state.emittedReadable = false; - debug2("on readable", state.length, state.reading); + debug3("on readable", state.length, state.reading); if (state.length) { emitReadable(this); } else if (!state.reading) { @@ -46768,13 +46768,13 @@ var require_stream_readable = __commonJS({ } } function nReadingNextTick(self2) { - debug2("readable nexttick read 0"); + debug3("readable nexttick read 0"); self2.read(0); } Readable2.prototype.resume = function() { var state = this._readableState; if (!state.flowing) { - debug2("resume"); + debug3("resume"); state.flowing = !state.readableListening; resume(this, state); } @@ -46788,7 +46788,7 @@ var require_stream_readable = __commonJS({ } } function resume_(stream2, state) { - debug2("resume", state.reading); + debug3("resume", state.reading); if (!state.reading) { stream2.read(0); } @@ -46798,9 +46798,9 @@ var require_stream_readable = __commonJS({ if (state.flowing && !state.reading) stream2.read(0); } Readable2.prototype.pause = function() { - debug2("call pause flowing=%j", this._readableState.flowing); + debug3("call pause flowing=%j", this._readableState.flowing); if (this._readableState.flowing !== false) { - debug2("pause"); + debug3("pause"); this._readableState.flowing = false; this.emit("pause"); } @@ -46809,7 +46809,7 @@ var require_stream_readable = __commonJS({ }; function flow(stream2) { var state = stream2._readableState; - debug2("flow", state.flowing); + debug3("flow", state.flowing); while (state.flowing && stream2.read() !== null) { ; } @@ -46819,7 +46819,7 @@ var require_stream_readable = __commonJS({ var state = this._readableState; var paused = false; stream2.on("end", function() { - debug2("wrapped end"); + debug3("wrapped end"); if (state.decoder && !state.ended) { var chunk = state.decoder.end(); if (chunk && chunk.length) _this.push(chunk); @@ -46827,7 +46827,7 @@ var require_stream_readable = __commonJS({ _this.push(null); }); stream2.on("data", function(chunk) { - debug2("wrapped data"); + debug3("wrapped data"); if (state.decoder) chunk = state.decoder.write(chunk); if (state.objectMode && (chunk === null || chunk === void 0)) return; else if (!state.objectMode && (!chunk || !chunk.length)) return; @@ -46850,7 +46850,7 @@ var require_stream_readable = __commonJS({ stream2.on(kProxyEvents[n], this.emit.bind(this, kProxyEvents[n])); } this._read = function(n2) { - debug2("wrapped _read", n2); + debug3("wrapped _read", n2); if (paused) { paused = false; stream2.resume(); @@ -46924,14 +46924,14 @@ var require_stream_readable = __commonJS({ } function endReadable(stream2) { var state = stream2._readableState; - debug2("endReadable", state.endEmitted); + debug3("endReadable", state.endEmitted); if (!state.endEmitted) { state.ended = true; process.nextTick(endReadableNT, state, stream2); } } function endReadableNT(state, stream2) { - debug2("endReadableNT", state.endEmitted, state.length); + debug3("endReadableNT", state.endEmitted, state.length); if (!state.endEmitted && state.length === 0) { state.endEmitted = true; stream2.readable = false; @@ -47382,11 +47382,11 @@ var require_common = __commonJS({ let enableOverride = null; let namespacesCache; let enabledCache; - function debug2(...args) { - if (!debug2.enabled) { + function debug3(...args) { + if (!debug3.enabled) { return; } - const self2 = debug2; + const self2 = debug3; const curr = Number(/* @__PURE__ */ new Date()); const ms = curr - (prevTime || curr); self2.diff = ms; @@ -47416,12 +47416,12 @@ var require_common = __commonJS({ const logFn = self2.log || createDebug.log; logFn.apply(self2, args); } - debug2.namespace = namespace; - debug2.useColors = createDebug.useColors(); - debug2.color = createDebug.selectColor(namespace); - debug2.extend = extend; - debug2.destroy = createDebug.destroy; - Object.defineProperty(debug2, "enabled", { + debug3.namespace = namespace; + debug3.useColors = createDebug.useColors(); + debug3.color = createDebug.selectColor(namespace); + debug3.extend = extend; + debug3.destroy = createDebug.destroy; + Object.defineProperty(debug3, "enabled", { enumerable: true, configurable: false, get: () => { @@ -47439,9 +47439,9 @@ var require_common = __commonJS({ } }); if (typeof createDebug.init === "function") { - createDebug.init(debug2); + createDebug.init(debug3); } - return debug2; + return debug3; } function extend(namespace, delimiter) { const newDebug = createDebug(this.namespace + (typeof delimiter === "undefined" ? ":" : delimiter) + namespace); @@ -47966,11 +47966,11 @@ var require_node2 = __commonJS({ function load() { return process.env.DEBUG; } - function init(debug2) { - debug2.inspectOpts = {}; + function init(debug3) { + debug3.inspectOpts = {}; const keys = Object.keys(exports2.inspectOpts); for (let i = 0; i < keys.length; i++) { - debug2.inspectOpts[keys[i]] = exports2.inspectOpts[keys[i]]; + debug3.inspectOpts[keys[i]] = exports2.inspectOpts[keys[i]]; } } module2.exports = require_common()(exports2); @@ -48038,7 +48038,7 @@ var require_modem = __commonJS({ var url = require("url"); var ssh = require_ssh(); var HttpDuplex = require_http_duplex(); - var debug2 = require_src()("modem"); + var debug3 = require_src()("modem"); var utils = require_utils5(); var util = require("util"); var splitca = require_split_ca(); @@ -48258,20 +48258,20 @@ var require_modem = __commonJS({ callback(e); return; } - debug2("Sending: %s", util.inspect(options, { + debug3("Sending: %s", util.inspect(options, { showHidden: true, depth: null })); if (self2.connectionTimeout) { connectionTimeoutTimer = setTimeout(function() { - debug2("Connection Timeout of %s ms exceeded", self2.connectionTimeout); + debug3("Connection Timeout of %s ms exceeded", self2.connectionTimeout); req.destroy(); }, self2.connectionTimeout); } if (self2.timeout) { req.setTimeout(self2.timeout); req.on("timeout", function() { - debug2("Timeout of %s ms exceeded", self2.timeout); + debug3("Timeout of %s ms exceeded", self2.timeout); req.destroy(); }); } @@ -48311,7 +48311,7 @@ var require_modem = __commonJS({ res.on("end", function() { var buffer = Buffer.concat(chunks); var result = buffer.toString(); - debug2("Received: %s", result); + debug3("Received: %s", result); var json = utils.parseJSON(result) || buffer; if (finished === false) { finished = true; @@ -99333,6 +99333,10 @@ var import_tar_stream = __toESM(require_tar_stream2()); var ContainerRuntimeError = class extends Error { }; var RWX_ALL = 511; +var OUTPUT_PATH = "/home/dependabot/dependabot-updater/output"; +var REPO_CONTENTS_PATH = "/home/dependabot/dependabot-updater/repo"; +var REPO_HANDOFF_PATH = "/home/dependabot/dependabot-updater/repo-handoff"; +var SUMMARY_FILE_PATH = `${OUTPUT_PATH}/summary.md`; var ContainerService = { async storeInput(name, path, container, input) { const tar = (0, import_tar_stream.pack)(); @@ -99347,6 +99351,29 @@ var ContainerService = { await container.putArchive(tar, { path }); }, async run(container, command) { + await this.runPhase(container, "all", command); + return true; + }, + async runFileFetcher(container) { + await this.runPhase(container, "fetch"); + }, + async runFileUpdater(container, command) { + await this.runPhase(container, "update", command); + }, + updaterCommands(phase, command) { + const commands = [`mkdir -p ${OUTPUT_PATH}`]; + if (phase === "fetch") { + commands.push("$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files"); + commands.push(`cp -a ${REPO_CONTENTS_PATH}/. ${REPO_HANDOFF_PATH}/`); + } + if (phase === "all" || phase === "update") { + commands.push( + command === "graph" ? "$DEPENDABOT_HOME/dependabot-updater/bin/run update_graph" : "$DEPENDABOT_HOME/dependabot-updater/bin/run update_files" + ); + } + return commands; + }, + async runPhase(container, phase, command) { try { await container.start(); core4.info(`Started container ${container.id}`); @@ -99354,40 +99381,31 @@ var ContainerService = { const isDependabotContainer = containerInfo.Config?.Env?.some( (env) => env.startsWith("DEPENDABOT_JOB_ID=") ); - if (isDependabotContainer) { + if (!isDependabotContainer) { + const outcome = await container.wait(); + if (outcome.StatusCode !== 0) { + throw new Error(`Container exited with code ${outcome.StatusCode}`); + } + return; + } + await this.execCommand( + container, + ["/usr/sbin/update-ca-certificates"], + "root" + ); + if (phase === "fetch") { await this.execCommand( container, - ["/usr/sbin/update-ca-certificates"], + ["chown", "dependabot", REPO_CONTENTS_PATH, REPO_HANDOFF_PATH], "root" ); - const dependabotCommands = [ - "mkdir -p /home/dependabot/dependabot-updater/output", - "$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files" - ]; - if (command === "graph") { - dependabotCommands.push( - "$DEPENDABOT_HOME/dependabot-updater/bin/run update_graph" - ); - } else { - dependabotCommands.push( - "$DEPENDABOT_HOME/dependabot-updater/bin/run update_files" - ); - } - for (const cmd of dependabotCommands) { - await this.execCommand( - container, - ["/bin/sh", "-c", cmd], - "dependabot" - ); - } + } + for (const cmd of this.updaterCommands(phase, command)) { + await this.execCommand(container, ["/bin/sh", "-c", cmd], "dependabot"); + } + if (phase !== "fetch") { await this.extractJobSummary(container); - } else { - const outcome = await container.wait(); - if (outcome.StatusCode !== 0) { - throw new Error(`Container exited with code ${outcome.StatusCode}`); - } } - return true; } catch (error3) { core4.info(`Failure running container ${container.id}: ${error3}`); throw new ContainerRuntimeError( @@ -99431,15 +99449,10 @@ var ContainerService = { ); } }, - async extractJobSummary(container) { - const summaryPath = "/home/dependabot/dependabot-updater/output/summary.md"; - const stepSummaryPath = process.env.GITHUB_STEP_SUMMARY; - if (!stepSummaryPath) { - return; - } + async readFile(container, path) { try { - const archiveStream = await container.getArchive({ path: summaryPath }); - const content = await new Promise((resolve, reject) => { + const archiveStream = await container.getArchive({ path }); + return await new Promise((resolve, reject) => { const extractor = (0, import_tar_stream.extract)(); let data = ""; extractor.on("entry", (header, stream2, next) => { @@ -99453,12 +99466,20 @@ var ContainerService = { extractor.on("error", (err) => reject(err)); archiveStream.pipe(extractor); }); - if (content.length > 0) { - fs.appendFileSync(stepSummaryPath, content); - core4.info("Job summary written to GITHUB_STEP_SUMMARY"); - } } catch { - core4.debug("No job summary file found in container"); + core4.debug(`No file found in container at ${path}`); + return void 0; + } + }, + async extractJobSummary(container) { + const stepSummaryPath = process.env.GITHUB_STEP_SUMMARY; + if (!stepSummaryPath) { + return; + } + const content = await this.readFile(container, SUMMARY_FILE_PATH); + if (content && content.length > 0) { + fs.appendFileSync(stepSummaryPath, content); + core4.info("Job summary written to GITHUB_STEP_SUMMARY"); } } }; @@ -99509,13 +99530,14 @@ var ProxyBuilder = class { docker; proxyImage; cachedMode; - async run(jobId2, jobToken, dependabotApiUrl, credentials) { - const name = `dependabot-job-${jobId2}-proxy`; + async run(jobId2, jobToken, dependabotApiUrl, credentials, phase) { + const prefix = phase ? `dependabot-job-${jobId2}-${phase}` : `dependabot-job-${jobId2}`; + const name = `${prefix}-proxy`; const config = this.buildProxyConfig(credentials); const cert = config.ca.cert; - const externalNetworkName = `dependabot-job-${jobId2}-external-network`; + const externalNetworkName = `${prefix}-external-network`; const externalNetwork = await this.ensureNetwork(externalNetworkName, false); - const internalNetworkName = `dependabot-job-${jobId2}-internal-network`; + const internalNetworkName = `${prefix}-internal-network`; const internalNetwork = await this.ensureNetwork(internalNetworkName, true); const container = await this.createContainer( jobId2, @@ -99733,23 +99755,30 @@ var JOB_OUTPUT_FILENAME = "output.json"; var JOB_OUTPUT_PATH = "/home/dependabot/dependabot-updater/output"; var JOB_INPUT_FILENAME = "job.json"; var JOB_INPUT_PATH = `/home/dependabot/dependabot-updater`; -var REPO_CONTENTS_PATH = "/home/dependabot/dependabot-updater/repo"; +var REPO_CONTENTS_PATH2 = "/home/dependabot/dependabot-updater/repo"; +var REPO_HANDOFF_PATH2 = "/home/dependabot/dependabot-updater/repo-handoff"; var CA_CERT_INPUT_PATH2 = "/usr/local/share/ca-certificates"; var CA_CERT_FILENAME = "dbot-ca.crt"; var UPDATER_MAX_MEMORY = 8 * 1024 * 1024 * 1024; var UpdaterBuilder = class { - constructor(docker, jobParams, input, proxy, updaterImage) { + constructor(docker, jobParams, input, proxy, updaterImage, phase = "all", repoVolume, handoffVolume) { this.docker = docker; this.jobParams = jobParams; this.input = input; this.proxy = proxy; this.updaterImage = updaterImage; + this.phase = phase; + this.repoVolume = repoVolume; + this.handoffVolume = handoffVolume; } docker; jobParams; input; proxy; updaterImage; + phase; + repoVolume; + handoffVolume; async run(containerName) { const proxyUrl = await this.proxy.url(); const updaterSha = extractUpdaterSha(this.updaterImage); @@ -99760,14 +99789,13 @@ var UpdaterBuilder = class { `DEPENDABOT_JOB_PATH=${JOB_INPUT_PATH}/${JOB_INPUT_FILENAME}`, `DEPENDABOT_OPEN_TIMEOUT_IN_SECONDS=15`, `DEPENDABOT_OUTPUT_PATH=${JOB_OUTPUT_PATH}/${JOB_OUTPUT_FILENAME}`, - `DEPENDABOT_REPO_CONTENTS_PATH=${REPO_CONTENTS_PATH}`, + `DEPENDABOT_REPO_CONTENTS_PATH=${REPO_CONTENTS_PATH2}`, `DEPENDABOT_API_URL=${this.jobParams.dependabotApiDockerUrl}`, `SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt`, `http_proxy=${proxyUrl}`, `HTTP_PROXY=${proxyUrl}`, `https_proxy=${proxyUrl}`, `HTTPS_PROXY=${proxyUrl}`, - `UPDATER_ONE_CONTAINER=1`, `ENABLE_CONNECTIVITY_CHECK=${process.env.DEPENDABOT_ENABLE_CONNECTIVITY_CHECK || "1"}`, // Increase V8 heap size from the default ~2GB to 4GB. // The container memory limit (UPDATER_MAX_MEMORY) is 8GB, but Node.js @@ -99778,6 +99806,12 @@ var UpdaterBuilder = class { // See: https://github.com/dependabot/dependabot-core/issues/14596 `NODE_OPTIONS=--max-old-space-size=4096` ]; + if (this.phase === "all") { + envVars.push(`UPDATER_ONE_CONTAINER=1`); + } + if (this.phase === "update") { + envVars.push("DEPENDABOT_LOCAL_CHECKOUT_ONLY=true"); + } if (updaterSha !== null) { envVars.push(`DEPENDABOT_UPDATER_SHA=${updaterSha}`); } @@ -99797,7 +99831,23 @@ var UpdaterBuilder = class { Tty: true, HostConfig: { Memory: UPDATER_MAX_MEMORY, - NetworkMode: this.proxy.networkName + NetworkMode: this.proxy.networkName, + Mounts: [ + ...this.repoVolume ? [ + { + Type: "volume", + Source: this.repoVolume, + Target: REPO_CONTENTS_PATH2 + } + ] : [], + ...this.handoffVolume ? [ + { + Type: "volume", + Source: this.handoffVolume, + Target: REPO_HANDOFF_PATH2 + } + ] : [] + ] } }); await ContainerService.storeCert( @@ -99818,6 +99868,7 @@ var UpdaterBuilder = class { }; // src/updater.ts +var FEATURE_SPLIT_FETCH_UPDATE = "isolate_fetch_update"; var Updater = class { constructor(updaterImage, proxyImage, apiClient, details, credentials) { this.updaterImage = updaterImage; @@ -99838,6 +99889,96 @@ var Updater = class { * Execute an update job and report the result to Dependabot API. */ async runUpdater() { + if (this.splitPhasesEnabled()) { + return await this.runSplitPhaseUpdate(); + } + return await this.runSingleContainerUpdate(); + } + splitPhasesEnabled() { + const experiments = this.details.experiments || {}; + return experiments[FEATURE_SPLIT_FETCH_UPDATE] === true; + } + async runSingleContainerUpdate() { + const proxy = await this.startProxy(this.credentials); + try { + await this.runUpdate(proxy); + } catch (error3) { + await this.cleanupAfterFailure(proxy, error3); + throw error3; + } + await this.cleanup(proxy); + return true; + } + /** + * Run the job as two sequential containers, copying the fetched checkout + * from a fetch-only volume into the volume handed to the updater. + */ + async runSplitPhaseUpdate() { + const cloneVolume = await this.docker.createVolume({ + Name: `dependabot-job-${this.apiClient.params.jobId}-clone`, + Labels: { "dependabot-job-id": String(this.apiClient.params.jobId) } + }); + let handoffVolume; + try { + handoffVolume = await this.docker.createVolume({ + Name: `dependabot-job-${this.apiClient.params.jobId}-handoff`, + Labels: { "dependabot-job-id": String(this.apiClient.params.jobId) } + }); + } catch (error3) { + await this.removeRepoVolumes([cloneVolume], true); + throw error3; + } + const repoVolumes = [cloneVolume, handoffVolume]; + try { + await this.runFetchPhase(cloneVolume.name, handoffVolume.name); + await this.runUpdatePhase(handoffVolume.name); + } catch (error3) { + await this.removeRepoVolumes(repoVolumes, true); + throw error3; + } + await this.removeRepoVolumes(repoVolumes, false); + return true; + } + async runFetchPhase(cloneVolume, handoffVolume) { + core7.info(`Fetching files for job ${this.apiClient.params.jobId}`); + const proxy = await this.startProxy(this.credentials, "fetch"); + try { + const name = `dependabot-job-${this.apiClient.params.jobId}-file-fetcher`; + const container = await this.createContainer( + proxy, + name, + { job: this.details }, + "fetch", + cloneVolume, + handoffVolume + ); + await ContainerService.runFileFetcher(container); + } catch (error3) { + await this.cleanupAfterFailure(proxy, error3); + throw error3; + } + await this.cleanup(proxy); + } + async runUpdatePhase(repoVolume) { + core7.info(`Running update job ${this.apiClient.params.jobId}`); + const proxy = await this.startProxy(this.credentials, "update"); + try { + const name = `dependabot-job-${this.apiClient.params.jobId}-updater`; + const container = await this.createContainer( + proxy, + name, + { job: this.details }, + "update", + repoVolume + ); + await ContainerService.runFileUpdater(container, this.details.command); + } catch (error3) { + await this.cleanupAfterFailure(proxy, error3); + throw error3; + } + await this.cleanup(proxy); + } + async startProxy(credentials, phase) { const cachedMode = Object.hasOwn( this.details.experiments ?? {}, "proxy-cached" @@ -99851,27 +99992,17 @@ var Updater = class { this.apiClient.params.jobId, this.apiClient.getJobToken(), this.apiClient.params.dependabotApiUrl, - this.credentials + credentials, + phase ); await proxy.container.start(); try { await proxy.waitUntilReady(); - await this.runUpdate(proxy); } catch (error3) { - try { - await this.cleanup(proxy); - } catch (cleanupError) { - const cleanupErrors = cleanupError instanceof AggregateError ? cleanupError.errors : [cleanupError]; - for (const cleanupFailure of cleanupErrors) { - core7.info( - `Failed to clean up proxy after update failure: ${cleanupFailure}` - ); - } - } + await this.cleanupAfterFailure(proxy, error3); throw error3; } - await this.cleanup(proxy); - return true; + return proxy; } generateCredentialsMetadata() { const unique = /* @__PURE__ */ new Set(); @@ -99962,18 +100093,55 @@ var Updater = class { }); await ContainerService.run(container, this.details.command); } - async createContainer(proxy, containerName, input) { + async createContainer(proxy, containerName, input, phase = "all", repoVolume, handoffVolume) { return new UpdaterBuilder( this.docker, this.apiClient.params, input, proxy, - this.updaterImage + this.updaterImage, + phase, + repoVolume, + handoffVolume ).run(containerName); } async cleanup(proxy) { await proxy.shutdown(); } + async cleanupAfterFailure(proxy, originalError) { + try { + await this.cleanup(proxy); + } catch (cleanupError) { + const cleanupErrors = cleanupError instanceof AggregateError ? cleanupError.errors : [cleanupError]; + for (const cleanupFailure of cleanupErrors) { + core7.info( + `Failed to clean up proxy after update failure: ${cleanupFailure}` + ); + } + core7.debug(`Original updater failure: ${originalError}`); + } + } + async removeRepoVolumes(volumes, afterFailure) { + const results = await Promise.allSettled( + volumes.map(async (volume) => volume.remove()) + ); + const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason); + if (errors.length === 0) { + return; + } + if (afterFailure) { + for (const error3 of errors) { + core7.info( + `Failed to clean up repository volume after update failure: ${error3}` + ); + } + return; + } + if (errors.length === 1) { + throw errors[0]; + } + throw new AggregateError(errors, "Failed to clean up repository volumes"); + } }; // src/main.ts diff --git a/src/config-types.ts b/src/config-types.ts index b69795c12..14ee211b6 100644 --- a/src/config-types.ts +++ b/src/config-types.ts @@ -1,11 +1,5 @@ import {Credential, JobDetails} from './api-client' -export type FetchedFiles = { - base_commit_sha: string - dependency_files: any[] - base64_dependency_files: any[] -} - export type FileFetcherInput = { job: JobDetails } @@ -21,10 +15,6 @@ export type DependencyFile = { operation: string } -export type FileUpdaterInput = FetchedFiles & { - job: JobDetails -} - export type CertificateAuthority = { cert: string key: string diff --git a/src/container-service.ts b/src/container-service.ts index 05a50d410..234c4d593 100644 --- a/src/container-service.ts +++ b/src/container-service.ts @@ -2,7 +2,7 @@ import * as core from '@actions/core' import * as fs from 'fs' import {Container} from 'dockerode' import {pack, extract} from 'tar-stream' -import {FileFetcherInput, FileUpdaterInput, ProxyConfig} from './config-types' +import {FileFetcherInput, ProxyConfig} from './config-types' import {outStream, errStream} from './utils' export class ContainerRuntimeError extends Error {} @@ -10,12 +10,12 @@ export class ContainerRuntimeError extends Error {} const RWX_ALL = 0o777 const OUTPUT_PATH = '/home/dependabot/dependabot-updater/output' -const OUTPUT_FILE_PATH = `${OUTPUT_PATH}/output.json` +const REPO_CONTENTS_PATH = '/home/dependabot/dependabot-updater/repo' +const REPO_HANDOFF_PATH = '/home/dependabot/dependabot-updater/repo-handoff' const SUMMARY_FILE_PATH = `${OUTPUT_PATH}/summary.md` -// 'fetch' clones the repo and writes the handoff artifact, 'update' runs the -// package manager against that artifact. 'all' runs both in one container, -// which is the legacy UPDATER_ONE_CONTAINER behaviour. +// 'fetch' clones the repo into a shared volume, 'update' consumes that checkout, +// and 'all' preserves the legacy single-container behaviour. export type UpdaterPhase = 'fetch' | 'update' | 'all' export const ContainerService = { @@ -23,7 +23,7 @@ export const ContainerService = { name: string, path: string, container: Container, - input: FileFetcherInput | FileUpdaterInput | ProxyConfig + input: FileFetcherInput | ProxyConfig ): Promise { const tar = pack() tar.entry({name, mode: RWX_ALL}, JSON.stringify(input)) @@ -48,12 +48,8 @@ export const ContainerService = { return true }, - /** - * Run the fetch phase and return the raw contents of the handoff artifact - * written by the fetcher, or undefined if it produced no output. - */ - async runFileFetcher(container: Container): Promise { - return await this.runPhase(container, 'fetch') + async runFileFetcher(container: Container): Promise { + await this.runPhase(container, 'fetch') }, async runFileUpdater(container: Container, command?: string): Promise { @@ -63,8 +59,9 @@ export const ContainerService = { updaterCommands(phase: UpdaterPhase, command?: string): string[] { const commands = [`mkdir -p ${OUTPUT_PATH}`] - if (phase === 'all' || phase === 'fetch') { + if (phase === 'fetch') { commands.push('$DEPENDABOT_HOME/dependabot-updater/bin/run fetch_files') + commands.push(`cp -a ${REPO_CONTENTS_PATH}/. ${REPO_HANDOFF_PATH}/`) } if (phase === 'all' || phase === 'update') { @@ -82,7 +79,7 @@ export const ContainerService = { container: Container, phase: UpdaterPhase, command?: string - ): Promise { + ): Promise { try { // Start the container await container.start() @@ -100,7 +97,7 @@ export const ContainerService = { if (outcome.StatusCode !== 0) { throw new Error(`Container exited with code ${outcome.StatusCode}`) } - return undefined + return } // For dependabot containers, run CA certificates update as root first @@ -110,24 +107,26 @@ export const ContainerService = { 'root' ) + if (phase === 'fetch') { + await this.execCommand( + container, + ['chown', 'dependabot', REPO_CONTENTS_PATH, REPO_HANDOFF_PATH], + 'root' + ) + } + // Then run the dependabot commands as dependabot user for (const cmd of this.updaterCommands(phase, command)) { await this.execCommand(container, ['/bin/sh', '-c', cmd], 'dependabot') } - if (phase === 'fetch') { - // The fetch phase hands its file subset off to the update container, so - // read it out before this container is torn down. - return await this.readFile(container, OUTPUT_FILE_PATH) + if (phase !== 'fetch') { + // Extract job summary only after all commands have succeeded. + // This prevents malicious code executed during fetch_files from + // injecting content — our updater overwrites the file at the end + // of a successful run. + await this.extractJobSummary(container) } - - // Extract job summary only after all commands have succeeded. - // This prevents malicious code executed during fetch_files from - // injecting content — our updater overwrites the file at the end - // of a successful run. - await this.extractJobSummary(container) - - return undefined } catch (error) { core.info(`Failure running container ${container.id}: ${error}`) throw new ContainerRuntimeError( diff --git a/src/updater-builder.ts b/src/updater-builder.ts index cd7f84480..fe47e199f 100644 --- a/src/updater-builder.ts +++ b/src/updater-builder.ts @@ -1,7 +1,7 @@ import * as core from '@actions/core' import Docker, {Container} from 'dockerode' import {ContainerService, UpdaterPhase} from './container-service' -import {FileFetcherInput, FileUpdaterInput} from './config-types' +import {FileFetcherInput} from './config-types' import {JobParameters} from './inputs' import {Proxy} from './proxy' import {extractUpdaterSha} from './utils' @@ -11,6 +11,7 @@ const JOB_OUTPUT_PATH = '/home/dependabot/dependabot-updater/output' const JOB_INPUT_FILENAME = 'job.json' const JOB_INPUT_PATH = `/home/dependabot/dependabot-updater` const REPO_CONTENTS_PATH = '/home/dependabot/dependabot-updater/repo' +const REPO_HANDOFF_PATH = '/home/dependabot/dependabot-updater/repo-handoff' const CA_CERT_INPUT_PATH = '/usr/local/share/ca-certificates' const CA_CERT_FILENAME = 'dbot-ca.crt' const UPDATER_MAX_MEMORY = 8 * 1024 * 1024 * 1024 // 8GB in bytes @@ -19,11 +20,13 @@ export class UpdaterBuilder { constructor( private readonly docker: Docker, private readonly jobParams: JobParameters, - private readonly input: FileFetcherInput | FileUpdaterInput, + private readonly input: FileFetcherInput, private readonly proxy: Proxy, private readonly updaterImage: string, - private readonly phase: UpdaterPhase = 'all' + private readonly phase: UpdaterPhase = 'all', + private readonly repoVolume?: string, + private readonly handoffVolume?: string ) {} async run(containerName: string): Promise { @@ -63,6 +66,10 @@ export class UpdaterBuilder { envVars.push(`UPDATER_ONE_CONTAINER=1`) } + if (this.phase === 'update') { + envVars.push('DEPENDABOT_LOCAL_CHECKOUT_ONLY=true') + } + // Add DEPENDABOT_UPDATER_SHA if we successfully extracted a SHA if (updaterSha !== null) { envVars.push(`DEPENDABOT_UPDATER_SHA=${updaterSha}`) @@ -89,7 +96,27 @@ export class UpdaterBuilder { Tty: true, HostConfig: { Memory: UPDATER_MAX_MEMORY, - NetworkMode: this.proxy.networkName + NetworkMode: this.proxy.networkName, + Mounts: [ + ...(this.repoVolume + ? [ + { + Type: 'volume' as const, + Source: this.repoVolume, + Target: REPO_CONTENTS_PATH + } + ] + : []), + ...(this.handoffVolume + ? [ + { + Type: 'volume' as const, + Source: this.handoffVolume, + Target: REPO_HANDOFF_PATH + } + ] + : []) + ] } }) diff --git a/src/updater.ts b/src/updater.ts index de97958e1..42da7f8be 100644 --- a/src/updater.ts +++ b/src/updater.ts @@ -2,33 +2,17 @@ import * as core from '@actions/core' import Docker, {Container} from 'dockerode' import {JobDetails, ApiClient, Credential} from './api-client' import {ContainerService, UpdaterPhase} from './container-service' -import { - DependencyFile, - FetchedFiles, - FileUpdaterInput, - FileFetcherInput -} from './config-types' +import {FileFetcherInput} from './config-types' import {ProxyBuilder, Proxy} from './proxy' import {UpdaterBuilder} from './updater-builder' -import {base64DecodeDependencyFile} from './utils' // Experiment which opts a job into running the fetch and update phases in // separate containers, each with its own proxy and credential set. -const FEATURE_SPLIT_FETCH_UPDATE = 'split-fetch-update-containers' - -// Opt-in required alongside the experiment. The updater image does not yet -// expose standalone phase entrypoints — `bin/run fetch_files` is a no-op kept -// for backward compatibility, and `update_files` runs the file fetcher -// in-process — so there is no handoff artifact for the update container to -// consume. This keeps the split path unreachable for real jobs until an image -// providing those entrypoints ships. -const SPLIT_FETCH_UPDATE_ENV = 'DEPENDABOT_SPLIT_FETCH_UPDATE' - -export class UpdaterFetchError extends Error { - constructor(msg: string) { - super(msg) - Object.setPrototypeOf(this, UpdaterFetchError.prototype) - } +const FEATURE_SPLIT_FETCH_UPDATE = 'isolate_fetch_update' + +type RepoVolume = { + name: string + remove: () => Promise } export class Updater { @@ -60,19 +44,7 @@ export class Updater { const experiments = (this.details.experiments || {}) as { [key: string]: unknown } - - if (experiments[FEATURE_SPLIT_FETCH_UPDATE] !== true) { - return false - } - - if (process.env[SPLIT_FETCH_UPDATE_ENV] !== '1') { - core.info( - `The ${FEATURE_SPLIT_FETCH_UPDATE} experiment is enabled but the updater image does not yet support split phases, running both phases in one container` - ) - return false - } - - return true + return experiments[FEATURE_SPLIT_FETCH_UPDATE] === true } private async runSingleContainerUpdate(): Promise { @@ -80,25 +52,53 @@ export class Updater { try { await this.runUpdate(proxy) - return true - } finally { - await this.cleanup(proxy) + } catch (error) { + await this.cleanupAfterFailure(proxy, error) + throw error } + + await this.cleanup(proxy) + return true } /** - * Run the job as two sequential containers. The fetch container clones the - * repository behind a proxy holding the target-repo credential and hands off - * the file subset it selected. The update container then runs the package - * manager against only that subset, behind its own proxy. + * Run the job as two sequential containers, copying the fetched checkout + * from a fetch-only volume into the volume handed to the updater. */ private async runSplitPhaseUpdate(): Promise { - const files = await this.runFetchPhase() - await this.runUpdatePhase(files) + const cloneVolume = (await this.docker.createVolume({ + Name: `dependabot-job-${this.apiClient.params.jobId}-clone`, + Labels: {'dependabot-job-id': String(this.apiClient.params.jobId)} + })) as unknown as RepoVolume + let handoffVolume: RepoVolume + + try { + handoffVolume = (await this.docker.createVolume({ + Name: `dependabot-job-${this.apiClient.params.jobId}-handoff`, + Labels: {'dependabot-job-id': String(this.apiClient.params.jobId)} + })) as unknown as RepoVolume + } catch (error) { + await this.removeRepoVolumes([cloneVolume], true) + throw error + } + + const repoVolumes = [cloneVolume, handoffVolume] + try { + await this.runFetchPhase(cloneVolume.name, handoffVolume.name) + await this.runUpdatePhase(handoffVolume.name) + } catch (error) { + await this.removeRepoVolumes(repoVolumes, true) + throw error + } + + await this.removeRepoVolumes(repoVolumes, false) return true } - private async runFetchPhase(): Promise { + private async runFetchPhase( + cloneVolume: string, + handoffVolume: string + ): Promise { core.info(`Fetching files for job ${this.apiClient.params.jobId}`) const proxy = await this.startProxy(this.credentials, 'fetch') @@ -109,90 +109,42 @@ export class Updater { proxy, name, {job: this.details}, - 'fetch' + 'fetch', + cloneVolume, + handoffVolume ) - const output = await ContainerService.runFileFetcher(container) - if (!output) { - throw new UpdaterFetchError( - 'No output.json created by the fetcher container' - ) - } - - const fileFetcherOutput = JSON.parse(output) - - return { - base_commit_sha: fileFetcherOutput.base_commit_sha, - base64_dependency_files: fileFetcherOutput.base64_dependency_files, - dependency_files: fileFetcherOutput.base64_dependency_files.map( - (file: DependencyFile) => base64DecodeDependencyFile(file) - ) - } - } finally { - // Tear the fetch proxy and its networks down before the update phase - // starts, so the two phases never share a proxy or a network. - await this.cleanup(proxy) + await ContainerService.runFileFetcher(container) + } catch (error) { + await this.cleanupAfterFailure(proxy, error) + throw error } + + await this.cleanup(proxy) } - private async runUpdatePhase(files: FetchedFiles): Promise { + private async runUpdatePhase(repoVolume: string): Promise { core.info(`Running update job ${this.apiClient.params.jobId}`) - const proxy = await this.startProxy(this.updatePhaseCredentials(), 'update') + const proxy = await this.startProxy(this.credentials, 'update') try { const name = `dependabot-job-${this.apiClient.params.jobId}-updater` - const input: FileUpdaterInput = { - base_commit_sha: files.base_commit_sha, - base64_dependency_files: files.base64_dependency_files, - dependency_files: files.dependency_files, - job: this.details - } - const container = await this.createContainer(proxy, name, input, 'update') - - await ContainerService.runFileUpdater(container, this.details.command) - } finally { - await this.cleanup(proxy) - } - } - - /** - * The credential set handed to the update phase's proxy. Credentials which - * resolve to the target repository are dropped, since the update phase works - * from the file subset handed over by the fetch phase rather than the repo. - */ - private updatePhaseCredentials(): Credential[] { - const targetRepo = this.details.source?.repo - - const credentials = this.credentials.filter( - credential => !this.canReadTargetRepo(credential, targetRepo) - ) - - const dropped = this.credentials.length - credentials.length - if (dropped > 0) { - core.info( - `Excluding ${dropped} target-repo credential(s) from the update phase proxy` + const container = await this.createContainer( + proxy, + name, + {job: this.details}, + 'update', + repoVolume ) - } - - return credentials - } - - private canReadTargetRepo( - credential: Credential, - targetRepo: string | undefined - ): boolean { - if (credential.type !== 'git_source') { - return false - } - // A git_source credential scoped to a different repository does not resolve - // to the target repo, so it is kept for git-sourced dependencies. - if (credential.repo && targetRepo && credential.repo !== targetRepo) { - return false + await ContainerService.runFileUpdater(container, this.details.command) + } catch (error) { + await this.cleanupAfterFailure(proxy, error) + throw error } - return true + await this.cleanup(proxy) } private async startProxy( @@ -218,7 +170,12 @@ export class Updater { phase ) await proxy.container.start() - await proxy.waitUntilReady() + try { + await proxy.waitUntilReady() + } catch (error) { + await this.cleanupAfterFailure(proxy, error) + throw error + } return proxy } @@ -327,8 +284,10 @@ export class Updater { private async createContainer( proxy: Proxy, containerName: string, - input: FileFetcherInput | FileUpdaterInput, - phase: UpdaterPhase = 'all' + input: FileFetcherInput, + phase: UpdaterPhase = 'all', + repoVolume?: string, + handoffVolume?: string ): Promise { return new UpdaterBuilder( this.docker, @@ -336,11 +295,64 @@ export class Updater { input, proxy, this.updaterImage, - phase + phase, + repoVolume, + handoffVolume ).run(containerName) } private async cleanup(proxy: Proxy): Promise { await proxy.shutdown() } + + private async cleanupAfterFailure( + proxy: Proxy, + originalError: unknown + ): Promise { + try { + await this.cleanup(proxy) + } catch (cleanupError) { + const cleanupErrors = + cleanupError instanceof AggregateError + ? cleanupError.errors + : [cleanupError] + for (const cleanupFailure of cleanupErrors) { + core.info( + `Failed to clean up proxy after update failure: ${cleanupFailure}` + ) + } + core.debug(`Original updater failure: ${originalError}`) + } + } + + private async removeRepoVolumes( + volumes: RepoVolume[], + afterFailure: boolean + ): Promise { + const results = await Promise.allSettled( + volumes.map(async volume => volume.remove()) + ) + const errors = results + .filter(result => result.status === 'rejected') + .map(result => result.reason) + + if (errors.length === 0) { + return + } + + if (afterFailure) { + for (const error of errors) { + core.info( + `Failed to clean up repository volume after update failure: ${error}` + ) + } + return + } + + if (errors.length === 1) { + throw errors[0] + } + + throw new AggregateError(errors, 'Failed to clean up repository volumes') + } }