diff --git a/apps/api/src/routes/internal/dependency-scan.http.ts b/apps/api/src/routes/internal/dependency-scan.http.ts new file mode 100644 index 0000000000..b587dc8ef0 --- /dev/null +++ b/apps/api/src/routes/internal/dependency-scan.http.ts @@ -0,0 +1,38 @@ +import { HttpApiBuilder } from "effect/http-api" +import { CurrentTenant, DependencyScanForbiddenError, MapleInternalApi } from "@maple/domain/http" +import { Effect } from "effect" +import { requireAdmin } from "@maple/backend/services/auth/auth" +import { DependencyScanService } from "@maple/backend/services/dependency-scan/DependencyScanService" + +export const HttpDependencyScanLive = HttpApiBuilder.group(MapleInternalApi, "dependencyScan", (handlers) => + Effect.gen(function* () { + const scans = yield* DependencyScanService + + return handlers + .handle("listRepositories", () => + Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listRepositories(tenant.orgId)), + ) + .handle("scanRepository", ({ params }) => + Effect.flatMap(CurrentTenant.Context, (tenant) => + scans.requestScan(tenant.orgId, params.repositoryId), + ), + ) + .handle("listFindings", ({ query }) => + Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listFindings(tenant.orgId, query)), + ) + .handle("updateFinding", ({ params, payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + // Triage, like the code audit's, is an admin's call. + yield* requireAdmin( + tenant.roles, + () => + new DependencyScanForbiddenError({ + message: "Only org admins can triage findings", + }), + ) + return yield* scans.updateFinding(tenant.orgId, tenant.userId, params.findingId, payload) + }), + ) + }), +) diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index 20f1b1e669..95a2fb94a7 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -20,6 +20,7 @@ import { AwsOidcRouter } from "@/routes/aws-oidc.http" import { DiscoveryRouter, instanceApiV2, NotFoundRouter } from "@/routes/discovery.http" import { HttpDigestLive } from "@/routes/internal/digest.http" import { HttpCodeReviewLive } from "@/routes/internal/code-review.http" +import { HttpDependencyScanLive } from "@/routes/internal/dependency-scan.http" import { HttpErrorsLive } from "@/routes/internal/errors.http" import { HttpIntegrationsLive } from "@/routes/internal/integrations.http" import { IntegrationsCallbackRouter } from "@/routes/integrations-callback.http" @@ -145,6 +146,7 @@ const ApiInternalRoutes = HttpApiBuilder.layer(MapleInternalApi).pipe( HttpAiTriageLive, HttpBillingLive, HttpDemoLive, + HttpDependencyScanLive, HttpDigestLive, HttpOpenRouterKeyLive, ), diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 02e4bd8d35..57a8f9064f 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -1,4 +1,5 @@ import { RuntimeAdvisoryCatalog } from "@maple/backend/services/security/RuntimeAdvisoryCatalog" +import { DependencyScanService } from "@maple/backend/services/dependency-scan/DependencyScanService" import { RuntimeReleaseCatalog } from "@maple/backend/services/security/RuntimeReleaseCatalog" import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" import { Layer } from "effect" @@ -167,6 +168,7 @@ export const HttpServicesLive = Layer.mergeAll( QueryEngineService.layer, RuntimeReleaseCatalog.layer, RuntimeAdvisoryCatalog.layer, + DependencyScanService.layer, TelemetryReadService.layer, DashboardWidgetDataService.layer, AlertDestinationsService.layer, diff --git a/apps/api/src/vcs-sync-runtime.ts b/apps/api/src/vcs-sync-runtime.ts index c36b4bf8a1..f31f9d8143 100644 --- a/apps/api/src/vcs-sync-runtime.ts +++ b/apps/api/src/vcs-sync-runtime.ts @@ -23,6 +23,7 @@ import { PullRequestCommentSinkLive, } from "@maple/backend/services/pr-review/pull-request-comment-handler" import { CiTraceSinkLive } from "@maple/backend/services/integrations/ci-traces/CiTraceService" +import { DependencyScanSinkLive } from "@maple/backend/services/dependency-scan/DependencyScanner" import { summarizeCause } from "@maple/backend/platform/describe-cause" import type { QueueBatch } from "@maple/backend/platform/queue-batch" import type { Message } from "@cloudflare/workers-types" @@ -70,6 +71,8 @@ export const VcsSyncLive = VcsSyncService.layer.pipe( Layer.provide(PullRequestChecklistSinkLive.pipe(Layer.provide(PrReviewLive))), // Finished CI runs and jobs, written as spans into the org's traces. Layer.provide(CiTraceSinkLive), + // Lockfile scans for known vulnerable dependencies, on pushes and installation syncs. + Layer.provide(DependencyScanSinkLive), Layer.provide(Layer.mergeAll(EventBaseLive, EdgeCacheServiceLive)), ) diff --git a/knip.json b/knip.json index 076655857e..fa50d388a9 100644 --- a/knip.json +++ b/knip.json @@ -87,6 +87,10 @@ ], "ignoreBinaries": ["duckdb"] }, + "packages/backend": { + // The opt-in GitHub e2e test compares the dependency scanner with a locally installed osv-scanner. + "ignoreBinaries": ["osv-scanner"] + }, "packages/db": { "entry": ["scripts/**/*.ts", "effect-orm.config.ts"] }, diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts new file mode 100644 index 0000000000..defbecfb11 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts @@ -0,0 +1,202 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { DependencyFindingId } from "@maple/domain/dependency-scan" +import { GitCommitSha, VcsRepositoryId, type VcsSyncJob } from "@maple/domain/http" +import { OrgId, UserId } from "@maple/domain/primitives" +import { Effect, Layer, Schema } from "effect" +import { TestClock } from "effect/testing" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" +import { + recordingQueueLayer, + testRepoLayer, +} from "@maple/backend/services/integrations/vcs/__tests__/harness" +import { DependencyScanService } from "./DependencyScanService" +import { DependencyScanStore } from "./DependencyScanStore" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const org = Schema.decodeSync(OrgId)("org_deps") +const admin = Schema.decodeSync(UserId)("user_admin") +const sha = (digit: string) => Schema.decodeSync(GitCommitSha)(digit.repeat(40)) + +const serviceFor = (sent: Array) => { + const db = createTestDb(trackedDbs) + const store = DependencyScanStore.layer.pipe(Layer.provide(db.layer)) + const vcs = testRepoLayer(db) + return Layer.effect(DependencyScanService, DependencyScanService.make).pipe( + Layer.provideMerge(Layer.mergeAll(store, vcs, recordingQueueLayer(sent))), + ) +} + +/** One active installation with two repositories; returns the first one's id. */ +const seed = Effect.gen(function* () { + const vcs = yield* VcsRepository + const installation = yield* vcs.upsertInstallation({ + orgId: org, + provider: "github", + externalInstallationId: "42", + accountLogin: "octo", + accountType: "organization", + externalAccountId: "100", + accountAvatarUrl: null, + repositorySelection: "all", + installedByUserId: admin, + }) + const repo = (externalRepoId: string, name: string) => ({ + externalRepoId, + owner: "octo", + name, + fullName: `octo/${name}`, + defaultBranch: "main", + htmlUrl: `https://github.com/octo/${name}`, + isPrivate: true, + isArchived: false, + }) + yield* vcs.upsertRepositories(installation, [repo("8", "web"), repo("7", "api")]) + const repositories = yield* vcs.listRepositoriesByInstallation(installation.id, "active") + const api = repositories.find((repository) => repository.fullName === "octo/api") + if (api === undefined) return yield* Effect.die("seeding did not store octo/api") + return api +}) + +const recordScan = (repositoryId: VcsRepositoryId, commit: string, defaultBranchHead: boolean) => + Effect.gen(function* () { + const store = yield* DependencyScanStore + // Scans of one repository start at distinct times; "latest" is by start time. + yield* TestClock.adjust("1 second") + const scan = yield* store.startScan({ + orgId: org, + repositoryId, + commitSha: sha(commit), + ref: defaultBranchHead ? "main" : null, + trigger: defaultBranchHead ? "schedule" : "production", + defaultBranchHead, + }) + return yield* store.completeScan({ + orgId: org, + scanId: scan.id, + lockfiles: [ + { + path: "bun.lock", + kind: "bun-lock", + blobSha: commit, + packageCount: 1, + status: "parsed", + message: null, + }, + ], + packageCount: 1, + matches: [ + { + lockfilePath: "bun.lock", + ecosystem: "npm", + packageName: "lodash", + version: "4.17.15", + relation: "direct", + advisoryId: "GHSA-p6mc-m468-83gw", + aliases: ["CVE-2020-8203"], + severity: "high", + summary: "Prototype pollution in lodash", + fixedIn: "4.17.19", + }, + ], + }) + }) + +describe("DependencyScanService", () => { + it.effect("lists every connected repository with its latest tracked-branch scan", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const api = yield* seed + const before = yield* service.listRepositories(org) + assert.deepStrictEqual( + before.repositories.map((repository) => [repository.fullName, repository.latestScan]), + [ + ["octo/api", null], + ["octo/web", null], + ], + ) + yield* recordScan(api.id, "a", true) + const latest = yield* recordScan(api.id, "b", true) + // An older commit scanned because it runs in production is not the branch's latest scan. + yield* recordScan(api.id, "c", false) + const after = yield* service.listRepositories(org) + assert.strictEqual(after.repositories[0]?.latestScan?.id, latest.id) + assert.strictEqual(after.repositories[0]?.latestScan?.vulnerableCount, 1) + }).pipe(Effect.provide(serviceFor(sent))) + }) + + it.effect("queues a manual scan for a connected repository and refuses an unknown one", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const api = yield* seed + assert.deepStrictEqual(yield* service.requestScan(org, api.id), { queued: true }) + assert.deepStrictEqual(sent, [ + { + kind: "dependency-scan", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + trigger: "manual", + }, + ]) + const unknown = Schema.decodeSync(VcsRepositoryId)("00000000-0000-4000-8000-0000000000ff") + const error = yield* Effect.flip(service.requestScan(org, unknown)) + assert.strictEqual( + error._tag, + "@maple/http/dependency-scan/DependencyScanRepositoryNotFoundError", + ) + }).pipe(Effect.provide(serviceFor(sent))) + }) + + it.effect("hides the findings of a repository the org no longer has access to", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const vcs = yield* VcsRepository + const api = yield* seed + yield* recordScan(api.id, "a", true) + assert.strictEqual((yield* service.listFindings(org, {})).findings.length, 1) + const [finding] = (yield* service.listFindings(org, {})).findings + yield* vcs.markRepositoryRemoved(api.id) + assert.deepStrictEqual((yield* service.listFindings(org, {})).findings, []) + if (finding === undefined) return + const error = yield* Effect.flip( + service.updateFinding(org, admin, finding.id, { status: "dismissed", reason: "gone" }), + ) + assert.strictEqual(error._tag, "@maple/http/dependency-scan/DependencyFindingNotFoundError") + }).pipe(Effect.provide(serviceFor(sent))) + }) + + it.effect("names each finding's repository and records who dismissed it and why", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const api = yield* seed + yield* recordScan(api.id, "a", true) + const { findings } = yield* service.listFindings(org, {}) + assert.deepStrictEqual( + findings.map((finding) => [finding.repositoryFullName, finding.packageName, finding.status]), + [["octo/api", "lodash", "open"]], + ) + const [finding] = findings + if (finding === undefined) return + const dismissed = yield* service.updateFinding(org, admin, finding.id, { + status: "dismissed", + reason: "Only used in a build script", + }) + assert.deepStrictEqual( + [dismissed.status, dismissed.statusReason, dismissed.statusChangedBy], + ["dismissed", "Only used in a build script", admin], + ) + assert.deepStrictEqual((yield* service.listFindings(org, { status: "open" })).findings, []) + + const missing = Schema.decodeSync(DependencyFindingId)("0".repeat(32)) + const error = yield* Effect.flip(service.updateFinding(org, admin, missing, { status: "open" })) + assert.strictEqual(error._tag, "@maple/http/dependency-scan/DependencyFindingNotFoundError") + }).pipe(Effect.provide(serviceFor(sent))) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts new file mode 100644 index 0000000000..b06655ca55 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -0,0 +1,264 @@ +import type { DependencyFindingRow, DependencyScanRow } from "@maple/db/tables" +import type { DependencyFindingId } from "@maple/domain/dependency-scan" +import { + DependencyFindingNotFoundError, + type DependencyFindingsResponse, + type DependencyFindingUpdateRequest, + type DependencyFindingView, + DependencyScanPersistenceError, + type DependencyScanRepositoriesResponse, + DependencyScanRepositoryNotFoundError, + type DependencyScanSummary, + isInstallationProcessable, + type VcsRepo, + type VcsRepositoryId, +} from "@maple/domain/http" +import type { OrgId, UserId } from "@maple/domain/primitives" +import type { DependencyFindingStatus } from "@maple/domain/dependency-scan" +import { Context, Effect, Layer, Option } from "effect" +import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" +import { VcsSyncQueue } from "@maple/backend/services/integrations/vcs/VcsSyncQueue" +import { DependencyScanStore } from "./DependencyScanStore" + +// The Dependencies tab's reads and writes: repositories with their latest scan, findings with +// their repository named, triage, and "scan now", which goes through the VCS sync queue. + +/** Findings returned per read; the table sorts worst first, so the cut drops the least severe. */ +export const FINDINGS_LIMIT = 500 + +export interface DependencyScanServiceApi { + readonly listRepositories: ( + orgId: OrgId, + ) => Effect.Effect + readonly requestScan: ( + orgId: OrgId, + repositoryId: VcsRepositoryId, + ) => Effect.Effect< + { readonly queued: boolean }, + DependencyScanPersistenceError | DependencyScanRepositoryNotFoundError + > + readonly listFindings: ( + orgId: OrgId, + query: { + readonly status?: DependencyFindingStatus | undefined + readonly repositoryId?: VcsRepositoryId | undefined + }, + ) => Effect.Effect + readonly updateFinding: ( + orgId: OrgId, + actor: UserId, + findingId: DependencyFindingId, + request: DependencyFindingUpdateRequest, + ) => Effect.Effect +} + +const persistence = + (message: string) => + (error: { readonly message: string }): DependencyScanPersistenceError => + new DependencyScanPersistenceError({ message: `${message}: ${error.message}` }) + +export const scanSummaryOf = (row: DependencyScanRow): DependencyScanSummary => ({ + id: row.id, + commitSha: row.commitSha, + ref: row.ref, + trigger: row.trigger, + status: row.status, + lockfiles: row.lockfiles, + packageCount: row.packageCount, + vulnerableCount: row.vulnerableCount, + error: row.error, + startedAt: row.startedAt, + finishedAt: row.finishedAt, +}) + +export const findingViewOf = ( + row: DependencyFindingRow, + repositoryFullName: string, +): DependencyFindingView => ({ + id: row.id, + repositoryId: row.repositoryId, + repositoryFullName, + lockfilePath: row.lockfilePath, + ecosystem: row.ecosystem, + packageName: row.packageName, + advisoryId: row.advisoryId, + aliases: row.aliases, + severity: row.severity, + summary: row.summary, + versions: row.versions, + fixedIn: row.fixedIn, + relation: row.relation, + status: row.status, + firstSeenScanId: row.firstSeenScanId, + lastSeenScanId: row.lastSeenScanId, + firstSeenSha: row.firstSeenSha, + lastSeenSha: row.lastSeenSha, + fixedScanId: row.fixedScanId, + fixedSha: row.fixedSha, + statusReason: row.statusReason, + statusChangedBy: row.statusChangedBy, + statusChangedAt: row.statusChangedAt, + createdAt: row.createdAt, + updatedAt: row.updatedAt, +}) + +export class DependencyScanService extends Context.Service()( + "@maple/backend/services/dependency-scan/DependencyScanService", + { + make: Effect.gen(function* () { + const store = yield* DependencyScanStore + const vcs = yield* VcsRepository + const queue = yield* VcsSyncQueue + + /** The org's repositories under a processable installation, active ones only. */ + const connectedRepositories = Effect.fn("DependencyScanService.connectedRepositories")(function* ( + orgId: OrgId, + ) { + const installations = (yield* vcs.listInstallationsByOrg(orgId)).filter( + isInstallationProcessable, + ) + const repositories = yield* Effect.forEach(installations, (installation) => + vcs.listRepositoriesByInstallation(installation.id, "active"), + ) + return repositories.flat() + }) + + const listRepositories: DependencyScanServiceApi["listRepositories"] = Effect.fn( + "DependencyScanService.listRepositories", + )(function* (orgId) { + const repositories = yield* connectedRepositories(orgId).pipe( + Effect.mapError(persistence("Could not list repositories")), + ) + const branchScans = yield* store + .latestBranchScans({ orgId }) + .pipe(Effect.mapError(persistence("Could not list scans"))) + const latest = new Map(branchScans.map((scan) => [scan.repositoryId, scan])) + return { + repositories: repositories + .map((repository: VcsRepo) => { + const scan = latest.get(repository.id) + return { + id: repository.id, + fullName: repository.fullName, + htmlUrl: repository.htmlUrl, + trackedBranch: repository.trackedBranch ?? repository.defaultBranch, + latestScan: scan === undefined ? null : scanSummaryOf(scan), + } + }) + .sort((a, b) => a.fullName.localeCompare(b.fullName)), + } + }) + + const requestScan: DependencyScanServiceApi["requestScan"] = Effect.fn( + "DependencyScanService.requestScan", + )(function* (orgId, repositoryId) { + const notFound = new DependencyScanRepositoryNotFoundError({ + message: "This repository is not connected to the organization", + repositoryId, + }) + const repository = yield* vcs + .getRepositoryById(orgId, repositoryId) + .pipe(Effect.mapError(persistence("Could not read the repository"))) + if (Option.isNone(repository) || repository.value.status !== "active") return yield* notFound + const installation = yield* vcs + .getInstallationById(orgId, repository.value.installationId) + .pipe(Effect.mapError(persistence("Could not read the installation"))) + if (Option.isNone(installation) || !isInstallationProcessable(installation.value)) + return yield* notFound + yield* queue + .send({ + kind: "dependency-scan", + provider: repository.value.provider, + externalInstallationId: installation.value.externalInstallationId, + externalRepoId: repository.value.externalRepoId, + trigger: "manual", + }) + .pipe(Effect.mapError(persistence("Could not queue the scan"))) + return { queued: true } + }) + + const listFindings: DependencyScanServiceApi["listFindings"] = Effect.fn( + "DependencyScanService.listFindings", + )(function* (orgId, query) { + // Only repositories the org still has: a purged or removed one's findings are not shown. + const connected = yield* connectedRepositories(orgId).pipe( + Effect.mapError(persistence("Could not list repositories")), + ) + const names = new Map(connected.map((repository) => [repository.id, repository.fullName])) + const rows = yield* store + .listFindings({ + orgId, + limit: FINDINGS_LIMIT, + status: query.status, + repositoryId: query.repositoryId, + repositoryIds: [...names.keys()], + }) + .pipe(Effect.mapError(persistence("Could not list findings"))) + const findings = rows.flatMap((row) => { + const name = names.get(row.repositoryId) + return name === undefined ? [] : [findingViewOf(row, name)] + }) + yield* Effect.annotateCurrentSpan({ "result.rowCount": findings.length }) + return { findings } + }) + + const updateFinding: DependencyScanServiceApi["updateFinding"] = Effect.fn( + "DependencyScanService.updateFinding", + )(function* (orgId, actor, findingId, request) { + // A finding of a repository the org no longer has is not listed, so it cannot be triaged either. + const notFound = new DependencyFindingNotFoundError({ message: "No such finding", findingId }) + const existing = yield* store + .getFinding({ orgId, id: findingId }) + .pipe(Effect.mapError(persistence("Could not read the finding"))) + if (Option.isNone(existing)) return yield* notFound + const connected = yield* connectedRepositories(orgId).pipe( + Effect.mapError(persistence("Could not list repositories")), + ) + if (!connected.some((repository) => repository.id === existing.value.repositoryId)) + return yield* notFound + const row = yield* store + .setFindingStatus({ + orgId, + id: findingId, + status: request.status, + reason: request.reason ?? null, + actor, + }) + .pipe( + Effect.catchTags({ + "@maple/dependency-scan/DependencyFindingNotFoundError": () => + Effect.fail( + new DependencyFindingNotFoundError({ + message: "No such finding", + findingId, + }), + ), + "@maple/dependency-scan/DependencyScanPersistenceError": (error) => + Effect.fail(persistence("Could not update the finding")(error)), + }), + ) + const repository = yield* vcs + .getRepositoryById(orgId, row.repositoryId) + .pipe(Effect.mapError(persistence("Could not read the repository"))) + return findingViewOf( + row, + Option.match(repository, { + onNone: () => "unknown repository", + onSome: (r) => r.fullName, + }), + ) + }) + + return { + listRepositories, + requestScan, + listFindings, + updateFinding, + } satisfies DependencyScanServiceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide(Layer.mergeAll(DependencyScanStore.layer, VcsRepository.layer, VcsSyncQueue.layer)), + ) +} diff --git a/packages/backend/src/services/dependency-scan/DependencyScanStore.ts b/packages/backend/src/services/dependency-scan/DependencyScanStore.ts index d37839b943..40eeb89de4 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanStore.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanStore.ts @@ -116,10 +116,10 @@ export interface DependencyScanStoreApi { /** Most severe first, then most recently updated. */ readonly listFindings: (input: { readonly orgId: OrgId - readonly repositoryId?: VcsRepositoryId + readonly repositoryId?: VcsRepositoryId | undefined /** Only these repositories, so the limit is spent on them; an empty list reads nothing. */ - readonly repositoryIds?: ReadonlyArray - readonly status?: DependencyFindingStatus + readonly repositoryIds?: ReadonlyArray | undefined + readonly status?: DependencyFindingStatus | undefined readonly limit: number }) => Effect.Effect, DependencyScanPersistenceError> readonly getFinding: (input: { diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.github.e2e.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.github.e2e.test.ts new file mode 100644 index 0000000000..782beb7076 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.github.e2e.test.ts @@ -0,0 +1,250 @@ +// Opt-in live check: runs the dependency scanner on public GitHub repositories end to end (tree +// listing, lockfile selection, blob reads, parsing, OSV) and compares every lockfile it read with +// osv-scanner on a shallow clone of the same commit. Needs network, git and osv-scanner: +// DEPSCAN_GITHUB_E2E_REPOS=Makisuo/maple,cli/cli GITHUB_TOKEN=$(gh auth token) \ +// bunx vitest run src/services/dependency-scan/DependencyScanner.github.e2e.test.ts --silent=false +import { afterEach, assert, describe, it } from "@effect/vitest" +import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" +import { GitCommitSha, VcsInstallation, VcsRepo, type VcsRepositoryRef } from "@maple/domain/http" +import { Console, Effect, Layer, Option, Schema } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http" +import { spawnSync } from "node:child_process" +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" +import { DependencyScanner, DependencySources } from "./DependencyScanner" +import { DependencyScanStore } from "./DependencyScanStore" + +const repos = (process.env.DEPSCAN_GITHUB_E2E_REPOS ?? "").split(",").filter((repo) => repo !== "") +const token = process.env.GITHUB_TOKEN ?? "" + +const trackedDbs: Array = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const repoOf = (repo: VcsRepositoryRef) => `${repo.owner}/${repo.name}` + +const github = (fullName: string, path: string) => + Effect.gen(function* () { + const client = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk) + const response = yield* client.execute( + HttpClientRequest.get(`https://api.github.com/repos/${fullName}${path}`).pipe( + HttpClientRequest.bearerToken(token), + ), + ) + return yield* response.json + }).pipe(Effect.orDie, Effect.provide(FetchHttpClient.layer)) + +const Tree = Schema.Struct({ + truncated: Schema.Boolean, + tree: Schema.Array( + Schema.Struct({ + path: Schema.String, + type: Schema.String, + sha: Schema.String, + size: Schema.optionalKey(Schema.Number), + }), + ), +}) +const Blob = Schema.Struct({ content: Schema.String }) +const Repo = Schema.Struct({ default_branch: Schema.String }) +const Commit = Schema.Struct({ sha: GitCommitSha }) + +const source = { + resolveRef: (_installation: unknown, _repo: VcsRepositoryRef, ref: string) => + github(repoOf(_repo), `/commits/${ref}`).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Commit)), + Effect.map((commit) => Option.some(commit.sha)), + Effect.orDie, + ), + fetchTreeFiles: (_installation: unknown, _repo: VcsRepositoryRef, commitSha: string) => + github(repoOf(_repo), `/git/trees/${commitSha}?recursive=1`).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Tree)), + Effect.map((tree) => ({ + truncated: tree.truncated, + files: tree.tree.flatMap((entry) => + entry.type === "blob" + ? [{ path: entry.path, sha: entry.sha, size: entry.size ?? 0 }] + : [], + ), + })), + Effect.orDie, + ), + fetchBlobText: (_installation: unknown, _repo: VcsRepositoryRef, blobSha: string) => + github(repoOf(_repo), `/git/blobs/${blobSha}`).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Blob)), + Effect.map((blob) => Buffer.from(blob.content.replace(/\s/g, ""), "base64").toString("utf8")), + Effect.orDie, + ), +} + +interface OsvScannerOutput { + readonly results?: ReadonlyArray<{ + readonly packages?: ReadonlyArray<{ + readonly package: { readonly name: string; readonly version: string; readonly ecosystem: string } + readonly groups?: ReadonlyArray<{ + readonly ids: ReadonlyArray + readonly aliases?: ReadonlyArray + }> + }> + }> +} + +const osvScanner = (path: string): OsvScannerOutput => { + // osv-scanner exits non-zero when it finds something, so its status says nothing here. + const run = spawnSync("osv-scanner", ["scan", "source", "--format", "json", "--lockfile", path], { + maxBuffer: 256 * 1024 * 1024, + }) + return JSON.parse(run.stdout.toString() || "{}") +} + +const scanAndCompare = (fullName: string) => + Effect.gen(function* () { + const [owner = "", name = ""] = fullName.split("/") + const repoInfo = yield* github(fullName, "").pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Repo)), + Effect.orDie, + ) + const branch = repoInfo.default_branch + const installation = Schema.decodeUnknownSync(VcsInstallation)({ + id: "5d7f8a52-1c3e-4b6a-9b1e-2a3c4d5e6f71", + orgId: "org_harness", + provider: "github", + externalInstallationId: "0", + accountLogin: owner, + accountType: "organization", + externalAccountId: "0", + accountAvatarUrl: null, + repositorySelection: "all", + status: "active", + suspendedAt: null, + ciTracesEnabled: false, + installedByUserId: "user_harness", + createdAt: 0, + updatedAt: 0, + }) + const repository = Schema.decodeUnknownSync(VcsRepo)({ + id: "7f0c1a52-4d8e-4b6a-9b1e-2a3c4d5e6f70", + orgId: "org_harness", + provider: "github", + installationId: installation.id, + externalRepoId: "0", + owner, + name, + fullName, + defaultBranch: repoInfo.default_branch, + trackedBranch: branch, + htmlUrl: `https://github.com/${fullName}`, + isPrivate: false, + isArchived: false, + status: "active", + syncStatus: "ready", + lastSyncedAt: null, + lastSyncError: null, + prReviewEnabled: false, + createdAt: 0, + updatedAt: 0, + }) + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + const started = Date.now() + const outcome = yield* scanner.scan(installation, repository, { trigger: "manual" }) + if (outcome._tag !== "Scanned") return yield* Console.log(`${fullName}: ${outcome._tag}`) + const scan = outcome.scan + const matches = yield* store.scanMatches({ orgId: repository.orgId, scanId: scan.id }) + yield* Console.log( + `${fullName}@${scan.commitSha.slice(0, 12)} (${branch}): ${scan.lockfiles.length} lockfiles, ${scan.packageCount} packages, ${matches.length} vulnerable matches, ${Date.now() - started} ms`, + ) + + const dir = mkdtempSync(join(tmpdir(), "depscan-")) + spawnSync("git", [ + "clone", + "--quiet", + "--depth", + "1", + "--branch", + branch, + `https://github.com/${fullName}`, + dir, + ]) + const cloned = spawnSync("git", ["-C", dir, "rev-parse", "HEAD"]).stdout.toString().trim() + if (cloned !== scan.commitSha) yield* Console.log(` clone is at ${cloned}, comparison may differ`) + + const results = scan.lockfiles.map((lockfile) => { + if (lockfile.status !== "parsed") + return { + line: ` SKIP ${lockfile.path}: ${lockfile.status} (${lockfile.message})`, + ok: true, + } + const ours = new Map>>() + matches + .filter((match) => match.lockfilePath === lockfile.path) + .forEach((match) => { + const key = `${match.ecosystem}:${match.packageName}@${match.version.replace(/^v/, "")}` + ours.set(key, [...(ours.get(key) ?? []), new Set([match.advisoryId, ...match.aliases])]) + }) + const theirs = new Map( + (osvScanner(join(dir, lockfile.path)).results ?? []) + .flatMap((result) => result.packages ?? []) + .map((pkg) => [ + `${pkg.package.ecosystem}:${pkg.package.name}@${pkg.package.version.replace(/^v/, "")}`, + (pkg.groups ?? []).map((group) => new Set([...group.ids, ...(group.aliases ?? [])])), + ]), + ) + const keys = new Set([...ours.keys(), ...theirs.keys()]) + const diffs = [...keys].flatMap((key) => { + const mine = ours.get(key) ?? [] + const other = theirs.get(key) ?? [] + const missing = other.filter( + (group) => !mine.some((names) => [...group].some((id) => names.has(id))), + ) + const extra = mine.filter( + (names) => !other.some((group) => [...group].some((id) => names.has(id))), + ) + return missing.length === 0 && extra.length === 0 + ? [] + : [ + ` ${key}: only osv-scanner ${missing.map((g) => [...g][0]).join(",")}; only ours ${extra.map((n) => [...n][0]).join(",")}`, + ] + }) + const count = [...ours.values()].reduce((sum, list) => sum + list.length, 0) + const theirCount = [...theirs.values()].reduce((sum, list) => sum + list.length, 0) + return { + line: [ + ` ${diffs.length === 0 ? "MATCH" : "DIFF "} ${lockfile.path}: ${lockfile.packageCount} packages, ${count} advisories (osv-scanner ${theirCount})`, + ...diffs, + ].join("\n"), + ok: diffs.length === 0, + } + }) + yield* Effect.forEach(results, (result) => Console.log(result.line)) + return results.flatMap((result) => (result.ok ? [] : [result.line])) + }) + +const layer = () => { + const cache = EdgeCacheService.layer.pipe(Layer.provide(MemoryCacheBackendLive)) + const store = DependencyScanStore.layer.pipe(Layer.provide(createTestDb(trackedDbs).layer)) + return Layer.effect(DependencyScanner, DependencyScanner.make).pipe( + Layer.provide( + Layer.mergeAll( + PackageAdvisoryCatalog.layer.pipe(Layer.provide(cache)), + Layer.succeed(DependencySources, { resolve: () => Effect.succeed(source) }), + cache, + ), + ), + Layer.provideMerge(store), + ) +} + +describe.skipIf(repos.length === 0)("DependencyScanner on GitHub repositories", () => { + it.each(repos)( + "finds what osv-scanner finds in %s", + (fullName) => + Effect.gen(function* () { + const diffs = yield* scanAndCompare(fullName) + assert.deepStrictEqual(diffs, []) + }).pipe(Effect.provide(layer()), Effect.runPromise), + 600_000, + ) +}) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts new file mode 100644 index 0000000000..f7063aae3a --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts @@ -0,0 +1,388 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" +import { GitCommitSha, VcsInstallation, VcsRepo } from "@maple/domain/http" +import { Effect, Layer, Option, Schema } from "effect" +import { HttpClient, type HttpClientRequest, HttpClientResponse } from "effect/http" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import type { VcsTreeFile } from "@maple/backend/services/integrations/vcs/VcsProviderClient" +import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" +import osvFixture from "../security/__fixtures__/osv.json" +import { DependencyScanner, DependencySources, MAX_LOCKFILE_BYTES } from "./DependencyScanner" +import { DependencyScanStore } from "./DependencyScanStore" +import { MAX_LOCKFILES } from "./lockfile-selection" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const sha = (digit: string) => Schema.decodeSync(GitCommitSha)(digit.repeat(40)) + +const installation = Schema.decodeUnknownSync(VcsInstallation)({ + id: "5d7f8a52-1c3e-4b6a-9b1e-2a3c4d5e6f71", + orgId: "org_scan", + provider: "github", + externalInstallationId: "42", + accountLogin: "octo", + accountType: "organization", + externalAccountId: "1", + accountAvatarUrl: null, + repositorySelection: "all", + status: "active", + suspendedAt: null, + ciTracesEnabled: true, + installedByUserId: "user_scan", + createdAt: 0, + updatedAt: 0, +}) + +const repository = Schema.decodeUnknownSync(VcsRepo)({ + id: "7f0c1a52-4d8e-4b6a-9b1e-2a3c4d5e6f70", + orgId: "org_scan", + provider: "github", + installationId: installation.id, + externalRepoId: "7", + owner: "octo", + name: "shop", + fullName: "octo/shop", + defaultBranch: "main", + trackedBranch: null, + htmlUrl: "https://github.com/octo/shop", + isPrivate: true, + isArchived: false, + status: "active", + syncStatus: "ready", + lastSyncedAt: null, + lastSyncError: null, + prReviewEnabled: false, + createdAt: 0, + updatedAt: 0, +}) + +const packageLock = (lodash: string) => + JSON.stringify({ + name: "shop", + lockfileVersion: 3, + requires: true, + packages: { + "": { name: "shop", dependencies: { lodash }, devDependencies: { minimist: "1.2.5" } }, + "node_modules/lodash": { version: lodash }, + "node_modules/minimist": { version: "1.2.5", dev: true }, + }, + }) + +const GO_MOD = "module example.com/api\n\ngo 1.22\n\nrequire golang.org/x/net v0.7.0\n" + +const CARGO_LOCK = [ + "version = 3", + "", + "[[package]]", + 'name = "shop"', + 'version = "0.1.0"', + 'dependencies = ["time"]', + "", + "[[package]]", + 'name = "time"', + 'version = "0.1.43"', + 'source = "registry+https://github.com/rust-lang/crates.io-index"', + "", +].join("\n") + +interface World { + head: GitCommitSha + files: Map + truncated: boolean + osvDown: boolean + /** Blob shas read from the provider, in order. */ + readonly blobReads: Array +} + +const world = (files: Record): World => ({ + head: sha("a"), + files: new Map( + Object.entries(files).map(([path, content]) => [ + path, + { sha: `blob:${path}:${content.length}`, content }, + ]), + ), + truncated: false, + osvDown: false, + blobReads: [], +}) + +const source = (state: World) => ({ + resolveRef: () => Effect.sync(() => Option.some(state.head)), + fetchTreeFiles: () => + Effect.sync(() => ({ + truncated: state.truncated, + files: [...state.files].map(([path, blob]): VcsTreeFile => ({ + path, + sha: blob.sha, + size: blob.size ?? blob.content.length, + })), + })), + fetchBlobText: (_installation: unknown, _repo: unknown, blobSha: string) => + Effect.sync(() => { + state.blobReads.push(blobSha) + return [...state.files.values()].find((blob) => blob.sha === blobSha)?.content ?? "" + }), +}) + +const json = (request: HttpClientRequest.HttpClientRequest, body: unknown, status = 200) => + HttpClientResponse.fromWeb(request, new Response(JSON.stringify(body), { status })) + +interface Query { + readonly package: { readonly ecosystem: string; readonly name: string } + readonly version: string +} + +/** OSV as captured in the fixture: packages it does not list have no advisories. */ +const osv = (state: World) => + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.sync(() => { + if (state.osvDown) return json(request, { message: "unavailable" }, 503) + const url = new URL(request.url) + if (url.pathname === "/v1/querybatch") { + const body: { queries: ReadonlyArray } = + request.body._tag === "Uint8Array" + ? JSON.parse(new TextDecoder().decode(request.body.body)) + : { queries: [] } + return json(request, { + results: body.queries.map((query) => { + const index = osvFixture.queries.findIndex( + (known) => + known.package.ecosystem === query.package.ecosystem && + known.package.name === query.package.name && + known.version === query.version, + ) + return osvFixture.batch.results[index] ?? {} + }), + }) + } + const id = decodeURIComponent(url.pathname.replace("/v1/vulns/", "")) + const vuln = osvFixture.vulns.find((candidate) => candidate.id === id) + return vuln === undefined ? json(request, { message: "not found" }, 404) : json(request, vuln) + }), + ) + +const scannerFor = (state: World) => { + const store = DependencyScanStore.layer.pipe(Layer.provide(createTestDb(trackedDbs).layer)) + const cache = EdgeCacheService.layer.pipe(Layer.provide(MemoryCacheBackendLive)) + const catalog = Layer.effect(PackageAdvisoryCatalog, PackageAdvisoryCatalog.make).pipe( + Layer.provide(Layer.mergeAll(Layer.succeed(HttpClient.HttpClient, osv(state)), cache)), + ) + const sources = Layer.succeed(DependencySources, { resolve: () => Effect.succeed(source(state)) }) + return Layer.effect(DependencyScanner, DependencyScanner.make).pipe( + Layer.provide(Layer.mergeAll(catalog, sources, cache)), + Layer.provideMerge(store), + ) +} + +const MONOREPO = { + "package-lock.json": packageLock("4.17.15"), + "services/api/go.mod": GO_MOD, + "services/api/go.sum": "golang.org/x/net v0.7.0 h1:abc=\n", + "crates/core/Cargo.lock": CARGO_LOCK, + "web/node_modules/left-pad/package-lock.json": packageLock("4.17.15"), + "legacy/bun.lockb": "binary", + "README.md": "# shop", +} + +const findingsOf = Effect.gen(function* () { + const store = yield* DependencyScanStore + return yield* store.listFindings({ orgId: repository.orgId, limit: 500 }) +}) + +const keyOf = (finding: { + readonly packageName: string + readonly advisoryId: string + readonly status: string +}) => `${finding.packageName}:${finding.advisoryId}:${finding.status}` + +describe("DependencyScanner", () => { + it.effect( + "reads every lockfile of a monorepo at the tracked head and opens a finding per advisory", + () => { + const state = world(MONOREPO) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const outcome = yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual(outcome._tag, "Scanned") + if (outcome._tag !== "Scanned") return + assert.deepStrictEqual( + outcome.scan.lockfiles.map((lockfile) => [ + lockfile.path, + lockfile.status, + lockfile.packageCount, + ]), + [ + ["package-lock.json", "parsed", 2], + ["legacy/bun.lockb", "unsupported", 0], + ["crates/core/Cargo.lock", "parsed", 1], + ["services/api/go.mod", "parsed", 1], + ], + ) + assert.isTrue(outcome.scan.defaultBranchHead) + assert.strictEqual(outcome.scan.ref, "main") + assert.strictEqual(outcome.scan.commitSha, sha("a")) + + const findings = yield* findingsOf + const byPackage = (name: string) => findings.filter((finding) => finding.packageName === name) + assert.strictEqual(byPackage("lodash").length, 6) + assert.deepStrictEqual( + byPackage("minimist").map((finding) => [ + finding.severity, + finding.relation, + finding.fixedIn, + ]), + [["critical", "dev", "1.2.6"]], + ) + assert.isTrue( + byPackage("golang.org/x/net").every( + (finding) => finding.lockfilePath === "services/api/go.mod", + ), + ) + assert.deepStrictEqual( + byPackage("time").map((finding) => finding.relation), + ["direct"], + ) + // The go.sum beside go.mod and the vendored lockfile are never read. + assert.isFalse( + state.blobReads.some((blob) => blob.includes("go.sum") || blob.includes("node_modules")), + ) + }).pipe(Effect.provide(scannerFor(state))) + }, + ) + + it.effect("skips a push that changed no lockfile, and fixes the advisories an upgrade removes", () => { + const state = world(MONOREPO) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + + state.head = sha("b") + state.files.set("README.md", { sha: "readme-2", content: "# shop 2" }) + const unchanged = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(unchanged._tag, "Unchanged") + + state.head = sha("c") + const upgraded = packageLock("4.18.0") + state.files.set("package-lock.json", { sha: "lock-2", content: upgraded }) + const outcome = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(outcome._tag, "Scanned") + + const lodash = (yield* findingsOf).filter((finding) => finding.packageName === "lodash") + assert.strictEqual(lodash.length, 6) + assert.isTrue( + lodash.every((finding) => finding.status === "fixed" && finding.fixedSha === sha("c")), + ) + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.effect("skips an unchanged push in a repository with more lockfiles than a scan reads", () => { + const many = Object.fromEntries( + Array.from({ length: MAX_LOCKFILES + 2 }, (_, i) => [ + `pkg${i}/package-lock.json`, + packageLock("4.17.15"), + ]), + ) + const state = world(many) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const first = yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual( + first._tag === "Scanned" + ? first.scan.lockfiles.filter((lockfile) => lockfile.status === "skipped").length + : 0, + 2, + ) + state.head = sha("b") + const again = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(again._tag, "Unchanged") + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.effect("reads a lockfile's packages from the parse cache on a rescan", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual(state.blobReads.length, 1) + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual(state.blobReads.length, 1) + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.live("records a failed scan and keeps the findings when OSV cannot be read", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + const before = (yield* findingsOf).map(keyOf).sort() + + state.osvDown = true + state.head = sha("d") + state.files.set("package-lock.json", { sha: "lock-3", content: packageLock("4.18.0") }) + const error = yield* Effect.flip(scanner.scan(installation, repository, { trigger: "push" })) + assert.strictEqual(error._tag, "@maple/dependency-scan/DependencyScanError") + + const [latest] = yield* store.listScans({ orgId: repository.orgId, limit: 1 }) + assert.strictEqual(latest?.status, "failed") + assert.deepStrictEqual((yield* findingsOf).map(keyOf).sort(), before) + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.effect( + "fails a scan whose tree listing was cut short instead of fixing the unlisted lockfiles' findings", + () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + state.truncated = true + state.files.delete("package-lock.json") + const error = yield* Effect.flip( + scanner.scan(installation, repository, { trigger: "schedule" }), + ) + assert.include(error.message, "too large") + assert.isTrue((yield* findingsOf).every((finding) => finding.status === "open")) + }).pipe(Effect.provide(scannerFor(state))) + }, + ) + + it.effect("records an oversized lockfile without reading it and keeps its findings open", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + state.files.set("package-lock.json", { sha: "huge", content: "", size: MAX_LOCKFILE_BYTES + 1 }) + const outcome = yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual( + outcome._tag === "Scanned" ? outcome.scan.lockfiles[0]?.status : undefined, + "too_large", + ) + assert.isFalse(state.blobReads.includes("huge")) + assert.isTrue((yield* findingsOf).every((finding) => finding.status === "open")) + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.effect("stores an older commit's matches without touching the findings", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + const outcome = yield* scanner.scan(installation, repository, { + trigger: "production", + commitSha: sha("e"), + }) + assert.strictEqual(outcome._tag, "Scanned") + if (outcome._tag !== "Scanned") return + assert.isFalse(outcome.scan.defaultBranchHead) + assert.isNull(outcome.scan.ref) + assert.strictEqual( + (yield* store.scanMatches({ orgId: repository.orgId, scanId: outcome.scan.id })).length, + 7, + ) + assert.deepStrictEqual(yield* findingsOf, []) + }).pipe(Effect.provide(scannerFor(state))) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts new file mode 100644 index 0000000000..90cfb21fed --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -0,0 +1,395 @@ +import { EdgeCacheService } from "@maple/cache" +import type { DependencyScanRow } from "@maple/db/tables" +import { DependencyRelation, type DependencyScanLockfile } from "@maple/domain/dependency-scan" +import { + type DependencyScanTrigger, + type GitCommitSha, + type UnknownVcsProviderError, + type VcsInstallation, + type VcsRepo, + VcsRepositoryId, + type VcsRepositoryRef, +} from "@maple/domain/http" +import { type PackageAdvisory, PackageEcosystem, packageKey } from "@maple/domain/package-advisories" +import { parseLockfile } from "@maple/lockfiles" +import { Cause, Context, Effect, Layer, Option, Schema } from "effect" +import type { + VcsProviderClient, + VcsTreeFile, +} from "@maple/backend/services/integrations/vcs/VcsProviderClient" +import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" +import { DependencyScanSink } from "@maple/backend/services/integrations/vcs/PullRequestEventSink" +import { summarizeCause } from "@maple/backend/platform/describe-cause" +import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" +import { type DependencyScanMatchInput, DependencyScanStore } from "./DependencyScanStore" +import { + MAX_LOCKFILES, + manifestsFor, + type SelectedLockfile, + sameLockfiles, + selectLockfiles, +} from "./lockfile-selection" + +// Reads a repository's lockfiles at one commit through the VCS provider, looks their packages up +// in OSV and stores the scan. A lockfile's parse is cached by its blob sha (and its manifests'), +// so a rescan for new advisories reads nothing from the provider but the tree. + +const PARSE_CACHE_BUCKET = "lockfile-packages" +const PARSE_CACHE_TTL_SECONDS = 7 * 24 * 60 * 60 +/** Larger lockfiles are recorded as too large rather than read into the Worker's memory. */ +export const MAX_LOCKFILE_BYTES = 15 * 1024 * 1024 +const READ_CONCURRENCY = 4 + +const LockedPackageRow = Schema.Struct({ + ecosystem: PackageEcosystem, + name: Schema.String, + version: Schema.String, + relation: DependencyRelation, +}) +type LockedPackageRow = typeof LockedPackageRow.Type + +const CachedLockfile = Schema.Struct({ + packages: Schema.Array(LockedPackageRow), + warnings: Schema.Array(Schema.String), +}) + +export class DependencyScanError extends Schema.TaggedError()( + "@maple/dependency-scan/DependencyScanError", + { message: Schema.String, repositoryId: VcsRepositoryId, cause: Schema.optionalKey(Schema.Defect()) }, +) {} + +export interface DependencyScanRequest { + readonly trigger: DependencyScanTrigger + /** Scan this commit instead of the tracked branch head; it then never reconciles findings. */ + readonly commitSha?: GitCommitSha | undefined +} + +export type DependencyScanOutcome = + | { readonly _tag: "Scanned"; readonly scan: DependencyScanRow } + /** A push that left every lockfile as the last scan read it. */ + | { readonly _tag: "Unchanged" } + /** The tracked branch has no commit yet. */ + | { readonly _tag: "EmptyRepository" } + +export interface DependencyScannerApi { + readonly scan: ( + installation: VcsInstallation, + repository: VcsRepo, + request: DependencyScanRequest, + ) => Effect.Effect +} + +/** The provider reads a scan makes; the registry's clients have them, tests stub only these. */ +export type DependencySource = Pick + +export class DependencySources extends Context.Service< + DependencySources, + { readonly resolve: (provider: string) => Effect.Effect } +>()("@maple/backend/services/dependency-scan/DependencySources") { + static readonly layer = Layer.effect( + this, + Effect.gen(function* () { + const registry = yield* VcsProviderRegistry + return { resolve: registry.resolve } + }), + ).pipe(Layer.provide(VcsProviderRegistry.layer)) +} + +interface ReadLockfile { + readonly lockfile: DependencyScanLockfile + readonly packages: ReadonlyArray +} + +export class DependencyScanner extends Context.Service()( + "@maple/backend/services/dependency-scan/DependencyScanner", + { + make: Effect.gen(function* () { + const sources = yield* DependencySources + const store = yield* DependencyScanStore + const catalog = yield* PackageAdvisoryCatalog + const edgeCache = yield* EdgeCacheService + + const lockfileRow = ( + selected: SelectedLockfile, + status: DependencyScanLockfile["status"], + packageCount: number, + message: string | null, + ): DependencyScanLockfile => ({ + path: selected.file.path, + kind: selected.kind, + blobSha: selected.file.sha, + packageCount, + status, + message, + }) + + /** Fetches and parses one lockfile; a format problem is the lockfile's status, a provider failure fails the scan. */ + const readLockfile = ( + provider: DependencySource, + installation: VcsInstallation, + repo: VcsRepositoryRef, + selected: SelectedLockfile, + tree: ReadonlyArray, + ) => { + if (selected.file.size > MAX_LOCKFILE_BYTES) + return Effect.succeed({ + lockfile: lockfileRow( + selected, + "too_large", + 0, + `${selected.file.size} bytes, over the ${MAX_LOCKFILE_BYTES} a scan reads`, + ), + packages: [], + }) + const manifests = manifestsFor(selected, tree) + const read = Effect.gen(function* () { + const content = yield* provider.fetchBlobText(installation, repo, selected.file.sha) + const manifestTexts = yield* Effect.forEach( + manifests, + (manifest) => + provider + .fetchBlobText(installation, repo, manifest.file.sha) + .pipe(Effect.map((text) => [manifest.relativePath, text] as const)), + { concurrency: READ_CONCURRENCY }, + ) + const parsed = yield* parseLockfile({ + path: selected.file.path, + content, + manifests: new Map(manifestTexts), + }) + return { packages: parsed.packages, warnings: parsed.warnings } + }) + return edgeCache + .getOrCompute( + { + bucket: PARSE_CACHE_BUCKET, + key: [selected.file.sha, ...manifests.map((manifest) => manifest.file.sha)].join( + ":", + ), + ttlSeconds: PARSE_CACHE_TTL_SECONDS, + schema: CachedLockfile, + }, + read, + ) + .pipe( + Effect.map(({ value }): ReadLockfile => ({ + lockfile: lockfileRow( + selected, + "parsed", + value.packages.length, + value.warnings.length === 0 ? null : value.warnings.join("; "), + ), + packages: value.packages, + })), + Effect.catchTags({ + "@maple/lockfiles/LockfileParseError": (error) => + Effect.succeed({ + lockfile: lockfileRow(selected, "failed", 0, error.message), + packages: [], + }), + "@maple/lockfiles/LockfileUnsupportedError": (error) => + Effect.succeed({ + lockfile: lockfileRow(selected, "unsupported", 0, error.message), + packages: [], + }), + }), + Effect.withSpan("DependencyScanner.readLockfile", { + attributes: { + "maple.depscan.lockfile.path": selected.file.path, + "maple.depscan.lockfile.kind": selected.kind, + }, + }), + ) + } + + const matchesOf = ( + reads: ReadonlyArray, + advisories: ReadonlyMap>, + ): ReadonlyArray => + reads.flatMap(({ lockfile, packages }) => + packages.flatMap((pkg) => + (advisories.get(packageKey(pkg)) ?? []).map((advisory): DependencyScanMatchInput => ({ + lockfilePath: lockfile.path, + ecosystem: pkg.ecosystem, + packageName: pkg.name, + version: pkg.version, + relation: pkg.relation, + advisoryId: advisory.id, + aliases: advisory.aliases, + severity: advisory.severity, + summary: advisory.summary, + fixedIn: advisory.fixedIn, + })), + ), + ) + + const scan: DependencyScannerApi["scan"] = Effect.fn("DependencyScanner.scan")( + function* (installation, repository, request) { + const orgId = repository.orgId + const ref = repository.trackedBranch ?? repository.defaultBranch + const repo: VcsRepositoryRef = { + externalRepoId: repository.externalRepoId, + owner: repository.owner, + name: repository.name, + } + yield* Effect.annotateCurrentSpan({ + orgId, + "vcs.repository.id": repository.id, + "maple.depscan.trigger": request.trigger, + }) + const scanError = (message: string) => (cause: unknown) => + new DependencyScanError({ message, repositoryId: repository.id, cause }) + const provider = yield* sources + .resolve(repository.provider) + .pipe(Effect.mapError(scanError("Unknown VCS provider"))) + const headOpt = + request.commitSha === undefined + ? yield* provider + .resolveRef(installation, repo, ref) + .pipe(Effect.mapError(scanError(`Could not resolve ${ref}`))) + : Option.some(request.commitSha) + if (Option.isNone(headOpt)) + return { _tag: "EmptyRepository" } satisfies DependencyScanOutcome + const commitSha = headOpt.value + const defaultBranchHead = request.commitSha === undefined + const tree = yield* provider + .fetchTreeFiles(installation, repo, commitSha) + .pipe(Effect.mapError(scanError("Could not list the repository tree"))) + const { selected, skipped } = selectLockfiles(tree.files) + yield* Effect.annotateCurrentSpan({ + "vcs.commit.sha": commitSha, + "maple.depscan.lockfile_count": selected.length, + "maple.depscan.lockfiles_skipped": skipped.length, + "maple.depscan.tree_truncated": tree.truncated, + }) + + if (request.trigger === "push") { + const latest = yield* store + .latestScan({ orgId, repositoryId: repository.id }) + .pipe(Effect.mapError(scanError("Could not read the last scan"))) + if ( + Option.isSome(latest) && + latest.value.defaultBranchHead && + !tree.truncated && + // Skipped lockfiles are on the last scan too, so they count toward "unchanged". + sameLockfiles(latest.value.lockfiles, [...selected, ...skipped]) + ) { + yield* Effect.annotateCurrentSpan({ "maple.depscan.outcome": "unchanged" }) + return { _tag: "Unchanged" } satisfies DependencyScanOutcome + } + } + + const started = yield* store + .startScan({ + orgId, + repositoryId: repository.id, + commitSha, + ref: defaultBranchHead ? ref : null, + trigger: request.trigger, + defaultBranchHead, + }) + .pipe(Effect.mapError(scanError("Could not record the scan"))) + yield* Effect.annotateCurrentSpan({ "maple.depscan.scan_id": started.id }) + + const run = Effect.gen(function* () { + // A cut-short listing would read as deleted lockfiles and mark their findings fixed. + if (tree.truncated) + return yield* new DependencyScanError({ + message: "The repository tree is too large to list in one read", + repositoryId: repository.id, + }) + const reads = yield* Effect.forEach( + selected, + (lockfile) => + readLockfile(provider, installation, repo, lockfile, tree.files).pipe( + Effect.mapError(scanError(`Could not read ${lockfile.file.path}`)), + ), + { concurrency: READ_CONCURRENCY }, + ) + const packages = reads.flatMap((read) => read.packages) + const advisories = yield* catalog + .advisories(packages) + .pipe(Effect.mapError(scanError("Could not read advisories"))) + const matches = matchesOf(reads, advisories) + yield* Effect.annotateCurrentSpan({ + "maple.depscan.package_count": packages.length, + "maple.depscan.match_count": matches.length, + }) + return yield* store + .completeScan({ + orgId, + scanId: started.id, + // A lockfile past the cap is recorded, so its findings are kept rather than read as deleted. + lockfiles: [ + ...reads.map((read) => read.lockfile), + ...skipped.map((lockfile) => + lockfileRow( + lockfile, + "skipped", + 0, + `Past the ${MAX_LOCKFILES} lockfiles a scan reads`, + ), + ), + ], + packageCount: packages.length, + matches, + }) + .pipe(Effect.mapError(scanError("Could not store the scan"))) + }) + // Any end but success settles the row, an interruption or a defect included. + const completed = yield* run.pipe( + Effect.onError((cause) => + store + .failScan({ + orgId, + scanId: started.id, + // A typed failure keeps its own message, which the next push compares against. + error: Option.match(Cause.findErrorOption(cause), { + onNone: () => summarizeCause(cause), + onSome: (error) => error.message, + }), + }) + .pipe(Effect.ignore), + ), + ) + yield* Effect.annotateCurrentSpan({ "maple.depscan.outcome": "scanned" }) + return { _tag: "Scanned", scan: completed } satisfies DependencyScanOutcome + }, + ) + + return { scan } satisfies DependencyScannerApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide( + Layer.mergeAll(DependencyScanStore.layer, PackageAdvisoryCatalog.layer, DependencySources.layer), + ), + ) +} + +/** Binds the VCS sync's scan requests to the scanner. A failed scan is recorded on its row and logged; the job stands. */ +export const DependencyScanSinkLive = Layer.effect( + DependencyScanSink, + Effect.gen(function* () { + const scanner = yield* DependencyScanner + return { + onScanRequested: (installation, repository, job) => + scanner + .scan(installation, repository, { trigger: job.trigger, commitSha: job.commitSha }) + .pipe( + Effect.asVoid, + Effect.catchCause((cause) => + Effect.logWarning("[DependencyScan] scan failed").pipe( + Effect.annotateLogs({ + orgId: repository.orgId, + repository: repository.fullName, + trigger: job.trigger, + cause: summarizeCause(cause), + }), + ), + ), + ), + } + }), +).pipe(Layer.provide(DependencyScanner.layer)) diff --git a/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts b/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts new file mode 100644 index 0000000000..50416fdf6f --- /dev/null +++ b/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest" +import { + MAX_LOCKFILES, + manifestsFor, + type SelectedLockfile, + sameLockfiles, + selectLockfiles, +} from "./lockfile-selection" + +const file = (path: string, sha = `sha-${path}`) => ({ path, sha, size: 100 }) + +describe("selectLockfiles", () => { + it("keeps lockfiles shallowest first and skips vendored, fixture and go.sum-beside-go.mod paths", () => { + const { selected, skipped } = selectLockfiles([ + file("services/api/go.sum"), + file("services/api/go.mod"), + file("tools/go.sum"), + file("README.md"), + file("web/node_modules/left-pad/package-lock.json"), + file("crates/core/Cargo.lock"), + file("package-lock.json"), + file("test/fixtures/yarn.lock"), + file("vendor/github.com/x/go.mod"), + ]) + expect(selected.map((lockfile) => [lockfile.file.path, lockfile.kind])).toEqual([ + ["package-lock.json", "package-lock"], + ["tools/go.sum", "go-sum"], + ["crates/core/Cargo.lock", "cargo-lock"], + ["services/api/go.mod", "go-mod"], + ]) + expect(skipped).toEqual([]) + }) + + it("caps the lockfiles a scan reads and counts the rest", () => { + const many = Array.from({ length: MAX_LOCKFILES + 3 }, (_, i) => file(`pkg${i}/bun.lock`)) + const { selected, skipped } = selectLockfiles(many) + expect(selected).toHaveLength(MAX_LOCKFILES) + expect(skipped).toHaveLength(3) + // Every lockfile is either read or recorded as skipped, never dropped. + expect(new Set([...selected, ...skipped].map((lockfile) => lockfile.file.path)).size).toBe( + many.length, + ) + }) +}) + +describe("manifestsFor", () => { + const tree = [ + file("web/yarn.lock"), + file("web/package.json"), + file("web/packages/ui/package.json"), + file("web/node_modules/react/package.json"), + file("api/package.json"), + ] + const lockfile = (path: string, kind: SelectedLockfile["kind"]): SelectedLockfile => ({ + file: file(path), + kind, + }) + + it("hands a yarn lockfile the package.json files under its directory, relative to it", () => { + expect(manifestsFor(lockfile("web/yarn.lock", "yarn-v1"), tree).map((m) => m.relativePath)).toEqual([ + "package.json", + "packages/ui/package.json", + ]) + }) + + it("reads no manifests for formats that record direct and dev dependencies themselves", () => { + expect(manifestsFor(lockfile("web/package-lock.json", "package-lock"), tree)).toEqual([]) + }) +}) + +describe("sameLockfiles", () => { + const previous = [ + { + path: "a/bun.lock", + kind: "bun-lock", + blobSha: "1", + packageCount: 3, + status: "parsed", + message: null, + }, + ] as const + it("matches only the same paths at the same blobs", () => { + const selected = (sha: string, path = "a/bun.lock"): ReadonlyArray => [ + { file: { path, sha, size: 1 }, kind: "bun-lock" }, + ] + expect(sameLockfiles(previous, selected("1"))).toBe(true) + expect(sameLockfiles(previous, selected("2"))).toBe(false) + expect(sameLockfiles(previous, selected("1", "b/bun.lock"))).toBe(false) + expect(sameLockfiles(previous, [])).toBe(false) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/lockfile-selection.ts b/packages/backend/src/services/dependency-scan/lockfile-selection.ts new file mode 100644 index 0000000000..cc2d22129d --- /dev/null +++ b/packages/backend/src/services/dependency-scan/lockfile-selection.ts @@ -0,0 +1,73 @@ +import type { DependencyScanLockfile } from "@maple/domain/dependency-scan" +import { isIgnoredLockfilePath, type LockfileKind, lockfileKindOf } from "@maple/lockfiles" +import type { VcsTreeFile } from "@maple/backend/services/integrations/vcs/VcsProviderClient" + +// Which files of a repository tree a scan reads. Pure, so the rules are testable without GitHub. + +/** Lockfiles read per scan; the rest are left out, deepest paths first. */ +export const MAX_LOCKFILES = 50 +/** Manifests read for one yarn lockfile, which says nothing about direct and dev dependencies. */ +export const MAX_MANIFESTS = 20 + +export interface SelectedLockfile { + readonly file: VcsTreeFile + readonly kind: LockfileKind +} + +export const directoryOf = (path: string): string => { + const slash = path.lastIndexOf("/") + return slash === -1 ? "" : path.slice(0, slash + 1) +} + +const depth = (path: string): number => path.split("/").length + +/** + * The tree's lockfiles, shallowest first, capped. A go.sum next to a go.mod is skipped: go.sum + * also lists versions the build never selects, which would only add false positives. + */ +export function selectLockfiles(files: ReadonlyArray): { + readonly selected: ReadonlyArray + /** Past the cap: recorded as skipped so their findings are kept, not read as deleted. */ + readonly skipped: ReadonlyArray +} { + const candidates = files.flatMap((file): ReadonlyArray => { + const kind = lockfileKindOf(file.path) + return kind === undefined || isIgnoredLockfilePath(file.path) ? [] : [{ file, kind }] + }) + const goModDirs = new Set( + candidates.filter((c) => c.kind === "go-mod").map((c) => directoryOf(c.file.path)), + ) + const kept = candidates + .filter((c) => c.kind !== "go-sum" || !goModDirs.has(directoryOf(c.file.path))) + .sort((a, b) => depth(a.file.path) - depth(b.file.path) || a.file.path.localeCompare(b.file.path)) + return { selected: kept.slice(0, MAX_LOCKFILES), skipped: kept.slice(MAX_LOCKFILES) } +} + +/** The package.json files under a yarn lockfile's directory, keyed relative to it. */ +export function manifestsFor( + lockfile: SelectedLockfile, + files: ReadonlyArray, +): ReadonlyArray<{ readonly relativePath: string; readonly file: VcsTreeFile }> { + if (lockfile.kind !== "yarn-v1" && lockfile.kind !== "yarn-berry") return [] + const dir = directoryOf(lockfile.file.path) + return files + .filter( + (file) => + file.path.startsWith(dir) && + (file.path === `${dir}package.json` || file.path.endsWith("/package.json")) && + !isIgnoredLockfilePath(file.path), + ) + .sort((a, b) => depth(a.path) - depth(b.path) || a.path.localeCompare(b.path)) + .slice(0, MAX_MANIFESTS) + .map((file) => ({ relativePath: file.path.slice(dir.length), file })) +} + +/** Whether a scan would read exactly the lockfile contents an earlier scan read. */ +export const sameLockfiles = ( + previous: ReadonlyArray, + selected: ReadonlyArray, +): boolean => + previous.length === selected.length && + selected.every(({ file }) => + previous.some((lockfile) => lockfile.path === file.path && lockfile.blobSha === file.sha), + ) diff --git a/packages/backend/src/services/integrations/vcs/PullRequestEventSink.ts b/packages/backend/src/services/integrations/vcs/PullRequestEventSink.ts index 5ff67cffc8..fcd847d288 100644 --- a/packages/backend/src/services/integrations/vcs/PullRequestEventSink.ts +++ b/packages/backend/src/services/integrations/vcs/PullRequestEventSink.ts @@ -1,8 +1,11 @@ import type { + DependencyScanJob, PullRequestChecklistJob, PullRequestCiApprovedJob, PullRequestCommentJob, PullRequestEventJob, + VcsInstallation, + VcsRepo, } from "@maple/domain/http" import type { OrgId } from "@maple/domain/primitives" import { Context, Effect, Layer } from "effect" @@ -114,3 +117,19 @@ export class PullRequestCiApprovalSink extends Context.Service< PullRequestCiApprovalSink, PullRequestCiApprovalSinkApi >()("@maple/backend/services/integrations/vcs/PullRequestCiApprovalSink") {} + +/** + * Where a repository's dependency scan request goes once its repository row is resolved. Optional + * in `VcsSyncService`: a root without the scanner neither enqueues nor runs scans. + */ +export interface DependencyScanSinkApi { + readonly onScanRequested: ( + installation: VcsInstallation, + repository: VcsRepo, + job: DependencyScanJob, + ) => Effect.Effect +} + +export class DependencyScanSink extends Context.Service()( + "@maple/backend/services/integrations/vcs/DependencyScanSink", +) {} diff --git a/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts b/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts index a12805bbf7..138a2581e3 100644 --- a/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts +++ b/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts @@ -55,6 +55,14 @@ export interface VcsCodeSearchMatch { readonly snippets: ReadonlyArray } +/** A file (blob) in a repository's tree at one commit. */ +export interface VcsTreeFile { + readonly path: string + /** The blob's sha: the same content always has the same sha. */ + readonly sha: string + readonly size: number +} + /** A text source file fetched from a repository at an explicit ref. */ export interface VcsSourceFile { readonly path: string @@ -493,6 +501,26 @@ export interface VcsProviderClient { VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRepositoryBlockedError > + /** Every file in the tree of a commit; `truncated` when the provider cut the listing short. */ + readonly fetchTreeFiles: ( + installation: VcsInstallation, + repo: VcsRepositoryRef, + commitSha: GitCommitSha, + ) => Effect.Effect< + { readonly files: ReadonlyArray; readonly truncated: boolean }, + VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRepositoryBlockedError + > + + /** A file's content as UTF-8 by its blob sha, without the size cap of `fetchSourceFile`. */ + readonly fetchBlobText: ( + installation: VcsInstallation, + repo: VcsRepositoryRef, + blobSha: string, + ) => Effect.Effect< + string, + VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRepositoryBlockedError + > + /** Fetch a UTF-8 source file. `Option.none` is an expected missing path/ref. */ readonly fetchSourceFile: ( installation: VcsInstallation, diff --git a/packages/backend/src/services/integrations/vcs/VcsSyncService.ts b/packages/backend/src/services/integrations/vcs/VcsSyncService.ts index 0701c10cb4..e82ba71310 100644 --- a/packages/backend/src/services/integrations/vcs/VcsSyncService.ts +++ b/packages/backend/src/services/integrations/vcs/VcsSyncService.ts @@ -25,10 +25,13 @@ import { type WorkflowJobAnnotation, type WorkflowJobJob, type WorkflowRunJob, + type DependencyScanJob, + type DependencyScanTrigger, } from "@maple/domain/http" import { Clock, Duration, Effect, Context, Layer, Option, Schema, Match } from "effect" import type { VcsProviderClient } from "./VcsProviderClient" import { + DependencyScanSink, PullRequestChecklistSink, PullRequestCiApprovalSink, PullRequestCommentSink, @@ -102,6 +105,25 @@ export class VcsSyncService extends Context.Service, + trigger: DependencyScanTrigger, + ) => + scanSink === undefined || externalRepoIds.length === 0 + ? Effect.void + : queue.sendBatch( + externalRepoIds.map((externalRepoId): VcsSyncJob => ({ + kind: "dependency-scan", + provider: installation.provider, + externalInstallationId: installation.externalInstallationId, + externalRepoId, + trigger, + })), + ) // The repo's single tracked branch, with the default-branch fallback for a // row whose `trackedBranch` was never set (legacy) — the one place the @@ -415,6 +437,7 @@ export class VcsSyncService extends Context.Service r.externalRepoId), + "schedule", + ) yield* Effect.annotateCurrentSpan({ "vcs.installation_sync.outcome": "handled", "vcs.installation_sync.reason": job.reason, @@ -827,6 +855,21 @@ export class VcsSyncService extends Context.Service handlePush(installation, job)), + Match.discriminator("kind")("dependency-scan", (job) => + handleDependencyScan(installation, job), + ), Match.discriminator("kind")("branch-event", (job) => handleBranchEvent(installation, job), ), diff --git a/packages/backend/src/services/integrations/vcs/__tests__/vcs.test.ts b/packages/backend/src/services/integrations/vcs/__tests__/vcs.test.ts index ff6d690cf1..49d6cf3840 100644 --- a/packages/backend/src/services/integrations/vcs/__tests__/vcs.test.ts +++ b/packages/backend/src/services/integrations/vcs/__tests__/vcs.test.ts @@ -47,7 +47,10 @@ import { VcsProviderRegistry, type VcsProviderRegistryApi, } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" -import { PullRequestEventSink } from "@maple/backend/services/integrations/vcs/PullRequestEventSink" +import { + DependencyScanSink, + PullRequestEventSink, +} from "@maple/backend/services/integrations/vcs/PullRequestEventSink" import { WorkflowEventSink } from "@maple/backend/services/integrations/vcs/WorkflowEventSink" import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" import { clampQueueDelaySeconds } from "@maple/backend/services/integrations/vcs/VcsSyncQueue" @@ -1827,6 +1830,8 @@ describe("VcsSyncService orchestrator", () => { readonly fetchReposError?: VcsRateLimitedError | VcsProviderError | VcsInstallationGoneError readonly branches?: ReadonlyArray<{ name: string; headSha: string | null }> readonly branchesTruncated?: boolean + /** Scan requests the dependency scanner hook received; absent leaves the hook out. */ + readonly scanRequests?: Array<{ repository: string; trigger: string; commitSha: string | undefined }> readonly fetchBranchesError?: | VcsProviderError | VcsInstallationGoneError @@ -1930,8 +1935,22 @@ describe("VcsSyncService orchestrator", () => { Effect.ignore, ), }) + const scanRequests = opts.scanRequests + const scanSink = + scanRequests === undefined + ? Layer.empty + : Layer.succeed(DependencyScanSink, { + onScanRequested: (_installation, repository, job) => + Effect.sync(() => { + scanRequests.push({ + repository: repository.fullName, + trigger: job.trigger, + commitSha: job.commitSha, + }) + }), + }) return Layer.effect(VcsSyncService, VcsSyncService.make).pipe( - Layer.provideMerge(Layer.mergeAll(repoLive, registry, queue, sink, workflowSink)), + Layer.provideMerge(Layer.mergeAll(repoLive, registry, queue, sink, workflowSink, scanSink)), ) } @@ -2613,6 +2632,98 @@ describe("VcsSyncService orchestrator", () => { }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, repos: oneRepo }))) }) + it.effect("with the scanner bound, a push to the tracked branch enqueues a scan of its head", () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + yield* seedInstallation(repo, asOrgId("org_orch")) + yield* upsertReposFor(repo, "42", oneRepo) + const push = (branch: string) => + svc.processMessage( + Schema.encodeSync(VcsSyncJob)({ + kind: "push", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + branch, + commits: [commit(SHA_A, 1)], + }), + ) + yield* push("feature/x") + yield* push("main") + assert.deepStrictEqual( + sent.filter((job) => job.kind === "dependency-scan"), + [ + { + kind: "dependency-scan", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + trigger: "push", + }, + ], + ) + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, repos: oneRepo, scanRequests: [] }))) + }) + + it.effect("without the scanner bound, a push enqueues no scan", () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + yield* seedInstallation(repo, asOrgId("org_orch")) + yield* upsertReposFor(repo, "42", oneRepo) + yield* svc.processMessage( + Schema.encodeSync(VcsSyncJob)({ + kind: "push", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + branch: "main", + commits: [commit(SHA_A, 1)], + }), + ) + assert.isFalse(sent.some((job) => job.kind === "dependency-scan")) + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, repos: oneRepo }))) + }) + + it.effect( + "hands a scan job to the scanner with its repository resolved, and drops one for an unknown repository", + () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + const scanRequests: Array<{ + repository: string + trigger: string + commitSha: string | undefined + }> = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + yield* seedInstallation(repo, asOrgId("org_orch")) + yield* upsertReposFor(repo, "42", oneRepo) + const scanJob = (externalRepoId: string) => + svc.processMessage( + Schema.encodeSync(VcsSyncJob)({ + kind: "dependency-scan", + provider: "github", + externalInstallationId: "42", + externalRepoId, + trigger: "manual", + }), + ) + yield* scanJob("7") + yield* scanJob("404") + assert.deepStrictEqual(scanRequests, [ + { repository: "octo/repo", trigger: "manual", commitSha: undefined }, + ]) + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, repos: oneRepo, scanRequests }))) + }, + ) + it.effect("drops a job for an unknown installation without persisting or failing", () => { const testDb = createTestDb(trackedDbs) const sent: Array = [] diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts index 485bedd7de..4cbc58f278 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts @@ -358,6 +358,29 @@ const GithubApiGitTreeListing = Schema.Struct({ ), }) +/** A recursive listing; `size` is set on blobs only, `truncated` when GitHub cut the listing short. */ +const GithubApiRecursiveTree = Schema.Struct({ + sha: Schema.String, + truncated: Schema.Boolean, + tree: Schema.Array( + Schema.Struct({ + path: Schema.String, + type: Schema.String, + sha: Schema.String, + size: Schema.optionalKey(Schema.Number), + }), + ), +}) +export type GithubRecursiveTree = typeof GithubApiRecursiveTree.Type + +const GithubApiBlob = Schema.Struct({ + sha: Schema.String, + size: Schema.Number, + encoding: Schema.String, + content: Schema.String, +}) +export type GithubBlob = typeof GithubApiBlob.Type + const GithubApiReviewCommentList = Schema.Array( Schema.Struct({ id: Schema.Number, @@ -562,6 +585,8 @@ const decodeCreatedComment = Schema.decodeUnknownEffect(GithubApiCreatedComment) const decodeCollaboratorPermission = Schema.decodeUnknownEffect(GithubApiCollaboratorPermission) const decodeGitCommit = Schema.decodeUnknownEffect(GithubApiGitCommit) const decodeGitTreeListing = Schema.decodeUnknownEffect(GithubApiGitTreeListing) +const decodeRecursiveTree = Schema.decodeUnknownEffect(GithubApiRecursiveTree) +const decodeBlob = Schema.decodeUnknownEffect(GithubApiBlob) const decodeComparison = Schema.decodeUnknownEffect(GithubApiComparison) const decodeReviewThreads = Schema.decodeUnknownEffect(GithubReviewThreadsResponse) const decodeMutation = Schema.decodeUnknownEffect(GithubMutationResponse) @@ -2136,6 +2161,50 @@ export class GithubAppClient extends Context.Service()( ) }) + /** Every entry under a commit's tree in one call; GitHub truncates past 100,000 entries or 7 MB. */ + const getRecursiveTree = Effect.fn("GithubAppClient.getRecursiveTree")(function* ( + externalInstallationId: string, + owner: string, + repo: string, + treeish: string, + ) { + const config = yield* resolveConfig + const token = yield* mintInstallationToken(externalInstallationId) + const response = yield* authedGet( + config, + token, + `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/trees/${encodeURIComponent(treeish)}?recursive=1`, + ) + if (!isOk(response)) return yield* failure(response, "Read tree", "repository") + return yield* decodeRecursiveTree(yield* parseJson(response, "Read tree")).pipe( + Effect.mapError( + (cause) => new GithubAppError({ message: "Unexpected tree payload", cause }), + ), + ) + }) + + /** One blob by its sha; unlike the contents API it serves files up to 100 MB. */ + const getBlob = Effect.fn("GithubAppClient.getBlob")(function* ( + externalInstallationId: string, + owner: string, + repo: string, + sha: string, + ) { + const config = yield* resolveConfig + const token = yield* mintInstallationToken(externalInstallationId) + const response = yield* authedGet( + config, + token, + `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/blobs/${encodeURIComponent(sha)}`, + ) + if (!isOk(response)) return yield* failure(response, "Read blob", "repository") + return yield* decodeBlob(yield* parseJson(response, "Read blob")).pipe( + Effect.mapError( + (cause) => new GithubAppError({ message: "Unexpected blob payload", cause }), + ), + ) + }) + // An Actions job id is its check run id. Reading annotations needs only `checks: read`, // which the App's `checks: write` covers. Paging stops at the first page holding a failure: // that is what CI traces want, and warnings before it should not hide it. @@ -2407,6 +2476,8 @@ export class GithubAppClient extends Context.Service()( upsertIssueComment, searchCode, getSourceFile, + getRecursiveTree, + getBlob, listCheckRunAnnotations, streamJobLog, canReadActions, diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts index b540526cd4..216c225bcb 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts @@ -1304,6 +1304,31 @@ export class GithubProvider extends Context.Service + client + .getRecursiveTree(installation.externalInstallationId, repo.owner, repo.name, commitSha) + .pipe( + Effect.map((listing) => ({ + truncated: listing.truncated, + files: listing.tree.flatMap((entry) => + entry.type === "blob" + ? [{ path: entry.path, sha: entry.sha, size: entry.size ?? 0 }] + : [], + ), + })), + Effect.mapError(toVcsCommitError), + ) + + const fetchBlobText: VcsProviderClient["fetchBlobText"] = (installation, repo, blobSha) => + client.getBlob(installation.externalInstallationId, repo.owner, repo.name, blobSha).pipe( + Effect.map((blob) => + blob.encoding === "base64" + ? Buffer.from(blob.content.replace(/\s/g, ""), "base64").toString("utf8") + : blob.content, + ), + Effect.mapError(toVcsCommitError), + ) + const resolveRef: VcsProviderClient["resolveRef"] = (installation, repo, ref) => client.getCommit(installation.externalInstallationId, repo.owner, repo.name, ref).pipe( // None when the ref is absent (404) or parses but names nothing (422). @@ -1873,6 +1898,8 @@ export class GithubProvider extends Context.Service -/** What started a scan; only a default-branch head reconciles findings. */ -export const DependencyScanTrigger = Schema.Literals(["push", "schedule", "manual", "production"]) -export type DependencyScanTrigger = typeof DependencyScanTrigger.Type +export { DependencyScanTrigger } export const DependencyScanStatus = Schema.Literals(["running", "completed", "failed"]) export type DependencyScanStatus = typeof DependencyScanStatus.Type @@ -78,7 +76,7 @@ export type DependencyFindingStatus = SecurityAuditFindingStatus export const DependencyFindingTriageStatus = Schema.Literals(["open", "dismissed", "accepted_risk"]) export type DependencyFindingTriageStatus = typeof DependencyFindingTriageStatus.Type -/** A finding on the wire; timestamps are epoch ms. */ +/** A finding on the wire, the row as it is; timestamps are epoch ms. */ export const DependencyFinding = Schema.Struct({ id: DependencyFindingId, repositoryId: VcsRepositoryId, @@ -88,21 +86,21 @@ export const DependencyFinding = Schema.Struct({ advisoryId: Schema.String, aliases: Schema.Array(Schema.String), severity: AdvisorySeverity, - summary: Schema.optionalKey(Schema.String), + summary: Schema.NullOr(Schema.String), /** The installed versions the advisory affects. */ versions: Schema.Array(Schema.String), - fixedIn: Schema.optionalKey(Schema.String), + fixedIn: Schema.NullOr(Schema.String), relation: DependencyRelation, status: DependencyFindingStatus, firstSeenScanId: DependencyScanId, lastSeenScanId: DependencyScanId, firstSeenSha: GitCommitSha, lastSeenSha: GitCommitSha, - fixedScanId: Schema.optionalKey(DependencyScanId), - fixedSha: Schema.optionalKey(GitCommitSha), - statusReason: Schema.optionalKey(Schema.String), - statusChangedBy: Schema.optionalKey(UserId), - statusChangedAt: Schema.optionalKey(Schema.Finite), + fixedScanId: Schema.NullOr(DependencyScanId), + fixedSha: Schema.NullOr(GitCommitSha), + statusReason: Schema.NullOr(Schema.String), + statusChangedBy: Schema.NullOr(UserId), + statusChangedAt: Schema.NullOr(Schema.Finite), createdAt: Schema.Finite, updatedAt: Schema.Finite, }) diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index 6795e6a5c5..719e8df081 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -11,6 +11,9 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/anomalies/AnomalyForbiddenError", "@maple/http/anomalies/AnomalyIncidentNotFoundError", "@maple/http/anomalies/AnomalyLinkedIssueNotFoundError", + "@maple/http/dependency-scan/DependencyFindingNotFoundError", + "@maple/http/dependency-scan/DependencyScanForbiddenError", + "@maple/http/dependency-scan/DependencyScanRepositoryNotFoundError", "@maple/http/errors/ActorNotFoundError", "@maple/http/errors/AlertDestinationInUseError", "@maple/http/errors/AlertDestinationNotFoundError", diff --git a/packages/domain/src/http/dependency-scan.ts b/packages/domain/src/http/dependency-scan.ts new file mode 100644 index 0000000000..1944b9ab68 --- /dev/null +++ b/packages/domain/src/http/dependency-scan.ts @@ -0,0 +1,133 @@ +import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" +import { Schema } from "effect" +import { + DependencyFinding, + DependencyFindingId, + DependencyFindingStatus, + DependencyFindingTriageStatus, + DependencyScanId, + DependencyScanLockfile, + DependencyScanStatus, + DependencyScanTrigger, +} from "../dependency-scan" +import { SessionAuthorization } from "./current-tenant" +import { GitCommitSha, VcsRepositoryId } from "./vcs" + +// The Security Center's Dependencies tab: known vulnerable packages in connected repositories' +// lockfiles. Dashboard only, served by the api Worker. + +export const DEPENDENCY_FINDING_REASON_MAX_LENGTH = 500 + +/** One scan as the tab shows it; timestamps are epoch ms. */ +export const DependencyScanSummary = Schema.Struct({ + id: DependencyScanId, + commitSha: GitCommitSha, + ref: Schema.NullOr(Schema.String), + trigger: DependencyScanTrigger, + status: DependencyScanStatus, + lockfiles: Schema.Array(DependencyScanLockfile), + packageCount: Schema.Int, + vulnerableCount: Schema.Int, + error: Schema.NullOr(Schema.String), + startedAt: Schema.Finite, + finishedAt: Schema.NullOr(Schema.Finite), +}) +export type DependencyScanSummary = typeof DependencyScanSummary.Type + +/** A connected repository and its latest scan of the tracked branch, null before the first. */ +export const DependencyScanRepository = Schema.Struct({ + id: VcsRepositoryId, + fullName: Schema.String, + htmlUrl: Schema.String, + trackedBranch: Schema.String, + latestScan: Schema.NullOr(DependencyScanSummary), +}) +export type DependencyScanRepository = typeof DependencyScanRepository.Type + +export const DependencyScanRepositoriesResponse = Schema.Struct({ + repositories: Schema.Array(DependencyScanRepository), +}) +export type DependencyScanRepositoriesResponse = typeof DependencyScanRepositoriesResponse.Type + +/** A finding with the repository named, so the table needs no second read. */ +export const DependencyFindingView = Schema.Struct({ + ...DependencyFinding.fields, + repositoryFullName: Schema.String, +}) +export type DependencyFindingView = typeof DependencyFindingView.Type + +export const DependencyFindingsResponse = Schema.Struct({ findings: Schema.Array(DependencyFindingView) }) +export type DependencyFindingsResponse = typeof DependencyFindingsResponse.Type + +export const DependencyFindingsQuery = Schema.Struct({ + status: Schema.optional(DependencyFindingStatus), + repositoryId: Schema.optional(VcsRepositoryId), +}) + +export const DependencyFindingUpdateRequest = Schema.Struct({ + status: DependencyFindingTriageStatus, + reason: Schema.optionalKey(Schema.String.check(Schema.isMaxLength(DEPENDENCY_FINDING_REASON_MAX_LENGTH))), +}) +export type DependencyFindingUpdateRequest = typeof DependencyFindingUpdateRequest.Type + +export const DependencyScanQueued = Schema.Struct({ queued: Schema.Boolean }) + +export class DependencyScanPersistenceError extends Schema.TaggedError()( + "@maple/http/dependency-scan/DependencyScanPersistenceError", + { message: Schema.String }, + { httpApiStatus: 503 }, +) {} + +export class DependencyScanForbiddenError extends Schema.TaggedError()( + "@maple/http/dependency-scan/DependencyScanForbiddenError", + { message: Schema.String }, + { httpApiStatus: 403 }, +) {} + +export class DependencyFindingNotFoundError extends Schema.TaggedError()( + "@maple/http/dependency-scan/DependencyFindingNotFoundError", + { message: Schema.String, findingId: DependencyFindingId }, + { httpApiStatus: 404 }, +) {} + +export class DependencyScanRepositoryNotFoundError extends Schema.TaggedError()( + "@maple/http/dependency-scan/DependencyScanRepositoryNotFoundError", + { message: Schema.String, repositoryId: VcsRepositoryId }, + { httpApiStatus: 404 }, +) {} + +export class DependencyScanApiGroup extends HttpApiGroup.make("dependencyScan") + .add( + HttpApiEndpoint.get("listRepositories", "/repositories", { + success: DependencyScanRepositoriesResponse, + error: DependencyScanPersistenceError, + }), + ) + .add( + HttpApiEndpoint.post("scanRepository", "/repositories/:repositoryId/scan", { + params: { repositoryId: VcsRepositoryId }, + success: DependencyScanQueued, + error: [DependencyScanPersistenceError, DependencyScanRepositoryNotFoundError], + }), + ) + .add( + HttpApiEndpoint.get("listFindings", "/findings", { + query: DependencyFindingsQuery, + success: DependencyFindingsResponse, + error: DependencyScanPersistenceError, + }), + ) + .add( + HttpApiEndpoint.patch("updateFinding", "/findings/:findingId", { + params: { findingId: DependencyFindingId }, + payload: DependencyFindingUpdateRequest, + success: DependencyFindingView, + error: [ + DependencyScanPersistenceError, + DependencyScanForbiddenError, + DependencyFindingNotFoundError, + ], + }), + ) + .prefix("/internal/dependency-scan") + .middleware(SessionAuthorization) {} diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index 4d279c9c6a..12ff73adcb 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -3,6 +3,7 @@ export * from "./ai-api" export * from "./internal-api" export * from "./ai-models" export * from "./ai-sessions" +export * from "./dependency-scan" export * from "./ai-triage" export * from "./openrouter-key" export * from "./incident-triage" diff --git a/packages/domain/src/http/internal-api.ts b/packages/domain/src/http/internal-api.ts index a325cc06f7..d2c824478b 100644 --- a/packages/domain/src/http/internal-api.ts +++ b/packages/domain/src/http/internal-api.ts @@ -5,6 +5,7 @@ import { AiTriageApiGroup } from "./ai-triage" import { BillingApiGroup } from "./billing" import { CodeReviewApiGroup } from "./code-review" import { DemoApiGroup } from "./demo" +import { DependencyScanApiGroup } from "./dependency-scan" import { DigestApiGroup } from "./digest" import { ErrorsApiGroup } from "./errors" import { IntegrationsApiGroup } from "./integrations" @@ -49,6 +50,7 @@ export class MapleInternalApi extends HttpApi.make("MapleInternalApi") .add(BillingApiGroup) .add(CodeReviewApiGroup) .add(DemoApiGroup) + .add(DependencyScanApiGroup) .add(DigestApiGroup) .add(ErrorsApiGroup) .add(IntegrationsApiGroup) diff --git a/packages/domain/src/http/vcs.ts b/packages/domain/src/http/vcs.ts index 4536836d82..3eb9449722 100644 --- a/packages/domain/src/http/vcs.ts +++ b/packages/domain/src/http/vcs.ts @@ -740,6 +740,22 @@ export const PullRequestHead = Schema.Struct({ }) export type PullRequestHead = Schema.Schema.Type +/** What started a dependency scan; only a scan of the tracked branch head reconciles findings. */ +export const DependencyScanTrigger = Schema.Literals(["push", "schedule", "manual", "production"]) +export type DependencyScanTrigger = typeof DependencyScanTrigger.Type + +// Scan one repository's lockfiles. Without `commitSha` the scanner reads the tracked branch's +// head at scan time, so a push's truncated commit list never decides what is scanned. +export const DependencyScanJob = Schema.Struct({ + kind: Schema.Literal("dependency-scan"), + provider: VcsProviderId, + externalInstallationId: Schema.String, + externalRepoId: Schema.String, + trigger: DependencyScanTrigger, + commitSha: Schema.optionalKey(GitCommitSha), +}) +export type DependencyScanJob = Schema.Schema.Type + export const VcsSyncJob = Schema.Union([ InstallationSyncJob, SyncCommitsJob, @@ -752,6 +768,7 @@ export const VcsSyncJob = Schema.Union([ PullRequestCiApprovedJob, WorkflowRunJob, WorkflowJobJob, + DependencyScanJob, ]) export type VcsSyncJob = Schema.Schema.Type