From a29c0110e995fd0eb2b03bcbd9b1a882208fb505 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 00:35:52 +0100 Subject: [PATCH 1/6] feat(security): scan connected repositories' lockfiles for vulnerable dependencies DependencyScanner reads a repository's tree at its tracked branch head (or at a given commit), picks the lockfiles (skipping node_modules, vendor and fixture paths, and a go.sum beside its go.mod), reads them by blob sha, parses them with @maple/lockfiles and looks the packages up in OSV. A lockfile's parse is edge-cached by blob sha, so a rescan for new advisories reads only the tree. Scans run as a new dependency-scan job on the VCS sync queue: a push to the tracked branch enqueues one (skipped when no lockfile changed), every installation sync (on connect and every 12 hours) enqueues one per repository, and the dashboard's "Scan now" enqueues one. A tree GitHub cut short, an OSV outage or a provider failure fails the scan and leaves the findings as they were, so missing data never reads as a fix. The dashboard reads it through /internal/dependency-scan: repositories with their latest scan, findings with their repository named, triage (admins only) and "Scan now". DependencyScanner.github.e2e.test.ts runs the scanner end to end against public GitHub repositories and compares each lockfile with osv-scanner on a clone of the same commit. Maple, cli/cli, ripgrep, vercel/swr and formik (go.mod, Cargo.lock, package-lock, pnpm-lock, yarn v1, bun.lock) all match. --- .../routes/internal/dependency-scan.http.ts | 38 ++ apps/api/src/runtime/http-graph.ts | 2 + apps/api/src/runtime/service-graph.ts | 2 + apps/api/src/vcs-sync-runtime.ts | 3 + .../DependencyScanService.test.ts | 183 +++++++++ .../dependency-scan/DependencyScanService.ts | 262 +++++++++++++ .../dependency-scan/DependencyScanStore.ts | 4 +- .../DependencyScanner.github.e2e.test.ts | 250 ++++++++++++ .../dependency-scan/DependencyScanner.test.ts | 364 +++++++++++++++++ .../dependency-scan/DependencyScanner.ts | 368 ++++++++++++++++++ .../lockfile-selection.test.ts | 87 +++++ .../dependency-scan/lockfile-selection.ts | 72 ++++ .../integrations/vcs/PullRequestEventSink.ts | 19 + .../integrations/vcs/VcsProviderClient.ts | 28 ++ .../integrations/vcs/VcsSyncService.ts | 48 ++- .../integrations/vcs/__tests__/vcs.test.ts | 115 +++++- .../vcs/vendor/github/GithubAppClient.ts | 71 ++++ .../vcs/vendor/github/GithubProvider.ts | 27 ++ packages/domain/src/dependency-scan.ts | 22 +- packages/domain/src/http/dependency-scan.ts | 133 +++++++ packages/domain/src/http/index.ts | 1 + packages/domain/src/http/internal-api.ts | 2 + packages/domain/src/http/vcs.ts | 17 + 23 files changed, 2101 insertions(+), 17 deletions(-) create mode 100644 apps/api/src/routes/internal/dependency-scan.http.ts create mode 100644 packages/backend/src/services/dependency-scan/DependencyScanService.test.ts create mode 100644 packages/backend/src/services/dependency-scan/DependencyScanService.ts create mode 100644 packages/backend/src/services/dependency-scan/DependencyScanner.github.e2e.test.ts create mode 100644 packages/backend/src/services/dependency-scan/DependencyScanner.test.ts create mode 100644 packages/backend/src/services/dependency-scan/DependencyScanner.ts create mode 100644 packages/backend/src/services/dependency-scan/lockfile-selection.test.ts create mode 100644 packages/backend/src/services/dependency-scan/lockfile-selection.ts create mode 100644 packages/domain/src/http/dependency-scan.ts diff --git a/apps/api/src/routes/internal/dependency-scan.http.ts b/apps/api/src/routes/internal/dependency-scan.http.ts new file mode 100644 index 0000000000..b587dc8ef0 --- /dev/null +++ b/apps/api/src/routes/internal/dependency-scan.http.ts @@ -0,0 +1,38 @@ +import { HttpApiBuilder } from "effect/http-api" +import { CurrentTenant, DependencyScanForbiddenError, MapleInternalApi } from "@maple/domain/http" +import { Effect } from "effect" +import { requireAdmin } from "@maple/backend/services/auth/auth" +import { DependencyScanService } from "@maple/backend/services/dependency-scan/DependencyScanService" + +export const HttpDependencyScanLive = HttpApiBuilder.group(MapleInternalApi, "dependencyScan", (handlers) => + Effect.gen(function* () { + const scans = yield* DependencyScanService + + return handlers + .handle("listRepositories", () => + Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listRepositories(tenant.orgId)), + ) + .handle("scanRepository", ({ params }) => + Effect.flatMap(CurrentTenant.Context, (tenant) => + scans.requestScan(tenant.orgId, params.repositoryId), + ), + ) + .handle("listFindings", ({ query }) => + Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listFindings(tenant.orgId, query)), + ) + .handle("updateFinding", ({ params, payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + // Triage, like the code audit's, is an admin's call. + yield* requireAdmin( + tenant.roles, + () => + new DependencyScanForbiddenError({ + message: "Only org admins can triage findings", + }), + ) + return yield* scans.updateFinding(tenant.orgId, tenant.userId, params.findingId, payload) + }), + ) + }), +) diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index 20f1b1e669..95a2fb94a7 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -20,6 +20,7 @@ import { AwsOidcRouter } from "@/routes/aws-oidc.http" import { DiscoveryRouter, instanceApiV2, NotFoundRouter } from "@/routes/discovery.http" import { HttpDigestLive } from "@/routes/internal/digest.http" import { HttpCodeReviewLive } from "@/routes/internal/code-review.http" +import { HttpDependencyScanLive } from "@/routes/internal/dependency-scan.http" import { HttpErrorsLive } from "@/routes/internal/errors.http" import { HttpIntegrationsLive } from "@/routes/internal/integrations.http" import { IntegrationsCallbackRouter } from "@/routes/integrations-callback.http" @@ -145,6 +146,7 @@ const ApiInternalRoutes = HttpApiBuilder.layer(MapleInternalApi).pipe( HttpAiTriageLive, HttpBillingLive, HttpDemoLive, + HttpDependencyScanLive, HttpDigestLive, HttpOpenRouterKeyLive, ), diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 02e4bd8d35..57a8f9064f 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -1,4 +1,5 @@ import { RuntimeAdvisoryCatalog } from "@maple/backend/services/security/RuntimeAdvisoryCatalog" +import { DependencyScanService } from "@maple/backend/services/dependency-scan/DependencyScanService" import { RuntimeReleaseCatalog } from "@maple/backend/services/security/RuntimeReleaseCatalog" import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" import { Layer } from "effect" @@ -167,6 +168,7 @@ export const HttpServicesLive = Layer.mergeAll( QueryEngineService.layer, RuntimeReleaseCatalog.layer, RuntimeAdvisoryCatalog.layer, + DependencyScanService.layer, TelemetryReadService.layer, DashboardWidgetDataService.layer, AlertDestinationsService.layer, diff --git a/apps/api/src/vcs-sync-runtime.ts b/apps/api/src/vcs-sync-runtime.ts index c36b4bf8a1..f31f9d8143 100644 --- a/apps/api/src/vcs-sync-runtime.ts +++ b/apps/api/src/vcs-sync-runtime.ts @@ -23,6 +23,7 @@ import { PullRequestCommentSinkLive, } from "@maple/backend/services/pr-review/pull-request-comment-handler" import { CiTraceSinkLive } from "@maple/backend/services/integrations/ci-traces/CiTraceService" +import { DependencyScanSinkLive } from "@maple/backend/services/dependency-scan/DependencyScanner" import { summarizeCause } from "@maple/backend/platform/describe-cause" import type { QueueBatch } from "@maple/backend/platform/queue-batch" import type { Message } from "@cloudflare/workers-types" @@ -70,6 +71,8 @@ export const VcsSyncLive = VcsSyncService.layer.pipe( Layer.provide(PullRequestChecklistSinkLive.pipe(Layer.provide(PrReviewLive))), // Finished CI runs and jobs, written as spans into the org's traces. Layer.provide(CiTraceSinkLive), + // Lockfile scans for known vulnerable dependencies, on pushes and installation syncs. + Layer.provide(DependencyScanSinkLive), Layer.provide(Layer.mergeAll(EventBaseLive, EdgeCacheServiceLive)), ) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts new file mode 100644 index 0000000000..105e8d6398 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts @@ -0,0 +1,183 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { DependencyFindingId } from "@maple/domain/dependency-scan" +import { GitCommitSha, VcsRepositoryId, type VcsSyncJob } from "@maple/domain/http" +import { OrgId, UserId } from "@maple/domain/primitives" +import { Effect, Layer, Schema } from "effect" +import { TestClock } from "effect/testing" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" +import { + recordingQueueLayer, + testRepoLayer, +} from "@maple/backend/services/integrations/vcs/__tests__/harness" +import { DependencyScanService } from "./DependencyScanService" +import { DependencyScanStore } from "./DependencyScanStore" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const org = Schema.decodeSync(OrgId)("org_deps") +const admin = Schema.decodeSync(UserId)("user_admin") +const sha = (digit: string) => Schema.decodeSync(GitCommitSha)(digit.repeat(40)) + +const serviceFor = (sent: Array) => { + const db = createTestDb(trackedDbs) + const store = DependencyScanStore.layer.pipe(Layer.provide(db.layer)) + const vcs = testRepoLayer(db) + return Layer.effect(DependencyScanService, DependencyScanService.make).pipe( + Layer.provideMerge(Layer.mergeAll(store, vcs, recordingQueueLayer(sent))), + ) +} + +/** One active installation with two repositories; returns the first one's id. */ +const seed = Effect.gen(function* () { + const vcs = yield* VcsRepository + const installation = yield* vcs.upsertInstallation({ + orgId: org, + provider: "github", + externalInstallationId: "42", + accountLogin: "octo", + accountType: "organization", + externalAccountId: "100", + accountAvatarUrl: null, + repositorySelection: "all", + installedByUserId: admin, + }) + const repo = (externalRepoId: string, name: string) => ({ + externalRepoId, + owner: "octo", + name, + fullName: `octo/${name}`, + defaultBranch: "main", + htmlUrl: `https://github.com/octo/${name}`, + isPrivate: true, + isArchived: false, + }) + yield* vcs.upsertRepositories(installation, [repo("8", "web"), repo("7", "api")]) + const repositories = yield* vcs.listRepositoriesByInstallation(installation.id, "active") + const api = repositories.find((repository) => repository.fullName === "octo/api") + if (api === undefined) return yield* Effect.die("seeding did not store octo/api") + return api +}) + +const recordScan = (repositoryId: VcsRepositoryId, commit: string, defaultBranchHead: boolean) => + Effect.gen(function* () { + const store = yield* DependencyScanStore + // Scans of one repository start at distinct times; "latest" is by start time. + yield* TestClock.adjust("1 second") + const scan = yield* store.startScan({ + orgId: org, + repositoryId, + commitSha: sha(commit), + ref: defaultBranchHead ? "main" : null, + trigger: defaultBranchHead ? "schedule" : "production", + defaultBranchHead, + }) + return yield* store.completeScan({ + orgId: org, + scanId: scan.id, + lockfiles: [ + { + path: "bun.lock", + kind: "bun-lock", + blobSha: commit, + packageCount: 1, + status: "parsed", + message: null, + }, + ], + packageCount: 1, + matches: [ + { + lockfilePath: "bun.lock", + ecosystem: "npm", + packageName: "lodash", + version: "4.17.15", + relation: "direct", + advisoryId: "GHSA-p6mc-m468-83gw", + aliases: ["CVE-2020-8203"], + severity: "high", + summary: "Prototype pollution in lodash", + fixedIn: "4.17.19", + }, + ], + }) + }) + +describe("DependencyScanService", () => { + it.effect("lists every connected repository with its latest tracked-branch scan", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const api = yield* seed + const before = yield* service.listRepositories(org) + assert.deepStrictEqual( + before.repositories.map((repository) => [repository.fullName, repository.latestScan]), + [ + ["octo/api", null], + ["octo/web", null], + ], + ) + yield* recordScan(api.id, "a", true) + const latest = yield* recordScan(api.id, "b", true) + // An older commit scanned because it runs in production is not the branch's latest scan. + yield* recordScan(api.id, "c", false) + const after = yield* service.listRepositories(org) + assert.strictEqual(after.repositories[0]?.latestScan?.id, latest.id) + assert.strictEqual(after.repositories[0]?.latestScan?.vulnerableCount, 1) + }).pipe(Effect.provide(serviceFor(sent))) + }) + + it.effect("queues a manual scan for a connected repository and refuses an unknown one", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const api = yield* seed + assert.deepStrictEqual(yield* service.requestScan(org, api.id), { queued: true }) + assert.deepStrictEqual(sent, [ + { + kind: "dependency-scan", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + trigger: "manual", + }, + ]) + const unknown = Schema.decodeSync(VcsRepositoryId)("00000000-0000-4000-8000-0000000000ff") + const error = yield* Effect.flip(service.requestScan(org, unknown)) + assert.strictEqual( + error._tag, + "@maple/http/dependency-scan/DependencyScanRepositoryNotFoundError", + ) + }).pipe(Effect.provide(serviceFor(sent))) + }) + + it.effect("names each finding's repository and records who dismissed it and why", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const api = yield* seed + yield* recordScan(api.id, "a", true) + const { findings } = yield* service.listFindings(org, {}) + assert.deepStrictEqual( + findings.map((finding) => [finding.repositoryFullName, finding.packageName, finding.status]), + [["octo/api", "lodash", "open"]], + ) + const [finding] = findings + if (finding === undefined) return + const dismissed = yield* service.updateFinding(org, admin, finding.id, { + status: "dismissed", + reason: "Only used in a build script", + }) + assert.deepStrictEqual( + [dismissed.status, dismissed.statusReason, dismissed.statusChangedBy], + ["dismissed", "Only used in a build script", admin], + ) + assert.deepStrictEqual((yield* service.listFindings(org, { status: "open" })).findings, []) + + const missing = Schema.decodeSync(DependencyFindingId)("0".repeat(32)) + const error = yield* Effect.flip(service.updateFinding(org, admin, missing, { status: "open" })) + assert.strictEqual(error._tag, "@maple/http/dependency-scan/DependencyFindingNotFoundError") + }).pipe(Effect.provide(serviceFor(sent))) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts new file mode 100644 index 0000000000..419c6333ea --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -0,0 +1,262 @@ +import type { DependencyFindingRow, DependencyScanRow } from "@maple/db/tables" +import type { DependencyFindingId } from "@maple/domain/dependency-scan" +import { + DependencyFindingNotFoundError, + type DependencyFindingsResponse, + type DependencyFindingUpdateRequest, + type DependencyFindingView, + DependencyScanPersistenceError, + type DependencyScanRepositoriesResponse, + DependencyScanRepositoryNotFoundError, + type DependencyScanSummary, + isInstallationProcessable, + type VcsRepo, + type VcsRepositoryId, +} from "@maple/domain/http" +import type { OrgId, UserId } from "@maple/domain/primitives" +import type { DependencyFindingStatus } from "@maple/domain/dependency-scan" +import { Context, Effect, Layer, Option } from "effect" +import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" +import { VcsSyncQueue } from "@maple/backend/services/integrations/vcs/VcsSyncQueue" +import { DependencyScanStore } from "./DependencyScanStore" + +// The Dependencies tab's reads and writes: repositories with their latest scan, findings with +// their repository named, triage, and "scan now", which goes through the VCS sync queue. + +/** Findings returned per read; the table sorts worst first, so the cut drops the least severe. */ +export const FINDINGS_LIMIT = 500 +/** Scans read to find each repository's latest. */ +const SCANS_LIMIT = 500 + +export interface DependencyScanServiceApi { + readonly listRepositories: ( + orgId: OrgId, + ) => Effect.Effect + readonly requestScan: ( + orgId: OrgId, + repositoryId: VcsRepositoryId, + ) => Effect.Effect< + { readonly queued: boolean }, + DependencyScanPersistenceError | DependencyScanRepositoryNotFoundError + > + readonly listFindings: ( + orgId: OrgId, + query: { + readonly status?: DependencyFindingStatus | undefined + readonly repositoryId?: VcsRepositoryId | undefined + }, + ) => Effect.Effect + readonly updateFinding: ( + orgId: OrgId, + actor: UserId, + findingId: DependencyFindingId, + request: DependencyFindingUpdateRequest, + ) => Effect.Effect +} + +const persistence = + (message: string) => + (error: { readonly message: string }): DependencyScanPersistenceError => + new DependencyScanPersistenceError({ message: `${message}: ${error.message}` }) + +export const scanSummaryOf = (row: DependencyScanRow): DependencyScanSummary => ({ + id: row.id, + commitSha: row.commitSha, + ref: row.ref, + trigger: row.trigger, + status: row.status, + lockfiles: row.lockfiles, + packageCount: row.packageCount, + vulnerableCount: row.vulnerableCount, + error: row.error, + startedAt: row.startedAt, + finishedAt: row.finishedAt, +}) + +export const findingViewOf = ( + row: DependencyFindingRow, + repositoryFullName: string, +): DependencyFindingView => ({ + id: row.id, + repositoryId: row.repositoryId, + repositoryFullName, + lockfilePath: row.lockfilePath, + ecosystem: row.ecosystem, + packageName: row.packageName, + advisoryId: row.advisoryId, + aliases: row.aliases, + severity: row.severity, + summary: row.summary, + versions: row.versions, + fixedIn: row.fixedIn, + relation: row.relation, + status: row.status, + firstSeenScanId: row.firstSeenScanId, + lastSeenScanId: row.lastSeenScanId, + firstSeenSha: row.firstSeenSha, + lastSeenSha: row.lastSeenSha, + fixedScanId: row.fixedScanId, + fixedSha: row.fixedSha, + statusReason: row.statusReason, + statusChangedBy: row.statusChangedBy, + statusChangedAt: row.statusChangedAt, + createdAt: row.createdAt, + updatedAt: row.updatedAt, +}) + +export class DependencyScanService extends Context.Service()( + "@maple/backend/services/dependency-scan/DependencyScanService", + { + make: Effect.gen(function* () { + const store = yield* DependencyScanStore + const vcs = yield* VcsRepository + const queue = yield* VcsSyncQueue + + /** The org's repositories under a processable installation, active ones only. */ + const connectedRepositories = Effect.fn("DependencyScanService.connectedRepositories")(function* ( + orgId: OrgId, + ) { + const installations = (yield* vcs.listInstallationsByOrg(orgId)).filter( + isInstallationProcessable, + ) + const repositories = yield* Effect.forEach(installations, (installation) => + vcs.listRepositoriesByInstallation(installation.id, "active"), + ) + return repositories.flat() + }) + + const listRepositories: DependencyScanServiceApi["listRepositories"] = Effect.fn( + "DependencyScanService.listRepositories", + )(function* (orgId) { + const repositories = yield* connectedRepositories(orgId).pipe( + Effect.mapError(persistence("Could not list repositories")), + ) + const scans = yield* store + .listScans({ orgId, limit: SCANS_LIMIT }) + .pipe(Effect.mapError(persistence("Could not list scans"))) + // Newest first, so the first tracked-branch scan per repository is its latest. + const latest = scans + .filter((scan) => scan.defaultBranchHead) + .reduce((byRepository, scan) => { + if (!byRepository.has(scan.repositoryId)) byRepository.set(scan.repositoryId, scan) + return byRepository + }, new Map()) + return { + repositories: repositories + .map((repository: VcsRepo) => { + const scan = latest.get(repository.id) + return { + id: repository.id, + fullName: repository.fullName, + htmlUrl: repository.htmlUrl, + trackedBranch: repository.trackedBranch ?? repository.defaultBranch, + latestScan: scan === undefined ? null : scanSummaryOf(scan), + } + }) + .sort((a, b) => a.fullName.localeCompare(b.fullName)), + } + }) + + const requestScan: DependencyScanServiceApi["requestScan"] = Effect.fn( + "DependencyScanService.requestScan", + )(function* (orgId, repositoryId) { + const notFound = new DependencyScanRepositoryNotFoundError({ + message: "This repository is not connected to the organization", + repositoryId, + }) + const repository = yield* vcs + .getRepositoryById(orgId, repositoryId) + .pipe(Effect.mapError(persistence("Could not read the repository"))) + if (Option.isNone(repository) || repository.value.status !== "active") return yield* notFound + const installation = yield* vcs + .getInstallationById(orgId, repository.value.installationId) + .pipe(Effect.mapError(persistence("Could not read the installation"))) + if (Option.isNone(installation) || !isInstallationProcessable(installation.value)) + return yield* notFound + yield* queue + .send({ + kind: "dependency-scan", + provider: repository.value.provider, + externalInstallationId: installation.value.externalInstallationId, + externalRepoId: repository.value.externalRepoId, + trigger: "manual", + }) + .pipe(Effect.mapError(persistence("Could not queue the scan"))) + return { queued: true } + }) + + const listFindings: DependencyScanServiceApi["listFindings"] = Effect.fn( + "DependencyScanService.listFindings", + )(function* (orgId, query) { + const rows = yield* store + .listFindings({ + orgId, + limit: FINDINGS_LIMIT, + status: query.status, + repositoryId: query.repositoryId, + }) + .pipe(Effect.mapError(persistence("Could not list findings"))) + const repositories = yield* vcs + .getRepositoriesByIds( + orgId, + rows.map((row) => row.repositoryId), + ) + .pipe(Effect.mapError(persistence("Could not read repositories"))) + const names = new Map(repositories.map((repository) => [repository.id, repository.fullName])) + yield* Effect.annotateCurrentSpan({ "result.rowCount": rows.length }) + return { + findings: rows.map((row) => + findingViewOf(row, names.get(row.repositoryId) ?? "unknown repository"), + ), + } + }) + + const updateFinding: DependencyScanServiceApi["updateFinding"] = Effect.fn( + "DependencyScanService.updateFinding", + )(function* (orgId, actor, findingId, request) { + const row = yield* store + .setFindingStatus({ + orgId, + id: findingId, + status: request.status, + reason: request.reason ?? null, + actor, + }) + .pipe( + Effect.catchTags({ + "@maple/dependency-scan/DependencyFindingNotFoundError": () => + Effect.fail( + new DependencyFindingNotFoundError({ + message: "No such finding", + findingId, + }), + ), + "@maple/dependency-scan/DependencyScanPersistenceError": (error) => + Effect.fail(persistence("Could not update the finding")(error)), + }), + ) + const repository = yield* vcs + .getRepositoryById(orgId, row.repositoryId) + .pipe(Effect.mapError(persistence("Could not read the repository"))) + return findingViewOf( + row, + Option.match(repository, { + onNone: () => "unknown repository", + onSome: (r) => r.fullName, + }), + ) + }) + + return { + listRepositories, + requestScan, + listFindings, + updateFinding, + } satisfies DependencyScanServiceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide(Layer.mergeAll(DependencyScanStore.layer, VcsRepository.layer, VcsSyncQueue.layer)), + ) +} diff --git a/packages/backend/src/services/dependency-scan/DependencyScanStore.ts b/packages/backend/src/services/dependency-scan/DependencyScanStore.ts index 76efd8f793..0e24773734 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanStore.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanStore.ts @@ -116,8 +116,8 @@ export interface DependencyScanStoreApi { /** Most severe first, then most recently updated. */ readonly listFindings: (input: { readonly orgId: OrgId - readonly repositoryId?: VcsRepositoryId - readonly status?: DependencyFindingStatus + readonly repositoryId?: VcsRepositoryId | undefined + readonly status?: DependencyFindingStatus | undefined readonly limit: number }) => Effect.Effect, DependencyScanPersistenceError> readonly getFinding: (input: { diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.github.e2e.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.github.e2e.test.ts new file mode 100644 index 0000000000..782beb7076 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.github.e2e.test.ts @@ -0,0 +1,250 @@ +// Opt-in live check: runs the dependency scanner on public GitHub repositories end to end (tree +// listing, lockfile selection, blob reads, parsing, OSV) and compares every lockfile it read with +// osv-scanner on a shallow clone of the same commit. Needs network, git and osv-scanner: +// DEPSCAN_GITHUB_E2E_REPOS=Makisuo/maple,cli/cli GITHUB_TOKEN=$(gh auth token) \ +// bunx vitest run src/services/dependency-scan/DependencyScanner.github.e2e.test.ts --silent=false +import { afterEach, assert, describe, it } from "@effect/vitest" +import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" +import { GitCommitSha, VcsInstallation, VcsRepo, type VcsRepositoryRef } from "@maple/domain/http" +import { Console, Effect, Layer, Option, Schema } from "effect" +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http" +import { spawnSync } from "node:child_process" +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" +import { DependencyScanner, DependencySources } from "./DependencyScanner" +import { DependencyScanStore } from "./DependencyScanStore" + +const repos = (process.env.DEPSCAN_GITHUB_E2E_REPOS ?? "").split(",").filter((repo) => repo !== "") +const token = process.env.GITHUB_TOKEN ?? "" + +const trackedDbs: Array = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const repoOf = (repo: VcsRepositoryRef) => `${repo.owner}/${repo.name}` + +const github = (fullName: string, path: string) => + Effect.gen(function* () { + const client = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk) + const response = yield* client.execute( + HttpClientRequest.get(`https://api.github.com/repos/${fullName}${path}`).pipe( + HttpClientRequest.bearerToken(token), + ), + ) + return yield* response.json + }).pipe(Effect.orDie, Effect.provide(FetchHttpClient.layer)) + +const Tree = Schema.Struct({ + truncated: Schema.Boolean, + tree: Schema.Array( + Schema.Struct({ + path: Schema.String, + type: Schema.String, + sha: Schema.String, + size: Schema.optionalKey(Schema.Number), + }), + ), +}) +const Blob = Schema.Struct({ content: Schema.String }) +const Repo = Schema.Struct({ default_branch: Schema.String }) +const Commit = Schema.Struct({ sha: GitCommitSha }) + +const source = { + resolveRef: (_installation: unknown, _repo: VcsRepositoryRef, ref: string) => + github(repoOf(_repo), `/commits/${ref}`).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Commit)), + Effect.map((commit) => Option.some(commit.sha)), + Effect.orDie, + ), + fetchTreeFiles: (_installation: unknown, _repo: VcsRepositoryRef, commitSha: string) => + github(repoOf(_repo), `/git/trees/${commitSha}?recursive=1`).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Tree)), + Effect.map((tree) => ({ + truncated: tree.truncated, + files: tree.tree.flatMap((entry) => + entry.type === "blob" + ? [{ path: entry.path, sha: entry.sha, size: entry.size ?? 0 }] + : [], + ), + })), + Effect.orDie, + ), + fetchBlobText: (_installation: unknown, _repo: VcsRepositoryRef, blobSha: string) => + github(repoOf(_repo), `/git/blobs/${blobSha}`).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Blob)), + Effect.map((blob) => Buffer.from(blob.content.replace(/\s/g, ""), "base64").toString("utf8")), + Effect.orDie, + ), +} + +interface OsvScannerOutput { + readonly results?: ReadonlyArray<{ + readonly packages?: ReadonlyArray<{ + readonly package: { readonly name: string; readonly version: string; readonly ecosystem: string } + readonly groups?: ReadonlyArray<{ + readonly ids: ReadonlyArray + readonly aliases?: ReadonlyArray + }> + }> + }> +} + +const osvScanner = (path: string): OsvScannerOutput => { + // osv-scanner exits non-zero when it finds something, so its status says nothing here. + const run = spawnSync("osv-scanner", ["scan", "source", "--format", "json", "--lockfile", path], { + maxBuffer: 256 * 1024 * 1024, + }) + return JSON.parse(run.stdout.toString() || "{}") +} + +const scanAndCompare = (fullName: string) => + Effect.gen(function* () { + const [owner = "", name = ""] = fullName.split("/") + const repoInfo = yield* github(fullName, "").pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Repo)), + Effect.orDie, + ) + const branch = repoInfo.default_branch + const installation = Schema.decodeUnknownSync(VcsInstallation)({ + id: "5d7f8a52-1c3e-4b6a-9b1e-2a3c4d5e6f71", + orgId: "org_harness", + provider: "github", + externalInstallationId: "0", + accountLogin: owner, + accountType: "organization", + externalAccountId: "0", + accountAvatarUrl: null, + repositorySelection: "all", + status: "active", + suspendedAt: null, + ciTracesEnabled: false, + installedByUserId: "user_harness", + createdAt: 0, + updatedAt: 0, + }) + const repository = Schema.decodeUnknownSync(VcsRepo)({ + id: "7f0c1a52-4d8e-4b6a-9b1e-2a3c4d5e6f70", + orgId: "org_harness", + provider: "github", + installationId: installation.id, + externalRepoId: "0", + owner, + name, + fullName, + defaultBranch: repoInfo.default_branch, + trackedBranch: branch, + htmlUrl: `https://github.com/${fullName}`, + isPrivate: false, + isArchived: false, + status: "active", + syncStatus: "ready", + lastSyncedAt: null, + lastSyncError: null, + prReviewEnabled: false, + createdAt: 0, + updatedAt: 0, + }) + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + const started = Date.now() + const outcome = yield* scanner.scan(installation, repository, { trigger: "manual" }) + if (outcome._tag !== "Scanned") return yield* Console.log(`${fullName}: ${outcome._tag}`) + const scan = outcome.scan + const matches = yield* store.scanMatches({ orgId: repository.orgId, scanId: scan.id }) + yield* Console.log( + `${fullName}@${scan.commitSha.slice(0, 12)} (${branch}): ${scan.lockfiles.length} lockfiles, ${scan.packageCount} packages, ${matches.length} vulnerable matches, ${Date.now() - started} ms`, + ) + + const dir = mkdtempSync(join(tmpdir(), "depscan-")) + spawnSync("git", [ + "clone", + "--quiet", + "--depth", + "1", + "--branch", + branch, + `https://github.com/${fullName}`, + dir, + ]) + const cloned = spawnSync("git", ["-C", dir, "rev-parse", "HEAD"]).stdout.toString().trim() + if (cloned !== scan.commitSha) yield* Console.log(` clone is at ${cloned}, comparison may differ`) + + const results = scan.lockfiles.map((lockfile) => { + if (lockfile.status !== "parsed") + return { + line: ` SKIP ${lockfile.path}: ${lockfile.status} (${lockfile.message})`, + ok: true, + } + const ours = new Map>>() + matches + .filter((match) => match.lockfilePath === lockfile.path) + .forEach((match) => { + const key = `${match.ecosystem}:${match.packageName}@${match.version.replace(/^v/, "")}` + ours.set(key, [...(ours.get(key) ?? []), new Set([match.advisoryId, ...match.aliases])]) + }) + const theirs = new Map( + (osvScanner(join(dir, lockfile.path)).results ?? []) + .flatMap((result) => result.packages ?? []) + .map((pkg) => [ + `${pkg.package.ecosystem}:${pkg.package.name}@${pkg.package.version.replace(/^v/, "")}`, + (pkg.groups ?? []).map((group) => new Set([...group.ids, ...(group.aliases ?? [])])), + ]), + ) + const keys = new Set([...ours.keys(), ...theirs.keys()]) + const diffs = [...keys].flatMap((key) => { + const mine = ours.get(key) ?? [] + const other = theirs.get(key) ?? [] + const missing = other.filter( + (group) => !mine.some((names) => [...group].some((id) => names.has(id))), + ) + const extra = mine.filter( + (names) => !other.some((group) => [...group].some((id) => names.has(id))), + ) + return missing.length === 0 && extra.length === 0 + ? [] + : [ + ` ${key}: only osv-scanner ${missing.map((g) => [...g][0]).join(",")}; only ours ${extra.map((n) => [...n][0]).join(",")}`, + ] + }) + const count = [...ours.values()].reduce((sum, list) => sum + list.length, 0) + const theirCount = [...theirs.values()].reduce((sum, list) => sum + list.length, 0) + return { + line: [ + ` ${diffs.length === 0 ? "MATCH" : "DIFF "} ${lockfile.path}: ${lockfile.packageCount} packages, ${count} advisories (osv-scanner ${theirCount})`, + ...diffs, + ].join("\n"), + ok: diffs.length === 0, + } + }) + yield* Effect.forEach(results, (result) => Console.log(result.line)) + return results.flatMap((result) => (result.ok ? [] : [result.line])) + }) + +const layer = () => { + const cache = EdgeCacheService.layer.pipe(Layer.provide(MemoryCacheBackendLive)) + const store = DependencyScanStore.layer.pipe(Layer.provide(createTestDb(trackedDbs).layer)) + return Layer.effect(DependencyScanner, DependencyScanner.make).pipe( + Layer.provide( + Layer.mergeAll( + PackageAdvisoryCatalog.layer.pipe(Layer.provide(cache)), + Layer.succeed(DependencySources, { resolve: () => Effect.succeed(source) }), + cache, + ), + ), + Layer.provideMerge(store), + ) +} + +describe.skipIf(repos.length === 0)("DependencyScanner on GitHub repositories", () => { + it.each(repos)( + "finds what osv-scanner finds in %s", + (fullName) => + Effect.gen(function* () { + const diffs = yield* scanAndCompare(fullName) + assert.deepStrictEqual(diffs, []) + }).pipe(Effect.provide(layer()), Effect.runPromise), + 600_000, + ) +}) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts new file mode 100644 index 0000000000..fe964ee32b --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts @@ -0,0 +1,364 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" +import { GitCommitSha, VcsInstallation, VcsRepo } from "@maple/domain/http" +import { Effect, Layer, Option, Schema } from "effect" +import { HttpClient, type HttpClientRequest, HttpClientResponse } from "effect/http" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import type { VcsTreeFile } from "@maple/backend/services/integrations/vcs/VcsProviderClient" +import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" +import osvFixture from "../security/__fixtures__/osv.json" +import { DependencyScanner, DependencySources, MAX_LOCKFILE_BYTES } from "./DependencyScanner" +import { DependencyScanStore } from "./DependencyScanStore" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const sha = (digit: string) => Schema.decodeSync(GitCommitSha)(digit.repeat(40)) + +const installation = Schema.decodeUnknownSync(VcsInstallation)({ + id: "5d7f8a52-1c3e-4b6a-9b1e-2a3c4d5e6f71", + orgId: "org_scan", + provider: "github", + externalInstallationId: "42", + accountLogin: "octo", + accountType: "organization", + externalAccountId: "1", + accountAvatarUrl: null, + repositorySelection: "all", + status: "active", + suspendedAt: null, + ciTracesEnabled: true, + installedByUserId: "user_scan", + createdAt: 0, + updatedAt: 0, +}) + +const repository = Schema.decodeUnknownSync(VcsRepo)({ + id: "7f0c1a52-4d8e-4b6a-9b1e-2a3c4d5e6f70", + orgId: "org_scan", + provider: "github", + installationId: installation.id, + externalRepoId: "7", + owner: "octo", + name: "shop", + fullName: "octo/shop", + defaultBranch: "main", + trackedBranch: null, + htmlUrl: "https://github.com/octo/shop", + isPrivate: true, + isArchived: false, + status: "active", + syncStatus: "ready", + lastSyncedAt: null, + lastSyncError: null, + prReviewEnabled: false, + createdAt: 0, + updatedAt: 0, +}) + +const packageLock = (lodash: string) => + JSON.stringify({ + name: "shop", + lockfileVersion: 3, + requires: true, + packages: { + "": { name: "shop", dependencies: { lodash }, devDependencies: { minimist: "1.2.5" } }, + "node_modules/lodash": { version: lodash }, + "node_modules/minimist": { version: "1.2.5", dev: true }, + }, + }) + +const GO_MOD = "module example.com/api\n\ngo 1.22\n\nrequire golang.org/x/net v0.7.0\n" + +const CARGO_LOCK = [ + "version = 3", + "", + "[[package]]", + 'name = "shop"', + 'version = "0.1.0"', + 'dependencies = ["time"]', + "", + "[[package]]", + 'name = "time"', + 'version = "0.1.43"', + 'source = "registry+https://github.com/rust-lang/crates.io-index"', + "", +].join("\n") + +interface World { + head: GitCommitSha + files: Map + truncated: boolean + osvDown: boolean + /** Blob shas read from the provider, in order. */ + readonly blobReads: Array +} + +const world = (files: Record): World => ({ + head: sha("a"), + files: new Map( + Object.entries(files).map(([path, content]) => [ + path, + { sha: `blob:${path}:${content.length}`, content }, + ]), + ), + truncated: false, + osvDown: false, + blobReads: [], +}) + +const source = (state: World) => ({ + resolveRef: () => Effect.sync(() => Option.some(state.head)), + fetchTreeFiles: () => + Effect.sync(() => ({ + truncated: state.truncated, + files: [...state.files].map(([path, blob]): VcsTreeFile => ({ + path, + sha: blob.sha, + size: blob.size ?? blob.content.length, + })), + })), + fetchBlobText: (_installation: unknown, _repo: unknown, blobSha: string) => + Effect.sync(() => { + state.blobReads.push(blobSha) + return [...state.files.values()].find((blob) => blob.sha === blobSha)?.content ?? "" + }), +}) + +const json = (request: HttpClientRequest.HttpClientRequest, body: unknown, status = 200) => + HttpClientResponse.fromWeb(request, new Response(JSON.stringify(body), { status })) + +interface Query { + readonly package: { readonly ecosystem: string; readonly name: string } + readonly version: string +} + +/** OSV as captured in the fixture: packages it does not list have no advisories. */ +const osv = (state: World) => + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.sync(() => { + if (state.osvDown) return json(request, { message: "unavailable" }, 503) + const url = new URL(request.url) + if (url.pathname === "/v1/querybatch") { + const body: { queries: ReadonlyArray } = + request.body._tag === "Uint8Array" + ? JSON.parse(new TextDecoder().decode(request.body.body)) + : { queries: [] } + return json(request, { + results: body.queries.map((query) => { + const index = osvFixture.queries.findIndex( + (known) => + known.package.ecosystem === query.package.ecosystem && + known.package.name === query.package.name && + known.version === query.version, + ) + return osvFixture.batch.results[index] ?? {} + }), + }) + } + const id = decodeURIComponent(url.pathname.replace("/v1/vulns/", "")) + const vuln = osvFixture.vulns.find((candidate) => candidate.id === id) + return vuln === undefined ? json(request, { message: "not found" }, 404) : json(request, vuln) + }), + ) + +const scannerFor = (state: World) => { + const store = DependencyScanStore.layer.pipe(Layer.provide(createTestDb(trackedDbs).layer)) + const cache = EdgeCacheService.layer.pipe(Layer.provide(MemoryCacheBackendLive)) + const catalog = Layer.effect(PackageAdvisoryCatalog, PackageAdvisoryCatalog.make).pipe( + Layer.provide(Layer.mergeAll(Layer.succeed(HttpClient.HttpClient, osv(state)), cache)), + ) + const sources = Layer.succeed(DependencySources, { resolve: () => Effect.succeed(source(state)) }) + return Layer.effect(DependencyScanner, DependencyScanner.make).pipe( + Layer.provide(Layer.mergeAll(catalog, sources, cache)), + Layer.provideMerge(store), + ) +} + +const MONOREPO = { + "package-lock.json": packageLock("4.17.15"), + "services/api/go.mod": GO_MOD, + "services/api/go.sum": "golang.org/x/net v0.7.0 h1:abc=\n", + "crates/core/Cargo.lock": CARGO_LOCK, + "web/node_modules/left-pad/package-lock.json": packageLock("4.17.15"), + "legacy/bun.lockb": "binary", + "README.md": "# shop", +} + +const findingsOf = Effect.gen(function* () { + const store = yield* DependencyScanStore + return yield* store.listFindings({ orgId: repository.orgId, limit: 500 }) +}) + +const keyOf = (finding: { + readonly packageName: string + readonly advisoryId: string + readonly status: string +}) => `${finding.packageName}:${finding.advisoryId}:${finding.status}` + +describe("DependencyScanner", () => { + it.effect( + "reads every lockfile of a monorepo at the tracked head and opens a finding per advisory", + () => { + const state = world(MONOREPO) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const outcome = yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual(outcome._tag, "Scanned") + if (outcome._tag !== "Scanned") return + assert.deepStrictEqual( + outcome.scan.lockfiles.map((lockfile) => [ + lockfile.path, + lockfile.status, + lockfile.packageCount, + ]), + [ + ["package-lock.json", "parsed", 2], + ["legacy/bun.lockb", "unsupported", 0], + ["crates/core/Cargo.lock", "parsed", 1], + ["services/api/go.mod", "parsed", 1], + ], + ) + assert.isTrue(outcome.scan.defaultBranchHead) + assert.strictEqual(outcome.scan.ref, "main") + assert.strictEqual(outcome.scan.commitSha, sha("a")) + + const findings = yield* findingsOf + const byPackage = (name: string) => findings.filter((finding) => finding.packageName === name) + assert.strictEqual(byPackage("lodash").length, 6) + assert.deepStrictEqual( + byPackage("minimist").map((finding) => [ + finding.severity, + finding.relation, + finding.fixedIn, + ]), + [["critical", "dev", "1.2.6"]], + ) + assert.isTrue( + byPackage("golang.org/x/net").every( + (finding) => finding.lockfilePath === "services/api/go.mod", + ), + ) + assert.deepStrictEqual( + byPackage("time").map((finding) => finding.relation), + ["direct"], + ) + // The go.sum beside go.mod and the vendored lockfile are never read. + assert.isFalse( + state.blobReads.some((blob) => blob.includes("go.sum") || blob.includes("node_modules")), + ) + }).pipe(Effect.provide(scannerFor(state))) + }, + ) + + it.effect("skips a push that changed no lockfile, and fixes the advisories an upgrade removes", () => { + const state = world(MONOREPO) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + + state.head = sha("b") + state.files.set("README.md", { sha: "readme-2", content: "# shop 2" }) + const unchanged = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(unchanged._tag, "Unchanged") + + state.head = sha("c") + const upgraded = packageLock("4.18.0") + state.files.set("package-lock.json", { sha: "lock-2", content: upgraded }) + const outcome = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(outcome._tag, "Scanned") + + const lodash = (yield* findingsOf).filter((finding) => finding.packageName === "lodash") + assert.strictEqual(lodash.length, 6) + assert.isTrue( + lodash.every((finding) => finding.status === "fixed" && finding.fixedSha === sha("c")), + ) + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.effect("reads a lockfile's packages from the parse cache on a rescan", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual(state.blobReads.length, 1) + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual(state.blobReads.length, 1) + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.live("records a failed scan and keeps the findings when OSV cannot be read", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + const before = (yield* findingsOf).map(keyOf).sort() + + state.osvDown = true + state.head = sha("d") + state.files.set("package-lock.json", { sha: "lock-3", content: packageLock("4.18.0") }) + const error = yield* Effect.flip(scanner.scan(installation, repository, { trigger: "push" })) + assert.strictEqual(error._tag, "@maple/dependency-scan/DependencyScanError") + + const [latest] = yield* store.listScans({ orgId: repository.orgId, limit: 1 }) + assert.strictEqual(latest?.status, "failed") + assert.deepStrictEqual((yield* findingsOf).map(keyOf).sort(), before) + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.effect( + "fails a scan whose tree listing was cut short instead of fixing the unlisted lockfiles' findings", + () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + state.truncated = true + state.files.delete("package-lock.json") + const error = yield* Effect.flip( + scanner.scan(installation, repository, { trigger: "schedule" }), + ) + assert.include(error.message, "too large") + assert.isTrue((yield* findingsOf).every((finding) => finding.status === "open")) + }).pipe(Effect.provide(scannerFor(state))) + }, + ) + + it.effect("records an oversized lockfile without reading it and keeps its findings open", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + yield* scanner.scan(installation, repository, { trigger: "schedule" }) + state.files.set("package-lock.json", { sha: "huge", content: "", size: MAX_LOCKFILE_BYTES + 1 }) + const outcome = yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual( + outcome._tag === "Scanned" ? outcome.scan.lockfiles[0]?.status : undefined, + "too_large", + ) + assert.isFalse(state.blobReads.includes("huge")) + assert.isTrue((yield* findingsOf).every((finding) => finding.status === "open")) + }).pipe(Effect.provide(scannerFor(state))) + }) + + it.effect("stores an older commit's matches without touching the findings", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + const outcome = yield* scanner.scan(installation, repository, { + trigger: "production", + commitSha: sha("e"), + }) + assert.strictEqual(outcome._tag, "Scanned") + if (outcome._tag !== "Scanned") return + assert.isFalse(outcome.scan.defaultBranchHead) + assert.isNull(outcome.scan.ref) + assert.strictEqual( + (yield* store.scanMatches({ orgId: repository.orgId, scanId: outcome.scan.id })).length, + 7, + ) + assert.deepStrictEqual(yield* findingsOf, []) + }).pipe(Effect.provide(scannerFor(state))) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts new file mode 100644 index 0000000000..4f2f84f729 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -0,0 +1,368 @@ +import { EdgeCacheService } from "@maple/cache" +import type { DependencyScanRow } from "@maple/db/tables" +import { DependencyRelation, type DependencyScanLockfile } from "@maple/domain/dependency-scan" +import { + type DependencyScanTrigger, + type GitCommitSha, + type UnknownVcsProviderError, + type VcsInstallation, + type VcsRepo, + VcsRepositoryId, + type VcsRepositoryRef, +} from "@maple/domain/http" +import { type PackageAdvisory, PackageEcosystem, packageKey } from "@maple/domain/package-advisories" +import { parseLockfile } from "@maple/lockfiles" +import { Context, Effect, Layer, Option, Schema } from "effect" +import type { + VcsProviderClient, + VcsTreeFile, +} from "@maple/backend/services/integrations/vcs/VcsProviderClient" +import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" +import { DependencyScanSink } from "@maple/backend/services/integrations/vcs/PullRequestEventSink" +import { summarizeCause } from "@maple/backend/platform/describe-cause" +import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" +import { type DependencyScanMatchInput, DependencyScanStore } from "./DependencyScanStore" +import { manifestsFor, type SelectedLockfile, sameLockfiles, selectLockfiles } from "./lockfile-selection" + +// Reads a repository's lockfiles at one commit through the VCS provider, looks their packages up +// in OSV and stores the scan. A lockfile's parse is cached by its blob sha (and its manifests'), +// so a rescan for new advisories reads nothing from the provider but the tree. + +const PARSE_CACHE_BUCKET = "lockfile-packages" +const PARSE_CACHE_TTL_SECONDS = 7 * 24 * 60 * 60 +/** Larger lockfiles are recorded as too large rather than read into the Worker's memory. */ +export const MAX_LOCKFILE_BYTES = 15 * 1024 * 1024 +const READ_CONCURRENCY = 4 + +const LockedPackageRow = Schema.Struct({ + ecosystem: PackageEcosystem, + name: Schema.String, + version: Schema.String, + relation: DependencyRelation, +}) +type LockedPackageRow = typeof LockedPackageRow.Type + +const CachedLockfile = Schema.Struct({ + packages: Schema.Array(LockedPackageRow), + warnings: Schema.Array(Schema.String), +}) + +export class DependencyScanError extends Schema.TaggedError()( + "@maple/dependency-scan/DependencyScanError", + { message: Schema.String, repositoryId: VcsRepositoryId, cause: Schema.optionalKey(Schema.Defect()) }, +) {} + +export interface DependencyScanRequest { + readonly trigger: DependencyScanTrigger + /** Scan this commit instead of the tracked branch head; it then never reconciles findings. */ + readonly commitSha?: GitCommitSha | undefined +} + +export type DependencyScanOutcome = + | { readonly _tag: "Scanned"; readonly scan: DependencyScanRow } + /** A push that left every lockfile as the last scan read it. */ + | { readonly _tag: "Unchanged" } + /** The tracked branch has no commit yet. */ + | { readonly _tag: "EmptyRepository" } + +export interface DependencyScannerApi { + readonly scan: ( + installation: VcsInstallation, + repository: VcsRepo, + request: DependencyScanRequest, + ) => Effect.Effect +} + +/** The provider reads a scan makes; the registry's clients have them, tests stub only these. */ +export type DependencySource = Pick + +export class DependencySources extends Context.Service< + DependencySources, + { readonly resolve: (provider: string) => Effect.Effect } +>()("@maple/backend/services/dependency-scan/DependencySources") { + static readonly layer = Layer.effect( + this, + Effect.gen(function* () { + const registry = yield* VcsProviderRegistry + return { resolve: registry.resolve } + }), + ).pipe(Layer.provide(VcsProviderRegistry.layer)) +} + +interface ReadLockfile { + readonly lockfile: DependencyScanLockfile + readonly packages: ReadonlyArray +} + +export class DependencyScanner extends Context.Service()( + "@maple/backend/services/dependency-scan/DependencyScanner", + { + make: Effect.gen(function* () { + const sources = yield* DependencySources + const store = yield* DependencyScanStore + const catalog = yield* PackageAdvisoryCatalog + const edgeCache = yield* EdgeCacheService + + const lockfileRow = ( + selected: SelectedLockfile, + status: DependencyScanLockfile["status"], + packageCount: number, + message: string | null, + ): DependencyScanLockfile => ({ + path: selected.file.path, + kind: selected.kind, + blobSha: selected.file.sha, + packageCount, + status, + message, + }) + + /** Fetches and parses one lockfile; a format problem is the lockfile's status, a provider failure fails the scan. */ + const readLockfile = ( + provider: DependencySource, + installation: VcsInstallation, + repo: VcsRepositoryRef, + selected: SelectedLockfile, + tree: ReadonlyArray, + ) => { + if (selected.file.size > MAX_LOCKFILE_BYTES) + return Effect.succeed({ + lockfile: lockfileRow( + selected, + "too_large", + 0, + `${selected.file.size} bytes, over the ${MAX_LOCKFILE_BYTES} a scan reads`, + ), + packages: [], + }) + const manifests = manifestsFor(selected, tree) + const read = Effect.gen(function* () { + const content = yield* provider.fetchBlobText(installation, repo, selected.file.sha) + const manifestTexts = yield* Effect.forEach( + manifests, + (manifest) => + provider + .fetchBlobText(installation, repo, manifest.file.sha) + .pipe(Effect.map((text) => [manifest.relativePath, text] as const)), + { concurrency: READ_CONCURRENCY }, + ) + const parsed = yield* parseLockfile({ + path: selected.file.path, + content, + manifests: new Map(manifestTexts), + }) + return { packages: parsed.packages, warnings: parsed.warnings } + }) + return edgeCache + .getOrCompute( + { + bucket: PARSE_CACHE_BUCKET, + key: [selected.file.sha, ...manifests.map((manifest) => manifest.file.sha)].join( + ":", + ), + ttlSeconds: PARSE_CACHE_TTL_SECONDS, + schema: CachedLockfile, + }, + read, + ) + .pipe( + Effect.map(({ value }): ReadLockfile => ({ + lockfile: lockfileRow( + selected, + "parsed", + value.packages.length, + value.warnings.length === 0 ? null : value.warnings.join("; "), + ), + packages: value.packages, + })), + Effect.catchTags({ + "@maple/lockfiles/LockfileParseError": (error) => + Effect.succeed({ + lockfile: lockfileRow(selected, "failed", 0, error.message), + packages: [], + }), + "@maple/lockfiles/LockfileUnsupportedError": (error) => + Effect.succeed({ + lockfile: lockfileRow(selected, "unsupported", 0, error.message), + packages: [], + }), + }), + Effect.withSpan("DependencyScanner.readLockfile", { + attributes: { + "maple.depscan.lockfile.path": selected.file.path, + "maple.depscan.lockfile.kind": selected.kind, + }, + }), + ) + } + + const matchesOf = ( + reads: ReadonlyArray, + advisories: ReadonlyMap>, + ): ReadonlyArray => + reads.flatMap(({ lockfile, packages }) => + packages.flatMap((pkg) => + (advisories.get(packageKey(pkg)) ?? []).map((advisory): DependencyScanMatchInput => ({ + lockfilePath: lockfile.path, + ecosystem: pkg.ecosystem, + packageName: pkg.name, + version: pkg.version, + relation: pkg.relation, + advisoryId: advisory.id, + aliases: advisory.aliases, + severity: advisory.severity, + summary: advisory.summary, + fixedIn: advisory.fixedIn, + })), + ), + ) + + const scan: DependencyScannerApi["scan"] = Effect.fn("DependencyScanner.scan")( + function* (installation, repository, request) { + const orgId = repository.orgId + const ref = repository.trackedBranch ?? repository.defaultBranch + const repo: VcsRepositoryRef = { + externalRepoId: repository.externalRepoId, + owner: repository.owner, + name: repository.name, + } + yield* Effect.annotateCurrentSpan({ + orgId, + "vcs.repository.id": repository.id, + "maple.depscan.trigger": request.trigger, + }) + const scanError = (message: string) => (cause: unknown) => + new DependencyScanError({ message, repositoryId: repository.id, cause }) + const provider = yield* sources + .resolve(repository.provider) + .pipe(Effect.mapError(scanError("Unknown VCS provider"))) + const headOpt = + request.commitSha === undefined + ? yield* provider + .resolveRef(installation, repo, ref) + .pipe(Effect.mapError(scanError(`Could not resolve ${ref}`))) + : Option.some(request.commitSha) + if (Option.isNone(headOpt)) + return { _tag: "EmptyRepository" } satisfies DependencyScanOutcome + const commitSha = headOpt.value + const defaultBranchHead = request.commitSha === undefined + const tree = yield* provider + .fetchTreeFiles(installation, repo, commitSha) + .pipe(Effect.mapError(scanError("Could not list the repository tree"))) + const { selected, skipped } = selectLockfiles(tree.files) + yield* Effect.annotateCurrentSpan({ + "vcs.commit.sha": commitSha, + "maple.depscan.lockfile_count": selected.length, + "maple.depscan.lockfiles_skipped": skipped, + "maple.depscan.tree_truncated": tree.truncated, + }) + + if (request.trigger === "push") { + const latest = yield* store + .latestScan({ orgId, repositoryId: repository.id }) + .pipe(Effect.mapError(scanError("Could not read the last scan"))) + if ( + Option.isSome(latest) && + latest.value.defaultBranchHead && + !tree.truncated && + sameLockfiles(latest.value.lockfiles, selected) + ) { + yield* Effect.annotateCurrentSpan({ "maple.depscan.outcome": "unchanged" }) + return { _tag: "Unchanged" } satisfies DependencyScanOutcome + } + } + + const started = yield* store + .startScan({ + orgId, + repositoryId: repository.id, + commitSha, + ref: defaultBranchHead ? ref : null, + trigger: request.trigger, + defaultBranchHead, + }) + .pipe(Effect.mapError(scanError("Could not record the scan"))) + yield* Effect.annotateCurrentSpan({ "maple.depscan.scan_id": started.id }) + + const run = Effect.gen(function* () { + // A cut-short listing would read as deleted lockfiles and mark their findings fixed. + if (tree.truncated) + return yield* new DependencyScanError({ + message: "The repository tree is too large to list in one read", + repositoryId: repository.id, + }) + const reads = yield* Effect.forEach( + selected, + (lockfile) => + readLockfile(provider, installation, repo, lockfile, tree.files).pipe( + Effect.mapError(scanError(`Could not read ${lockfile.file.path}`)), + ), + { concurrency: READ_CONCURRENCY }, + ) + const packages = reads.flatMap((read) => read.packages) + const advisories = yield* catalog + .advisories(packages) + .pipe(Effect.mapError(scanError("Could not read advisories"))) + const matches = matchesOf(reads, advisories) + yield* Effect.annotateCurrentSpan({ + "maple.depscan.package_count": packages.length, + "maple.depscan.match_count": matches.length, + }) + return yield* store + .completeScan({ + orgId, + scanId: started.id, + lockfiles: reads.map((read) => read.lockfile), + packageCount: packages.length, + matches, + }) + .pipe(Effect.mapError(scanError("Could not store the scan"))) + }) + const completed = yield* run.pipe( + Effect.tapError((error) => + store + .failScan({ orgId, scanId: started.id, error: error.message }) + .pipe(Effect.ignore), + ), + ) + yield* Effect.annotateCurrentSpan({ "maple.depscan.outcome": "scanned" }) + return { _tag: "Scanned", scan: completed } satisfies DependencyScanOutcome + }, + ) + + return { scan } satisfies DependencyScannerApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide( + Layer.mergeAll(DependencyScanStore.layer, PackageAdvisoryCatalog.layer, DependencySources.layer), + ), + ) +} + +/** Binds the VCS sync's scan requests to the scanner. A failed scan is recorded on its row and logged; the job stands. */ +export const DependencyScanSinkLive = Layer.effect( + DependencyScanSink, + Effect.gen(function* () { + const scanner = yield* DependencyScanner + return { + onScanRequested: (installation, repository, job) => + scanner + .scan(installation, repository, { trigger: job.trigger, commitSha: job.commitSha }) + .pipe( + Effect.asVoid, + Effect.catchCause((cause) => + Effect.logWarning("[DependencyScan] scan failed").pipe( + Effect.annotateLogs({ + orgId: repository.orgId, + repository: repository.fullName, + trigger: job.trigger, + cause: summarizeCause(cause), + }), + ), + ), + ), + } + }), +).pipe(Layer.provide(DependencyScanner.layer)) diff --git a/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts b/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts new file mode 100644 index 0000000000..ef2a6eba02 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest" +import { + MAX_LOCKFILES, + manifestsFor, + type SelectedLockfile, + sameLockfiles, + selectLockfiles, +} from "./lockfile-selection" + +const file = (path: string, sha = `sha-${path}`) => ({ path, sha, size: 100 }) + +describe("selectLockfiles", () => { + it("keeps lockfiles shallowest first and skips vendored, fixture and go.sum-beside-go.mod paths", () => { + const { selected, skipped } = selectLockfiles([ + file("services/api/go.sum"), + file("services/api/go.mod"), + file("tools/go.sum"), + file("README.md"), + file("web/node_modules/left-pad/package-lock.json"), + file("crates/core/Cargo.lock"), + file("package-lock.json"), + file("test/fixtures/yarn.lock"), + file("vendor/github.com/x/go.mod"), + ]) + expect(selected.map((lockfile) => [lockfile.file.path, lockfile.kind])).toEqual([ + ["package-lock.json", "package-lock"], + ["tools/go.sum", "go-sum"], + ["crates/core/Cargo.lock", "cargo-lock"], + ["services/api/go.mod", "go-mod"], + ]) + expect(skipped).toBe(0) + }) + + it("caps the lockfiles a scan reads and counts the rest", () => { + const many = Array.from({ length: MAX_LOCKFILES + 3 }, (_, i) => file(`pkg${i}/bun.lock`)) + const { selected, skipped } = selectLockfiles(many) + expect(selected).toHaveLength(MAX_LOCKFILES) + expect(skipped).toBe(3) + }) +}) + +describe("manifestsFor", () => { + const tree = [ + file("web/yarn.lock"), + file("web/package.json"), + file("web/packages/ui/package.json"), + file("web/node_modules/react/package.json"), + file("api/package.json"), + ] + const lockfile = (path: string, kind: SelectedLockfile["kind"]): SelectedLockfile => ({ + file: file(path), + kind, + }) + + it("hands a yarn lockfile the package.json files under its directory, relative to it", () => { + expect(manifestsFor(lockfile("web/yarn.lock", "yarn-v1"), tree).map((m) => m.relativePath)).toEqual([ + "package.json", + "packages/ui/package.json", + ]) + }) + + it("reads no manifests for formats that record direct and dev dependencies themselves", () => { + expect(manifestsFor(lockfile("web/package-lock.json", "package-lock"), tree)).toEqual([]) + }) +}) + +describe("sameLockfiles", () => { + const previous = [ + { + path: "a/bun.lock", + kind: "bun-lock", + blobSha: "1", + packageCount: 3, + status: "parsed", + message: null, + }, + ] as const + it("matches only the same paths at the same blobs", () => { + const selected = (sha: string, path = "a/bun.lock"): ReadonlyArray => [ + { file: { path, sha, size: 1 }, kind: "bun-lock" }, + ] + expect(sameLockfiles(previous, selected("1"))).toBe(true) + expect(sameLockfiles(previous, selected("2"))).toBe(false) + expect(sameLockfiles(previous, selected("1", "b/bun.lock"))).toBe(false) + expect(sameLockfiles(previous, [])).toBe(false) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/lockfile-selection.ts b/packages/backend/src/services/dependency-scan/lockfile-selection.ts new file mode 100644 index 0000000000..b22ff97635 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/lockfile-selection.ts @@ -0,0 +1,72 @@ +import type { DependencyScanLockfile } from "@maple/domain/dependency-scan" +import { isIgnoredLockfilePath, type LockfileKind, lockfileKindOf } from "@maple/lockfiles" +import type { VcsTreeFile } from "@maple/backend/services/integrations/vcs/VcsProviderClient" + +// Which files of a repository tree a scan reads. Pure, so the rules are testable without GitHub. + +/** Lockfiles read per scan; the rest are left out, deepest paths first. */ +export const MAX_LOCKFILES = 50 +/** Manifests read for one yarn lockfile, which says nothing about direct and dev dependencies. */ +export const MAX_MANIFESTS = 50 + +export interface SelectedLockfile { + readonly file: VcsTreeFile + readonly kind: LockfileKind +} + +export const directoryOf = (path: string): string => { + const slash = path.lastIndexOf("/") + return slash === -1 ? "" : path.slice(0, slash + 1) +} + +const depth = (path: string): number => path.split("/").length + +/** + * The tree's lockfiles, shallowest first, capped. A go.sum next to a go.mod is skipped: go.sum + * also lists versions the build never selects, which would only add false positives. + */ +export function selectLockfiles(files: ReadonlyArray): { + readonly selected: ReadonlyArray + readonly skipped: number +} { + const candidates = files.flatMap((file): ReadonlyArray => { + const kind = lockfileKindOf(file.path) + return kind === undefined || isIgnoredLockfilePath(file.path) ? [] : [{ file, kind }] + }) + const goModDirs = new Set( + candidates.filter((c) => c.kind === "go-mod").map((c) => directoryOf(c.file.path)), + ) + const kept = candidates + .filter((c) => c.kind !== "go-sum" || !goModDirs.has(directoryOf(c.file.path))) + .sort((a, b) => depth(a.file.path) - depth(b.file.path) || a.file.path.localeCompare(b.file.path)) + return { selected: kept.slice(0, MAX_LOCKFILES), skipped: Math.max(0, kept.length - MAX_LOCKFILES) } +} + +/** The package.json files under a yarn lockfile's directory, keyed relative to it. */ +export function manifestsFor( + lockfile: SelectedLockfile, + files: ReadonlyArray, +): ReadonlyArray<{ readonly relativePath: string; readonly file: VcsTreeFile }> { + if (lockfile.kind !== "yarn-v1" && lockfile.kind !== "yarn-berry") return [] + const dir = directoryOf(lockfile.file.path) + return files + .filter( + (file) => + file.path.startsWith(dir) && + (file.path === `${dir}package.json` || file.path.endsWith("/package.json")) && + !isIgnoredLockfilePath(file.path), + ) + .sort((a, b) => depth(a.path) - depth(b.path) || a.path.localeCompare(b.path)) + .slice(0, MAX_MANIFESTS) + .map((file) => ({ relativePath: file.path.slice(dir.length), file })) +} + +/** Whether a scan would read exactly the lockfile contents an earlier scan read. */ +export const sameLockfiles = ( + previous: ReadonlyArray, + selected: ReadonlyArray, +): boolean => + previous.length === selected.length && + selected.every(({ file }) => + previous.some((lockfile) => lockfile.path === file.path && lockfile.blobSha === file.sha), + ) diff --git a/packages/backend/src/services/integrations/vcs/PullRequestEventSink.ts b/packages/backend/src/services/integrations/vcs/PullRequestEventSink.ts index 5ff67cffc8..fcd847d288 100644 --- a/packages/backend/src/services/integrations/vcs/PullRequestEventSink.ts +++ b/packages/backend/src/services/integrations/vcs/PullRequestEventSink.ts @@ -1,8 +1,11 @@ import type { + DependencyScanJob, PullRequestChecklistJob, PullRequestCiApprovedJob, PullRequestCommentJob, PullRequestEventJob, + VcsInstallation, + VcsRepo, } from "@maple/domain/http" import type { OrgId } from "@maple/domain/primitives" import { Context, Effect, Layer } from "effect" @@ -114,3 +117,19 @@ export class PullRequestCiApprovalSink extends Context.Service< PullRequestCiApprovalSink, PullRequestCiApprovalSinkApi >()("@maple/backend/services/integrations/vcs/PullRequestCiApprovalSink") {} + +/** + * Where a repository's dependency scan request goes once its repository row is resolved. Optional + * in `VcsSyncService`: a root without the scanner neither enqueues nor runs scans. + */ +export interface DependencyScanSinkApi { + readonly onScanRequested: ( + installation: VcsInstallation, + repository: VcsRepo, + job: DependencyScanJob, + ) => Effect.Effect +} + +export class DependencyScanSink extends Context.Service()( + "@maple/backend/services/integrations/vcs/DependencyScanSink", +) {} diff --git a/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts b/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts index a12805bbf7..138a2581e3 100644 --- a/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts +++ b/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts @@ -55,6 +55,14 @@ export interface VcsCodeSearchMatch { readonly snippets: ReadonlyArray } +/** A file (blob) in a repository's tree at one commit. */ +export interface VcsTreeFile { + readonly path: string + /** The blob's sha: the same content always has the same sha. */ + readonly sha: string + readonly size: number +} + /** A text source file fetched from a repository at an explicit ref. */ export interface VcsSourceFile { readonly path: string @@ -493,6 +501,26 @@ export interface VcsProviderClient { VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRepositoryBlockedError > + /** Every file in the tree of a commit; `truncated` when the provider cut the listing short. */ + readonly fetchTreeFiles: ( + installation: VcsInstallation, + repo: VcsRepositoryRef, + commitSha: GitCommitSha, + ) => Effect.Effect< + { readonly files: ReadonlyArray; readonly truncated: boolean }, + VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRepositoryBlockedError + > + + /** A file's content as UTF-8 by its blob sha, without the size cap of `fetchSourceFile`. */ + readonly fetchBlobText: ( + installation: VcsInstallation, + repo: VcsRepositoryRef, + blobSha: string, + ) => Effect.Effect< + string, + VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRepositoryBlockedError + > + /** Fetch a UTF-8 source file. `Option.none` is an expected missing path/ref. */ readonly fetchSourceFile: ( installation: VcsInstallation, diff --git a/packages/backend/src/services/integrations/vcs/VcsSyncService.ts b/packages/backend/src/services/integrations/vcs/VcsSyncService.ts index 0701c10cb4..e82ba71310 100644 --- a/packages/backend/src/services/integrations/vcs/VcsSyncService.ts +++ b/packages/backend/src/services/integrations/vcs/VcsSyncService.ts @@ -25,10 +25,13 @@ import { type WorkflowJobAnnotation, type WorkflowJobJob, type WorkflowRunJob, + type DependencyScanJob, + type DependencyScanTrigger, } from "@maple/domain/http" import { Clock, Duration, Effect, Context, Layer, Option, Schema, Match } from "effect" import type { VcsProviderClient } from "./VcsProviderClient" import { + DependencyScanSink, PullRequestChecklistSink, PullRequestCiApprovalSink, PullRequestCommentSink, @@ -102,6 +105,25 @@ export class VcsSyncService extends Context.Service, + trigger: DependencyScanTrigger, + ) => + scanSink === undefined || externalRepoIds.length === 0 + ? Effect.void + : queue.sendBatch( + externalRepoIds.map((externalRepoId): VcsSyncJob => ({ + kind: "dependency-scan", + provider: installation.provider, + externalInstallationId: installation.externalInstallationId, + externalRepoId, + trigger, + })), + ) // The repo's single tracked branch, with the default-branch fallback for a // row whose `trackedBranch` was never set (legacy) — the one place the @@ -415,6 +437,7 @@ export class VcsSyncService extends Context.Service r.externalRepoId), + "schedule", + ) yield* Effect.annotateCurrentSpan({ "vcs.installation_sync.outcome": "handled", "vcs.installation_sync.reason": job.reason, @@ -827,6 +855,21 @@ export class VcsSyncService extends Context.Service handlePush(installation, job)), + Match.discriminator("kind")("dependency-scan", (job) => + handleDependencyScan(installation, job), + ), Match.discriminator("kind")("branch-event", (job) => handleBranchEvent(installation, job), ), diff --git a/packages/backend/src/services/integrations/vcs/__tests__/vcs.test.ts b/packages/backend/src/services/integrations/vcs/__tests__/vcs.test.ts index ff6d690cf1..49d6cf3840 100644 --- a/packages/backend/src/services/integrations/vcs/__tests__/vcs.test.ts +++ b/packages/backend/src/services/integrations/vcs/__tests__/vcs.test.ts @@ -47,7 +47,10 @@ import { VcsProviderRegistry, type VcsProviderRegistryApi, } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" -import { PullRequestEventSink } from "@maple/backend/services/integrations/vcs/PullRequestEventSink" +import { + DependencyScanSink, + PullRequestEventSink, +} from "@maple/backend/services/integrations/vcs/PullRequestEventSink" import { WorkflowEventSink } from "@maple/backend/services/integrations/vcs/WorkflowEventSink" import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" import { clampQueueDelaySeconds } from "@maple/backend/services/integrations/vcs/VcsSyncQueue" @@ -1827,6 +1830,8 @@ describe("VcsSyncService orchestrator", () => { readonly fetchReposError?: VcsRateLimitedError | VcsProviderError | VcsInstallationGoneError readonly branches?: ReadonlyArray<{ name: string; headSha: string | null }> readonly branchesTruncated?: boolean + /** Scan requests the dependency scanner hook received; absent leaves the hook out. */ + readonly scanRequests?: Array<{ repository: string; trigger: string; commitSha: string | undefined }> readonly fetchBranchesError?: | VcsProviderError | VcsInstallationGoneError @@ -1930,8 +1935,22 @@ describe("VcsSyncService orchestrator", () => { Effect.ignore, ), }) + const scanRequests = opts.scanRequests + const scanSink = + scanRequests === undefined + ? Layer.empty + : Layer.succeed(DependencyScanSink, { + onScanRequested: (_installation, repository, job) => + Effect.sync(() => { + scanRequests.push({ + repository: repository.fullName, + trigger: job.trigger, + commitSha: job.commitSha, + }) + }), + }) return Layer.effect(VcsSyncService, VcsSyncService.make).pipe( - Layer.provideMerge(Layer.mergeAll(repoLive, registry, queue, sink, workflowSink)), + Layer.provideMerge(Layer.mergeAll(repoLive, registry, queue, sink, workflowSink, scanSink)), ) } @@ -2613,6 +2632,98 @@ describe("VcsSyncService orchestrator", () => { }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, repos: oneRepo }))) }) + it.effect("with the scanner bound, a push to the tracked branch enqueues a scan of its head", () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + yield* seedInstallation(repo, asOrgId("org_orch")) + yield* upsertReposFor(repo, "42", oneRepo) + const push = (branch: string) => + svc.processMessage( + Schema.encodeSync(VcsSyncJob)({ + kind: "push", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + branch, + commits: [commit(SHA_A, 1)], + }), + ) + yield* push("feature/x") + yield* push("main") + assert.deepStrictEqual( + sent.filter((job) => job.kind === "dependency-scan"), + [ + { + kind: "dependency-scan", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + trigger: "push", + }, + ], + ) + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, repos: oneRepo, scanRequests: [] }))) + }) + + it.effect("without the scanner bound, a push enqueues no scan", () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + yield* seedInstallation(repo, asOrgId("org_orch")) + yield* upsertReposFor(repo, "42", oneRepo) + yield* svc.processMessage( + Schema.encodeSync(VcsSyncJob)({ + kind: "push", + provider: "github", + externalInstallationId: "42", + externalRepoId: "7", + branch: "main", + commits: [commit(SHA_A, 1)], + }), + ) + assert.isFalse(sent.some((job) => job.kind === "dependency-scan")) + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, repos: oneRepo }))) + }) + + it.effect( + "hands a scan job to the scanner with its repository resolved, and drops one for an unknown repository", + () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + const scanRequests: Array<{ + repository: string + trigger: string + commitSha: string | undefined + }> = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + yield* seedInstallation(repo, asOrgId("org_orch")) + yield* upsertReposFor(repo, "42", oneRepo) + const scanJob = (externalRepoId: string) => + svc.processMessage( + Schema.encodeSync(VcsSyncJob)({ + kind: "dependency-scan", + provider: "github", + externalInstallationId: "42", + externalRepoId, + trigger: "manual", + }), + ) + yield* scanJob("7") + yield* scanJob("404") + assert.deepStrictEqual(scanRequests, [ + { repository: "octo/repo", trigger: "manual", commitSha: undefined }, + ]) + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, repos: oneRepo, scanRequests }))) + }, + ) + it.effect("drops a job for an unknown installation without persisting or failing", () => { const testDb = createTestDb(trackedDbs) const sent: Array = [] diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts index 485bedd7de..4cbc58f278 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts @@ -358,6 +358,29 @@ const GithubApiGitTreeListing = Schema.Struct({ ), }) +/** A recursive listing; `size` is set on blobs only, `truncated` when GitHub cut the listing short. */ +const GithubApiRecursiveTree = Schema.Struct({ + sha: Schema.String, + truncated: Schema.Boolean, + tree: Schema.Array( + Schema.Struct({ + path: Schema.String, + type: Schema.String, + sha: Schema.String, + size: Schema.optionalKey(Schema.Number), + }), + ), +}) +export type GithubRecursiveTree = typeof GithubApiRecursiveTree.Type + +const GithubApiBlob = Schema.Struct({ + sha: Schema.String, + size: Schema.Number, + encoding: Schema.String, + content: Schema.String, +}) +export type GithubBlob = typeof GithubApiBlob.Type + const GithubApiReviewCommentList = Schema.Array( Schema.Struct({ id: Schema.Number, @@ -562,6 +585,8 @@ const decodeCreatedComment = Schema.decodeUnknownEffect(GithubApiCreatedComment) const decodeCollaboratorPermission = Schema.decodeUnknownEffect(GithubApiCollaboratorPermission) const decodeGitCommit = Schema.decodeUnknownEffect(GithubApiGitCommit) const decodeGitTreeListing = Schema.decodeUnknownEffect(GithubApiGitTreeListing) +const decodeRecursiveTree = Schema.decodeUnknownEffect(GithubApiRecursiveTree) +const decodeBlob = Schema.decodeUnknownEffect(GithubApiBlob) const decodeComparison = Schema.decodeUnknownEffect(GithubApiComparison) const decodeReviewThreads = Schema.decodeUnknownEffect(GithubReviewThreadsResponse) const decodeMutation = Schema.decodeUnknownEffect(GithubMutationResponse) @@ -2136,6 +2161,50 @@ export class GithubAppClient extends Context.Service()( ) }) + /** Every entry under a commit's tree in one call; GitHub truncates past 100,000 entries or 7 MB. */ + const getRecursiveTree = Effect.fn("GithubAppClient.getRecursiveTree")(function* ( + externalInstallationId: string, + owner: string, + repo: string, + treeish: string, + ) { + const config = yield* resolveConfig + const token = yield* mintInstallationToken(externalInstallationId) + const response = yield* authedGet( + config, + token, + `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/trees/${encodeURIComponent(treeish)}?recursive=1`, + ) + if (!isOk(response)) return yield* failure(response, "Read tree", "repository") + return yield* decodeRecursiveTree(yield* parseJson(response, "Read tree")).pipe( + Effect.mapError( + (cause) => new GithubAppError({ message: "Unexpected tree payload", cause }), + ), + ) + }) + + /** One blob by its sha; unlike the contents API it serves files up to 100 MB. */ + const getBlob = Effect.fn("GithubAppClient.getBlob")(function* ( + externalInstallationId: string, + owner: string, + repo: string, + sha: string, + ) { + const config = yield* resolveConfig + const token = yield* mintInstallationToken(externalInstallationId) + const response = yield* authedGet( + config, + token, + `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/blobs/${encodeURIComponent(sha)}`, + ) + if (!isOk(response)) return yield* failure(response, "Read blob", "repository") + return yield* decodeBlob(yield* parseJson(response, "Read blob")).pipe( + Effect.mapError( + (cause) => new GithubAppError({ message: "Unexpected blob payload", cause }), + ), + ) + }) + // An Actions job id is its check run id. Reading annotations needs only `checks: read`, // which the App's `checks: write` covers. Paging stops at the first page holding a failure: // that is what CI traces want, and warnings before it should not hide it. @@ -2407,6 +2476,8 @@ export class GithubAppClient extends Context.Service()( upsertIssueComment, searchCode, getSourceFile, + getRecursiveTree, + getBlob, listCheckRunAnnotations, streamJobLog, canReadActions, diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts index b540526cd4..216c225bcb 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts @@ -1304,6 +1304,31 @@ export class GithubProvider extends Context.Service + client + .getRecursiveTree(installation.externalInstallationId, repo.owner, repo.name, commitSha) + .pipe( + Effect.map((listing) => ({ + truncated: listing.truncated, + files: listing.tree.flatMap((entry) => + entry.type === "blob" + ? [{ path: entry.path, sha: entry.sha, size: entry.size ?? 0 }] + : [], + ), + })), + Effect.mapError(toVcsCommitError), + ) + + const fetchBlobText: VcsProviderClient["fetchBlobText"] = (installation, repo, blobSha) => + client.getBlob(installation.externalInstallationId, repo.owner, repo.name, blobSha).pipe( + Effect.map((blob) => + blob.encoding === "base64" + ? Buffer.from(blob.content.replace(/\s/g, ""), "base64").toString("utf8") + : blob.content, + ), + Effect.mapError(toVcsCommitError), + ) + const resolveRef: VcsProviderClient["resolveRef"] = (installation, repo, ref) => client.getCommit(installation.externalInstallationId, repo.owner, repo.name, ref).pipe( // None when the ref is absent (404) or parses but names nothing (422). @@ -1873,6 +1898,8 @@ export class GithubProvider extends Context.Service -/** What started a scan; only a default-branch head reconciles findings. */ -export const DependencyScanTrigger = Schema.Literals(["push", "schedule", "manual", "production"]) -export type DependencyScanTrigger = typeof DependencyScanTrigger.Type +export { DependencyScanTrigger } export const DependencyScanStatus = Schema.Literals(["running", "completed", "failed"]) export type DependencyScanStatus = typeof DependencyScanStatus.Type @@ -78,7 +76,7 @@ export type DependencyFindingStatus = SecurityAuditFindingStatus export const DependencyFindingTriageStatus = Schema.Literals(["open", "dismissed", "accepted_risk"]) export type DependencyFindingTriageStatus = typeof DependencyFindingTriageStatus.Type -/** A finding on the wire; timestamps are epoch ms. */ +/** A finding on the wire, the row as it is; timestamps are epoch ms. */ export const DependencyFinding = Schema.Struct({ id: DependencyFindingId, repositoryId: VcsRepositoryId, @@ -88,21 +86,21 @@ export const DependencyFinding = Schema.Struct({ advisoryId: Schema.String, aliases: Schema.Array(Schema.String), severity: AdvisorySeverity, - summary: Schema.optionalKey(Schema.String), + summary: Schema.NullOr(Schema.String), /** The installed versions the advisory affects. */ versions: Schema.Array(Schema.String), - fixedIn: Schema.optionalKey(Schema.String), + fixedIn: Schema.NullOr(Schema.String), relation: DependencyRelation, status: DependencyFindingStatus, firstSeenScanId: DependencyScanId, lastSeenScanId: DependencyScanId, firstSeenSha: GitCommitSha, lastSeenSha: GitCommitSha, - fixedScanId: Schema.optionalKey(DependencyScanId), - fixedSha: Schema.optionalKey(GitCommitSha), - statusReason: Schema.optionalKey(Schema.String), - statusChangedBy: Schema.optionalKey(UserId), - statusChangedAt: Schema.optionalKey(Schema.Finite), + fixedScanId: Schema.NullOr(DependencyScanId), + fixedSha: Schema.NullOr(GitCommitSha), + statusReason: Schema.NullOr(Schema.String), + statusChangedBy: Schema.NullOr(UserId), + statusChangedAt: Schema.NullOr(Schema.Finite), createdAt: Schema.Finite, updatedAt: Schema.Finite, }) diff --git a/packages/domain/src/http/dependency-scan.ts b/packages/domain/src/http/dependency-scan.ts new file mode 100644 index 0000000000..1944b9ab68 --- /dev/null +++ b/packages/domain/src/http/dependency-scan.ts @@ -0,0 +1,133 @@ +import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" +import { Schema } from "effect" +import { + DependencyFinding, + DependencyFindingId, + DependencyFindingStatus, + DependencyFindingTriageStatus, + DependencyScanId, + DependencyScanLockfile, + DependencyScanStatus, + DependencyScanTrigger, +} from "../dependency-scan" +import { SessionAuthorization } from "./current-tenant" +import { GitCommitSha, VcsRepositoryId } from "./vcs" + +// The Security Center's Dependencies tab: known vulnerable packages in connected repositories' +// lockfiles. Dashboard only, served by the api Worker. + +export const DEPENDENCY_FINDING_REASON_MAX_LENGTH = 500 + +/** One scan as the tab shows it; timestamps are epoch ms. */ +export const DependencyScanSummary = Schema.Struct({ + id: DependencyScanId, + commitSha: GitCommitSha, + ref: Schema.NullOr(Schema.String), + trigger: DependencyScanTrigger, + status: DependencyScanStatus, + lockfiles: Schema.Array(DependencyScanLockfile), + packageCount: Schema.Int, + vulnerableCount: Schema.Int, + error: Schema.NullOr(Schema.String), + startedAt: Schema.Finite, + finishedAt: Schema.NullOr(Schema.Finite), +}) +export type DependencyScanSummary = typeof DependencyScanSummary.Type + +/** A connected repository and its latest scan of the tracked branch, null before the first. */ +export const DependencyScanRepository = Schema.Struct({ + id: VcsRepositoryId, + fullName: Schema.String, + htmlUrl: Schema.String, + trackedBranch: Schema.String, + latestScan: Schema.NullOr(DependencyScanSummary), +}) +export type DependencyScanRepository = typeof DependencyScanRepository.Type + +export const DependencyScanRepositoriesResponse = Schema.Struct({ + repositories: Schema.Array(DependencyScanRepository), +}) +export type DependencyScanRepositoriesResponse = typeof DependencyScanRepositoriesResponse.Type + +/** A finding with the repository named, so the table needs no second read. */ +export const DependencyFindingView = Schema.Struct({ + ...DependencyFinding.fields, + repositoryFullName: Schema.String, +}) +export type DependencyFindingView = typeof DependencyFindingView.Type + +export const DependencyFindingsResponse = Schema.Struct({ findings: Schema.Array(DependencyFindingView) }) +export type DependencyFindingsResponse = typeof DependencyFindingsResponse.Type + +export const DependencyFindingsQuery = Schema.Struct({ + status: Schema.optional(DependencyFindingStatus), + repositoryId: Schema.optional(VcsRepositoryId), +}) + +export const DependencyFindingUpdateRequest = Schema.Struct({ + status: DependencyFindingTriageStatus, + reason: Schema.optionalKey(Schema.String.check(Schema.isMaxLength(DEPENDENCY_FINDING_REASON_MAX_LENGTH))), +}) +export type DependencyFindingUpdateRequest = typeof DependencyFindingUpdateRequest.Type + +export const DependencyScanQueued = Schema.Struct({ queued: Schema.Boolean }) + +export class DependencyScanPersistenceError extends Schema.TaggedError()( + "@maple/http/dependency-scan/DependencyScanPersistenceError", + { message: Schema.String }, + { httpApiStatus: 503 }, +) {} + +export class DependencyScanForbiddenError extends Schema.TaggedError()( + "@maple/http/dependency-scan/DependencyScanForbiddenError", + { message: Schema.String }, + { httpApiStatus: 403 }, +) {} + +export class DependencyFindingNotFoundError extends Schema.TaggedError()( + "@maple/http/dependency-scan/DependencyFindingNotFoundError", + { message: Schema.String, findingId: DependencyFindingId }, + { httpApiStatus: 404 }, +) {} + +export class DependencyScanRepositoryNotFoundError extends Schema.TaggedError()( + "@maple/http/dependency-scan/DependencyScanRepositoryNotFoundError", + { message: Schema.String, repositoryId: VcsRepositoryId }, + { httpApiStatus: 404 }, +) {} + +export class DependencyScanApiGroup extends HttpApiGroup.make("dependencyScan") + .add( + HttpApiEndpoint.get("listRepositories", "/repositories", { + success: DependencyScanRepositoriesResponse, + error: DependencyScanPersistenceError, + }), + ) + .add( + HttpApiEndpoint.post("scanRepository", "/repositories/:repositoryId/scan", { + params: { repositoryId: VcsRepositoryId }, + success: DependencyScanQueued, + error: [DependencyScanPersistenceError, DependencyScanRepositoryNotFoundError], + }), + ) + .add( + HttpApiEndpoint.get("listFindings", "/findings", { + query: DependencyFindingsQuery, + success: DependencyFindingsResponse, + error: DependencyScanPersistenceError, + }), + ) + .add( + HttpApiEndpoint.patch("updateFinding", "/findings/:findingId", { + params: { findingId: DependencyFindingId }, + payload: DependencyFindingUpdateRequest, + success: DependencyFindingView, + error: [ + DependencyScanPersistenceError, + DependencyScanForbiddenError, + DependencyFindingNotFoundError, + ], + }), + ) + .prefix("/internal/dependency-scan") + .middleware(SessionAuthorization) {} diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index 4d279c9c6a..12ff73adcb 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -3,6 +3,7 @@ export * from "./ai-api" export * from "./internal-api" export * from "./ai-models" export * from "./ai-sessions" +export * from "./dependency-scan" export * from "./ai-triage" export * from "./openrouter-key" export * from "./incident-triage" diff --git a/packages/domain/src/http/internal-api.ts b/packages/domain/src/http/internal-api.ts index a325cc06f7..d2c824478b 100644 --- a/packages/domain/src/http/internal-api.ts +++ b/packages/domain/src/http/internal-api.ts @@ -5,6 +5,7 @@ import { AiTriageApiGroup } from "./ai-triage" import { BillingApiGroup } from "./billing" import { CodeReviewApiGroup } from "./code-review" import { DemoApiGroup } from "./demo" +import { DependencyScanApiGroup } from "./dependency-scan" import { DigestApiGroup } from "./digest" import { ErrorsApiGroup } from "./errors" import { IntegrationsApiGroup } from "./integrations" @@ -49,6 +50,7 @@ export class MapleInternalApi extends HttpApi.make("MapleInternalApi") .add(BillingApiGroup) .add(CodeReviewApiGroup) .add(DemoApiGroup) + .add(DependencyScanApiGroup) .add(DigestApiGroup) .add(ErrorsApiGroup) .add(IntegrationsApiGroup) diff --git a/packages/domain/src/http/vcs.ts b/packages/domain/src/http/vcs.ts index 4536836d82..3eb9449722 100644 --- a/packages/domain/src/http/vcs.ts +++ b/packages/domain/src/http/vcs.ts @@ -740,6 +740,22 @@ export const PullRequestHead = Schema.Struct({ }) export type PullRequestHead = Schema.Schema.Type +/** What started a dependency scan; only a scan of the tracked branch head reconciles findings. */ +export const DependencyScanTrigger = Schema.Literals(["push", "schedule", "manual", "production"]) +export type DependencyScanTrigger = typeof DependencyScanTrigger.Type + +// Scan one repository's lockfiles. Without `commitSha` the scanner reads the tracked branch's +// head at scan time, so a push's truncated commit list never decides what is scanned. +export const DependencyScanJob = Schema.Struct({ + kind: Schema.Literal("dependency-scan"), + provider: VcsProviderId, + externalInstallationId: Schema.String, + externalRepoId: Schema.String, + trigger: DependencyScanTrigger, + commitSha: Schema.optionalKey(GitCommitSha), +}) +export type DependencyScanJob = Schema.Schema.Type + export const VcsSyncJob = Schema.Union([ InstallationSyncJob, SyncCommitsJob, @@ -752,6 +768,7 @@ export const VcsSyncJob = Schema.Union([ PullRequestCiApprovedJob, WorkflowRunJob, WorkflowJobJob, + DependencyScanJob, ]) export type VcsSyncJob = Schema.Schema.Type From 1a5dacf94258ebf0bdf141c7b8ff647669f83f74 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 01:12:37 +0100 Subject: [PATCH 2/6] fix(security): lockfiles past the scan cap are recorded as skipped A monorepo's lockfiles past the 50 a scan reads were left off the scan, so reconciling read them as deleted and marked their open findings fixed. They are now recorded with status skipped, which keeps their findings. --- .../dependency-scan/DependencyScanner.ts | 23 ++++++++++++++++--- .../lockfile-selection.test.ts | 8 +++++-- .../dependency-scan/lockfile-selection.ts | 7 +++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts index 4f2f84f729..6e5d6d2999 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -22,7 +22,13 @@ import { DependencyScanSink } from "@maple/backend/services/integrations/vcs/Pul import { summarizeCause } from "@maple/backend/platform/describe-cause" import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" import { type DependencyScanMatchInput, DependencyScanStore } from "./DependencyScanStore" -import { manifestsFor, type SelectedLockfile, sameLockfiles, selectLockfiles } from "./lockfile-selection" +import { + MAX_LOCKFILES, + manifestsFor, + type SelectedLockfile, + sameLockfiles, + selectLockfiles, +} from "./lockfile-selection" // Reads a repository's lockfiles at one commit through the VCS provider, looks their packages up // in OSV and stores the scan. A lockfile's parse is cached by its blob sha (and its manifests'), @@ -253,7 +259,7 @@ export class DependencyScanner extends Context.Service read.lockfile), + // A lockfile past the cap is recorded, so its findings are kept rather than read as deleted. + lockfiles: [ + ...reads.map((read) => read.lockfile), + ...skipped.map((lockfile) => + lockfileRow( + lockfile, + "skipped", + 0, + `Past the ${MAX_LOCKFILES} lockfiles a scan reads`, + ), + ), + ], packageCount: packages.length, matches, }) diff --git a/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts b/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts index ef2a6eba02..50416fdf6f 100644 --- a/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts +++ b/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts @@ -28,14 +28,18 @@ describe("selectLockfiles", () => { ["crates/core/Cargo.lock", "cargo-lock"], ["services/api/go.mod", "go-mod"], ]) - expect(skipped).toBe(0) + expect(skipped).toEqual([]) }) it("caps the lockfiles a scan reads and counts the rest", () => { const many = Array.from({ length: MAX_LOCKFILES + 3 }, (_, i) => file(`pkg${i}/bun.lock`)) const { selected, skipped } = selectLockfiles(many) expect(selected).toHaveLength(MAX_LOCKFILES) - expect(skipped).toBe(3) + expect(skipped).toHaveLength(3) + // Every lockfile is either read or recorded as skipped, never dropped. + expect(new Set([...selected, ...skipped].map((lockfile) => lockfile.file.path)).size).toBe( + many.length, + ) }) }) diff --git a/packages/backend/src/services/dependency-scan/lockfile-selection.ts b/packages/backend/src/services/dependency-scan/lockfile-selection.ts index b22ff97635..cc2d22129d 100644 --- a/packages/backend/src/services/dependency-scan/lockfile-selection.ts +++ b/packages/backend/src/services/dependency-scan/lockfile-selection.ts @@ -7,7 +7,7 @@ import type { VcsTreeFile } from "@maple/backend/services/integrations/vcs/VcsPr /** Lockfiles read per scan; the rest are left out, deepest paths first. */ export const MAX_LOCKFILES = 50 /** Manifests read for one yarn lockfile, which says nothing about direct and dev dependencies. */ -export const MAX_MANIFESTS = 50 +export const MAX_MANIFESTS = 20 export interface SelectedLockfile { readonly file: VcsTreeFile @@ -27,7 +27,8 @@ const depth = (path: string): number => path.split("/").length */ export function selectLockfiles(files: ReadonlyArray): { readonly selected: ReadonlyArray - readonly skipped: number + /** Past the cap: recorded as skipped so their findings are kept, not read as deleted. */ + readonly skipped: ReadonlyArray } { const candidates = files.flatMap((file): ReadonlyArray => { const kind = lockfileKindOf(file.path) @@ -39,7 +40,7 @@ export function selectLockfiles(files: ReadonlyArray): { const kept = candidates .filter((c) => c.kind !== "go-sum" || !goModDirs.has(directoryOf(c.file.path))) .sort((a, b) => depth(a.file.path) - depth(b.file.path) || a.file.path.localeCompare(b.file.path)) - return { selected: kept.slice(0, MAX_LOCKFILES), skipped: Math.max(0, kept.length - MAX_LOCKFILES) } + return { selected: kept.slice(0, MAX_LOCKFILES), skipped: kept.slice(MAX_LOCKFILES) } } /** The package.json files under a yarn lockfile's directory, keyed relative to it. */ From 7912bdb66b3f96c3883cc70e1d8f5038edfc7b55 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 01:22:02 +0100 Subject: [PATCH 3/6] fix(security): branch scans read directly, pushes past the cap dedupe - The repositories list read each repository's latest scan out of the org's 500 newest, so a quiet repository could read as never scanned; it now reads each repository's newest branch scan directly. - A push compared the last scan's lockfiles, which now include skipped ones, with the read ones only, so an unchanged push past the cap rescanned. - knip: the opt-in GitHub e2e test runs a locally installed osv-scanner. --- knip.json | 4 ++++ .../dependency-scan/DependencyScanService.ts | 14 +++-------- .../dependency-scan/DependencyScanner.test.ts | 24 +++++++++++++++++++ .../dependency-scan/DependencyScanner.ts | 3 ++- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/knip.json b/knip.json index 076655857e..fa50d388a9 100644 --- a/knip.json +++ b/knip.json @@ -87,6 +87,10 @@ ], "ignoreBinaries": ["duckdb"] }, + "packages/backend": { + // The opt-in GitHub e2e test compares the dependency scanner with a locally installed osv-scanner. + "ignoreBinaries": ["osv-scanner"] + }, "packages/db": { "entry": ["scripts/**/*.ts", "effect-orm.config.ts"] }, diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index 419c6333ea..c5e68bab4b 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -25,8 +25,6 @@ import { DependencyScanStore } from "./DependencyScanStore" /** Findings returned per read; the table sorts worst first, so the cut drops the least severe. */ export const FINDINGS_LIMIT = 500 -/** Scans read to find each repository's latest. */ -const SCANS_LIMIT = 500 export interface DependencyScanServiceApi { readonly listRepositories: ( @@ -131,16 +129,10 @@ export class DependencyScanService extends Context.Service scan.defaultBranchHead) - .reduce((byRepository, scan) => { - if (!byRepository.has(scan.repositoryId)) byRepository.set(scan.repositoryId, scan) - return byRepository - }, new Map()) + const latest = new Map(branchScans.map((scan) => [scan.repositoryId, scan])) return { repositories: repositories .map((repository: VcsRepo) => { diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts index fe964ee32b..f7063aae3a 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts @@ -9,6 +9,7 @@ import { PackageAdvisoryCatalog } from "@maple/backend/services/security/Package import osvFixture from "../security/__fixtures__/osv.json" import { DependencyScanner, DependencySources, MAX_LOCKFILE_BYTES } from "./DependencyScanner" import { DependencyScanStore } from "./DependencyScanStore" +import { MAX_LOCKFILES } from "./lockfile-selection" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) @@ -276,6 +277,29 @@ describe("DependencyScanner", () => { }).pipe(Effect.provide(scannerFor(state))) }) + it.effect("skips an unchanged push in a repository with more lockfiles than a scan reads", () => { + const many = Object.fromEntries( + Array.from({ length: MAX_LOCKFILES + 2 }, (_, i) => [ + `pkg${i}/package-lock.json`, + packageLock("4.17.15"), + ]), + ) + const state = world(many) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const first = yield* scanner.scan(installation, repository, { trigger: "schedule" }) + assert.strictEqual( + first._tag === "Scanned" + ? first.scan.lockfiles.filter((lockfile) => lockfile.status === "skipped").length + : 0, + 2, + ) + state.head = sha("b") + const again = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(again._tag, "Unchanged") + }).pipe(Effect.provide(scannerFor(state))) + }) + it.effect("reads a lockfile's packages from the parse cache on a rescan", () => { const state = world({ "package-lock.json": packageLock("4.17.15") }) return Effect.gen(function* () { diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts index 6e5d6d2999..d53a487720 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -271,7 +271,8 @@ export class DependencyScanner extends Context.Service Date: Sun, 1 Nov 2026 01:29:38 +0100 Subject: [PATCH 4/6] chore(domain): register the dependency scan API's anticipated errors Regenerated with scripts/gen-anticipated-errors.ts: the three 4xx errors of /internal/dependency-scan. --- packages/domain/src/generated/anticipated-error-identifiers.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index 6795e6a5c5..719e8df081 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -11,6 +11,9 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/anomalies/AnomalyForbiddenError", "@maple/http/anomalies/AnomalyIncidentNotFoundError", "@maple/http/anomalies/AnomalyLinkedIssueNotFoundError", + "@maple/http/dependency-scan/DependencyFindingNotFoundError", + "@maple/http/dependency-scan/DependencyScanForbiddenError", + "@maple/http/dependency-scan/DependencyScanRepositoryNotFoundError", "@maple/http/errors/ActorNotFoundError", "@maple/http/errors/AlertDestinationInUseError", "@maple/http/errors/AlertDestinationNotFoundError", From 370afe76a2e073cca3d1b32e68ba54084ea38cd8 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 01:38:39 +0100 Subject: [PATCH 5/6] fix(security): findings of a purged or removed repository are not listed Purging a repository or its installation leaves its dependency findings behind; they were listed as "unknown repository" forever. Only repositories the org still has access to show findings (a re-granted one shows them again). --- .../DependencyScanService.test.ts | 13 +++++++++++++ .../dependency-scan/DependencyScanService.ts | 19 ++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts index 105e8d6398..776f43278e 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts @@ -152,6 +152,19 @@ describe("DependencyScanService", () => { }).pipe(Effect.provide(serviceFor(sent))) }) + it.effect("hides the findings of a repository the org no longer has access to", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const vcs = yield* VcsRepository + const api = yield* seed + yield* recordScan(api.id, "a", true) + assert.strictEqual((yield* service.listFindings(org, {})).findings.length, 1) + yield* vcs.markRepositoryRemoved(api.id) + assert.deepStrictEqual((yield* service.listFindings(org, {})).findings, []) + }).pipe(Effect.provide(serviceFor(sent))) + }) + it.effect("names each finding's repository and records who dismissed it and why", () => { const sent: Array = [] return Effect.gen(function* () { diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index c5e68bab4b..2597d0fbe5 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -194,13 +194,18 @@ export class DependencyScanService extends Context.Service row.repositoryId), ) .pipe(Effect.mapError(persistence("Could not read repositories"))) - const names = new Map(repositories.map((repository) => [repository.id, repository.fullName])) - yield* Effect.annotateCurrentSpan({ "result.rowCount": rows.length }) - return { - findings: rows.map((row) => - findingViewOf(row, names.get(row.repositoryId) ?? "unknown repository"), - ), - } + // A purged or removed repository's findings stay in the table but are not shown. + const names = new Map( + repositories + .filter((repository) => repository.status === "active") + .map((repository) => [repository.id, repository.fullName]), + ) + const findings = rows.flatMap((row) => { + const name = names.get(row.repositoryId) + return name === undefined ? [] : [findingViewOf(row, name)] + }) + yield* Effect.annotateCurrentSpan({ "result.rowCount": findings.length }) + return { findings } }) const updateFinding: DependencyScanServiceApi["updateFinding"] = Effect.fn( From f416695d5203719b67091b487877e551b663c92b Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 04:10:39 +0100 Subject: [PATCH 6/6] fix(security): spend the findings row limit on connected repositories Removed and purged repositories' findings were dropped after the 500-row limit, so they could push active findings out of the list. The query now reads only the connected repositories' findings. --- .../dependency-scan/DependencyScanService.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index 2597d0fbe5..bd00bcc133 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -180,26 +180,20 @@ export class DependencyScanService extends Context.Service [repository.id, repository.fullName])) const rows = yield* store .listFindings({ orgId, limit: FINDINGS_LIMIT, status: query.status, repositoryId: query.repositoryId, + repositoryIds: [...names.keys()], }) .pipe(Effect.mapError(persistence("Could not list findings"))) - const repositories = yield* vcs - .getRepositoriesByIds( - orgId, - rows.map((row) => row.repositoryId), - ) - .pipe(Effect.mapError(persistence("Could not read repositories"))) - // A purged or removed repository's findings stay in the table but are not shown. - const names = new Map( - repositories - .filter((repository) => repository.status === "active") - .map((repository) => [repository.id, repository.fullName]), - ) const findings = rows.flatMap((row) => { const name = names.get(row.repositoryId) return name === undefined ? [] : [findingViewOf(row, name)]