From 3fdac25577af7c0505a76ae7f84d99380cd55cdd Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 01:01:17 +0100 Subject: [PATCH 01/15] feat(security): Dependencies tab, ranked by what runs in production The Security Center gets a Dependencies tab: a stat rail of open findings by severity, each connected repository with its last scan (lockfiles that were unsupported, too large, skipped or failed say why) and a Scan now button, and the vulnerable packages table with status and repository filters and a sheet for triage (dismiss or accept the risk with a reason, admins only). Findings are joined to telemetry: runningCommitsQuery reads which commit each service served traffic from over the last day (service overview rollups, vcs.ref.head.revision), the commits map to repositories through the synced commit history, and each finding lists the services running a commit whose scan matched it. The table ranks by severity, how the code depends on the package and that traffic, so a medium advisory in a busy service outranks a high one nothing deploys. Running commits not scanned yet (or whose scan failed over an hour ago) are queued, ten per read. Without commit telemetry the tab says how to add it. Review fixes to the scanner and catalog: blobs are read raw, lockfiles past the cap are recorded as skipped instead of read as deleted, a commit already scanned or being scanned is not read again, a tree too large to list fails once instead of on every push, the parse cache key carries a parser version, OSV paging is capped and a scan past the advisory cap stores what it read without reconciling. --- .../routes/internal/dependency-scan.http.ts | 4 +- apps/web/src/lab/entries/security.ts | 45 +- .../src/lab/security-dependencies-fixture.ts | 419 ++++++++++++++++++ apps/web/src/lab/security-lab/scenarios.ts | 116 +++++ apps/web/src/page/registry.ts | 2 + .../src/page/security/dependencies/command.ts | 74 ++++ .../dependencies/dependencies.scene.test.ts | 387 ++++++++++++++++ .../src/page/security/dependencies/derive.ts | 161 +++++++ .../src/page/security/dependencies/index.ts | 24 + .../src/page/security/dependencies/model.ts | 81 ++++ .../security/dependencies/subscription.ts | 33 ++ .../src/page/security/dependencies/update.ts | 245 ++++++++++ .../security/dependencies/view-findings.ts | 365 +++++++++++++++ .../dependencies/view-repositories.ts | 182 ++++++++ .../page/security/dependencies/view-sheet.ts | 279 ++++++++++++ .../page/security/dependencies/view-states.ts | 130 ++++++ .../src/page/security/dependencies/view.ts | 57 +++ apps/web/src/page/security/kit/layout.ts | 13 +- apps/web/src/route/app-route.ts | 1 + apps/web/src/route/route.test.ts | 1 + apps/web/src/route/routers.ts | 1 + packages/backend/package.json | 2 +- .../backend/scripts/depscan-vs-osv-scanner.ts | 2 +- .../DependencyScanService.test.ts | 193 +++++++- .../dependency-scan/DependencyScanService.ts | 356 +++++++++++++-- .../dependency-scan/DependencyScanner.test.ts | 24 + .../dependency-scan/DependencyScanner.ts | 96 +++- .../lockfile-selection.test.ts | 8 +- .../dependency-scan/lockfile-selection.ts | 7 +- .../dependency-scan/production.test.ts | 51 +++ .../services/dependency-scan/production.ts | 134 ++++++ .../vcs/vendor/github/GithubAppClient.ts | 32 +- .../vcs/vendor/github/GithubProvider.ts | 11 +- .../security/PackageAdvisoryCatalog.test.ts | 27 +- .../security/PackageAdvisoryCatalog.ts | 75 ++-- packages/domain/src/http/dependency-scan.ts | 41 +- .../src/__sql_baseline__/catalog.sql | 56 +++ .../query-engine/src/benchmark/builders.ts | 7 + packages/query-engine/src/ch/index.ts | 3 + .../query-engine/src/ch/queries/releases.ts | 38 ++ 40 files changed, 3641 insertions(+), 142 deletions(-) create mode 100644 apps/web/src/lab/security-dependencies-fixture.ts create mode 100644 apps/web/src/page/security/dependencies/command.ts create mode 100644 apps/web/src/page/security/dependencies/dependencies.scene.test.ts create mode 100644 apps/web/src/page/security/dependencies/derive.ts create mode 100644 apps/web/src/page/security/dependencies/index.ts create mode 100644 apps/web/src/page/security/dependencies/model.ts create mode 100644 apps/web/src/page/security/dependencies/subscription.ts create mode 100644 apps/web/src/page/security/dependencies/update.ts create mode 100644 apps/web/src/page/security/dependencies/view-findings.ts create mode 100644 apps/web/src/page/security/dependencies/view-repositories.ts create mode 100644 apps/web/src/page/security/dependencies/view-sheet.ts create mode 100644 apps/web/src/page/security/dependencies/view-states.ts create mode 100644 apps/web/src/page/security/dependencies/view.ts create mode 100644 packages/backend/src/services/dependency-scan/production.test.ts create mode 100644 packages/backend/src/services/dependency-scan/production.ts diff --git a/apps/api/src/routes/internal/dependency-scan.http.ts b/apps/api/src/routes/internal/dependency-scan.http.ts index b587dc8ef0..0cc364ddfd 100644 --- a/apps/api/src/routes/internal/dependency-scan.http.ts +++ b/apps/api/src/routes/internal/dependency-scan.http.ts @@ -10,7 +10,7 @@ export const HttpDependencyScanLive = HttpApiBuilder.group(MapleInternalApi, "de return handlers .handle("listRepositories", () => - Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listRepositories(tenant.orgId)), + Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listRepositories(tenant)), ) .handle("scanRepository", ({ params }) => Effect.flatMap(CurrentTenant.Context, (tenant) => @@ -18,7 +18,7 @@ export const HttpDependencyScanLive = HttpApiBuilder.group(MapleInternalApi, "de ), ) .handle("listFindings", ({ query }) => - Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listFindings(tenant.orgId, query)), + Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listFindings(tenant, query)), ) .handle("updateFinding", ({ params, payload }) => Effect.gen(function* () { diff --git a/apps/web/src/lab/entries/security.ts b/apps/web/src/lab/entries/security.ts index 5a001f52e9..15b5ffa95a 100644 --- a/apps/web/src/lab/entries/security.ts +++ b/apps/web/src/lab/entries/security.ts @@ -2,6 +2,9 @@ import { Clock, Effect, Option, Schema } from "effect" import { Command } from "foldkit" import type { Html, HtmlBuilder } from "foldkit/html" import { defineMessageUnion } from "foldkit/message" +import * as DependenciesPage from "../../page/security/dependencies/model" +import * as DependenciesUpdate from "../../page/security/dependencies/update" +import { view as dependenciesView } from "../../page/security/dependencies/view" import * as FindingPage from "../../page/security/finding/model" import * as FindingUpdate from "../../page/security/finding/update" import { breadcrumbsOf as findingCrumbs, view as findingView } from "../../page/security/finding/view" @@ -27,6 +30,7 @@ const Model = Schema.Struct({ run: Schema.NullOr(RunPage.Model), finding: Schema.NullOr(FindingPage.Model), telemetry: Schema.NullOr(TelemetryPage.Model), + dependencies: Schema.NullOr(DependenciesPage.Model), }) type Model = typeof Model.Type @@ -35,6 +39,7 @@ const Message = defineMessageUnion({ GotRunMessage: { message: RunPage.Message }, GotFindingMessage: { message: FindingPage.Message }, GotTelemetryMessage: { message: TelemetryPage.Message }, + GotDependenciesMessage: { message: DependenciesPage.Message }, }) type Message = typeof Message.Type @@ -54,7 +59,14 @@ const flags = Effect.all({ const init = (input: typeof Flags.Type) => { const built = build(input.scenario, input.nowMs) - const empty: Model = { scenario: input.scenario, tabs: null, run: null, finding: null, telemetry: null } + const empty: Model = { + scenario: input.scenario, + tabs: null, + run: null, + finding: null, + telemetry: null, + dependencies: null, + } switch (built._tag) { case "Tabs": return { @@ -84,6 +96,13 @@ const init = (input: typeof Flags.Type) => { Message.GotTelemetryMessage({ message }), ), } + case "Dependencies": + return { + model: { ...empty, dependencies: built.step.model }, + commands: Command.mapMessages(built.step.commands, (message) => + Message.GotDependenciesMessage({ message }), + ), + } } } @@ -129,6 +148,16 @@ const update = (model: Model, message: Message) => ), } }, + GotDependenciesMessage: ({ message: inner }) => { + if (model.dependencies === null) return { model } + const next = DependenciesUpdate.update(model.dependencies, inner, context) + return { + model: { ...model, dependencies: next.model }, + commands: Command.mapMessages((next.commands ?? []).filter(isLocalCommand), (child) => + Message.GotDependenciesMessage({ message: child }), + ), + } + }, }) const view = (model: Model, h: HtmlBuilder): Html => @@ -164,7 +193,15 @@ const view = (model: Model, h: HtmlBuilder): Html => viewInputs: { context }, toParentMessage: (message) => Message.GotTelemetryMessage({ message }), }) - : h.empty + : model.dependencies !== null + ? h.submodel({ + slotId: "security-lab-dependencies", + model: model.dependencies, + view: dependenciesView, + viewInputs: { context }, + toParentMessage: (message) => Message.GotDependenciesMessage({ message }), + }) + : h.empty export const lab = defineShellLab("Security Center", { Model, @@ -184,5 +221,7 @@ export const lab = defineShellLab("Security Center", { ? findingCrumbs(model.finding) : model.telemetry !== null ? tabsCrumbs(model.telemetry.tab) - : [], + : model.dependencies !== null + ? tabsCrumbs("dependencies") + : [], }) diff --git a/apps/web/src/lab/security-dependencies-fixture.ts b/apps/web/src/lab/security-dependencies-fixture.ts new file mode 100644 index 0000000000..a1c0d09969 --- /dev/null +++ b/apps/web/src/lab/security-dependencies-fixture.ts @@ -0,0 +1,419 @@ +import { DependencyFindingId, DependencyScanId } from "@maple/domain/dependency-scan" +import { + type DependencyFindingsResponse, + type DependencyFindingView, + type DependencyScanRepositoriesResponse, + type DependencyScanRepository, + GitCommitSha, + VcsRepositoryId, +} from "@maple/domain/http" +import { Schema } from "effect" + +// Fixtures for the Security Center's Dependencies tab, for `/lab/security` and the page's scene +// tests. Built relative to `now` so relative times read the same. + +const HOUR = 3_600_000 +const DAY = 24 * HOUR + +const repositoryId = Schema.decodeSync(VcsRepositoryId) +const scanId = Schema.decodeSync(DependencyScanId) +const findingId = Schema.decodeSync(DependencyFindingId) +const sha = Schema.decodeSync(GitCommitSha) + +export const REPOSITORY_IDS = { + payments: repositoryId("6f1c2d3e-4a5b-4c6d-8e7f-901a2b3c4d5e"), + web: repositoryId("7a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d"), + cli: repositoryId("8b3c4d5e-6f7a-4b8c-9d0e-1f2a3b4c5d6e"), +} as const + +const SCAN_IDS = { + payments: scanId("1a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d"), + web: scanId("2b3c4d5e-6f7a-4b8c-9d0e-1f2a3b4c5d6e"), + cli: scanId("3c4d5e6f-7a8b-4c9d-8e0f-2a3b4c5d6e7f"), + older: scanId("4d5e6f7a-8b9c-4d0e-9f1a-3b4c5d6e7f8a"), +} as const + +const SHAS = { + payments: sha("4e1d9c2a7b3f8e6d5c4b3a2918f7e6d5c4b3a291"), + web: sha("9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0b"), + cli: sha("c0ffee1234567890abcdef1234567890abcdef12"), + older: sha("0123456789abcdef0123456789abcdef01234567"), +} as const + +export const PAYMENTS_ID = REPOSITORY_IDS.payments + +const service = (serviceName: string, commitSha: GitCommitSha, spanCount: number) => ({ + serviceName, + environment: "production", + commitSha, + spanCount, +}) + +/** What production ran over the last day: payments-api and web are deployed, cli is not. */ +const RUNNING = { + payments: [ + service("api-gateway", SHAS.payments, 2_400_000), + service("payments-worker", SHAS.payments, 180_000), + ], + web: [service("web-frontend", SHAS.web, 640_000)], +} as const + +const payments = (nowMs: number): DependencyScanRepository => ({ + id: REPOSITORY_IDS.payments, + fullName: "acme/payments-api", + htmlUrl: "https://github.com/acme/payments-api", + runningCommits: [{ commitSha: SHAS.payments, scanStatus: "completed", services: RUNNING.payments }], + trackedBranch: "main", + latestScan: { + id: SCAN_IDS.payments, + commitSha: SHAS.payments, + ref: "refs/heads/main", + trigger: "push", + status: "completed", + lockfiles: [ + { + path: "pnpm-lock.yaml", + kind: "pnpm-lock", + blobSha: "a1", + packageCount: 1184, + status: "parsed", + message: null, + }, + { + path: "services/ledger/go.sum", + kind: "go-sum", + blobSha: "a2", + packageCount: 96, + status: "parsed", + message: null, + }, + { + path: "tools/migrate/Cargo.lock", + kind: "cargo-lock", + blobSha: "a3", + packageCount: 0, + status: "failed", + message: "Line 212: expected a [[package]] table.", + }, + ], + packageCount: 1280, + vulnerableCount: 5, + error: null, + startedAt: nowMs - 2 * HOUR - 40_000, + finishedAt: nowMs - 2 * HOUR, + }, +}) + +const web = (nowMs: number): DependencyScanRepository => ({ + id: REPOSITORY_IDS.web, + fullName: "acme/web", + htmlUrl: "https://github.com/acme/web", + runningCommits: [{ commitSha: SHAS.web, scanStatus: "completed", services: RUNNING.web }], + trackedBranch: "main", + latestScan: { + id: SCAN_IDS.web, + commitSha: SHAS.web, + ref: "refs/heads/main", + trigger: "push", + status: "completed", + lockfiles: [ + { + path: "bun.lockb", + kind: "bun-lockb", + blobSha: "b1", + packageCount: 0, + status: "unsupported", + message: + "Binary bun.lockb is not readable. Run `bun install --save-text-lockfile` and commit bun.lock.", + }, + { + path: "docs/package-lock.json", + kind: "package-lock", + blobSha: "b2", + packageCount: 412, + status: "parsed", + message: null, + }, + ], + packageCount: 412, + vulnerableCount: 2, + error: null, + startedAt: nowMs - DAY - 30_000, + finishedAt: nowMs - DAY, + }, +}) + +const cli = (nowMs: number): DependencyScanRepository => ({ + id: REPOSITORY_IDS.cli, + fullName: "acme/cli", + htmlUrl: "https://github.com/acme/cli", + runningCommits: [], + trackedBranch: "release", + latestScan: { + id: SCAN_IDS.cli, + commitSha: SHAS.cli, + ref: "refs/heads/release", + trigger: "schedule", + status: "completed", + lockfiles: [ + { + path: "Cargo.lock", + kind: "cargo-lock", + blobSha: "c1", + packageCount: 214, + status: "parsed", + message: null, + }, + ], + packageCount: 214, + vulnerableCount: 1, + error: null, + startedAt: nowMs - 3 * DAY - 20_000, + finishedAt: nowMs - 3 * DAY, + }, +}) + +export const repositoriesOf = (nowMs: number): DependencyScanRepositoriesResponse => ({ + repositories: [payments(nowMs), web(nowMs), cli(nowMs)], +}) + +/** Connected, before the repository's first scan. */ +export const neverScannedOf = (nowMs: number): DependencyScanRepositoriesResponse => ({ + repositories: [{ ...payments(nowMs), latestScan: null }], +}) + +export const noRepositories: DependencyScanRepositoriesResponse = { repositories: [] } + +/** The payments repository scanned once more, later, as a completed Scan now would leave it. */ +export const rescannedOf = (nowMs: number): DependencyScanRepositoriesResponse => { + const [first, ...rest] = repositoriesOf(nowMs).repositories + return first === undefined || first.latestScan === null + ? repositoriesOf(nowMs) + : { + repositories: [ + { + ...first, + latestScan: { + ...first.latestScan, + id: SCAN_IDS.older, + commitSha: SHAS.older, + finishedAt: nowMs, + }, + }, + ...rest, + ], + } +} + +export const FINDING_IDS = { + axios: findingId("0a1b2c3d4e5f60718293a4b5c6d7e8f9"), + xnet: findingId("1b2c3d4e5f60718293a4b5c6d7e8f90a"), + vite: findingId("2c3d4e5f60718293a4b5c6d7e8f90a1b"), + tokio: findingId("3d4e5f60718293a4b5c6d7e8f90a1b2c"), + marked: findingId("4e5f60718293a4b5c6d7e8f90a1b2c3d"), + lodash: findingId("5f60718293a4b5c6d7e8f90a1b2c3d4e"), + jwt: findingId("60718293a4b5c6d7e8f90a1b2c3d4e5f"), +} as const + +type FindingSeed = Pick< + DependencyFindingView, + | "id" + | "repositoryId" + | "repositoryFullName" + | "lockfilePath" + | "ecosystem" + | "packageName" + | "advisoryId" + | "aliases" + | "severity" + | "summary" + | "versions" + | "fixedIn" + | "relation" +> + +const finding = ( + nowMs: number, + seed: FindingSeed, + firstSeenAgoMs: number, + production: DependencyFindingView["production"] = [], +): DependencyFindingView => { + const scan = + seed.repositoryId === REPOSITORY_IDS.payments + ? "payments" + : seed.repositoryId === REPOSITORY_IDS.web + ? "web" + : "cli" + return { + ...seed, + status: "open", + firstSeenScanId: SCAN_IDS.older, + lastSeenScanId: SCAN_IDS[scan], + firstSeenSha: SHAS.older, + lastSeenSha: SHAS[scan], + fixedScanId: null, + fixedSha: null, + statusReason: null, + statusChangedBy: null, + statusChangedAt: null, + createdAt: nowMs - firstSeenAgoMs, + updatedAt: nowMs - 2 * HOUR, + production, + } +} + +const PAYMENTS = { repositoryId: REPOSITORY_IDS.payments, repositoryFullName: "acme/payments-api" } as const +const WEB = { repositoryId: REPOSITORY_IDS.web, repositoryFullName: "acme/web" } as const +const CLI = { repositoryId: REPOSITORY_IDS.cli, repositoryFullName: "acme/cli" } as const + +/** Worst first, as the server sorts them: one per severity, a dismissed and a fixed finding. */ +export const findingsOf = (nowMs: number): DependencyFindingsResponse => ({ + production: { available: true, runningServices: 4, linkedServices: 3 }, + findings: [ + finding( + nowMs, + { + ...PAYMENTS, + id: FINDING_IDS.axios, + lockfilePath: "pnpm-lock.yaml", + ecosystem: "npm", + packageName: "axios", + advisoryId: "GHSA-jr5f-v2jv-69x6", + aliases: ["CVE-2025-27152"], + severity: "critical", + summary: + "axios requests to absolute URLs ignore baseURL, so a crafted URL can leak credentials to another host.", + versions: ["1.7.4"], + fixedIn: "1.8.2", + relation: "direct", + }, + 9 * DAY, + RUNNING.payments, + ), + finding( + nowMs, + { + ...PAYMENTS, + id: FINDING_IDS.xnet, + lockfilePath: "services/ledger/go.sum", + ecosystem: "Go", + packageName: "golang.org/x/net", + advisoryId: "GO-2025-3503", + aliases: ["CVE-2025-22870", "GHSA-qxp5-gwg8-xv66"], + severity: "high", + summary: "HTTP proxy bypass using IPv6 zone IDs.", + versions: ["0.33.0"], + fixedIn: "0.36.0", + relation: "transitive", + }, + 4 * DAY, + [RUNNING.payments[1]], + ), + finding( + nowMs, + { + ...PAYMENTS, + id: FINDING_IDS.vite, + lockfilePath: "pnpm-lock.yaml", + ecosystem: "npm", + packageName: "vite", + advisoryId: "GHSA-vg6x-rcgg-rjx6", + aliases: ["CVE-2025-24010"], + severity: "medium", + summary: + "Websites were able to send any requests to the development server and read the response.", + versions: ["5.4.10", "6.0.7"], + fixedIn: "6.0.9", + relation: "dev", + }, + 2 * DAY, + ), + finding( + nowMs, + { + ...CLI, + id: FINDING_IDS.tokio, + lockfilePath: "Cargo.lock", + ecosystem: "crates.io", + packageName: "tokio", + advisoryId: "RUSTSEC-2025-0023", + aliases: ["GHSA-rr8g-9fpq-6wmg"], + severity: "low", + summary: "Broadcast channel calls clone in parallel, but does not require Sync.", + versions: ["1.38.0"], + fixedIn: "1.38.2", + relation: "transitive", + }, + 20 * DAY, + ), + finding( + nowMs, + { + ...WEB, + id: FINDING_IDS.marked, + lockfilePath: "docs/package-lock.json", + ecosystem: "npm", + packageName: "marked", + advisoryId: "GHSA-4r62-v4vq-hr96", + aliases: [], + severity: "unknown", + summary: null, + versions: ["4.0.9"], + fixedIn: null, + relation: "unknown", + }, + DAY, + ), + { + ...finding( + nowMs, + { + ...WEB, + id: FINDING_IDS.lodash, + lockfilePath: "docs/package-lock.json", + ecosystem: "npm", + packageName: "lodash.template", + advisoryId: "GHSA-35jh-r3h4-6jhm", + aliases: ["CVE-2021-23337"], + severity: "high", + summary: "Command injection in lodash template.", + versions: ["4.5.0"], + fixedIn: null, + relation: "dev", + }, + 30 * DAY, + ), + status: "dismissed", + statusReason: "Only used by the docs build, never with user input.", + statusChangedAt: nowMs - 6 * DAY, + }, + { + ...finding( + nowMs, + { + ...PAYMENTS, + id: FINDING_IDS.jwt, + lockfilePath: "pnpm-lock.yaml", + ecosystem: "npm", + packageName: "jsonwebtoken", + advisoryId: "GHSA-8cf7-32gw-wr33", + aliases: ["CVE-2022-23539"], + severity: "high", + summary: "Insecure key types can be used for signature verification.", + versions: ["8.5.1"], + fixedIn: "9.0.0", + relation: "direct", + }, + 40 * DAY, + ), + status: "fixed", + fixedScanId: SCAN_IDS.payments, + fixedSha: SHAS.payments, + }, + ], +}) + +export const noFindings: DependencyFindingsResponse = { + findings: [], + production: { available: true, runningServices: 3, linkedServices: 3 }, +} diff --git a/apps/web/src/lab/security-lab/scenarios.ts b/apps/web/src/lab/security-lab/scenarios.ts index 018db34d5a..2c8782d742 100644 --- a/apps/web/src/lab/security-lab/scenarios.ts +++ b/apps/web/src/lab/security-lab/scenarios.ts @@ -1,5 +1,8 @@ import { OrgId } from "@maple/domain" import { + DependencyScanPersistenceError, + type DependencyFindingsResponse, + type DependencyScanRepositoriesResponse, SecurityAuditDisabledError, SecurityAuditForbiddenError, SecurityAuditTargetExistsError, @@ -10,6 +13,8 @@ import type { Command } from "foldkit" import { toLoadFailure } from "../../data/load-failure" import type { PageContext } from "../../page/contract" import * as AddTarget from "../../page/security/kit/add-target" +import * as DependenciesPage from "../../page/security/dependencies/model" +import * as DependenciesUpdate from "../../page/security/dependencies/update" import * as FindingPage from "../../page/security/finding/model" import * as FindingUpdate from "../../page/security/finding/update" import * as RunPage from "../../page/security/run/model" @@ -39,6 +44,14 @@ import { targetsOf, unitsOf, } from "../security-fixture" +import { + FINDING_IDS, + findingsOf as dependencyFindingsOf, + neverScannedOf, + noFindings, + noRepositories, + repositoriesOf, +} from "../security-dependencies-fixture" import { emptyExposureOf, exposureOf, runtimesOf } from "../security-telemetry-fixture" // Every Security Center state the lab renders, built by sending each page the fixture answers its @@ -68,6 +81,13 @@ export const SCENARIOS = [ "exposure", "exposure-empty", "runtimes", + "dependencies", + "dependencies-finding", + "dependencies-clean", + "dependencies-never-scanned", + "dependencies-no-github", + "dependencies-error", + "dependencies-loading", ] as const export const Scenario = Schema.Literals(SCENARIOS) export type Scenario = typeof Scenario.Type @@ -319,6 +339,56 @@ const exposure = (nowMs: number, data: TelemetryPage.Exposure): TelemetryStep => T.CompletedFetchExposure({ key: model.exposure.key ?? "", result: Result.succeed(data) }), ) +// DEPENDENCIES + +type DependenciesStep = Step +const D = DependenciesPage.Message + +type DependencyAnswers = Readonly<{ + repositories: DependencyScanRepositoriesResponse | "failed" + findings: DependencyFindingsResponse +}> + +const dependencies = ( + nowMs: number, + answers: DependencyAnswers | null, + then: ReadonlyArray<(model: DependenciesPage.Model) => DependenciesPage.Message> = [], +): DependenciesStep => { + const entered = DependenciesUpdate.init(AppRoute.SecurityDependencies(), context) + return fold( + { model: entered.model, commands: entered.commands ?? [] }, + (model, message) => DependenciesUpdate.update(model, message, context), + [ + () => D.ReadClock({ nowMs }), + () => D.CompletedReadIsAdmin({ isAdmin: true }), + ...(answers === null + ? [] + : [ + (model: DependenciesPage.Model) => + D.CompletedFetchRepositories({ + key: model.repositories.key ?? "", + result: + answers.repositories === "failed" + ? Result.fail( + toLoadFailure( + new DependencyScanPersistenceError({ + message: "Dependency scans are unavailable", + }), + ), + ) + : Result.succeed(answers.repositories), + }), + (model: DependenciesPage.Model) => + D.CompletedFetchFindings({ + key: model.findings.key ?? "", + result: Result.succeed(answers.findings), + }), + ]), + ...then, + ], + ) +} + // SCENARIOS export type Built = @@ -326,6 +396,7 @@ export type Built = | Readonly<{ _tag: "Run"; step: RunStep }> | Readonly<{ _tag: "Finding"; step: FindingStep }> | Readonly<{ _tag: "Telemetry"; step: TelemetryStep }> + | Readonly<{ _tag: "Dependencies"; step: DependenciesStep }> const overviewRoute = AppRoute.Security() @@ -450,5 +521,50 @@ export const build = (scenario: Scenario, nowMs: number): Built => { }), ), } + case "dependencies": + return { _tag: "Dependencies", step: populatedDependencies(nowMs) } + case "dependencies-finding": + return { + _tag: "Dependencies", + step: populatedDependencies(nowMs, [ + () => D.ClickedFinding({ findingId: FINDING_IDS.axios }), + ]), + } + case "dependencies-clean": + return { + _tag: "Dependencies", + step: dependencies(nowMs, { + repositories: repositoriesOf(nowMs), + findings: { + ...dependencyFindingsOf(nowMs), + findings: dependencyFindingsOf(nowMs).findings.filter( + (finding) => finding.status !== "open", + ), + }, + }), + } + case "dependencies-never-scanned": + return { + _tag: "Dependencies", + step: dependencies(nowMs, { repositories: neverScannedOf(nowMs), findings: noFindings }), + } + case "dependencies-no-github": + return { + _tag: "Dependencies", + step: dependencies(nowMs, { repositories: noRepositories, findings: noFindings }), + } + case "dependencies-error": + return { + _tag: "Dependencies", + step: dependencies(nowMs, { repositories: "failed", findings: noFindings }), + } + case "dependencies-loading": + return { _tag: "Dependencies", step: dependencies(nowMs, null) } } } + +const populatedDependencies = ( + nowMs: number, + then: ReadonlyArray<(model: DependenciesPage.Model) => DependenciesPage.Message> = [], +): DependenciesStep => + dependencies(nowMs, { repositories: repositoriesOf(nowMs), findings: dependencyFindingsOf(nowMs) }, then) diff --git a/apps/web/src/page/registry.ts b/apps/web/src/page/registry.ts index ba0868b657..b400b87a99 100644 --- a/apps/web/src/page/registry.ts +++ b/apps/web/src/page/registry.ts @@ -33,6 +33,7 @@ import * as DashboardTemplates from "./dashboards/templates" import * as DashboardView from "./dashboards/view" import * as DashboardWidget from "./dashboards/widget" import * as CodeReview from "./code-review" +import * as SecurityDependencies from "./security/dependencies" import * as SecurityFinding from "./security/finding" import * as SecurityRun from "./security/run" import * as SecurityTabs from "./security/tabs" @@ -152,6 +153,7 @@ export const pages = [ CodeReview.page, SecurityTabs.page, SecurityTelemetry.page, + SecurityDependencies.page, SecurityRun.page, SecurityFinding.page, AgentSessionsList.page, diff --git a/apps/web/src/page/security/dependencies/command.ts b/apps/web/src/page/security/dependencies/command.ts new file mode 100644 index 0000000000..b4a551ab36 --- /dev/null +++ b/apps/web/src/page/security/dependencies/command.ts @@ -0,0 +1,74 @@ +import { DependencyFindingId, DependencyFindingTriageStatus } from "@maple/domain/dependency-scan" +import { VcsRepositoryId } from "@maple/domain/http" +import { Clock, Effect, Schema } from "effect" +import { Command } from "foldkit" +import { toLoadFailure } from "../../../data/load-failure" +import { MapleInternalApiClient } from "../../../platform/api-clients" +import { readIsAdmin } from "../../alerts/shared/commands" +import { Message } from "./model" + +// The Dependencies tab's reads and writes on the api Worker. Names start with `SecurityApi` (and +// the clock's and session's with `ReadSecurity`) so the lab drops them. + +type Api = typeof MapleInternalApiClient.Service + +const call = (run: (api: Api) => Effect.Effect) => + Effect.gen(function* () { + const api = yield* MapleInternalApiClient + return yield* run(api) + }).pipe(Effect.mapError(toLoadFailure), Effect.result) + +export const ReadDependenciesClock = Command.define("ReadSecurityDependenciesClock", { + messages: [Message.ReadClock], + execute: Effect.map(Clock.currentTimeMillis, (nowMs) => Message.ReadClock({ nowMs })), +}) + +export const ReadDependenciesIsAdmin = Command.define("ReadSecurityDependenciesIsAdmin", { + messages: [Message.CompletedReadIsAdmin], + execute: Effect.map(readIsAdmin, (isAdmin) => Message.CompletedReadIsAdmin({ isAdmin })), +}) + +export const FetchRepositories = Command.define("SecurityApiDependencyRepositories", { + args: { key: Schema.String }, + messages: [Message.CompletedFetchRepositories], + execute: ({ key }) => + Effect.map( + call((api) => api.dependencyScan.listRepositories()), + (result) => Message.CompletedFetchRepositories({ key, result }), + ), +}) + +export const FetchFindings = Command.define("SecurityApiDependencyFindings", { + args: { key: Schema.String }, + messages: [Message.CompletedFetchFindings], + execute: ({ key }) => + Effect.map( + call((api) => api.dependencyScan.listFindings({ query: {} })), + (result) => Message.CompletedFetchFindings({ key, result }), + ), +}) + +export const ScanRepository = Command.define("SecurityApiScanRepository", { + args: { repositoryId: VcsRepositoryId }, + messages: [Message.CompletedScanRequest], + execute: ({ repositoryId }) => + Effect.map( + call((api) => api.dependencyScan.scanRepository({ params: { repositoryId } })), + (result) => Message.CompletedScanRequest({ repositoryId, result }), + ), +}) + +export const UpdateFinding = Command.define("SecurityApiUpdateDependencyFinding", { + args: { findingId: DependencyFindingId, status: DependencyFindingTriageStatus, reason: Schema.String }, + messages: [Message.CompletedTriage], + execute: ({ findingId, status, reason }) => + Effect.map( + call((api) => + api.dependencyScan.updateFinding({ + params: { findingId }, + payload: reason === "" ? { status } : { status, reason }, + }), + ), + (result) => Message.CompletedTriage({ result }), + ), +}) diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts new file mode 100644 index 0000000000..5e4ae85856 --- /dev/null +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -0,0 +1,387 @@ +import { OrgId } from "@maple/domain" +import { DependencyScanForbiddenError, DependencyScanPersistenceError } from "@maple/domain/http" +import type { DependencyFindingsResponse, DependencyScanRepositoriesResponse } from "@maple/domain/http" +import { Result, Schema } from "effect" +import { + click, + Command, + expect as expectView, + expectOutMessage, + given, + role, + scene, + text, + type, + withViewInputs, + Mount, +} from "foldkit/scene" +import { describe, expect, test } from "vitest" +import { toLoadFailure } from "../../../data/load-failure" +import { + FINDING_IDS, + findingsOf, + neverScannedOf, + noFindings, + noRepositories, + PAYMENTS_ID, + repositoriesOf, + rescannedOf, +} from "../../../lab/security-dependencies-fixture" +import { AppRoute, hrefOf } from "../../../route" +import * as Dialog from "../../../ui/dialog" +import * as Field from "../../../ui/field" +import * as Select from "../../../ui/select" +import { type PageContext, PageOutMessage } from "../../contract" +import { UpdateFinding } from "./command" +import { Message, type Model } from "./model" +import { init, update } from "./update" +import { view as pageView } from "./view" + +const context: PageContext = { + orgId: Schema.decodeSync(OrgId)("org_acme"), + pinnedNamespace: null, + timeZone: "UTC", +} +const NOW = Date.UTC(2026, 10, 1, 9, 0) +const route = AppRoute.SecurityDependencies() +const pageUpdate = (model: Model, message: Message) => update(model, message, context) +const view = withViewInputs(pageView, { context })() +const names = (commands: ReadonlyArray<{ readonly name: string }> | undefined) => + (commands ?? []).map(({ name }) => name) +const apply = (model: Model, messages: ReadonlyArray<(model: Model) => Message>): Model => + messages.reduce((current, toMessage) => pageUpdate(current, toMessage(current)).model, model) + +const loaded = ( + repositories: DependencyScanRepositoriesResponse = repositoriesOf(NOW), + findings: DependencyFindingsResponse = findingsOf(NOW), + isAdmin = true, +): Model => + apply(init(route, context).model, [ + () => Message.ReadClock({ nowMs: NOW }), + () => Message.CompletedReadIsAdmin({ isAdmin }), + (model) => + Message.CompletedFetchRepositories({ + key: model.repositories.key ?? "", + result: Result.succeed(repositories), + }), + (model) => + Message.CompletedFetchFindings({ + key: model.findings.key ?? "", + result: Result.succeed(findings), + }), + ]) + +const forbiddenFailure = toLoadFailure( + new DependencyScanForbiddenError({ message: "Only org admins can triage findings" }), +) +const forbidden = Result.fail(forbiddenFailure) + +const mounted = Mount.resolve(Dialog.MountDialog, Dialog.Message.CompletedMountDialog()) + +const opened = (model: Model, findingId = FINDING_IDS.axios): Model => + pageUpdate(model, Message.ClickedFinding({ findingId })).model + +describe("security dependencies tab", () => { + test("entering reads the repositories and the findings", () => { + expect(names(init(route, context).commands)).toEqual( + expect.arrayContaining(["SecurityApiDependencyRepositories", "SecurityApiDependencyFindings"]), + ) + }) + + test("while loading, the tab strip shows and the content is busy", () => { + scene( + { update: pageUpdate, view }, + given(init(route, context).model), + expectView(role("link", { name: /Dependencies/ })).toExist(), + expectView(text("Repositories")).toBeAbsent(), + ) + }) + + test("a failed read shows the error with a retry that reads again", () => { + const failed = apply(init(route, context).model, [ + (model) => + Message.CompletedFetchRepositories({ + key: model.repositories.key ?? "", + result: Result.fail(toLoadFailure(new Error("unexpected body"))), + }), + ]) + scene( + { update: pageUpdate, view }, + given(failed), + expectView(text("Couldn't load dependency scans")).toExist(), + expectView(role("button", { name: "Try again" })).toExist(), + ) + expect(names(pageUpdate(failed, Message.ClickedRetry()).commands)).toContain( + "SecurityApiDependencyRepositories", + ) + }) + + test("with no connected repository, the tab is the GitHub on-ramp", () => { + scene( + { update: pageUpdate, view }, + given(loaded(noRepositories, noFindings)), + expectView(text("Connect a repository to scan its dependencies")).toExist(), + expectView(role("link", { name: "Connect GitHub" })).toHaveAttr( + "href", + hrefOf(AppRoute.Integrations({ search: {} })), + ), + expectView(text("Vulnerable packages")).toBeAbsent(), + ) + }) + + test("connected but never scanned says the first scan is coming and offers Scan now", () => { + scene( + { update: pageUpdate, view }, + given(loaded(neverScannedOf(NOW), noFindings)), + expectView(text("First scan is on its way")).toExist(), + expectView(text("Not scanned yet")).toExist(), + expectView(text("Vulnerable packages")).toBeAbsent(), + ) + }) + + test("names the busiest service running each finding, and says when one is not deployed", () => { + scene( + { update: pageUpdate, view }, + given(loaded(repositoriesOf(NOW), findingsOf(NOW))), + expectView(text("api-gateway +1")).toExist(), + expectView(text("2.6M spans/day")).toExist(), + expectView(text("payments-worker")).toExist(), + expectView(text("Not running")).toExist(), + expectView(text(/vcs\.ref\.head\.revision/)).toBeAbsent(), + ) + }) + + test("explains why findings cannot be tied to production when no service reports its commit", () => { + scene( + { update: pageUpdate, view }, + given( + loaded(repositoriesOf(NOW), { + ...findingsOf(NOW), + findings: findingsOf(NOW).findings.map((finding) => ({ ...finding, production: [] })), + production: { available: true, runningServices: 0, linkedServices: 0 }, + }), + ), + expectView(text(/vcs\.ref\.head\.revision/)).toExist(), + expectView(text("Unknown")).toExist(), + expectView(text("api-gateway +1")).toBeAbsent(), + ) + }) + + test("scanned with nothing open shows the calm clean state", () => { + const settled = findingsOf(NOW).findings.filter((finding) => finding.status !== "open") + scene( + { update: pageUpdate, view }, + given(loaded(repositoriesOf(NOW), { ...findingsOf(NOW), findings: settled })), + expectView(text("No known vulnerable packages")).toExist(), + expectView(text("axios")).toBeAbsent(), + ) + }) + + test("the populated tab shows the rail, repository problems and open findings worst first", () => { + scene( + { update: pageUpdate, view }, + given(loaded()), + expectView(text("3 / 3")).toExist(), + expectView(text("open, and 1 unrated")).toExist(), + expectView(text(/commit bun\.lock/)).toExist(), + expectView(text("Line 212: expected a [[package]] table.")).toExist(), + expectView(role("link", { name: "CVE-2025-27152" })).toHaveAttr( + "href", + "https://osv.dev/vulnerability/GHSA-jr5f-v2jv-69x6", + ), + // An advisory with no CVE alias shows its own id. + expectView(role("link", { name: "RUSTSEC-2025-0023" })).toExist(), + expectView(text("No fix yet")).toExist(), + expectView(text("lodash.template")).toBeAbsent(), + expectView(text("jsonwebtoken")).toBeAbsent(), + ) + }) + + test("the status filter shows dismissed and fixed findings, and All shows every one", () => { + const status = (value: string) => () => + Message.GotStatusSelectMessage({ message: Select.Message.SelectedItem({ index: 0, value }) }) + scene( + { update: pageUpdate, view }, + given(apply(loaded(), [status("dismissed")])), + expectView(text("lodash.template")).toExist(), + expectView(text("axios")).toBeAbsent(), + ) + scene( + { update: pageUpdate, view }, + given(apply(loaded(), [status("all")])), + expectView(text("lodash.template")).toExist(), + expectView(text("jsonwebtoken")).toExist(), + expectView(text("axios")).toExist(), + ) + }) + + test("the repository filter keeps one repository's findings", () => { + const filtered = apply(loaded(), [ + () => + Message.GotRepositorySelectMessage({ + message: Select.Message.SelectedItem({ index: 1, value: PAYMENTS_ID }), + }), + ]) + scene( + { update: pageUpdate, view }, + given(filtered), + expectView(text("axios")).toExist(), + expectView(text("tokio")).toBeAbsent(), + ) + }) + + test("Scan now requests a scan, then shows it queued and polls until a newer scan finishes", () => { + const clicked = pageUpdate(loaded(), Message.ClickedScanNow({ repositoryId: PAYMENTS_ID })) + expect(names(clicked.commands)).toEqual(["SecurityApiScanRepository"]) + expect( + names(pageUpdate(clicked.model, Message.ClickedScanNow({ repositoryId: PAYMENTS_ID })).commands), + ).toEqual([]) + const queued = pageUpdate( + clicked.model, + Message.CompletedScanRequest({ + repositoryId: PAYMENTS_ID, + result: Result.succeed({ queued: true }), + }), + ) + expect(queued.outMessage).toMatchObject({ title: "Scan queued" }) + scene( + { update: pageUpdate, view }, + given(queued.model), + expectView(role("button", { name: "Scan queued" })).toHaveAttr("disabled", "true"), + ) + const ticked = pageUpdate(queued.model, Message.TickedScanPoll({ nowMs: NOW + 5_000 })) + expect(names(ticked.commands)).toEqual(["SecurityApiDependencyRepositories"]) + const unchanged = pageUpdate( + ticked.model, + Message.CompletedFetchRepositories({ + key: ticked.model.repositories.key ?? "", + result: Result.succeed(repositoriesOf(NOW)), + }), + ) + expect(unchanged.model.pendingScans).toHaveLength(1) + const again = pageUpdate(unchanged.model, Message.TickedScanPoll({ nowMs: NOW + 10_000 })) + const finished = pageUpdate( + again.model, + Message.CompletedFetchRepositories({ + key: again.model.repositories.key ?? "", + result: Result.succeed(rescannedOf(NOW)), + }), + ) + expect(finished.model.pendingScans).toEqual([]) + expect(names(finished.commands)).toEqual(["SecurityApiDependencyFindings"]) + }) + + test("a pending scan stops polling after two minutes", () => { + const queued = apply(loaded(), [ + () => Message.ClickedScanNow({ repositoryId: PAYMENTS_ID }), + () => + Message.CompletedScanRequest({ + repositoryId: PAYMENTS_ID, + result: Result.succeed({ queued: true }), + }), + ]) + const late = pageUpdate(queued, Message.TickedScanPoll({ nowMs: NOW + 121_000 })) + expect(late.model.pendingScans).toEqual([]) + expect(names(late.commands)).toEqual([]) + }) + + test("a refused scan says why and lets Scan now be pressed again", () => { + const refused = pageUpdate( + pageUpdate(loaded(), Message.ClickedScanNow({ repositoryId: PAYMENTS_ID })).model, + Message.CompletedScanRequest({ + repositoryId: PAYMENTS_ID, + result: Result.fail(toLoadFailure(new DependencyScanPersistenceError({ message: "down" }))), + }), + ) + expect(refused.outMessage).toMatchObject({ type: "error" }) + expect( + names(pageUpdate(refused.model, Message.ClickedScanNow({ repositoryId: PAYMENTS_ID })).commands), + ).toEqual(["SecurityApiScanRepository"]) + }) + + test("a row opens the finding's sheet with the advisory and where the package is locked", () => { + scene( + { update: pageUpdate, view }, + given(opened(loaded())), + mounted, + expectView(role("dialog")).toExist(), + expectView(text(/ignore baseURL/)).toExist(), + expectView(text("1.8.2")).toExist(), + expectView(text("pnpm-lock.yaml")).toExist(), + expectView(role("button", { name: "Accept risk" })).toExist(), + ) + }) + + test("Dismiss needs a reason, then writes the status with it", () => { + const write = UpdateFinding({ + findingId: FINDING_IDS.axios, + status: "dismissed", + reason: "Not reachable", + }) + scene( + { update: pageUpdate, view }, + given(opened(loaded())), + mounted, + click(role("button", { name: "Dismiss" })), + Command.expectNone(), + expectView(text("Add a reason. It is shown on the finding with your name.")).toExist(), + type(role("textbox", { name: "Reason" }), "Not reachable"), + click(role("button", { name: "Dismiss" })), + Command.expectHas(write), + // The server refuses a member's write with a 403; it comes back as an error toast. + Command.resolve(UpdateFinding, Message.CompletedTriage({ result: forbidden })), + expectOutMessage( + PageOutMessage.RequestedToast({ + type: "error", + title: forbiddenFailure.title, + description: forbiddenFailure.message, + }), + ), + ) + }) + + test("a dismissed finding is reopened without a reason", () => { + const dismissed = apply(loaded(), [ + () => + Message.GotStatusSelectMessage({ + message: Select.Message.SelectedItem({ index: 1, value: "dismissed" }), + }), + () => Message.ClickedFinding({ findingId: FINDING_IDS.lodash }), + ]) + scene( + { update: pageUpdate, view }, + given(dismissed), + mounted, + expectView(text("Only used by the docs build, never with user input.")).toExist(), + click(role("button", { name: "Reopen" })), + Command.expectHas(UpdateFinding({ findingId: FINDING_IDS.lodash, status: "open", reason: "" })), + Command.resolve(UpdateFinding, Message.CompletedTriage({ result: forbidden })), + ) + }) + + test("a member sees the finding but not the triage actions", () => { + scene( + { update: pageUpdate, view }, + given(opened(loaded(repositoriesOf(NOW), findingsOf(NOW), false))), + mounted, + expectView(text("Only org admins can change a finding's status.")).toExist(), + expectView(role("button", { name: "Dismiss" })).toBeAbsent(), + ) + }) + + test("a saved triage reads the findings again and says what changed", () => { + const finding = findingsOf(NOW).findings[0]! + const saving = apply(opened(loaded()), [ + () => + Message.GotReasonMessage({ message: Field.Message.ChangedValue({ value: "Not reachable" }) }), + () => Message.ClickedTriage({ status: "accepted_risk" }), + ]) + const saved = pageUpdate( + saving, + Message.CompletedTriage({ result: Result.succeed({ ...finding, status: "accepted_risk" }) }), + ) + expect(names(saved.commands)).toEqual(["SecurityApiDependencyFindings"]) + expect(saved.outMessage).toMatchObject({ title: "Marked accepted risk" }) + expect(saved.model.reason.value).toBe("") + }) +}) diff --git a/apps/web/src/page/security/dependencies/derive.ts b/apps/web/src/page/security/dependencies/derive.ts new file mode 100644 index 0000000000..56cc0c4e79 --- /dev/null +++ b/apps/web/src/page/security/dependencies/derive.ts @@ -0,0 +1,161 @@ +import type { AdvisorySeverity } from "@maple/domain/package-advisories" +import type { DependencyRelation } from "@maple/domain/dependency-scan" +import { + DEPENDENCY_FINDING_REASON_MAX_LENGTH, + type DependencyFindingView, + type DependencyProductionCoverage, + type DependencyScanRepository, + type VcsRepositoryId, +} from "@maple/domain/http" +import { Duration, Option } from "effect" +import { AsyncData } from "foldkit" +import type { LoadFailure } from "../../../data/load-failure" +import { type Model, type PendingScan, type StatusFilter, StatusFilter as StatusFilterSchema } from "./model" + +// What the Dependencies tab shows, read off its two slots. + +export const STATUS_FILTERS: ReadonlyArray = StatusFilterSchema.literals + +export const STATUS_FILTER_LABEL = { + open: "Open", + dismissed: "Dismissed", + accepted_risk: "Accepted risk", + fixed: "Fixed", + all: "All", +} satisfies Record + +/** Worst first; `unknown` is an advisory with no score, read as "Unrated". */ +export const SEVERITIES: ReadonlyArray = ["critical", "high", "medium", "low", "unknown"] + +export const SEVERITY_LABEL = { + critical: "Critical", + high: "High", + medium: "Medium", + low: "Low", + unknown: "Unrated", +} satisfies Record + +export const RELATION_LABEL = { + direct: "Direct", + dev: "Dev", + transitive: "Transitive", + unknown: "Unknown", +} satisfies Record + +/** A pending scan stops polling after this long, whether or not a newer scan showed up. */ +export const SCAN_POLL_LIMIT = Duration.minutes(2) + +export const repositoriesOf = (model: Model): ReadonlyArray => + Option.getOrNull(AsyncData.getData(model.repositories.data))?.repositories ?? [] + +export const allFindingsOf = (model: Model): ReadonlyArray => + Option.getOrNull(AsyncData.getData(model.findings.data))?.findings ?? [] + +export type PageState = + | Readonly<{ _tag: "Loading" }> + | Readonly<{ _tag: "Failed"; failure: LoadFailure }> + | Readonly<{ _tag: "NotConnected" }> + | Readonly<{ + _tag: "Ready" + repositories: ReadonlyArray + findings: ReadonlyArray + production: DependencyProductionCoverage + }> + +/** A failed read with nothing to show; a stale one keeps its data on screen. */ +const failureOf = (data: AsyncData.AsyncData): LoadFailure | null => + AsyncData.isFailure(data) ? Option.getOrNull(AsyncData.getError(data)) : null + +/** Either read failing fails the tab; it shows once both are in. */ +export const pageStateOf = (model: Model): PageState => { + const failure = failureOf(model.repositories.data) + const findingsFailure = failureOf(model.findings.data) + if (failure !== null) return { _tag: "Failed", failure } + if (findingsFailure !== null) return { _tag: "Failed", failure: findingsFailure } + const repositories = Option.getOrNull(AsyncData.getData(model.repositories.data)) + const findings = Option.getOrNull(AsyncData.getData(model.findings.data)) + if (repositories === null || findings === null) return { _tag: "Loading" } + if (repositories.repositories.length === 0) return { _tag: "NotConnected" } + return { + _tag: "Ready", + repositories: repositories.repositories, + findings: findings.findings, + production: findings.production, + } +} + +export const matchesStatus = (finding: DependencyFindingView, status: StatusFilter): boolean => + status === "all" || finding.status === status + +/** The table's rows: the server's order (worst first) under the two filters. */ +export const filteredFindings = ( + findings: ReadonlyArray, + status: StatusFilter, + repositoryId: VcsRepositoryId | null, +): ReadonlyArray => + findings.filter( + (finding) => + matchesStatus(finding, status) && + (repositoryId === null || finding.repositoryId === repositoryId), + ) + +export const statusFilterOf = (model: Model): StatusFilter => + STATUS_FILTERS.find((status) => status === model.statusSelect.value) ?? "open" + +export const repositoryFilterOf = (model: Model): VcsRepositoryId | null => + repositoriesOf(model).find((repository) => repository.id === model.repositorySelect.value)?.id ?? null + +export type SeverityCounts = Readonly> + +const NO_SEVERITIES = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 } satisfies SeverityCounts + +export const openSeverityCounts = (findings: ReadonlyArray): SeverityCounts => + findings + .filter((finding) => finding.status === "open") + .reduce( + (counts, finding) => ({ ...counts, [finding.severity]: counts[finding.severity] + 1 }), + NO_SEVERITIES, + ) + +export const scannedCount = (repositories: ReadonlyArray): number => + repositories.filter((repository) => repository.latestScan !== null).length + +export const isScanPending = (model: Model, repositoryId: VcsRepositoryId): boolean => + model.requestingScans.includes(repositoryId) || + model.pendingScans.some((pending) => pending.repositoryId === repositoryId) + +/** A pending scan is done once its repository shows a newer scan that has finished. */ +const isSettled = (pending: PendingScan, repositories: ReadonlyArray): boolean => { + const scan = repositories.find((repository) => repository.id === pending.repositoryId)?.latestScan ?? null + return scan !== null && scan.id !== pending.previousScanId && scan.status !== "running" +} + +/** The scans still to poll for, and whether any finished (so the findings are read again). */ +export const settlePendingScans = ( + pendingScans: ReadonlyArray, + repositories: ReadonlyArray, +): Readonly<{ pending: ReadonlyArray; hasSettled: boolean }> => { + const pending = pendingScans.filter((scan) => !isSettled(scan, repositories)) + return { pending, hasSettled: pending.length < pendingScans.length } +} + +export const unexpiredScans = ( + pendingScans: ReadonlyArray, + nowMs: number, +): ReadonlyArray => + pendingScans.filter((scan) => nowMs - scan.requestedAtMs < Duration.toMillis(SCAN_POLL_LIMIT)) + +export const selectedFindingOf = (model: Model): DependencyFindingView | null => + model.selectedId === null + ? null + : (allFindingsOf(model).find((finding) => finding.id === model.selectedId) ?? null) + +export const reasonErrorOf = (reason: string): string | null => + reason.trim() === "" + ? "Add a reason. It is shown on the finding with your name." + : reason.trim().length > DEPENDENCY_FINDING_REASON_MAX_LENGTH + ? `Keep the reason under ${DEPENDENCY_FINDING_REASON_MAX_LENGTH} characters.` + : null + +/** A commit as its first seven characters, as GitHub abbreviates it. */ +export const shortSha = (sha: string): string => sha.slice(0, 7) diff --git a/apps/web/src/page/security/dependencies/index.ts b/apps/web/src/page/security/dependencies/index.ts new file mode 100644 index 0000000000..a67e683560 --- /dev/null +++ b/apps/web/src/page/security/dependencies/index.ts @@ -0,0 +1,24 @@ +import { definePage } from "../../contract" +import { breadcrumbsOf } from "../kit/layout" +import { Message, Model } from "./model" +import { subscriptions } from "./subscription" +import { init, update } from "./update" +import { view } from "./view" + +/** + * `/security/dependencies`, in the Security frame: lockfile scans of the connected repositories, + * so no code-audit plan gates it. + */ +export const page = definePage( + "SecurityDependencies", + { Model, Message }, + { + routes: ["SecurityDependencies"], + key: () => "security-dependencies", + init, + update, + view, + subscriptions, + breadcrumbs: () => breadcrumbsOf("dependencies"), + }, +) diff --git a/apps/web/src/page/security/dependencies/model.ts b/apps/web/src/page/security/dependencies/model.ts new file mode 100644 index 0000000000..dec9585968 --- /dev/null +++ b/apps/web/src/page/security/dependencies/model.ts @@ -0,0 +1,81 @@ +import { + DependencyFindingsResponse, + DependencyFindingView, + DependencyScanRepositoriesResponse, + VcsRepositoryId, +} from "@maple/domain/http" +import { + DependencyFindingId, + DependencyFindingTriageStatus, + DependencyScanId, +} from "@maple/domain/dependency-scan" +import { Schema } from "effect" +import { defineMessageUnion } from "foldkit/message" +import { KeyedSchema } from "../../../data/keyed" +import { LoadFailure } from "../../../data/load-failure" +import * as Dialog from "../../../ui/dialog" +import * as Field from "../../../ui/field" +import * as Select from "../../../ui/select" + +// `/security/dependencies`: the connected repositories' latest lockfile scans and the vulnerable +// packages they found. Every finding is read once; the filters narrow it on the client. + +export const StatusFilter = Schema.Literals(["open", "dismissed", "accepted_risk", "fixed", "all"]) +export type StatusFilter = typeof StatusFilter.Type + +/** A scan someone asked for: polled until the repository's latest scan is a newer, finished one. */ +export const PendingScan = Schema.Struct({ + repositoryId: VcsRepositoryId, + requestedAtMs: Schema.Finite, + /** The latest scan when it was asked for; null before the first. */ + previousScanId: Schema.NullOr(DependencyScanId), +}) +export type PendingScan = typeof PendingScan.Type + +export const Model = Schema.Struct({ + nowMs: Schema.Finite, + isAdmin: Schema.Boolean, + repositories: KeyedSchema(DependencyScanRepositoriesResponse, LoadFailure), + findings: KeyedSchema(DependencyFindingsResponse, LoadFailure), + statusSelect: Select.Model, + repositorySelect: Select.Model, + /** Repositories whose Scan now request is in flight. */ + requestingScans: Schema.Array(VcsRepositoryId), + pendingScans: Schema.Array(PendingScan), + /** The finding the detail sheet shows. */ + selectedId: Schema.NullOr(DependencyFindingId), + sheet: Dialog.Model, + reason: Field.Model, + isReasonSubmitted: Schema.Boolean, + savingStatus: Schema.NullOr(DependencyFindingTriageStatus), +}) +export type Model = typeof Model.Type + +const result = (schema: S) => ({ + key: Schema.String, + result: Schema.Result(schema, LoadFailure), +}) + +export const Message = defineMessageUnion({ + ReadClock: { nowMs: Schema.Finite }, + CompletedReadIsAdmin: { isAdmin: Schema.Boolean }, + CompletedFetchRepositories: result(DependencyScanRepositoriesResponse), + CompletedFetchFindings: result(DependencyFindingsResponse), + ClickedRetry: {}, + ClickedScanNow: { repositoryId: VcsRepositoryId }, + CompletedScanRequest: { + repositoryId: VcsRepositoryId, + result: Schema.Result(Schema.Struct({ queued: Schema.Boolean }), LoadFailure), + }, + TickedScanPoll: { nowMs: Schema.Finite }, + GotStatusSelectMessage: { message: Select.Message }, + GotRepositorySelectMessage: { message: Select.Message }, + ClickedFinding: { findingId: DependencyFindingId }, + /** An advisory link inside a row: it opens OSV, not the row's sheet. */ + ClickedAdvisoryLink: {}, + GotSheetMessage: { message: Dialog.Message }, + GotReasonMessage: { message: Field.Message }, + ClickedTriage: { status: DependencyFindingTriageStatus }, + CompletedTriage: { result: Schema.Result(DependencyFindingView, LoadFailure) }, +}) +export type Message = typeof Message.Type diff --git a/apps/web/src/page/security/dependencies/subscription.ts b/apps/web/src/page/security/dependencies/subscription.ts new file mode 100644 index 0000000000..57e7005921 --- /dev/null +++ b/apps/web/src/page/security/dependencies/subscription.ts @@ -0,0 +1,33 @@ +import { Clock, Duration, Effect, Schema, Stream } from "effect" +import { Subscription } from "foldkit" +import { pollTicks } from "../../../data/poll" +import type { PageSubscriptionInput } from "../../contract" +import { Message, type Model } from "./model" + +/** While a scan someone asked for is pending, the repositories are read every 5 s. */ +const SCAN_POLL = Duration.seconds(5) +/** The clock is read every minute, so the last-scan times move. */ +const CLOCK = Duration.minutes(1) + +const clockTicks = (every: Duration.Input, toMessage: (nowMs: number) => A) => + pollTicks(every).pipe(Stream.mapEffect(() => Effect.map(Clock.currentTimeMillis, toMessage))) + +export const subscriptions = Subscription.make, Message>()((entry) => ({ + scanPoll: entry( + { isPending: Schema.Boolean }, + { + modelToDependencies: ({ model }) => ({ isPending: model.pendingScans.length > 0 }), + dependenciesToStream: ({ isPending }) => + isPending + ? clockTicks(SCAN_POLL, (nowMs) => Message.TickedScanPoll({ nowMs })) + : Stream.empty, + }, + ), + clock: entry( + {}, + { + modelToDependencies: () => ({}), + dependenciesToStream: () => clockTicks(CLOCK, (nowMs) => Message.ReadClock({ nowMs })), + }, + ), +})) diff --git a/apps/web/src/page/security/dependencies/update.ts b/apps/web/src/page/security/dependencies/update.ts new file mode 100644 index 0000000000..9d7fd6d009 --- /dev/null +++ b/apps/web/src/page/security/dependencies/update.ts @@ -0,0 +1,245 @@ +import type { DependencyFindingStatus, DependencyFindingTriageStatus } from "@maple/domain/dependency-scan" +import { Result } from "effect" +import { Command } from "foldkit" +import * as Keyed from "../../../data/keyed" +import { requestKey } from "../../../data/request-key" +import { isClerkAuthEnabled } from "../../../lib/services/common/auth-mode" +import type { AppRoute } from "../../../route" +import * as Dialog from "../../../ui/dialog" +import * as Field from "../../../ui/field" +import * as Select from "../../../ui/select" +import { type PageContext, PageOutMessage, type PageReturn } from "../../contract" +import { chainAll, slotStep } from "../../kit/keyed-step" +import { + FetchFindings, + FetchRepositories, + ReadDependenciesClock, + ReadDependenciesIsAdmin, + ScanRepository, + UpdateFinding, +} from "./command" +import { + isScanPending, + reasonErrorOf, + repositoriesOf, + selectedFindingOf, + settlePendingScans, + STATUS_FILTER_LABEL, + unexpiredScans, +} from "./derive" +import { Message, type Model } from "./model" + +type Return = PageReturn + +export type SecurityDependenciesRoute = Extract + +export const SHEET_ID = "security-dependency-finding" + +const toast = (type: "success" | "error", title: string, description = "") => + PageOutMessage.RequestedToast({ type, title, description }) + +const readRepositories = (model: Model, context: PageContext, mode: Keyed.ReadMode): Return => + slotStep( + model, + model.repositories, + (repositories) => ({ repositories }), + requestKey(context.orgId, "dependencyScan.listRepositories", {}), + mode, + (key) => FetchRepositories({ key }), + ) + +const readFindings = (model: Model, context: PageContext, mode: Keyed.ReadMode): Return => + slotStep( + model, + model.findings, + (findings) => ({ findings }), + requestKey(context.orgId, "dependencyScan.listFindings", {}), + mode, + (key) => FetchFindings({ key }), + ) + +const readAll = (model: Model, context: PageContext, mode: Keyed.ReadMode): Return => + chainAll(model, [ + (next) => readRepositories(next, context, mode), + (next) => readFindings(next, context, mode), + ]) + +export const initialModel = (): Model => ({ + nowMs: 0, + isAdmin: !isClerkAuthEnabled, + repositories: Keyed.idle(), + findings: Keyed.idle(), + statusSelect: Select.init("security-dependency-status", { value: "open", selectedIndex: 0 }), + repositorySelect: Select.init("security-dependency-repository", { value: "all", selectedIndex: 0 }), + requestingScans: [], + pendingScans: [], + selectedId: null, + sheet: Dialog.init(SHEET_ID, "sheet"), + reason: Field.init(), + isReasonSubmitted: false, + savingStatus: null, +}) + +export const init = (_route: SecurityDependenciesRoute, context: PageContext): Return => { + const loaded = readAll(initialModel(), context, "Load") + return { + ...loaded, + commands: [ + ReadDependenciesClock(), + ...(isClerkAuthEnabled ? [ReadDependenciesIsAdmin()] : []), + ...(loaded.commands ?? []), + ], + } +} + +const selectStep = ( + model: Model, + select: Select.Model, + message: Select.Message, + write: (select: Select.Model) => Partial, + wrap: (message: Select.Message) => Message, +): Return => { + const next = Select.update(select, message) + return { + model: { ...model, ...write(next.model) }, + commands: Command.mapMessages(next.commands ?? [], wrap), + } +} + +const sheetStep = (model: Model, result: ReturnType): Return => ({ + model: { ...model, sheet: result.model }, + commands: Command.mapMessages(result.commands ?? [], (message) => Message.GotSheetMessage({ message })), +}) + +/** A finished scan changes the findings, so they are read again. */ +const afterRepositories = (model: Model, context: PageContext): Return => { + if (model.pendingScans.length === 0) return { model } + const { pending, hasSettled } = settlePendingScans(model.pendingScans, repositoriesOf(model)) + const settled = { ...model, pendingScans: pending } + return hasSettled ? readFindings(settled, context, "Supersede") : { model: settled } +} + +const TRIAGE_TOAST = { + open: "Finding reopened", + dismissed: `Marked ${STATUS_FILTER_LABEL.dismissed.toLowerCase()}`, + accepted_risk: `Marked ${STATUS_FILTER_LABEL.accepted_risk.toLowerCase()}`, +} satisfies Record + +/** The answer's status as a triage one; a write never answers `fixed`, so it reads as open. */ +const triageStatusOf = (status: DependencyFindingStatus): DependencyFindingTriageStatus => + status === "fixed" ? "open" : status + +const triage = (model: Model, status: DependencyFindingTriageStatus): Return => { + const finding = selectedFindingOf(model) + if (finding === null || !model.isAdmin || model.savingStatus !== null) return { model } + const reason = model.reason.value.trim() + // Reopening needs no reason; setting a finding aside does. + if (status !== "open" && reasonErrorOf(reason) !== null) + return { model: { ...model, isReasonSubmitted: true } } + return { + model: { ...model, isReasonSubmitted: true, savingStatus: status }, + commands: [UpdateFinding({ findingId: finding.id, status, reason })], + } +} + +export const update = (model: Model, message: Message, context: PageContext): Return => + Message.match(message, { + ReadClock: ({ nowMs }) => ({ model: { ...model, nowMs } }), + CompletedReadIsAdmin: ({ isAdmin }) => ({ model: { ...model, isAdmin } }), + CompletedFetchRepositories: ({ key, result }) => + afterRepositories( + { ...model, repositories: Keyed.accept(model.repositories, key, result) }, + context, + ), + CompletedFetchFindings: ({ key, result }) => ({ + model: { ...model, findings: Keyed.accept(model.findings, key, result) }, + }), + ClickedRetry: () => readAll(model, context, "Refresh"), + ClickedScanNow: ({ repositoryId }) => + isScanPending(model, repositoryId) + ? { model } + : { + model: { ...model, requestingScans: [...model.requestingScans, repositoryId] }, + commands: [ScanRepository({ repositoryId })], + }, + CompletedScanRequest: ({ repositoryId, result }) => { + const requested = { + ...model, + requestingScans: model.requestingScans.filter((id) => id !== repositoryId), + } + return Result.match(result, { + onSuccess: () => { + const repository = repositoriesOf(model).find((entry) => entry.id === repositoryId) + const pending = { + repositoryId, + requestedAtMs: model.nowMs, + previousScanId: repository?.latestScan?.id ?? null, + } + return { + model: { ...requested, pendingScans: [...requested.pendingScans, pending] }, + outMessage: toast("success", "Scan queued", repository?.fullName ?? ""), + } + }, + onFailure: (failure) => ({ + model: requested, + outMessage: toast("error", failure.title, failure.message), + }), + }) + }, + TickedScanPoll: ({ nowMs }) => { + const pendingScans = unexpiredScans(model.pendingScans, nowMs) + const next = { ...model, nowMs, pendingScans } + return pendingScans.length === 0 ? { model: next } : readRepositories(next, context, "Supersede") + }, + GotStatusSelectMessage: ({ message: inner }) => + selectStep( + model, + model.statusSelect, + inner, + (statusSelect) => ({ statusSelect }), + (next) => Message.GotStatusSelectMessage({ message: next }), + ), + GotRepositorySelectMessage: ({ message: inner }) => + selectStep( + model, + model.repositorySelect, + inner, + (repositorySelect) => ({ repositorySelect }), + (next) => Message.GotRepositorySelectMessage({ message: next }), + ), + ClickedFinding: ({ findingId }) => + sheetStep( + { ...model, selectedId: findingId, reason: Field.init(), isReasonSubmitted: false }, + Dialog.open(model.sheet), + ), + ClickedAdvisoryLink: () => ({ model }), + GotSheetMessage: ({ message: inner }) => sheetStep(model, Dialog.update(model.sheet, inner)), + GotReasonMessage: ({ message: inner }) => ({ + model: { ...model, reason: Field.update(model.reason, inner).model }, + }), + ClickedTriage: ({ status }) => triage(model, status), + CompletedTriage: ({ result }) => + Result.match(result, { + onSuccess: (finding) => ({ + ...readFindings( + { + ...model, + savingStatus: null, + isReasonSubmitted: false, + reason: Field.init(), + }, + context, + "Supersede", + ), + outMessage: toast( + "success", + TRIAGE_TOAST[triageStatusOf(finding.status)], + finding.packageName, + ), + }), + onFailure: (failure) => ({ + model: { ...model, savingStatus: null }, + outMessage: toast("error", failure.title, failure.message), + }), + }), + }) diff --git a/apps/web/src/page/security/dependencies/view-findings.ts b/apps/web/src/page/security/dependencies/view-findings.ts new file mode 100644 index 0000000000..1bbd4ca76f --- /dev/null +++ b/apps/web/src/page/security/dependencies/view-findings.ts @@ -0,0 +1,365 @@ +import type { + DependencyFindingView, + DependencyProductionCoverage, + DependencyScanRepository, +} from "@maple/domain/http" +import { type AdvisorySeverity, displayAdvisoryId } from "@maple/domain/package-advisories" +import { countLabel, formatRate } from "@maple/ui/lib/format" +import { cn } from "@maple/ui/lib/utils" +import type { Html, HtmlBuilder } from "foldkit/html" +import { badge } from "../../../ui/display" +import { empty, emptyDescription, emptyHeader, emptyMedia, emptyTitle } from "../../../ui/empty" +import { icon } from "../../../ui/icon" +import { middleTruncate } from "../../../ui/middle-truncate" +import * as Select from "../../../ui/select" +import * as SelectView from "../../../ui/select-view" +import { table, tableBody, tableCell, tableHead, tableHeader, tableRow } from "../../../ui/table" +import { textLink } from "../../../ui/text-link" +import { listToolbar } from "../../kit/list-toolbar" +import { nothingInWindow } from "../../kit/nothing-in-window" +import { relativeTime } from "../../kit/relative-time" +import { securitySeverityBadge } from "../kit/parts" +import { + filteredFindings, + matchesStatus, + RELATION_LABEL, + repositoryFilterOf, + SEVERITY_LABEL, + STATUS_FILTER_LABEL, + STATUS_FILTERS, + statusFilterOf, +} from "./derive" +import { Message, type Model } from "./model" + +// The findings table: one row per advisory on a package in a lockfile, worst first as the server +// sorts them, under a status and a repository filter. + +type H = HtmlBuilder +type Clock = Readonly<{ nowMs: number; timeZone: string }> + +/** The code audit's severity badge; an advisory with no score reads as "Unrated". */ +export const dependencySeverityBadge = (h: H, severity: AdvisorySeverity, className?: string): Html => + severity === "unknown" + ? badge(h, { variant: "muted", className: cn("text-muted-foreground", className) }, [ + SEVERITY_LABEL.unknown, + ]) + : securitySeverityBadge(h, severity, className) + +export const advisoryHref = (advisoryId: string): string => + `https://osv.dev/vulnerability/${encodeURIComponent(advisoryId)}` + +export const advisoryLink = (h: H, finding: DependencyFindingView, className?: string): Html => + textLink( + h, + { + className: cn("font-mono text-xs", className), + href: advisoryHref(finding.advisoryId), + attributes: [ + h.Target("_blank"), + h.Rel("noopener noreferrer"), + h.OnClick(Message.ClickedAdvisoryLink(), { propagation: "Stop" }), + ], + }, + [displayAdvisoryId({ id: finding.advisoryId, aliases: finding.aliases })], + ) + +export const versionsLabel = (finding: DependencyFindingView): string => + finding.versions.length === 0 + ? finding.packageName + : `${finding.packageName}@${finding.versions.join(", ")}` + +const item = (value: string, label: string): SelectView.SelectEntry => ({ + _tag: "Item", + item: { value, label }, +}) + +const filterSelect = ( + h: H, + select: Select.Model, + entries: ReadonlyArray, + label: string, + renderValue: (value: string) => string, + wrap: (message: Select.Message) => Message, +): Html => + h.submodel({ + slotId: select.id, + model: select, + view: SelectView.view, + viewInputs: { + entries, + size: "sm", + ariaLabel: label, + className: "h-7 w-auto min-w-28 text-xs", + renderValue: (value) => [ + h.span([h.Class("text-muted-foreground")], [`${label} `]), + renderValue(value), + ], + toView: (parts) => h.div([h.Class("contents")], parts), + }, + toParentMessage: wrap, + }) + +const toolbar = ( + h: H, + model: Model, + findings: ReadonlyArray, + repositories: ReadonlyArray, +): Html => { + const statusLabel = (value: string) => + STATUS_FILTER_LABEL[STATUS_FILTERS.find((status) => status === value) ?? "open"] + const repositoryLabel = (value: string) => + repositories.find((repository) => repository.id === value)?.fullName ?? "All repositories" + return listToolbar(h, { + framed: true, + leading: h.h2([h.Class("px-1 text-sm font-medium")], ["Vulnerable packages"]), + trailing: [ + filterSelect( + h, + model.statusSelect, + STATUS_FILTERS.map((status) => + item( + status, + `${STATUS_FILTER_LABEL[status]} (${findings.filter((finding) => matchesStatus(finding, status)).length})`, + ), + ), + "Status", + statusLabel, + (message) => Message.GotStatusSelectMessage({ message }), + ), + // One repository needs no repository filter. + ...(repositories.length < 2 + ? [] + : [ + filterSelect( + h, + model.repositorySelect, + [ + item("all", "All repositories"), + ...repositories.map((repository) => item(repository.id, repository.fullName)), + ], + "Repository", + repositoryLabel, + (message) => Message.GotRepositorySelectMessage({ message }), + ), + ]), + ], + }) +} + +const muted = (h: H, text: string, className?: string): Html => + h.span([h.Class(cn("text-xs text-muted-foreground", className))], [text]) + +/** Spans a finding's running services served over the last day. */ +export const productionSpans = (finding: DependencyFindingView): number => + finding.production.reduce((sum, service) => sum + service.spanCount, 0) + +/** The busiest service running the finding, and how many more; unknown without commit telemetry. */ +const productionCell = ( + h: H, + finding: DependencyFindingView, + production: DependencyProductionCoverage, +): Html => { + const [busiest] = finding.production + if (busiest === undefined) + return muted(h, production.available && production.linkedServices > 0 ? "Not running" : "Unknown") + const others = new Set(finding.production.map((service) => service.serviceName)).size - 1 + return h.span( + [h.Class("flex min-w-0 max-w-48 flex-col")], + [ + h.span( + [h.Class("truncate text-sm"), h.Title(busiest.serviceName)], + [others > 0 ? `${busiest.serviceName} +${others}` : busiest.serviceName], + ), + muted(h, `${formatRate(productionSpans(finding))} spans/day`, "tabular-nums"), + ], + ) +} + +/** + * Said when findings cannot be tied to what runs: no service reports the commit it was built from, + * or none of those commits belongs to a connected repository. + */ +const productionHint = (h: H, production: DependencyProductionCoverage): ReadonlyArray => { + const text = !production.available + ? "Production traffic could not be read, so findings are ranked by severity alone." + : production.runningServices === 0 + ? "No service reports the commit it runs (vcs.ref.head.revision), so findings cannot be tied to production. Set it to rank them by traffic." + : production.linkedServices === 0 + ? "The commits your services run are not in a connected repository's tracked branch, so findings cannot be tied to production." + : null + return text === null + ? [] + : [h.p([h.Class("border-b bg-surface-sunken px-4 py-2 text-xs text-muted-foreground")], [text])] +} + +const findingRow = ( + h: H, + finding: DependencyFindingView, + isSelected: boolean, + clock: Clock, + production: DependencyProductionCoverage, +): Html => { + const open = Message.ClickedFinding({ findingId: finding.id }) + return tableRow( + h, + [ + tableCell(h, [dependencySeverityBadge(h, finding.severity)], "w-24"), + tableCell(h, [ + h.button( + [ + h.Type("button"), + h.OnClick(open, { propagation: "Stop" }), + h.Class( + "flex min-w-0 max-w-72 flex-col items-start text-left focus-visible:underline focus-visible:outline-none", + ), + ], + [ + h.span( + [h.Class("truncate font-mono text-sm"), h.Title(versionsLabel(finding))], + [finding.packageName], + ), + muted( + h, + `${finding.ecosystem} ยท ${finding.versions.join(", ")}`, + "truncate font-mono", + ), + ], + ), + ]), + tableCell(h, [advisoryLink(h, finding)]), + tableCell(h, [productionCell(h, finding, production)]), + tableCell(h, [ + finding.fixedIn === null + ? muted(h, "No fix yet") + : h.span([h.Class("font-mono text-xs tabular-nums")], [finding.fixedIn]), + ]), + tableCell(h, [muted(h, RELATION_LABEL[finding.relation])], "hidden md:table-cell"), + tableCell( + h, + [ + h.span( + [h.Class("flex min-w-0 max-w-64 flex-col")], + [ + h.span( + [h.Class("truncate text-sm"), h.Title(finding.repositoryFullName)], + [finding.repositoryFullName], + ), + middleTruncate(h, { + text: finding.lockfilePath, + tail: Math.min(24, finding.lockfilePath.length), + mono: true, + className: "text-xs text-muted-foreground", + }), + ], + ), + ], + "hidden lg:table-cell", + ), + tableCell( + h, + [ + ...(finding.status === "open" + ? [] + : [ + badge(h, { variant: "meta", className: "mr-2" }, [ + STATUS_FILTER_LABEL[finding.status], + ]), + ]), + relativeTime(h, { + value: finding.createdAt, + nowMs: clock.nowMs, + timeZone: clock.timeZone, + variant: "orDate", + tooltip: "title", + className: "whitespace-nowrap text-xs text-muted-foreground", + }), + ], + "whitespace-nowrap text-right", + ), + ], + cn( + "cursor-pointer hover:bg-surface-sunken active:bg-surface-sunken", + isSelected && "bg-surface-strong", + ), + [h.Key(finding.id), h.OnClick(open)], + ) +} + +const findingsTable = ( + h: H, + rows: ReadonlyArray, + selectedId: string | null, + clock: Clock, + production: DependencyProductionCoverage, +): Html => + table(h, { ariaLabel: "Vulnerable packages" }, [ + tableHeader(h, [ + tableRow(h, [ + tableHead(h, ["Severity"]), + tableHead(h, ["Package"]), + tableHead(h, ["Advisory"]), + tableHead(h, ["In production"]), + tableHead(h, ["Fixed in"]), + tableHead(h, ["Relation"], "hidden md:table-cell"), + tableHead(h, ["Location"], "hidden lg:table-cell"), + tableHead(h, ["First seen"], "text-right"), + ]), + ]), + tableBody( + h, + rows.map((finding) => findingRow(h, finding, finding.id === selectedId, clock, production)), + ), + ]) + +const packagesChecked = (repositories: ReadonlyArray): number => + repositories.reduce((sum, repository) => sum + (repository.latestScan?.packageCount ?? 0), 0) + +/** Scanned and nothing open: a calm line, not an alarm. */ +const cleanState = (h: H, repositories: ReadonlyArray): Html => + empty( + h, + [ + emptyHeader(h, [ + emptyMedia(h, { variant: "icon" }, [icon(h, "ShieldCheckIcon")]), + emptyTitle(h, ["No known vulnerable packages"]), + emptyDescription(h, [ + `The latest scans checked ${countLabel(packagesChecked(repositories), "package")} against OSV.dev. New advisories are matched on every scan.`, + ]), + ]), + ], + "md:py-14", + ) + +export const findingsSection = ( + h: H, + model: Model, + state: Readonly<{ + findings: ReadonlyArray + repositories: ReadonlyArray + production: DependencyProductionCoverage + }>, + clock: Clock, +): Html => { + const { findings, repositories, production } = state + const status = statusFilterOf(model) + const repositoryId = repositoryFilterOf(model) + const rows = filteredFindings(findings, status, repositoryId) + const content = + rows.length > 0 + ? findingsTable(h, rows, model.selectedId, clock, production) + : status === "open" && repositoryId === null + ? cleanState(h, repositories) + : h.div( + [h.Class("p-4")], + [ + nothingInWindow(h, { + title: "No findings match", + description: `No ${STATUS_FILTER_LABEL[status].toLowerCase()} findings for these filters.`, + }), + ], + ) + return h.section( + [h.AriaLabel("Vulnerable packages"), h.Class("overflow-hidden rounded-md border bg-card")], + [toolbar(h, model, findings, repositories), ...productionHint(h, production), content], + ) +} diff --git a/apps/web/src/page/security/dependencies/view-repositories.ts b/apps/web/src/page/security/dependencies/view-repositories.ts new file mode 100644 index 0000000000..b181ff3ac4 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view-repositories.ts @@ -0,0 +1,182 @@ +import type { DependencyLockfileStatus, DependencyScanLockfile } from "@maple/domain/dependency-scan" +import type { DependencyScanRepository, DependencyScanSummary } from "@maple/domain/http" +import { countLabel } from "@maple/ui/lib/format" +import type { Html, HtmlBuilder } from "foldkit/html" +import { button } from "../../../ui/button" +import { badge } from "../../../ui/display" +import { icon } from "../../../ui/icon" +import { panel, panelHeader } from "../../../ui/panel" +import { relativeTime } from "../../kit/relative-time" +import { repositoryName } from "../kit/parts" +import { isScanPending, shortSha } from "./derive" +import { Message, type Model } from "./model" + +// Each connected repository: its tracked branch, the latest scan, the lockfiles it could not read +// and a Scan now button. + +type H = HtmlBuilder +type Clock = Readonly<{ nowMs: number; timeZone: string }> + +/** Said when the scanner left no message of its own. */ +const LOCKFILE_FALLBACK = { + parsed: "", + unsupported: "This lockfile format is not supported yet.", + too_large: "Too large to scan.", + failed: "Could not be read.", + skipped: "Past the lockfiles a scan reads.", +} satisfies Record + +const LOCKFILE_STATUS_LABEL = { + parsed: "Parsed", + unsupported: "Unsupported", + too_large: "Too large", + failed: "Failed", + skipped: "Skipped", +} satisfies Record + +const lockfileIssue = (h: H, lockfile: DependencyScanLockfile): Html => + h.li( + [h.Key(lockfile.path), h.Class("flex min-w-0 flex-wrap items-baseline gap-x-2 gap-y-0.5 text-xs")], + [ + h.span([h.Class("font-mono text-foreground")], [lockfile.path]), + badge(h, { variant: lockfile.status === "failed" ? "warn" : "muted" }, [ + LOCKFILE_STATUS_LABEL[lockfile.status], + ]), + h.span( + [h.Class("min-w-0 text-muted-foreground")], + [lockfile.message ?? LOCKFILE_FALLBACK[lockfile.status]], + ), + ], + ) + +const dot = (h: H): Html => h.span([h.AriaHidden(true), h.Class("text-foreground-faint")], ["ยท"]) + +const scanSummary = (h: H, scan: DependencyScanSummary, clock: Clock): Html => + h.span( + [h.Class("flex min-w-0 flex-wrap items-center gap-x-1.5 text-xs text-muted-foreground")], + [ + scan.status === "running" ? "Scan started" : "Scanned", + relativeTime(h, { + value: scan.finishedAt ?? scan.startedAt, + nowMs: clock.nowMs, + timeZone: clock.timeZone, + variant: "orDate", + tooltip: "title", + }), + dot(h), + h.span([h.Class("font-mono"), h.Title(scan.commitSha)], [shortSha(scan.commitSha)]), + dot(h), + countLabel(scan.lockfiles.length, "lockfile"), + dot(h), + countLabel(scan.packageCount, "package"), + ], + ) + +export const scanButton = (h: H, model: Model, repository: DependencyScanRepository): Html => { + const isRequesting = model.requestingScans.includes(repository.id) + const isQueued = !isRequesting && isScanPending(model, repository.id) + const isRunning = repository.latestScan?.status === "running" + return button( + h, + { + variant: "outline", + size: "sm", + isLoading: isRequesting, + isDisabled: isQueued || isRunning, + attributes: [h.OnClick(Message.ClickedScanNow({ repositoryId: repository.id }))], + }, + [isQueued ? "Scan queued" : isRunning ? "Scanning" : "Scan now"], + ) +} + +const scanState = (h: H, scan: DependencyScanSummary | null): ReadonlyArray => + scan === null + ? [badge(h, { variant: "muted" }, ["Not scanned yet"])] + : scan.status === "failed" + ? [badge(h, { variant: "crit" }, ["Scan failed"])] + : [] + +const repositoryRow = (h: H, model: Model, repository: DependencyScanRepository, clock: Clock): Html => { + const scan = repository.latestScan + const issues = (scan?.lockfiles ?? []).filter((lockfile) => lockfile.status !== "parsed") + return h.li( + [h.Key(repository.id), h.Class("flex flex-col gap-2 px-4 py-3")], + [ + h.div( + [h.Class("flex flex-wrap items-center justify-between gap-x-4 gap-y-2")], + [ + h.div( + [h.Class("flex min-w-0 flex-col gap-0.5")], + [ + h.span( + [h.Class("flex min-w-0 items-center gap-2")], + [ + icon(h, "GithubIcon", { + size: 14, + className: "shrink-0 text-muted-foreground", + }), + h.a( + [ + h.Href(repository.htmlUrl), + h.Target("_blank"), + h.Rel("noopener noreferrer"), + h.Class( + "min-w-0 text-sm font-medium hover:underline active:underline", + ), + ], + [repositoryName(h, repository.fullName)], + ), + h.span( + [h.Class("shrink-0 font-mono text-xs text-muted-foreground")], + [repository.trackedBranch], + ), + ], + ), + scan === null + ? h.span( + [h.Class("text-xs text-muted-foreground")], + ["Scanned on the next push, or now."], + ) + : scanSummary(h, scan, clock), + ], + ), + h.div( + [h.Class("flex shrink-0 items-center gap-2")], + [...scanState(h, scan), scanButton(h, model, repository)], + ), + ], + ), + ...(scan?.error === null || scan?.error === undefined + ? [] + : [h.p([h.Class("text-xs text-severity-error")], [scan.error])]), + ...(issues.length === 0 + ? [] + : [ + h.ul( + [h.Class("flex flex-col gap-1 pl-5.5")], + issues.map((lockfile) => lockfileIssue(h, lockfile)), + ), + ]), + ], + ) +} + +export const repositoriesPanel = ( + h: H, + model: Model, + repositories: ReadonlyArray, + clock: Clock, +): Html => + panel(h, { className: "overflow-hidden" }, [ + panelHeader(h, { + title: "Repositories", + action: h.span( + [h.Class("text-xs text-muted-foreground")], + ["Tracked branch, rescanned on every push"], + ), + }), + h.ul( + [h.Class("divide-y divide-border")], + repositories.map((repository) => repositoryRow(h, model, repository, clock)), + ), + ]) diff --git a/apps/web/src/page/security/dependencies/view-sheet.ts b/apps/web/src/page/security/dependencies/view-sheet.ts new file mode 100644 index 0000000000..4d175f6ec2 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view-sheet.ts @@ -0,0 +1,279 @@ +import type { DependencyFindingView } from "@maple/domain/http" +import { displayAdvisoryId } from "@maple/domain/package-advisories" +import { formatRate } from "@maple/ui/lib/format" +import type { Html, HtmlBuilder } from "foldkit/html" +import { button } from "../../../ui/button" +import { detailRailGroup, detailRailRow } from "../../../ui/detail-rail" +import { badge } from "../../../ui/display" +import * as DialogView from "../../../ui/dialog-view" +import * as Field from "../../../ui/field" +import { textarea } from "../../../ui/textarea" +import { relativeTime } from "../../kit/relative-time" +import { sheetDetailHeader } from "../../kit/sheet-detail-header" +import { RELATION_LABEL, reasonErrorOf, selectedFindingOf, shortSha, STATUS_FILTER_LABEL } from "./derive" +import { Message, type Model } from "./model" +import { advisoryLink, dependencySeverityBadge } from "./view-findings" + +// The finding's sheet: what the advisory says, where the package is locked, and its triage. + +type H = HtmlBuilder +type Clock = Readonly<{ nowMs: number; timeZone: string }> + +const value = (h: H, text: string, mono = false): Html => + h.span([h.Class(mono ? "truncate font-mono text-xs" : "truncate text-sm"), h.Title(text)], [text]) + +const at = (h: H, sha: string, epochMs: number, clock: Clock): Html => + h.span( + [h.Class("flex min-w-0 items-center gap-1.5 text-sm")], + [ + h.span([h.Class("font-mono text-xs"), h.Title(sha)], [shortSha(sha)]), + relativeTime(h, { + value: epochMs, + nowMs: clock.nowMs, + timeZone: clock.timeZone, + variant: "orDate", + tooltip: "title", + className: "text-xs text-muted-foreground", + }), + ], + ) + +/** The services running a commit that carries the finding, busiest first. */ +const production = (h: H, finding: DependencyFindingView): ReadonlyArray => + finding.production.length === 0 + ? [] + : [ + detailRailGroup( + h, + "Running in production", + // The environment labels the row, so a long service name keeps the value's width. + finding.production.map((service) => + detailRailRow(h, { label: service.environment }, [ + h.span( + [h.Class("flex min-w-0 items-center justify-end gap-1.5 text-xs")], + [ + h.span( + [h.Class("truncate text-sm"), h.Title(service.serviceName)], + [service.serviceName], + ), + h.span( + [ + h.Class("shrink-0 font-mono text-muted-foreground"), + h.Title(service.commitSha), + ], + [shortSha(service.commitSha)], + ), + h.span( + [h.Class("shrink-0 tabular-nums text-muted-foreground")], + [`${formatRate(service.spanCount)} spans/day`], + ), + ], + ), + ]), + ), + ), + ] + +/** The ids besides the one the advisory link already shows. */ +const aliasesBesides = (finding: DependencyFindingView): ReadonlyArray => { + const shown = displayAdvisoryId({ id: finding.advisoryId, aliases: finding.aliases }) + return [finding.advisoryId, ...finding.aliases].filter((name) => name !== shown) +} + +const details = (h: H, finding: DependencyFindingView, clock: Clock): Html => + detailRailGroup(h, "Details", [ + detailRailRow(h, { label: "Advisory" }, [advisoryLink(h, finding)]), + ...(aliasesBesides(finding).length === 0 + ? [] + : [detailRailRow(h, { label: "Aliases" }, [value(h, aliasesBesides(finding).join(", "), true)])]), + detailRailRow(h, { label: "Versions" }, [value(h, finding.versions.join(", "), true)]), + detailRailRow(h, { label: "Fixed in" }, [ + finding.fixedIn === null ? value(h, "No fix yet") : value(h, finding.fixedIn, true), + ]), + detailRailRow(h, { label: "Relation" }, [value(h, RELATION_LABEL[finding.relation])]), + detailRailRow(h, { label: "Repository" }, [value(h, finding.repositoryFullName)]), + detailRailRow(h, { label: "Lockfile" }, [value(h, finding.lockfilePath, true)]), + detailRailRow(h, { label: "First seen" }, [at(h, finding.firstSeenSha, finding.createdAt, clock)]), + detailRailRow(h, { label: "Last seen" }, [at(h, finding.lastSeenSha, finding.updatedAt, clock)]), + ...(finding.fixedSha === null + ? [] + : [ + detailRailRow(h, { label: "Fixed at" }, [ + h.span( + [h.Class("font-mono text-xs"), h.Title(finding.fixedSha)], + [shortSha(finding.fixedSha)], + ), + ]), + ]), + ]) + +const REASON_IDS: Field.FieldIds = { + control: "dependency-triage-reason", + label: "dependency-triage-reason-label", + error: "dependency-triage-reason-error", + description: "dependency-triage-reason-description", +} + +const reasonField = (h: H, model: Model): Html => { + const error = model.isReasonSubmitted ? reasonErrorOf(model.reason.value) : null + const state = Field.stateOf(model.reason, error !== null) + return Field.field(h, { state }, [ + Field.fieldLabel(h, { ids: REASON_IDS, state }, "Reason"), + textarea(h, { + id: REASON_IDS.control, + attributes: [ + h.Placeholder("Why is this not a risk here?"), + h.AriaLabelledBy(REASON_IDS.label), + h.AriaDescribedBy(error === null ? (REASON_IDS.description ?? "") : REASON_IDS.error), + h.AriaInvalid(error !== null), + h.Rows(3), + h.Value(model.reason.value), + h.OnInput((next) => + Message.GotReasonMessage({ message: Field.Message.ChangedValue({ value: next }) }), + ), + h.OnBlur(Message.GotReasonMessage({ message: Field.Message.BlurredControl() })), + ], + }), + error === null + ? Field.fieldDescription(h, { ids: REASON_IDS, state }, "Required to dismiss or accept the risk.") + : Field.fieldError(h, { ids: REASON_IDS, state, attributes: [h.Role("alert")] }, error), + ]) +} + +const note = (h: H, text: string): Html => h.p([h.Class("text-xs text-muted-foreground")], [text]) + +const triage = (h: H, model: Model, finding: DependencyFindingView, clock: Clock): Html => { + if (!model.isAdmin) + return detailRailGroup(h, "Triage", [note(h, "Only org admins can change a finding's status.")]) + if (finding.status === "fixed") + return detailRailGroup(h, "Triage", [ + note(h, "A later scan no longer finds this package version. It reopens if it comes back."), + ]) + if (finding.status !== "open") + return detailRailGroup(h, "Triage", [ + h.div( + [h.Class("flex flex-col gap-3")], + [ + h.p( + [h.Class("text-sm")], + [ + `${STATUS_FILTER_LABEL[finding.status]}`, + ...(finding.statusChangedAt === null + ? [] + : [ + " ", + relativeTime(h, { + value: finding.statusChangedAt, + nowMs: clock.nowMs, + timeZone: clock.timeZone, + variant: "orDate", + tooltip: "title", + className: "text-muted-foreground", + }), + ]), + ], + ), + ...(finding.statusReason === null + ? [] + : [ + h.p( + [h.Class("font-prose text-sm text-muted-foreground")], + [finding.statusReason], + ), + ]), + button( + h, + { + variant: "outline", + isLoading: model.savingStatus === "open", + className: "self-start", + attributes: [h.OnClick(Message.ClickedTriage({ status: "open" }))], + }, + ["Reopen"], + ), + ], + ), + ]) + return detailRailGroup(h, "Triage", [ + h.div( + [h.Class("flex flex-col gap-3")], + [ + reasonField(h, model), + h.div( + [h.Class("flex gap-2")], + [ + button( + h, + { + variant: "outline", + isLoading: model.savingStatus === "dismissed", + isDisabled: model.savingStatus !== null, + attributes: [h.OnClick(Message.ClickedTriage({ status: "dismissed" }))], + }, + ["Dismiss"], + ), + button( + h, + { + variant: "outline", + isLoading: model.savingStatus === "accepted_risk", + isDisabled: model.savingStatus !== null, + attributes: [h.OnClick(Message.ClickedTriage({ status: "accepted_risk" }))], + }, + ["Accept risk"], + ), + ], + ), + ], + ), + ]) +} + +const content = (h: H, model: Model, finding: DependencyFindingView, clock: Clock): ReadonlyArray => [ + sheetDetailHeader(h, { + dialogId: model.sheet.id, + kind: [finding.ecosystem, finding.repositoryFullName], + title: [finding.packageName], + adornment: + finding.status === "open" + ? null + : badge(h, { variant: "meta" }, [STATUS_FILTER_LABEL[finding.status]]), + description: finding.summary ?? `A known vulnerability in ${finding.packageName}`, + meta: [dependencySeverityBadge(h, finding.severity), advisoryLink(h, finding)], + }), + DialogView.dialogPanel( + h, + "sheet", + model.sheet.id, + [ + h.p( + [h.Class("font-prose text-sm text-pretty")], + [finding.summary ?? "The advisory publishes no summary. Read it on OSV.dev."], + ), + ...production(h, finding), + details(h, finding, clock), + triage(h, model, finding, clock), + ], + "flex flex-col gap-6", + ), +] + +/** The sheet; its content renders only while a finding is picked and still in the list. */ +export const findingSheet = (h: H, model: Model, clock: Clock): Html => + h.submodel({ + slotId: `${model.sheet.id}-sheet`, + model: model.sheet, + view: DialogView.view, + viewInputs: { + className: "p-0 sm:max-w-[480px]", + side: "right", + hasTrigger: false, + isPopupFocused: true, + toView: ({ portal }) => portal, + toContent: () => { + const finding = selectedFindingOf(model) + return finding === null ? [] : content(h, model, finding, clock) + }, + }, + toParentMessage: (message) => Message.GotSheetMessage({ message }), + }) diff --git a/apps/web/src/page/security/dependencies/view-states.ts b/apps/web/src/page/security/dependencies/view-states.ts new file mode 100644 index 0000000000..dc56ff1616 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view-states.ts @@ -0,0 +1,130 @@ +import type { DependencyFindingView, DependencyScanRepository } from "@maple/domain/http" +import type { Html, HtmlBuilder } from "foldkit/html" +import { AppRoute, hrefOf } from "../../../route" +import { linkButton } from "../../../ui/button" +import { empty, emptyContent, emptyDescription, emptyHeader, emptyMedia, emptyTitle } from "../../../ui/empty" +import { icon } from "../../../ui/icon" +import { textLink } from "../../../ui/text-link" +import { statRail, statRailItem } from "../../kit/stat-rail" +import { placeholder } from "../kit/parts" +import { openSeverityCounts, scannedCount } from "./derive" +import type { Message, Model } from "./model" +import { scanButton } from "./view-repositories" + +// The Dependencies tab's rail and its whole-tab states: loading, no GitHub, first scan pending. + +type H = HtmlBuilder + +export const dependenciesLoading = (h: H): ReadonlyArray => [ + h.div( + [h.AriaBusy(true), h.AriaLabel("Loading"), h.Class("grid grid-cols-2 gap-px md:grid-cols-5")], + Array.from({ length: 5 }, () => placeholder(h, "h-24")), + ), + placeholder(h, "h-32 w-full"), + placeholder(h, "h-64 w-full"), +] + +/** No connected repository: the GitHub on-ramp the code audits use. */ +export const notConnected = (h: H): Html => + empty( + h, + [ + emptyHeader(h, [ + emptyMedia(h, { variant: "icon" }, [icon(h, "GithubIcon")]), + emptyTitle(h, ["Connect a repository to scan its dependencies"]), + emptyDescription(h, [ + "Maple reads the lockfiles on each repository's tracked branch, matches every package against OSV.dev and rescans on every push.", + ]), + ]), + emptyContent(h, [ + linkButton(h, { href: hrefOf(AppRoute.Integrations({ search: {} })) }, [ + icon(h, "GithubIcon"), + "Connect GitHub", + ]), + h.p( + [h.Class("text-xs text-muted-foreground")], + ["npm, pnpm, Yarn, Bun, Go and Cargo lockfiles. Read-only, code is never executed."], + ), + ]), + ], + "rounded-md border md:py-16", + ) + +/** Connected, never scanned: the first scan follows the connection or the next push. */ +export const firstScan = ( + h: H, + model: Model, + repositories: ReadonlyArray, +): Html => { + const only = repositories.length === 1 ? (repositories[0] ?? null) : null + return empty( + h, + [ + emptyHeader(h, [ + emptyMedia(h, { variant: "icon" }, [icon(h, "CubeIcon")]), + emptyTitle(h, ["First scan is on its way"]), + emptyDescription(h, [ + "Each connected repository is scanned shortly after it is connected and again on every push to its tracked branch.", + ]), + ]), + ...(only === null ? [] : [emptyContent(h, [scanButton(h, model, only)])]), + ], + "rounded-md border md:py-14", + ) +} + +export const severityRail = ( + h: H, + findings: ReadonlyArray, + repositories: ReadonlyArray, +): Html => { + const counts = openSeverityCounts(findings) + return statRail(h, { columns: 5 }, [ + statRailItem(h, { + eyebrow: "Critical", + value: String(counts.critical), + tone: counts.critical > 0 ? "crit" : undefined, + compact: true, + subline: "open, fix first", + }), + statRailItem(h, { + eyebrow: "High", + value: String(counts.high), + tone: counts.high > 0 ? "warn" : undefined, + compact: true, + subline: "open", + }), + statRailItem(h, { eyebrow: "Medium", value: String(counts.medium), compact: true, subline: "open" }), + statRailItem(h, { + eyebrow: "Low", + value: String(counts.low), + compact: true, + subline: counts.unknown > 0 ? `open, and ${counts.unknown} unrated` : "open", + }), + statRailItem(h, { + eyebrow: "Repositories", + value: `${scannedCount(repositories)} / ${repositories.length}`, + compact: true, + subline: "scanned / connected", + }), + ]) +} + +export const footnote = (h: H): Html => + h.p( + [h.Class("text-xs text-muted-foreground")], + [ + "Advisories from ", + textLink( + h, + { + tone: "muted", + className: "underline", + href: "https://osv.dev", + attributes: [h.Target("_blank"), h.Rel("noopener noreferrer")], + }, + ["OSV.dev"], + ), + ". Findings are kept per lockfile and package, and marked fixed when a scan no longer finds them.", + ], + ) diff --git a/apps/web/src/page/security/dependencies/view.ts b/apps/web/src/page/security/dependencies/view.ts new file mode 100644 index 0000000000..bf721c0b86 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view.ts @@ -0,0 +1,57 @@ +import { Submodel } from "foldkit" +import type { Html, HtmlBuilder } from "foldkit/html" +import { errorState } from "../../../ui/error-state" +import type { PageViewInputs } from "../../contract" +import { securityLayout } from "../kit/layout" +import { pageStateOf, scannedCount } from "./derive" +import { Message, type Model } from "./model" +import { findingsSection } from "./view-findings" +import { repositoriesPanel } from "./view-repositories" +import { findingSheet } from "./view-sheet" +import { dependenciesLoading, firstScan, footnote, notConnected, severityRail } from "./view-states" + +// The Dependencies tab inside the Security frame. No plan gate: it reads lockfiles. + +type H = HtmlBuilder +type Clock = Readonly<{ nowMs: number; timeZone: string }> + +const body = (h: H, model: Model, clock: Clock): ReadonlyArray => { + const state = pageStateOf(model) + switch (state._tag) { + case "Loading": + return dependenciesLoading(h) + case "Failed": + return [ + errorState(h, { + failure: state.failure, + title: "Couldn't load dependency scans", + onRetry: Message.ClickedRetry(), + }), + ] + case "NotConnected": + return [notConnected(h)] + case "Ready": + return scannedCount(state.repositories) === 0 + ? [ + firstScan(h, model, state.repositories), + repositoriesPanel(h, model, state.repositories, clock), + ] + : [ + severityRail(h, state.findings, state.repositories), + repositoriesPanel(h, model, state.repositories, clock), + h.div( + [h.Class("flex flex-col gap-2")], + [findingsSection(h, model, state, clock), footnote(h)], + ), + ] + } +} + +export const view = Submodel.defineView((model, inputs, h) => { + const clock: Clock = { nowMs: model.nowMs, timeZone: inputs.context.timeZone } + return securityLayout( + h, + { active: "dependencies", overlay: [findingSheet(h, model, clock)] }, + body(h, model, clock), + ) +}) diff --git a/apps/web/src/page/security/kit/layout.ts b/apps/web/src/page/security/kit/layout.ts index ff243f9f8d..9d5d1d24fc 100644 --- a/apps/web/src/page/security/kit/layout.ts +++ b/apps/web/src/page/security/kit/layout.ts @@ -6,9 +6,10 @@ import { pageTitle } from "../../../ui/page-layout" import { dashboardPage } from "../../kit/dashboard-page" import { linkTabStrip } from "../../kit/underline-tabs" // The frame of the Security tabs: title block, the tab strip, content. Overview, Findings and Runs -// are the code audits' page; Exposure and Runtimes read the org's telemetry. +// are the code audits' page; Exposure and Runtimes read the org's telemetry; Dependencies reads +// the connected repositories' lockfiles. -export type SecurityTab = "overview" | "findings" | "runs" | "exposure" | "runtimes" +export type SecurityTab = "overview" | "findings" | "runs" | "exposure" | "runtimes" | "dependencies" const TABS: ReadonlyArray<{ tab: SecurityTab; label: string; icon: IconName; href: string }> = [ { tab: "overview", label: "Overview", icon: "ShieldCheckIcon", href: hrefOf(AppRoute.Security()) }, @@ -26,6 +27,12 @@ const TABS: ReadonlyArray<{ tab: SecurityTab; label: string; icon: IconName; hre }, { tab: "exposure", label: "Exposure", icon: "GlobeIcon", href: hrefOf(AppRoute.SecurityExposure()) }, { tab: "runtimes", label: "Runtimes", icon: "MicrochipIcon", href: hrefOf(AppRoute.SecurityRuntimes()) }, + { + tab: "dependencies", + label: "Dependencies", + icon: "CubeIcon", + href: hrefOf(AppRoute.SecurityDependencies()), + }, ] export const breadcrumbsOf = (tab: SecurityTab): ReadonlyArray => { @@ -59,6 +66,8 @@ const DESCRIPTION = { exposure: "Scanners probing your HTTP services over the last 7 days, read from server spans.", runtimes: "The language runtime each service reports, checked for known vulnerabilities and end-of-life dates.", + dependencies: + "Known vulnerable packages in your connected repositories' lockfiles, rescanned on every push.", } satisfies Record /** The title block: the page `

` and its one line of context, like every other page header. */ diff --git a/apps/web/src/route/app-route.ts b/apps/web/src/route/app-route.ts index aec4e460a8..39438035d0 100644 --- a/apps/web/src/route/app-route.ts +++ b/apps/web/src/route/app-route.ts @@ -49,6 +49,7 @@ export const AppRoute = defineRouteUnion({ SecurityRuns: { search: Search.SecurityRunsSearch }, SecurityExposure: {}, SecurityRuntimes: {}, + SecurityDependencies: {}, SecurityRun: { runId: SecurityAuditRunId }, SecurityFinding: { targetId: SecurityAuditTargetId, fingerprint: Schema.String }, Dashboard: { dashboardId: DashboardId, search: Search.DashboardSearch }, diff --git a/apps/web/src/route/route.test.ts b/apps/web/src/route/route.test.ts index ba7509c7ec..d178d646e2 100644 --- a/apps/web/src/route/route.test.ts +++ b/apps/web/src/route/route.test.ts @@ -144,6 +144,7 @@ describe("routes", () => { expect(parse("/security/runs")._tag).toBe("SecurityRuns") expect(parse("/security/exposure")._tag).toBe("SecurityExposure") expect(parse("/security/runtimes")._tag).toBe("SecurityRuntimes") + expect(parse("/security/dependencies")._tag).toBe("SecurityDependencies") expect(parse("/security/runs/run_9f3c2a1e7b")).toMatchObject({ _tag: "SecurityRun", runId: "run_9f3c2a1e7b", diff --git a/apps/web/src/route/routers.ts b/apps/web/src/route/routers.ts index 77cf931252..e449ec87bc 100644 --- a/apps/web/src/route/routers.ts +++ b/apps/web/src/route/routers.ts @@ -150,6 +150,7 @@ const routers = { ), SecurityExposure: pipe(path("security", "exposure"), Route.mapTo(AppRoute.SecurityExposure)), SecurityRuntimes: pipe(path("security", "runtimes"), Route.mapTo(AppRoute.SecurityRuntimes)), + SecurityDependencies: pipe(path("security", "dependencies"), Route.mapTo(AppRoute.SecurityDependencies)), SecurityRun: pipe( path("security", "runs"), slash(schemaSegment("runId", SecurityAuditRunId)), diff --git a/packages/backend/package.json b/packages/backend/package.json index 02b1c8bcdb..10f8a70608 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -34,10 +34,10 @@ "@maple/chat-platform": "workspace:*", "@maple/db": "workspace:*", "@maple/domain": "workspace:*", - "@maple/lockfiles": "workspace:*", "@maple/email": "workspace:*", "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", + "@maple/lockfiles": "workspace:*", "@maple/prometheus-otlp": "workspace:*", "@maple/query-engine": "workspace:*", "@maple/query-engine-integrations": "workspace:*", diff --git a/packages/backend/scripts/depscan-vs-osv-scanner.ts b/packages/backend/scripts/depscan-vs-osv-scanner.ts index fa6e807658..25ac01c690 100644 --- a/packages/backend/scripts/depscan-vs-osv-scanner.ts +++ b/packages/backend/scripts/depscan-vs-osv-scanner.ts @@ -40,7 +40,7 @@ const compare = (path: string) => const content = yield* Effect.promise(() => Bun.file(path).text()) const parsed = yield* parseLockfile({ path: basename(path), content, manifests }) const catalog = yield* PackageAdvisoryCatalog - const ours = yield* catalog.advisories(parsed.packages) + const { byPackage: ours } = yield* catalog.advisories(parsed.packages) const theirs = new Map( (osvScanner(path).results ?? []) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts index 105e8d6398..0d2e7cc050 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts @@ -10,7 +10,14 @@ import { recordingQueueLayer, testRepoLayer, } from "@maple/backend/services/integrations/vcs/__tests__/harness" -import { DependencyScanService } from "./DependencyScanService" +import { systemTenant } from "@maple/backend/services/alerts/system-tenant" +import { + DependencyScanService, + PRODUCTION_SCANS_PER_READ, + type RunningCommitRow, + RunningCommits, + RunningCommitsReadError, +} from "./DependencyScanService" import { DependencyScanStore } from "./DependencyScanStore" const trackedDbs: TestDb[] = [] @@ -20,12 +27,25 @@ const org = Schema.decodeSync(OrgId)("org_deps") const admin = Schema.decodeSync(UserId)("user_admin") const sha = (digit: string) => Schema.decodeSync(GitCommitSha)(digit.repeat(40)) -const serviceFor = (sent: Array) => { +const tenant = systemTenant(org) + +/** What telemetry says services ran; `undefined` makes the read fail. */ +interface Telemetry { + rows: ReadonlyArray | undefined +} + +const serviceFor = (sent: Array, telemetry: Telemetry = { rows: [] }) => { const db = createTestDb(trackedDbs) const store = DependencyScanStore.layer.pipe(Layer.provide(db.layer)) const vcs = testRepoLayer(db) + const running = Layer.succeed(RunningCommits, { + read: () => + telemetry.rows === undefined + ? Effect.fail(new RunningCommitsReadError({ message: "warehouse down", cause: undefined })) + : Effect.succeed(telemetry.rows), + }) return Layer.effect(DependencyScanService, DependencyScanService.make).pipe( - Layer.provideMerge(Layer.mergeAll(store, vcs, recordingQueueLayer(sent))), + Layer.provideMerge(Layer.mergeAll(store, vcs, recordingQueueLayer(sent), running)), ) } @@ -110,7 +130,7 @@ describe("DependencyScanService", () => { return Effect.gen(function* () { const service = yield* DependencyScanService const api = yield* seed - const before = yield* service.listRepositories(org) + const before = yield* service.listRepositories(tenant) assert.deepStrictEqual( before.repositories.map((repository) => [repository.fullName, repository.latestScan]), [ @@ -122,7 +142,7 @@ describe("DependencyScanService", () => { const latest = yield* recordScan(api.id, "b", true) // An older commit scanned because it runs in production is not the branch's latest scan. yield* recordScan(api.id, "c", false) - const after = yield* service.listRepositories(org) + const after = yield* service.listRepositories(tenant) assert.strictEqual(after.repositories[0]?.latestScan?.id, latest.id) assert.strictEqual(after.repositories[0]?.latestScan?.vulnerableCount, 1) }).pipe(Effect.provide(serviceFor(sent))) @@ -158,7 +178,7 @@ describe("DependencyScanService", () => { const service = yield* DependencyScanService const api = yield* seed yield* recordScan(api.id, "a", true) - const { findings } = yield* service.listFindings(org, {}) + const { findings } = yield* service.listFindings(tenant, {}) assert.deepStrictEqual( findings.map((finding) => [finding.repositoryFullName, finding.packageName, finding.status]), [["octo/api", "lodash", "open"]], @@ -173,7 +193,7 @@ describe("DependencyScanService", () => { [dismissed.status, dismissed.statusReason, dismissed.statusChangedBy], ["dismissed", "Only used in a build script", admin], ) - assert.deepStrictEqual((yield* service.listFindings(org, { status: "open" })).findings, []) + assert.deepStrictEqual((yield* service.listFindings(tenant, { status: "open" })).findings, []) const missing = Schema.decodeSync(DependencyFindingId)("0".repeat(32)) const error = yield* Effect.flip(service.updateFinding(org, admin, missing, { status: "open" })) @@ -181,3 +201,162 @@ describe("DependencyScanService", () => { }).pipe(Effect.provide(serviceFor(sent))) }) }) + +const commitOf = (digit: string) => ({ + sha: digit.repeat(40), + message: `commit ${digit}`, + authorName: null, + authorEmail: null, + authorLogin: null, + authorAvatarUrl: null, + authoredAt: null, + committedAt: 1, + htmlUrl: `https://github.com/octo/api/commit/${digit.repeat(40)}`, +}) + +const running = ( + serviceName: string, + environment: string, + commit: string, + spanCount: number, +): RunningCommitRow => ({ + serviceName, + environment, + commitSha: commit.length === 1 ? commit.repeat(40) : commit, + spanCount, +}) + +describe("DependencyScanService production", () => { + it.effect( + "joins findings to the services running a commit whose scan matched them, and queues unscanned commits", + () => { + const sent: Array = [] + const telemetry: Telemetry = { + rows: [ + running("api-server", "production", "c", 9000), + running("worker", "production", "d", 50), + running("api-server", "staging", "a", 100), + // Not a commit of a connected repository, and a placeholder revision. + running("billing", "production", "f", 400), + running("legacy", "production", "unknown", 10), + ], + } + return Effect.gen(function* () { + const service = yield* DependencyScanService + const vcs = yield* VcsRepository + const api = yield* seed + yield* vcs.upsertCommits(api, [commitOf("a"), commitOf("c"), commitOf("d")]) + yield* recordScan(api.id, "c", false) + yield* recordScan(api.id, "a", true) + + const { findings, production } = yield* service.listFindings(tenant, {}) + // Services, not rows: api-server runs two commits and counts once. + assert.deepStrictEqual(production, { available: true, runningServices: 4, linkedServices: 2 }) + assert.deepStrictEqual( + findings[0]?.production?.map((s) => [ + s.serviceName, + s.environment, + s.commitSha.slice(0, 1), + s.spanCount, + ]), + [ + ["api-server", "production", "c", 9000], + ["api-server", "staging", "a", 100], + ], + ) + assert.deepStrictEqual( + sent.map((job) => + job.kind === "dependency-scan" ? [job.trigger, job.commitSha] : [job.kind], + ), + [["production", sha("d")]], + ) + + const { repositories } = yield* service.listRepositories(tenant) + assert.deepStrictEqual( + repositories[0]?.runningCommits?.map((commit) => [ + commit.commitSha.slice(0, 1), + commit.scanStatus, + ]), + [ + ["c", "completed"], + ["d", null], + ["a", "completed"], + ], + ) + }).pipe(Effect.provide(serviceFor(sent, telemetry))) + }, + ) + + it.effect( + "retries a running commit whose scan failed an hour ago, and keeps its older completed scan's matches", + () => { + const sent: Array = [] + const telemetry: Telemetry = { rows: [running("api-server", "production", "c", 9000)] } + return Effect.gen(function* () { + const service = yield* DependencyScanService + const store = yield* DependencyScanStore + const vcs = yield* VcsRepository + const api = yield* seed + yield* vcs.upsertCommits(api, [commitOf("c")]) + yield* recordScan(api.id, "a", true) + yield* recordScan(api.id, "c", false) + yield* TestClock.adjust("1 second") + const failed = yield* store.startScan({ + orgId: org, + repositoryId: api.id, + commitSha: sha("c"), + ref: null, + trigger: "production", + defaultBranchHead: false, + }) + yield* store.failScan({ orgId: org, scanId: failed.id, error: "GitHub answered 502" }) + + const { findings } = yield* service.listFindings(tenant, {}) + assert.deepStrictEqual( + findings.map((finding) => finding.production?.map((s) => s.serviceName)), + [["api-server"]], + ) + assert.deepStrictEqual(sent, []) + + yield* TestClock.adjust("61 minutes") + yield* service.listRepositories(tenant) + assert.deepStrictEqual( + sent.map((job) => (job.kind === "dependency-scan" ? job.commitSha : undefined)), + [sha("c")], + ) + }).pipe(Effect.provide(serviceFor(sent, telemetry))) + }, + ) + + it.effect("queues at most a handful of unscanned commits per read", () => { + const sent: Array = [] + const digits = "0123456789abcdef".split("") + const telemetry: Telemetry = { + rows: digits.map((digit, i) => running(`svc-${i}`, "production", digit, 100 - i)), + } + return Effect.gen(function* () { + const service = yield* DependencyScanService + const vcs = yield* VcsRepository + const api = yield* seed + yield* vcs.upsertCommits(api, digits.map(commitOf)) + yield* service.listRepositories(tenant) + assert.strictEqual(sent.length, PRODUCTION_SCANS_PER_READ) + assert.strictEqual(sent[0]?.kind === "dependency-scan" ? sent[0].commitSha : undefined, sha("0")) + }).pipe(Effect.provide(serviceFor(sent, telemetry))) + }) + + it.effect("still lists findings, unranked by traffic, when telemetry cannot be read", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const api = yield* seed + yield* recordScan(api.id, "a", true) + const { findings, production } = yield* service.listFindings(tenant, {}) + assert.deepStrictEqual(production, { available: false, runningServices: 0, linkedServices: 0 }) + assert.deepStrictEqual( + findings.map((finding) => [finding.packageName, finding.production]), + [["lodash", []]], + ) + }).pipe(Effect.provide(serviceFor(sent, { rows: undefined }))) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index 419c6333ea..b4133cc75e 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -1,4 +1,4 @@ -import type { DependencyFindingRow, DependencyScanRow } from "@maple/db/tables" +import type { DependencyFindingRow, DependencyScanMatchRow, DependencyScanRow } from "@maple/db/tables" import type { DependencyFindingId } from "@maple/domain/dependency-scan" import { DependencyFindingNotFoundError, @@ -9,28 +9,139 @@ import { type DependencyScanRepositoriesResponse, DependencyScanRepositoryNotFoundError, type DependencyScanSummary, + type DependencyRunningCommit, + type DependencyRunningService, + GitCommitSha, isInstallationProcessable, + type VcsInstallation, type VcsRepo, type VcsRepositoryId, + type VcsSyncJob, } from "@maple/domain/http" import type { OrgId, UserId } from "@maple/domain/primitives" import type { DependencyFindingStatus } from "@maple/domain/dependency-scan" -import { Context, Effect, Layer, Option } from "effect" +import { Clock, Context, Effect, Layer, Option, Schema } from "effect" +import { CH, formatWarehouseDateTime } from "@maple/query-engine" +import type { TenantContext } from "@maple/backend/services/auth/AuthService" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" import { VcsSyncQueue } from "@maple/backend/services/integrations/vcs/VcsSyncQueue" import { DependencyScanStore } from "./DependencyScanStore" +import { + findingKey, + latestCompletedPerCommit, + latestScanPerCommit, + needsScan, + productionByFinding, + type RunningService, + riskScore, + scanAt, +} from "./production" // The Dependencies tab's reads and writes: repositories with their latest scan, findings with // their repository named, triage, and "scan now", which goes through the VCS sync queue. /** Findings returned per read; the table sorts worst first, so the cut drops the least severe. */ export const FINDINGS_LIMIT = 500 -/** Scans read to find each repository's latest. */ -const SCANS_LIMIT = 500 +/** Production is what services served over the last day. */ +const RUNNING_WINDOW_MS = 24 * 60 * 60 * 1000 +/** Unscanned running commits queued per read, busiest first; the next read queues more. */ +export const PRODUCTION_SCANS_PER_READ = 10 + +export type RunningCommitRow = typeof CH.runningCommitsRowSchema.Type + +export class RunningCommitsReadError extends Schema.TaggedError()( + "@maple/dependency-scan/RunningCommitsReadError", + { message: Schema.String, cause: Schema.Defect() }, +) {} + +/** Which commit each service served traffic from in a window, busiest first. */ +export class RunningCommits extends Context.Service< + RunningCommits, + { + readonly read: ( + tenant: TenantContext, + window: { readonly startMs: number; readonly endMs: number }, + ) => Effect.Effect, RunningCommitsReadError> + } +>()("@maple/backend/services/dependency-scan/RunningCommits") { + static readonly layer = Layer.effect( + this, + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + return { + read: (tenant, window) => + warehouse + .compiledQuery( + tenant, + CH.compile( + CH.runningCommitsQuery(), + { + orgId: tenant.orgId, + startTime: formatWarehouseDateTime(window.startMs), + endTime: formatWarehouseDateTime(window.endMs), + }, + { rowSchema: CH.runningCommitsRowSchema }, + ), + { profile: "list", context: "dependencyRunningCommits" }, + ) + .pipe( + Effect.mapError( + (cause) => + new RunningCommitsReadError({ + message: "Could not read running commits", + cause, + }), + ), + ), + } + }), + ).pipe(Layer.provide(WarehouseQueryService.layer)) +} + +interface Connected { + readonly installation: VcsInstallation + readonly repository: VcsRepo +} + +interface Production { + readonly available: boolean + /** Running (service, environment, commit) rows whose commit belongs to a connected repository. */ + readonly running: ReadonlyArray + /** Distinct services reporting a commit. */ + readonly runningServices: number + /** Latest scan per running commit, any status: what the repository list shows. */ + readonly latest: ReadonlyMap + /** The matches of each running commit's latest completed scan. */ + readonly matches: ReadonlyArray + readonly completed: ReadonlyMap +} + +const NO_PRODUCTION: Production = { + available: false, + running: [], + runningServices: 0, + latest: new Map(), + matches: [], + completed: new Map(), +} + +const distinctServices = (services: ReadonlyArray<{ readonly serviceName: string }>): number => + new Set(services.map((service) => service.serviceName)).size + +const decodeSha = Schema.decodeUnknownOption(GitCommitSha) + +const toRunningService = (service: RunningService): DependencyRunningService => ({ + serviceName: service.serviceName, + environment: service.environment, + commitSha: service.commitSha, + spanCount: service.spanCount, +}) export interface DependencyScanServiceApi { + /** Each repository with its latest scan and the commits services run from it. */ readonly listRepositories: ( - orgId: OrgId, + tenant: TenantContext, ) => Effect.Effect readonly requestScan: ( orgId: OrgId, @@ -39,8 +150,9 @@ export interface DependencyScanServiceApi { { readonly queued: boolean }, DependencyScanPersistenceError | DependencyScanRepositoryNotFoundError > + /** Most urgent first; production comes from telemetry and is left out when it cannot be read. */ readonly listFindings: ( - orgId: OrgId, + tenant: TenantContext, query: { readonly status?: DependencyFindingStatus | undefined readonly repositoryId?: VcsRepositoryId | undefined @@ -56,8 +168,8 @@ export interface DependencyScanServiceApi { const persistence = (message: string) => - (error: { readonly message: string }): DependencyScanPersistenceError => - new DependencyScanPersistenceError({ message: `${message}: ${error.message}` }) + (error: { readonly _tag: string; readonly message: string }): DependencyScanPersistenceError => + new DependencyScanPersistenceError({ message: `${message}: ${error.message}`, cause: error._tag }) export const scanSummaryOf = (row: DependencyScanRow): DependencyScanSummary => ({ id: row.id, @@ -111,6 +223,7 @@ export class DependencyScanService extends Context.Service - vcs.listRepositoriesByInstallation(installation.id, "active"), + vcs + .listRepositoriesByInstallation(installation.id, "active") + .pipe( + Effect.map((list) => + list.map((repository): Connected => ({ installation, repository })), + ), + ), ) return repositories.flat() }) + /** + * Queues a scan of each running commit that needs one, busiest first and capped. A queue + * failure is logged here: the read it runs beside still answers. + */ + const queueProductionScans = Effect.fn("DependencyScanService.queueProductionScans")(function* ( + connected: ReadonlyArray, + production: Production, + ) { + const now = yield* Clock.currentTimeMillis + // Rows come busiest first, so a commit's first row is its busiest. + const unscanned = production.running + .filter((service) => needsScan(production.latest, service, now)) + .filter( + (service, index, all) => + all.findIndex( + (other) => + other.repositoryId === service.repositoryId && + other.commitSha === service.commitSha, + ) === index, + ) + .slice(0, PRODUCTION_SCANS_PER_READ) + const jobs = unscanned.flatMap((service): ReadonlyArray => { + const owner = connected.find((entry) => entry.repository.id === service.repositoryId) + return owner === undefined + ? [] + : [ + { + kind: "dependency-scan", + provider: owner.repository.provider, + externalInstallationId: owner.installation.externalInstallationId, + externalRepoId: owner.repository.externalRepoId, + trigger: "production", + commitSha: service.commitSha, + }, + ] + }) + yield* Effect.annotateCurrentSpan({ "maple.depscan.production_queued": jobs.length }) + if (jobs.length === 0) return + yield* queue + .sendBatch(jobs) + .pipe( + Effect.catch((error) => + Effect.logWarning("[DependencyScan] could not queue production scans").pipe( + Effect.annotateLogs({ error: error.message }), + ), + ), + ) + }) + + /** The commits services ran over the last day, joined to connected repositories and their scans. */ + const readProduction = Effect.fn("DependencyScanService.readProduction")(function* ( + tenant: TenantContext, + connected: ReadonlyArray, + ) { + const now = yield* Clock.currentTimeMillis + const rows = yield* runningCommits.read(tenant, { + startMs: now - RUNNING_WINDOW_MS, + endMs: now, + }) + const withSha = rows.flatMap((row) => + Option.match(decodeSha(row.commitSha), { + onNone: () => [], + onSome: (sha) => [{ ...row, sha }], + }), + ) + const connectedIds = new Set(connected.map((entry) => entry.repository.id)) + const commits = yield* vcs.findCommitsByShas( + tenant.orgId, + withSha.map((row) => row.sha), + ) + const repositoryOf = new Map( + commits + .filter((commit) => connectedIds.has(commit.repositoryId)) + .map((commit) => [commit.sha, commit.repositoryId] as const), + ) + const running = withSha.flatMap((row): ReadonlyArray => { + const repositoryId = repositoryOf.get(row.sha) + return repositoryId === undefined + ? [] + : [ + { + repositoryId, + commitSha: row.sha, + serviceName: row.serviceName, + environment: row.environment, + spanCount: row.spanCount, + }, + ] + }) + const scans = yield* store.scansAtCommits({ + orgId: tenant.orgId, + repositoryIds: running.map((service) => service.repositoryId), + commitShas: running.map((service) => service.commitSha), + }) + const completed = latestCompletedPerCommit(scans) + const matches = yield* store.matchesOfScans({ + orgId: tenant.orgId, + scanIds: [...completed.values()].map((scan) => scan.id), + }) + const production: Production = { + available: true, + running, + runningServices: distinctServices(rows), + latest: latestScanPerCommit(scans), + matches, + completed, + } + yield* Effect.annotateCurrentSpan({ + "maple.depscan.running_services": production.runningServices, + "maple.depscan.linked_services": distinctServices(running), + }) + yield* queueProductionScans(connected, production) + return production + }) + + /** Production when telemetry and the scans can be read; the tab still lists findings without it. */ + const productionOrNone = (tenant: TenantContext, connected: ReadonlyArray) => + readProduction(tenant, connected).pipe( + Effect.catch((error) => + Effect.logWarning("[DependencyScan] production read failed").pipe( + Effect.annotateLogs({ orgId: tenant.orgId, error: error.message }), + Effect.as(NO_PRODUCTION), + ), + ), + ) + const listRepositories: DependencyScanServiceApi["listRepositories"] = Effect.fn( "DependencyScanService.listRepositories", - )(function* (orgId) { - const repositories = yield* connectedRepositories(orgId).pipe( + )(function* (tenant) { + const orgId = tenant.orgId + const connected = yield* connectedRepositories(orgId).pipe( Effect.mapError(persistence("Could not list repositories")), ) - const scans = yield* store - .listScans({ orgId, limit: SCANS_LIMIT }) + const branchScans = yield* store + .latestBranchScans({ orgId }) .pipe(Effect.mapError(persistence("Could not list scans"))) - // Newest first, so the first tracked-branch scan per repository is its latest. - const latest = scans - .filter((scan) => scan.defaultBranchHead) - .reduce((byRepository, scan) => { - if (!byRepository.has(scan.repositoryId)) byRepository.set(scan.repositoryId, scan) - return byRepository - }, new Map()) + const latest = new Map(branchScans.map((scan) => [scan.repositoryId, scan])) + const production = yield* productionOrNone(tenant, connected) + const runningCommitsOf = ( + repositoryId: VcsRepositoryId, + ): ReadonlyArray => { + const services = production.running.filter( + (service) => service.repositoryId === repositoryId, + ) + const shas = [...new Set(services.map((service) => service.commitSha))] + return shas.map((commitSha) => ({ + commitSha, + scanStatus: scanAt(production.latest, { repositoryId, commitSha })?.status ?? null, + services: services + .filter((service) => service.commitSha === commitSha) + .map(toRunningService), + })) + } return { - repositories: repositories - .map((repository: VcsRepo) => { + repositories: connected + .map(({ repository }) => { const scan = latest.get(repository.id) return { id: repository.id, @@ -151,6 +407,7 @@ export class DependencyScanService extends Context.Service a.fullName.localeCompare(b.fullName)), @@ -187,7 +444,8 @@ export class DependencyScanService extends Context.Service row.repositoryId), - ) - .pipe(Effect.mapError(persistence("Could not read repositories"))) - const names = new Map(repositories.map((repository) => [repository.id, repository.fullName])) - yield* Effect.annotateCurrentSpan({ "result.rowCount": rows.length }) + const connected = yield* connectedRepositories(orgId).pipe( + Effect.mapError(persistence("Could not list repositories")), + ) + const production = yield* productionOrNone(tenant, connected) + const byFinding = productionByFinding( + production.running, + production.completed, + production.matches, + ) + const names = new Map(connected.map(({ repository }) => [repository.id, repository.fullName])) + const ranked = rows + .map((row, order) => { + const services = byFinding.get(findingKey(row)) ?? [] + const spans = services.reduce((sum, service) => sum + service.spanCount, 0) + return { row, order, services, score: riskScore(row, spans) } + }) + .sort((a, b) => b.score - a.score || a.order - b.order) + yield* Effect.annotateCurrentSpan({ + "result.rowCount": rows.length, + "maple.depscan.findings_in_production": ranked.filter( + (entry) => entry.services.length > 0, + ).length, + }) return { - findings: rows.map((row) => - findingViewOf(row, names.get(row.repositoryId) ?? "unknown repository"), - ), + findings: ranked.map(({ row, services }) => ({ + ...findingViewOf(row, names.get(row.repositoryId) ?? "unknown repository"), + production: services.map(toRunningService), + })), + production: { + available: production.available, + runningServices: production.runningServices, + linkedServices: distinctServices(production.running), + }, } }) @@ -257,6 +536,13 @@ export class DependencyScanService extends Context.Service { const scanner = yield* DependencyScanner yield* scanner.scan(installation, repository, { trigger: "schedule" }) + // A production scan of an older commit in between does not hide the branch's last scan. + yield* scanner.scan(installation, repository, { trigger: "production", commitSha: sha("9") }) state.head = sha("b") state.files.set("README.md", { sha: "readme-2", content: "# shop 2" }) const unchanged = yield* scanner.scan(installation, repository, { trigger: "push" }) @@ -325,6 +327,20 @@ describe("DependencyScanner", () => { }, ) + it.effect("records a tree too large to list once, not on every push", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + state.truncated = true + yield* Effect.flip(scanner.scan(installation, repository, { trigger: "push" })) + state.head = sha("b") + const again = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(again._tag, "Unchanged") + assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 1) + }).pipe(Effect.provide(scannerFor(state))) + }) + it.effect("records an oversized lockfile without reading it and keeps its findings open", () => { const state = world({ "package-lock.json": packageLock("4.17.15") }) return Effect.gen(function* () { @@ -359,6 +375,14 @@ describe("DependencyScanner", () => { 7, ) assert.deepStrictEqual(yield* findingsOf, []) + // A commit's content never changes, so asking again reads nothing. + const reads = state.blobReads.length + const again = yield* scanner.scan(installation, repository, { + trigger: "production", + commitSha: sha("e"), + }) + assert.strictEqual(again._tag, "Unchanged") + assert.strictEqual(state.blobReads.length, reads) }).pipe(Effect.provide(scannerFor(state))) }) }) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts index 4f2f84f729..f532f49518 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -12,27 +12,40 @@ import { } from "@maple/domain/http" import { type PackageAdvisory, PackageEcosystem, packageKey } from "@maple/domain/package-advisories" import { parseLockfile } from "@maple/lockfiles" -import { Context, Effect, Layer, Option, Schema } from "effect" +import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import type { VcsProviderClient, VcsTreeFile, } from "@maple/backend/services/integrations/vcs/VcsProviderClient" import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" import { DependencyScanSink } from "@maple/backend/services/integrations/vcs/PullRequestEventSink" -import { summarizeCause } from "@maple/backend/platform/describe-cause" -import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" +import { + MAX_VULNERABILITIES, + PackageAdvisoryCatalog, +} from "@maple/backend/services/security/PackageAdvisoryCatalog" import { type DependencyScanMatchInput, DependencyScanStore } from "./DependencyScanStore" -import { manifestsFor, type SelectedLockfile, sameLockfiles, selectLockfiles } from "./lockfile-selection" +import { + MAX_LOCKFILES, + manifestsFor, + type SelectedLockfile, + sameLockfiles, + selectLockfiles, +} from "./lockfile-selection" +import { RESCAN_AFTER_MS } from "./production" // Reads a repository's lockfiles at one commit through the VCS provider, looks their packages up // in OSV and stores the scan. A lockfile's parse is cached by its blob sha (and its manifests'), // so a rescan for new advisories reads nothing from the provider but the tree. const PARSE_CACHE_BUCKET = "lockfile-packages" +/** Part of the parse cache key: bump it when a parser fix should reach lockfiles already cached. */ +const PARSER_VERSION = 1 const PARSE_CACHE_TTL_SECONDS = 7 * 24 * 60 * 60 /** Larger lockfiles are recorded as too large rather than read into the Worker's memory. */ -export const MAX_LOCKFILE_BYTES = 15 * 1024 * 1024 -const READ_CONCURRENCY = 4 +export const MAX_LOCKFILE_BYTES = 8 * 1024 * 1024 +/** Two lockfiles at a time keeps a monorepo's largest pair well inside a Worker's memory. */ +const READ_CONCURRENCY = 2 +const MANIFEST_CONCURRENCY = 4 const LockedPackageRow = Schema.Struct({ ecosystem: PackageEcosystem, @@ -60,7 +73,7 @@ export interface DependencyScanRequest { export type DependencyScanOutcome = | { readonly _tag: "Scanned"; readonly scan: DependencyScanRow } - /** A push that left every lockfile as the last scan read it. */ + /** A push that left every lockfile as the last scan read it, or a commit already scanned. */ | { readonly _tag: "Unchanged" } /** The tracked branch has no commit yet. */ | { readonly _tag: "EmptyRepository" } @@ -89,6 +102,8 @@ export class DependencySources extends Context.Service< ).pipe(Layer.provide(VcsProviderRegistry.layer)) } +const TREE_TOO_LARGE = "The repository tree is too large to list in one read" + interface ReadLockfile { readonly lockfile: DependencyScanLockfile readonly packages: ReadonlyArray @@ -144,7 +159,7 @@ export class DependencyScanner extends Context.Service [manifest.relativePath, text] as const)), - { concurrency: READ_CONCURRENCY }, + { concurrency: MANIFEST_CONCURRENCY }, ) const parsed = yield* parseLockfile({ path: selected.file.path, @@ -157,9 +172,11 @@ export class DependencyScanner extends Context.Service manifest.file.sha)].join( - ":", - ), + key: [ + `v${PARSER_VERSION}`, + selected.file.sha, + ...manifests.map((manifest) => manifest.file.sha), + ].join(":"), ttlSeconds: PARSE_CACHE_TTL_SECONDS, schema: CachedLockfile, }, @@ -246,6 +263,24 @@ export class DependencyScanner extends Context.Service + scan.status === "completed" || + (scan.status === "running" && now - scan.startedAt <= RESCAN_AFTER_MS), + ) + if (covered) return { _tag: "Unchanged" } satisfies DependencyScanOutcome + } + const tree = yield* provider .fetchTreeFiles(installation, repo, commitSha) .pipe(Effect.mapError(scanError("Could not list the repository tree"))) @@ -253,17 +288,16 @@ export class DependencyScanner extends Context.Service read.lockfile), + lockfiles: [ + ...reads.map((read) => read.lockfile), + ...skipped.map((lockfile) => + lockfileRow( + lockfile, + "skipped", + 0, + `Past the ${MAX_LOCKFILES} lockfiles a scan reads`, + ), + ), + ], packageCount: packages.length, matches, + incomplete: + advisories.unread === 0 + ? undefined + : `${advisories.unread} known vulnerabilities past the ${MAX_VULNERABILITIES} a scan reads were left out`, }) .pipe(Effect.mapError(scanError("Could not store the scan"))) }) @@ -352,13 +409,14 @@ export const DependencyScanSinkLive = Layer.effect( .scan(installation, repository, { trigger: job.trigger, commitSha: job.commitSha }) .pipe( Effect.asVoid, - Effect.catchCause((cause) => + // The scan row records the failure; a defect still fails the job so it shows. + Effect.catch((error) => Effect.logWarning("[DependencyScan] scan failed").pipe( Effect.annotateLogs({ orgId: repository.orgId, repository: repository.fullName, trigger: job.trigger, - cause: summarizeCause(cause), + error: error.message, }), ), ), diff --git a/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts b/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts index ef2a6eba02..50416fdf6f 100644 --- a/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts +++ b/packages/backend/src/services/dependency-scan/lockfile-selection.test.ts @@ -28,14 +28,18 @@ describe("selectLockfiles", () => { ["crates/core/Cargo.lock", "cargo-lock"], ["services/api/go.mod", "go-mod"], ]) - expect(skipped).toBe(0) + expect(skipped).toEqual([]) }) it("caps the lockfiles a scan reads and counts the rest", () => { const many = Array.from({ length: MAX_LOCKFILES + 3 }, (_, i) => file(`pkg${i}/bun.lock`)) const { selected, skipped } = selectLockfiles(many) expect(selected).toHaveLength(MAX_LOCKFILES) - expect(skipped).toBe(3) + expect(skipped).toHaveLength(3) + // Every lockfile is either read or recorded as skipped, never dropped. + expect(new Set([...selected, ...skipped].map((lockfile) => lockfile.file.path)).size).toBe( + many.length, + ) }) }) diff --git a/packages/backend/src/services/dependency-scan/lockfile-selection.ts b/packages/backend/src/services/dependency-scan/lockfile-selection.ts index b22ff97635..cc2d22129d 100644 --- a/packages/backend/src/services/dependency-scan/lockfile-selection.ts +++ b/packages/backend/src/services/dependency-scan/lockfile-selection.ts @@ -7,7 +7,7 @@ import type { VcsTreeFile } from "@maple/backend/services/integrations/vcs/VcsPr /** Lockfiles read per scan; the rest are left out, deepest paths first. */ export const MAX_LOCKFILES = 50 /** Manifests read for one yarn lockfile, which says nothing about direct and dev dependencies. */ -export const MAX_MANIFESTS = 50 +export const MAX_MANIFESTS = 20 export interface SelectedLockfile { readonly file: VcsTreeFile @@ -27,7 +27,8 @@ const depth = (path: string): number => path.split("/").length */ export function selectLockfiles(files: ReadonlyArray): { readonly selected: ReadonlyArray - readonly skipped: number + /** Past the cap: recorded as skipped so their findings are kept, not read as deleted. */ + readonly skipped: ReadonlyArray } { const candidates = files.flatMap((file): ReadonlyArray => { const kind = lockfileKindOf(file.path) @@ -39,7 +40,7 @@ export function selectLockfiles(files: ReadonlyArray): { const kept = candidates .filter((c) => c.kind !== "go-sum" || !goModDirs.has(directoryOf(c.file.path))) .sort((a, b) => depth(a.file.path) - depth(b.file.path) || a.file.path.localeCompare(b.file.path)) - return { selected: kept.slice(0, MAX_LOCKFILES), skipped: Math.max(0, kept.length - MAX_LOCKFILES) } + return { selected: kept.slice(0, MAX_LOCKFILES), skipped: kept.slice(MAX_LOCKFILES) } } /** The package.json files under a yarn lockfile's directory, keyed relative to it. */ diff --git a/packages/backend/src/services/dependency-scan/production.test.ts b/packages/backend/src/services/dependency-scan/production.test.ts new file mode 100644 index 0000000000..a3546b01fc --- /dev/null +++ b/packages/backend/src/services/dependency-scan/production.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest" +import { riskScore } from "./production" + +const rank = ( + findings: ReadonlyArray<{ + readonly name: string + readonly severity: "critical" | "high" | "medium" | "low" | "unknown" + readonly relation: "direct" | "dev" | "transitive" | "unknown" + readonly spans: number + }>, +) => + [...findings].sort((a, b) => riskScore(b, b.spans) - riskScore(a, a.spans)).map((finding) => finding.name) + +describe("riskScore", () => { + it("puts a medium advisory a busy service runs above a high one nothing deploys", () => { + expect( + rank([ + { name: "high, not deployed", severity: "high", relation: "direct", spans: 0 }, + { name: "medium, in production", severity: "medium", relation: "transitive", spans: 50_000 }, + ]), + ).toEqual(["medium, in production", "high, not deployed"]) + }) + + it("ranks the same advisory by the traffic that runs it", () => { + expect( + rank([ + { name: "quiet", severity: "high", relation: "transitive", spans: 10 }, + { name: "busy", severity: "high", relation: "transitive", spans: 1_000_000 }, + { name: "undeployed", severity: "high", relation: "transitive", spans: 0 }, + ]), + ).toEqual(["busy", "quiet", "undeployed"]) + }) + + it("keeps a critical advisory first even when it is only a direct dependency nothing deploys", () => { + expect( + rank([ + { name: "low, busy", severity: "low", relation: "transitive", spans: 1_000_000 }, + { name: "critical, undeployed", severity: "critical", relation: "direct", spans: 0 }, + ]), + ).toEqual(["critical, undeployed", "low, busy"]) + }) + + it("ranks a dev-only dependency below the same advisory shipped to users", () => { + expect( + rank([ + { name: "dev", severity: "high", relation: "dev", spans: 0 }, + { name: "transitive", severity: "high", relation: "transitive", spans: 0 }, + ]), + ).toEqual(["transitive", "dev"]) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/production.ts b/packages/backend/src/services/dependency-scan/production.ts new file mode 100644 index 0000000000..ddaaaeef35 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/production.ts @@ -0,0 +1,134 @@ +import type { DependencyScanMatchRow, DependencyScanRow } from "@maple/db/tables" +import type { DependencyRelation } from "@maple/domain/dependency-scan" +import type { GitCommitSha, VcsRepositoryId } from "@maple/domain/http" +import type { AdvisorySeverity } from "@maple/domain/package-advisories" + +// Which findings the code running in production carries, and how the findings rank by it. Pure: +// the service reads the running commits, their scans and matches, and hands them in. + +/** One service serving traffic from a commit of a connected repository. */ +export interface RunningService { + readonly repositoryId: VcsRepositoryId + readonly commitSha: GitCommitSha + readonly serviceName: string + readonly environment: string + readonly spanCount: number +} + +export interface FindingKeyFields { + readonly repositoryId: VcsRepositoryId + readonly lockfilePath: string + readonly ecosystem: string + readonly packageName: string + readonly advisoryId: string +} + +export const findingKey = (finding: FindingKeyFields): string => + [ + finding.repositoryId, + finding.lockfilePath, + finding.ecosystem, + finding.packageName, + finding.advisoryId, + ].join("\u0000") + +const commitKey = (repositoryId: string, commitSha: string) => `${repositoryId}@${commitSha}` + +/** The latest scan per (repository, commit), from scans listed newest first. */ +export const latestScanPerCommit = ( + scans: ReadonlyArray, +): ReadonlyMap => + scans.reduce((latest, scan) => { + const key = commitKey(scan.repositoryId, scan.commitSha) + if (!latest.has(key)) latest.set(key, scan) + return latest + }, new Map()) + +/** The latest completed scan per (repository, commit): what the matches come from. */ +export const latestCompletedPerCommit = ( + scans: ReadonlyArray, +): ReadonlyMap => + latestScanPerCommit(scans.filter((scan) => scan.status === "completed")) + +/** A failed or stuck scan is retried once it is this old, so one bad read does not stick. */ +export const RESCAN_AFTER_MS = 60 * 60 * 1000 + +/** Whether a running commit should be scanned: never scanned, or its last scan failed or stalled. */ +export const needsScan = ( + latest: ReadonlyMap, + running: Pick, + nowMs: number, +): boolean => { + const scan = scanAt(latest, running) + if (scan === undefined) return true + return scan.status !== "completed" && nowMs - scan.startedAt > RESCAN_AFTER_MS +} + +export const scanAt = ( + latest: ReadonlyMap, + running: Pick, +): DependencyScanRow | undefined => latest.get(commitKey(running.repositoryId, running.commitSha)) + +/** + * The running services per finding key: a service counts when the latest completed scan of the + * commit it runs (`completed`, from `latestCompletedPerCommit`) matched the finding. Busiest first. + */ +export function productionByFinding( + running: ReadonlyArray, + completed: ReadonlyMap, + matches: ReadonlyArray, +): ReadonlyMap> { + const scanById = new Map([...completed.values()].map((scan) => [scan.id, scan])) + const servicesByCommit = running.reduce((byCommit, service) => { + const key = commitKey(service.repositoryId, service.commitSha) + byCommit.set(key, [...(byCommit.get(key) ?? []), service]) + return byCommit + }, new Map>()) + const byFinding = matches.reduce((acc, match) => { + const scan = scanById.get(match.scanId) + if (scan === undefined) return acc + const key = findingKey({ ...match, repositoryId: scan.repositoryId }) + const services = servicesByCommit.get(commitKey(scan.repositoryId, scan.commitSha)) ?? [] + const known = acc.get(key) ?? [] + const added = services.filter( + (service) => + !known.some( + (other) => + other.serviceName === service.serviceName && + other.environment === service.environment && + other.commitSha === service.commitSha, + ), + ) + acc.set(key, [...known, ...added]) + return acc + }, new Map>()) + return new Map( + [...byFinding].map(([key, services]) => [ + key, + [...services].sort((a, b) => b.spanCount - a.spanCount), + ]), + ) +} + +const SEVERITY_WEIGHT = { critical: 8, high: 4, medium: 2, low: 1, unknown: 1 } satisfies Record< + AdvisorySeverity, + number +> + +/** A dev-only dependency rarely ships; a direct one is the team's own choice to upgrade. */ +const RELATION_WEIGHT = { direct: 1.5, transitive: 1, unknown: 1, dev: 0.25 } satisfies Record< + DependencyRelation, + number +> + +/** + * How urgent a finding is: its severity, how the code depends on the package, and the traffic of + * the services running it. A finding in production outranks the same one that is not deployed. + */ +export const riskScore = ( + finding: { readonly severity: AdvisorySeverity; readonly relation: DependencyRelation }, + productionSpans: number, +): number => + SEVERITY_WEIGHT[finding.severity] * + RELATION_WEIGHT[finding.relation] * + (productionSpans > 0 ? 2 + Math.log10(1 + productionSpans) : 1) diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts index 4cbc58f278..9c3ead13d1 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts @@ -373,14 +373,6 @@ const GithubApiRecursiveTree = Schema.Struct({ }) export type GithubRecursiveTree = typeof GithubApiRecursiveTree.Type -const GithubApiBlob = Schema.Struct({ - sha: Schema.String, - size: Schema.Number, - encoding: Schema.String, - content: Schema.String, -}) -export type GithubBlob = typeof GithubApiBlob.Type - const GithubApiReviewCommentList = Schema.Array( Schema.Struct({ id: Schema.Number, @@ -586,7 +578,6 @@ const decodeCollaboratorPermission = Schema.decodeUnknownEffect(GithubApiCollabo const decodeGitCommit = Schema.decodeUnknownEffect(GithubApiGitCommit) const decodeGitTreeListing = Schema.decodeUnknownEffect(GithubApiGitTreeListing) const decodeRecursiveTree = Schema.decodeUnknownEffect(GithubApiRecursiveTree) -const decodeBlob = Schema.decodeUnknownEffect(GithubApiBlob) const decodeComparison = Schema.decodeUnknownEffect(GithubApiComparison) const decodeReviewThreads = Schema.decodeUnknownEffect(GithubReviewThreadsResponse) const decodeMutation = Schema.decodeUnknownEffect(GithubMutationResponse) @@ -935,14 +926,19 @@ export class GithubAppClient extends Context.Service()( return grantsActions(yield* installationToken(externalInstallationId)) }) - const authedGet = (_config: ResolvedAppConfig, token: string, url: string) => + const authedGet = ( + _config: ResolvedAppConfig, + token: string, + url: string, + accept = "application/vnd.github+json", + ) => rateLimitedFetch( tracedFetch( url, { headers: { authorization: `token ${token}`, - accept: "application/vnd.github+json", + accept, "x-github-api-version": GITHUB_API_VERSION, "user-agent": USER_AGENT, }, @@ -2183,8 +2179,11 @@ export class GithubAppClient extends Context.Service()( ) }) - /** One blob by its sha; unlike the contents API it serves files up to 100 MB. */ - const getBlob = Effect.fn("GithubAppClient.getBlob")(function* ( + /** + * One blob's bytes as text, by its sha; unlike the contents API it serves files up to 100 MB. + * Read raw, so a large lockfile is never held as base64 JSON as well. + */ + const getBlobText = Effect.fn("GithubAppClient.getBlobText")(function* ( externalInstallationId: string, owner: string, repo: string, @@ -2196,11 +2195,12 @@ export class GithubAppClient extends Context.Service()( config, token, `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/blobs/${encodeURIComponent(sha)}`, + "application/vnd.github.raw+json", ) if (!isOk(response)) return yield* failure(response, "Read blob", "repository") - return yield* decodeBlob(yield* parseJson(response, "Read blob")).pipe( + return yield* response.text.pipe( Effect.mapError( - (cause) => new GithubAppError({ message: "Unexpected blob payload", cause }), + (cause) => new GithubAppError({ message: "Could not read the blob", cause }), ), ) }) @@ -2477,7 +2477,7 @@ export class GithubAppClient extends Context.Service()( searchCode, getSourceFile, getRecursiveTree, - getBlob, + getBlobText, listCheckRunAnnotations, streamJobLog, canReadActions, diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts index 216c225bcb..89e8fbf92e 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts @@ -1320,14 +1320,9 @@ export class GithubProvider extends Context.Service - client.getBlob(installation.externalInstallationId, repo.owner, repo.name, blobSha).pipe( - Effect.map((blob) => - blob.encoding === "base64" - ? Buffer.from(blob.content.replace(/\s/g, ""), "base64").toString("utf8") - : blob.content, - ), - Effect.mapError(toVcsCommitError), - ) + client + .getBlobText(installation.externalInstallationId, repo.owner, repo.name, blobSha) + .pipe(Effect.mapError(toVcsCommitError)) const resolveRef: VcsProviderClient["resolveRef"] = (installation, repo, ref) => client.getCommit(installation.externalInstallationId, repo.owner, repo.name, ref).pipe( diff --git a/packages/backend/src/services/security/PackageAdvisoryCatalog.test.ts b/packages/backend/src/services/security/PackageAdvisoryCatalog.test.ts index 2e39a894f1..51196d45eb 100644 --- a/packages/backend/src/services/security/PackageAdvisoryCatalog.test.ts +++ b/packages/backend/src/services/security/PackageAdvisoryCatalog.test.ts @@ -53,7 +53,11 @@ const osv = (fake: FakeOsv) => const id = decodeURIComponent(url.pathname.replace("/v1/vulns/", "")) fake.requests.push(`vuln:${id}`) const vuln = fixture.vulns.find((candidate) => candidate.id === id) - return vuln === undefined ? json(request, { message: "not found" }, 404) : json(request, vuln) + if (vuln !== undefined) return json(request, vuln) + // Synthetic ids a test invents answer with a bare record. + return id.startsWith("GHSA-many-") + ? json(request, { id, aliases: [] }) + : json(request, { message: "not found" }, 404) }), ) @@ -88,7 +92,7 @@ describe("PackageAdvisoryCatalog", () => { const fake: FakeOsv = { requests: [] } return Effect.gen(function* () { const catalog = yield* PackageAdvisoryCatalog - const first = yield* catalog.advisories([...refs, ...refs]) + const { byPackage: first } = yield* catalog.advisories([...refs, ...refs]) assert.deepStrictEqual( first .get(keyOf("minimist")) @@ -105,7 +109,7 @@ describe("PackageAdvisoryCatalog", () => { assert.strictEqual(reads.length, fixture.vulns.length) fake.requests.length = 0 - const second = yield* catalog.advisories(refs) + const { byPackage: second } = yield* catalog.advisories(refs) assert.deepStrictEqual(fake.requests, [`querybatch:${refs.length}`]) assert.deepStrictEqual([...second.keys()], [...first.keys()]) }).pipe(Effect.provide(catalogWith(fake))) @@ -123,7 +127,7 @@ describe("PackageAdvisoryCatalog", () => { } return Effect.gen(function* () { const catalog = yield* PackageAdvisoryCatalog - const advisories = yield* catalog.advisories(refs) + const { byPackage: advisories } = yield* catalog.advisories(refs) assert.deepStrictEqual( advisories .get(keyOf("lodash")) @@ -148,9 +152,9 @@ describe("PackageAdvisoryCatalog", () => { }).pipe(Effect.provide(catalogWith(fake))) }) - it.effect("refuses more distinct vulnerabilities than a scan reads", () => { - const many = Array.from({ length: MAX_VULNERABILITIES + 1 }, (_, i) => ({ - id: `GHSA-many-${i}`, + it.effect("reads at most the capped number of vulnerabilities and counts the rest as unread", () => { + const many = Array.from({ length: MAX_VULNERABILITIES + 3 }, (_, i) => ({ + id: `GHSA-many-${String(i).padStart(4, "0")}`, modified: "1", })) const fake: FakeOsv = { @@ -159,9 +163,12 @@ describe("PackageAdvisoryCatalog", () => { } return Effect.gen(function* () { const catalog = yield* PackageAdvisoryCatalog - const error = yield* Effect.flip(catalog.advisories(refs)) - assert.strictEqual(error._tag, "@maple/security/PackageAdvisoryCatalogError") - assert.isFalse(fake.requests.some((request) => request.startsWith("vuln:"))) + const { unread } = yield* catalog.advisories(refs) + assert.strictEqual(unread, 3) + assert.strictEqual( + fake.requests.filter((request) => request.startsWith("vuln:")).length, + MAX_VULNERABILITIES, + ) }).pipe(Effect.provide(catalogWith(fake))) }) }) diff --git a/packages/backend/src/services/security/PackageAdvisoryCatalog.ts b/packages/backend/src/services/security/PackageAdvisoryCatalog.ts index 26ba1ea6f1..7cbcc61e74 100644 --- a/packages/backend/src/services/security/PackageAdvisoryCatalog.ts +++ b/packages/backend/src/services/security/PackageAdvisoryCatalog.ts @@ -15,9 +15,11 @@ const CACHE_BUCKET = "osv-vulns" const CACHE_TTL_SECONDS = 7 * 24 * 60 * 60 const CACHE_READ_TIMEOUT_MS = 150 const REQUEST_TIMEOUT = "15 seconds" -/** Distinct vulnerabilities read per call; a scan past it fails rather than store a partial answer. */ +/** Distinct vulnerabilities read per call; past it the answer says how many it left unread. */ export const MAX_VULNERABILITIES = 500 const CONCURRENCY = 16 +/** Pages followed per query past the first; a package with more than a page of advisories is rare. */ +const MAX_EXTRA_PAGES = 5 const RETRY = { times: 2, schedule: Schedule.exponential("250 millis") } as const const QueryBatchResponse = Schema.Struct({ @@ -44,11 +46,16 @@ export class PackageAdvisoryCatalogError extends Schema.TaggedError, - ) => Effect.Effect>, PackageAdvisoryCatalogError> + readonly advisories: (packages: ReadonlyArray) => Effect.Effect< + { + readonly byPackage: ReadonlyMap> + readonly unread: number + }, + PackageAdvisoryCatalogError + > } interface Pending { @@ -113,27 +120,36 @@ export class PackageAdvisoryCatalog extends Context.Service< const queryAll = ( pending: ReadonlyArray, found: ReadonlyMap>, + depth = 0, ): Effect.Effect>, PackageAdvisoryCatalogError> => pending.length === 0 ? Effect.succeed(found) - : Effect.forEach(Arr.chunksOf(pending, BATCH_SIZE), queryBatch, { concurrency: 2 }).pipe( - Effect.flatMap((batches) => { - const pages = batches.flat() - const next = new Map(found) - pages.forEach((page) => { - const key = packageKey(page.ref) - next.set(key, [...(next.get(key) ?? []), ...page.vulns]) - }) - return queryAll( - pages.flatMap((page) => - page.nextPageToken === undefined - ? [] - : [{ ref: page.ref, pageToken: page.nextPageToken }], - ), - next, - ) - }), - ) + : depth > MAX_EXTRA_PAGES + ? Effect.fail( + new PackageAdvisoryCatalogError({ + message: `OSV kept paging past ${MAX_EXTRA_PAGES} pages`, + cause: undefined, + }), + ) + : Effect.forEach(Arr.chunksOf(pending, BATCH_SIZE), queryBatch, { concurrency: 2 }).pipe( + Effect.flatMap((batches) => { + const pages = batches.flat() + const next = new Map(found) + pages.forEach((page) => { + const key = packageKey(page.ref) + next.set(key, [...(next.get(key) ?? []), ...page.vulns]) + }) + return queryAll( + pages.flatMap((page) => + page.nextPageToken === undefined + ? [] + : [{ ref: page.ref, pageToken: page.nextPageToken }], + ), + next, + depth + 1, + ) + }), + ) const fetchRecord = (vuln: VulnRef) => edgeCache @@ -172,19 +188,15 @@ export class PackageAdvisoryCatalog extends Context.Service< "maple.depscan.package_count": distinct.length, "maple.depscan.vulnerability_count": vulns.length, }) - if (vulns.length > MAX_VULNERABILITIES) { - return yield* new PackageAdvisoryCatalogError({ - message: `${vulns.length} known vulnerabilities, more than the ${MAX_VULNERABILITIES} a scan reads`, - cause: undefined, - }) - } + // Sorted, so the same vulnerabilities are read each time the cap bites. + const read = [...vulns].sort((a, b) => a.id.localeCompare(b.id)).slice(0, MAX_VULNERABILITIES) const records = new Map( - (yield* Effect.forEach(vulns, fetchRecord, { concurrency: CONCURRENCY })).map((record) => [ + (yield* Effect.forEach(read, fetchRecord, { concurrency: CONCURRENCY })).map((record) => [ record.id, record, ]), ) - return new Map( + const byPackage = new Map( distinct.flatMap((ref) => { const matched = (found.get(packageKey(ref)) ?? []).flatMap((vuln) => { const record = records.get(vuln.id) @@ -194,6 +206,7 @@ export class PackageAdvisoryCatalog extends Context.Service< return list.length === 0 ? [] : [[packageKey(ref), list] as const] }), ) + return { byPackage, unread: vulns.length - read.length } }) return { advisories } satisfies PackageAdvisoryCatalogApi diff --git a/packages/domain/src/http/dependency-scan.ts b/packages/domain/src/http/dependency-scan.ts index 1944b9ab68..fbd4b0d5bb 100644 --- a/packages/domain/src/http/dependency-scan.ts +++ b/packages/domain/src/http/dependency-scan.ts @@ -34,6 +34,23 @@ export const DependencyScanSummary = Schema.Struct({ }) export type DependencyScanSummary = typeof DependencyScanSummary.Type +/** A service serving traffic from one commit over the last day; spans are what it served. */ +export const DependencyRunningService = Schema.Struct({ + serviceName: Schema.String, + environment: Schema.String, + commitSha: GitCommitSha, + spanCount: Schema.Finite, +}) +export type DependencyRunningService = typeof DependencyRunningService.Type + +/** A commit of the repository that services run, and the scan that read it (null until one does). */ +export const DependencyRunningCommit = Schema.Struct({ + commitSha: GitCommitSha, + scanStatus: Schema.NullOr(DependencyScanStatus), + services: Schema.Array(DependencyRunningService), +}) +export type DependencyRunningCommit = typeof DependencyRunningCommit.Type + /** A connected repository and its latest scan of the tracked branch, null before the first. */ export const DependencyScanRepository = Schema.Struct({ id: VcsRepositoryId, @@ -41,6 +58,8 @@ export const DependencyScanRepository = Schema.Struct({ htmlUrl: Schema.String, trackedBranch: Schema.String, latestScan: Schema.NullOr(DependencyScanSummary), + /** Commits services ran over the last day, busiest first. */ + runningCommits: Schema.Array(DependencyRunningCommit), }) export type DependencyScanRepository = typeof DependencyScanRepository.Type @@ -53,10 +72,27 @@ export type DependencyScanRepositoriesResponse = typeof DependencyScanRepositori export const DependencyFindingView = Schema.Struct({ ...DependencyFinding.fields, repositoryFullName: Schema.String, + /** Services whose running commit carries the finding, busiest first; empty when none does. */ + production: Schema.Array(DependencyRunningService), }) export type DependencyFindingView = typeof DependencyFindingView.Type -export const DependencyFindingsResponse = Schema.Struct({ findings: Schema.Array(DependencyFindingView) }) +/** How much of production the findings could be joined to. */ +export const DependencyProductionCoverage = Schema.Struct({ + /** False when telemetry could not be read; the findings are then ranked without it. */ + available: Schema.Boolean, + /** Services reporting a commit (`vcs.ref.head.revision`) over the last day. */ + runningServices: Schema.Int, + /** Of those, the ones whose commit belongs to a connected repository. */ + linkedServices: Schema.Int, +}) +export type DependencyProductionCoverage = typeof DependencyProductionCoverage.Type + +/** Findings most urgent first: severity, how the code depends on the package, production traffic. */ +export const DependencyFindingsResponse = Schema.Struct({ + findings: Schema.Array(DependencyFindingView), + production: DependencyProductionCoverage, +}) export type DependencyFindingsResponse = typeof DependencyFindingsResponse.Type export const DependencyFindingsQuery = Schema.Struct({ @@ -74,7 +110,8 @@ export const DependencyScanQueued = Schema.Struct({ queued: Schema.Boolean }) export class DependencyScanPersistenceError extends Schema.TaggedError()( "@maple/http/dependency-scan/DependencyScanPersistenceError", - { message: Schema.String }, + /** `cause` names the failure behind it (its tag), never its details. */ + { message: Schema.String, cause: Schema.optionalKey(Schema.String) }, { httpApiStatus: 503 }, ) {} diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index c9e17a3cba..7ccd04413f 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -4086,6 +4086,62 @@ SELECT LIMIT 5000 FORMAT JSON +-- builder:releases:runningCommitsQuery:default [cfa733c1] +SELECT + service_windows.bServiceName AS serviceName, + service_windows.bEnvironment AS environment, + service_windows.bCommitSha AS commitSha, + sum(service_windows.bSpanCount) AS spanCount + FROM ( +SELECT + toStartOfHour(service_overview_spans.Timestamp) AS bBucket, + service_overview_spans.ServiceName AS bServiceName, + service_overview_spans.ServiceNamespace AS bServiceNamespace, + service_overview_spans.DeploymentEnv AS bEnvironment, + service_overview_spans.CommitSha AS bCommitSha, + count() AS bSpanCount, + sum(service_overview_spans.SampleRate) AS bEstimatedSpanCount, + countIf(service_overview_spans.StatusCode = 'Error') AS bErrorCount, + sumIf(service_overview_spans.SampleRate, service_overview_spans.StatusCode = 'Error') AS bEstimatedErrorCount, + sum(toFloat64(Duration)) AS bDurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS bDurationQuantiles, + min(service_overview_spans.Timestamp) AS bFirstSeen, + countIf((service_overview_spans.StatusCode != 'Error' AND service_overview_spans.Duration < 500000000)) AS bApdexSatisfiedCount, + countIf(((service_overview_spans.StatusCode != 'Error' AND service_overview_spans.Duration >= 500000000) AND service_overview_spans.Duration < 2000000000)) AS bApdexToleratingCount + FROM service_overview_spans + WHERE service_overview_spans.OrgId = 'org_sql_catalog' + AND service_overview_spans.Timestamp >= '2026-01-01 10:30:00' + AND service_overview_spans.Timestamp <= '2026-01-03 14:15:00' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + GROUP BY bBucket, bServiceName, bServiceNamespace, bEnvironment, bCommitSha +UNION ALL +SELECT + service_overview_hourly.Hour AS bBucket, + service_overview_hourly.ServiceName AS bServiceName, + service_overview_hourly.ServiceNamespace AS bServiceNamespace, + service_overview_hourly.DeploymentEnv AS bEnvironment, + service_overview_hourly.CommitSha AS bCommitSha, + sum(service_overview_hourly.SpanCount) AS bSpanCount, + sum(service_overview_hourly.EstimatedSpanCount) AS bEstimatedSpanCount, + sum(service_overview_hourly.ErrorCount) AS bErrorCount, + sum(service_overview_hourly.EstimatedErrorCount) AS bEstimatedErrorCount, + sum(service_overview_hourly.DurationSum) AS bDurationSum, + quantilesTDigestMergeState(0.5, 0.95, 0.99)(DurationQuantiles) AS bDurationQuantiles, + min(service_overview_hourly.FirstSeen) AS bFirstSeen, + sum(service_overview_hourly.ApdexSatisfiedCount) AS bApdexSatisfiedCount, + sum(service_overview_hourly.ApdexToleratingCount) AS bApdexToleratingCount + FROM service_overview_hourly + WHERE service_overview_hourly.OrgId = 'org_sql_catalog' + AND service_overview_hourly.Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND service_overview_hourly.Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + GROUP BY bBucket, bServiceName, bServiceNamespace, bEnvironment, bCommitSha +) AS service_windows + WHERE service_windows.bCommitSha NOT IN ('', 'unknown', 'N/A') + GROUP BY serviceName, environment, commitSha + ORDER BY spanCount DESC + LIMIT 200 + FORMAT JSON + -- builder:releases:serviceDeploymentsQuery:hourInterior [ae47e5db] SELECT ranked.serviceName AS serviceName, diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index 0a23a579a6..c49412a128 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -1759,6 +1759,13 @@ export const builderFixtures: ReadonlyArray = [ window, ), }, + // Dependency scanner: which commit each service runs, ranked by traffic. + { + module: "releases", + name: "runningCommitsQuery", + label: "default", + compile: () => CH.compileUnsafe(CH.runningCommitsQuery(), window), + }, { module: "releases", name: "releasesTimelineQuery", diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 7ac613e241..e74afc78e6 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -340,6 +340,9 @@ export { // Queries โ€” Releases export { releasesListQuery, + runningCommitsQuery, + runningCommitsRowSchema, + RUNNING_COMMITS_CAP, releasesListRowSchema, releasesTimelineQuery, releaseErrorFingerprintsQuery, diff --git a/packages/query-engine/src/ch/queries/releases.ts b/packages/query-engine/src/ch/queries/releases.ts index e7bbc305a2..f28fd5752d 100644 --- a/packages/query-engine/src/ch/queries/releases.ts +++ b/packages/query-engine/src/ch/queries/releases.ts @@ -126,6 +126,44 @@ export function releasesListQuery(opts: ReleasesListOpts = {}) { .format("JSON") } +// Running commits +// +// Which commit each service served traffic from in the window, for the dependency scanner: a +// vulnerable package matters most in the commit a busy service is running. + +/** At most this many (service, environment, commit) rows, busiest first. */ +export const RUNNING_COMMITS_CAP = 200 + +export interface RunningCommitsOutput { + readonly serviceName: string + readonly environment: string + readonly commitSha: string + readonly spanCount: number +} + +export const runningCommitsRowSchema = Schema.Struct({ + serviceName: Schema.String, + environment: Schema.String, + commitSha: Schema.String, + spanCount: CHNumber, +}) satisfies CompiledQueryRowSchema + +/** The releases list without its latency and error columns, ordered by traffic. */ +export function runningCommitsQuery() { + return serviceOverviewWindows({}) + .select(($) => ({ + serviceName: $.bServiceName, + environment: $.bEnvironment, + commitSha: $.bCommitSha, + spanCount: CH.sum($.bSpanCount), + })) + .where(($) => [CH.notInList($.bCommitSha, PLACEHOLDER_COMMIT_SHAS)]) + .groupBy("serviceName", "environment", "commitSha") + .orderBy(["spanCount", "desc"]) + .limit(RUNNING_COMMITS_CAP) + .format("JSON") +} + // Releases timeline // // The org-wide sibling of `serviceReleasesTimelineQuery`: per bucket, per From 7e6481f3c59a1b5d92445364a7d346172471dcc4 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 01:08:21 +0100 Subject: [PATCH 02/15] fix(security): every finding view carries its production services Making production required left updateFinding's response without it; the triage response now carries an empty list (the page re-reads findings after a triage). Also drops an unused severity list knip reported. --- apps/web/src/page/security/dependencies/derive.ts | 2 -- .../dependency-scan/DependencyScanService.ts | 15 +++++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/web/src/page/security/dependencies/derive.ts b/apps/web/src/page/security/dependencies/derive.ts index 56cc0c4e79..0a39a393e1 100644 --- a/apps/web/src/page/security/dependencies/derive.ts +++ b/apps/web/src/page/security/dependencies/derive.ts @@ -24,8 +24,6 @@ export const STATUS_FILTER_LABEL = { all: "All", } satisfies Record -/** Worst first; `unknown` is an advisory with no score, read as "Unrated". */ -export const SEVERITIES: ReadonlyArray = ["critical", "high", "medium", "low", "unknown"] export const SEVERITY_LABEL = { critical: "Critical", diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index b4133cc75e..de702ccf2d 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -188,7 +188,9 @@ export const scanSummaryOf = (row: DependencyScanRow): DependencyScanSummary => export const findingViewOf = ( row: DependencyFindingRow, repositoryFullName: string, + production: ReadonlyArray, ): DependencyFindingView => ({ + production, id: row.id, repositoryId: row.repositoryId, repositoryFullName, @@ -478,10 +480,13 @@ export class DependencyScanService extends Context.Service ({ - ...findingViewOf(row, names.get(row.repositoryId) ?? "unknown repository"), - production: services.map(toRunningService), - })), + findings: ranked.map(({ row, services }) => + findingViewOf( + row, + names.get(row.repositoryId) ?? "unknown repository", + services.map(toRunningService), + ), + ), production: { available: production.available, runningServices: production.runningServices, @@ -517,12 +522,14 @@ export class DependencyScanService extends Context.Service "unknown repository", onSome: (r) => r.fullName, }), + [], ) }) From 349710efbab7409a38b00a2c356e6adabb297348 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 01:20:15 +0100 Subject: [PATCH 03/15] fix(web): Dependencies tab copy and pending production scans - The All filter's empty state reads "No findings", not "No all findings". - A finding in a repository whose running commit is still being scanned reads "Scanning" instead of "Not running". - The triage reason is capped untrimmed, as the API checks it. --- .../dependencies/dependencies.scene.test.ts | 37 +++++++++++++++++++ .../src/page/security/dependencies/derive.ts | 4 +- .../security/dependencies/view-findings.ts | 28 ++++++++++++-- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts index 5e4ae85856..70ad75cbf4 100644 --- a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -33,6 +33,7 @@ import * as Field from "../../../ui/field" import * as Select from "../../../ui/select" import { type PageContext, PageOutMessage } from "../../contract" import { UpdateFinding } from "./command" +import { reasonErrorOf } from "./derive" import { Message, type Model } from "./model" import { init, update } from "./update" import { view as pageView } from "./view" @@ -151,6 +152,42 @@ describe("security dependencies tab", () => { ) }) + test("says a finding is being scanned, not absent, while a running commit's scan is pending", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.id === PAYMENTS_ID + ? { + ...repository, + runningCommits: repository.runningCommits.map((commit) => ({ + ...commit, + scanStatus: null, + })), + } + : repository, + ), + }, + { + ...findingsOf(NOW), + findings: findingsOf(NOW).findings.map((finding) => ({ ...finding, production: [] })), + }, + ), + ), + expectView(text("Scanning")).toExist(), + expectView(text("api-gateway +1")).toBeAbsent(), + ) + }) + + test("caps a triage reason as the API does, untrimmed", () => { + expect(reasonErrorOf(" ".repeat(10) + "x".repeat(495))).not.toBeNull() + expect(reasonErrorOf("x".repeat(500))).toBeNull() + expect(reasonErrorOf(" ")).not.toBeNull() + }) + test("explains why findings cannot be tied to production when no service reports its commit", () => { scene( { update: pageUpdate, view }, diff --git a/apps/web/src/page/security/dependencies/derive.ts b/apps/web/src/page/security/dependencies/derive.ts index 0a39a393e1..5cffef62a6 100644 --- a/apps/web/src/page/security/dependencies/derive.ts +++ b/apps/web/src/page/security/dependencies/derive.ts @@ -24,7 +24,6 @@ export const STATUS_FILTER_LABEL = { all: "All", } satisfies Record - export const SEVERITY_LABEL = { critical: "Critical", high: "High", @@ -151,7 +150,8 @@ export const selectedFindingOf = (model: Model): DependencyFindingView | null => export const reasonErrorOf = (reason: string): string | null => reason.trim() === "" ? "Add a reason. It is shown on the finding with your name." - : reason.trim().length > DEPENDENCY_FINDING_REASON_MAX_LENGTH + : // The API checks the reason as sent, so the cap applies to it untrimmed. + reason.length > DEPENDENCY_FINDING_REASON_MAX_LENGTH ? `Keep the reason under ${DEPENDENCY_FINDING_REASON_MAX_LENGTH} characters.` : null diff --git a/apps/web/src/page/security/dependencies/view-findings.ts b/apps/web/src/page/security/dependencies/view-findings.ts index 1bbd4ca76f..945bdbab9a 100644 --- a/apps/web/src/page/security/dependencies/view-findings.ts +++ b/apps/web/src/page/security/dependencies/view-findings.ts @@ -149,6 +149,16 @@ const toolbar = ( const muted = (h: H, text: string, className?: string): Html => h.span([h.Class(cn("text-xs text-muted-foreground", className))], [text]) +/** Repositories with a running commit whose scan has not completed. */ +const pendingRepositories = (repositories: ReadonlyArray): ReadonlySet => + new Set( + repositories + .filter((repository) => + repository.runningCommits.some((commit) => commit.scanStatus !== "completed"), + ) + .map((repository) => repository.id), + ) + /** Spans a finding's running services served over the last day. */ export const productionSpans = (finding: DependencyFindingView): number => finding.production.reduce((sum, service) => sum + service.spanCount, 0) @@ -158,8 +168,11 @@ const productionCell = ( h: H, finding: DependencyFindingView, production: DependencyProductionCoverage, + pending: ReadonlySet, ): Html => { const [busiest] = finding.production + // A running commit not scanned yet may carry the finding: no answer until its scan lands. + if (busiest === undefined && pending.has(finding.repositoryId)) return muted(h, "Scanning") if (busiest === undefined) return muted(h, production.available && production.linkedServices > 0 ? "Not running" : "Unknown") const others = new Set(finding.production.map((service) => service.serviceName)).size - 1 @@ -198,6 +211,7 @@ const findingRow = ( isSelected: boolean, clock: Clock, production: DependencyProductionCoverage, + pending: ReadonlySet, ): Html => { const open = Message.ClickedFinding({ findingId: finding.id }) return tableRow( @@ -227,7 +241,7 @@ const findingRow = ( ), ]), tableCell(h, [advisoryLink(h, finding)]), - tableCell(h, [productionCell(h, finding, production)]), + tableCell(h, [productionCell(h, finding, production, pending)]), tableCell(h, [ finding.fixedIn === null ? muted(h, "No fix yet") @@ -291,6 +305,7 @@ const findingsTable = ( selectedId: string | null, clock: Clock, production: DependencyProductionCoverage, + pending: ReadonlySet, ): Html => table(h, { ariaLabel: "Vulnerable packages" }, [ tableHeader(h, [ @@ -307,7 +322,9 @@ const findingsTable = ( ]), tableBody( h, - rows.map((finding) => findingRow(h, finding, finding.id === selectedId, clock, production)), + rows.map((finding) => + findingRow(h, finding, finding.id === selectedId, clock, production, pending), + ), ), ]) @@ -346,7 +363,7 @@ export const findingsSection = ( const rows = filteredFindings(findings, status, repositoryId) const content = rows.length > 0 - ? findingsTable(h, rows, model.selectedId, clock, production) + ? findingsTable(h, rows, model.selectedId, clock, production, pendingRepositories(repositories)) : status === "open" && repositoryId === null ? cleanState(h, repositories) : h.div( @@ -354,7 +371,10 @@ export const findingsSection = ( [ nothingInWindow(h, { title: "No findings match", - description: `No ${STATUS_FILTER_LABEL[status].toLowerCase()} findings for these filters.`, + description: + status === "all" + ? "No findings for these filters." + : `No ${STATUS_FILTER_LABEL[status].toLowerCase()} findings for these filters.`, }), ], ) From 56c64846ed2f75b7cd8300509c86ba37ddcd86d7 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 01:24:35 +0100 Subject: [PATCH 04/15] perf(security): one running-commits warehouse read per page, not two The Dependencies tab's two reads (repositories and findings) each ran the same running-commits query: 34 reads for 17 page loads on the preview. The read is edge-cached per org for a minute, so the second one hits it. --- .../dependency-scan/DependencyScanService.ts | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index de702ccf2d..9d9828b1e3 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -20,6 +20,7 @@ import { } from "@maple/domain/http" import type { OrgId, UserId } from "@maple/domain/primitives" import type { DependencyFindingStatus } from "@maple/domain/dependency-scan" +import { EdgeCacheService } from "@maple/cache" import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import { CH, formatWarehouseDateTime } from "@maple/query-engine" import type { TenantContext } from "@maple/backend/services/auth/AuthService" @@ -49,6 +50,10 @@ const RUNNING_WINDOW_MS = 24 * 60 * 60 * 1000 export const PRODUCTION_SCANS_PER_READ = 10 export type RunningCommitRow = typeof CH.runningCommitsRowSchema.Type +const RunningCommitRows = Schema.Array(CH.runningCommitsRowSchema) +const RUNNING_COMMITS_CACHE_BUCKET = "dependency-running-commits" +/** Running commits move with deploys, not by the second: one warehouse read per org a minute. */ +const RUNNING_COMMITS_CACHE_MS = 60_000 export class RunningCommitsReadError extends Schema.TaggedError()( "@maple/dependency-scan/RunningCommitsReadError", @@ -69,23 +74,34 @@ export class RunningCommits extends Context.Service< this, Effect.gen(function* () { const warehouse = yield* WarehouseQueryService + const edgeCache = yield* EdgeCacheService return { + // The tab's two reads land together: the first reads the warehouse, the second the cache. read: (tenant, window) => - warehouse - .compiledQuery( - tenant, - CH.compile( - CH.runningCommitsQuery(), - { - orgId: tenant.orgId, - startTime: formatWarehouseDateTime(window.startMs), - endTime: formatWarehouseDateTime(window.endMs), - }, - { rowSchema: CH.runningCommitsRowSchema }, + edgeCache + .getOrCompute( + { + bucket: RUNNING_COMMITS_CACHE_BUCKET, + key: `${tenant.orgId}:${Math.floor(window.endMs / RUNNING_COMMITS_CACHE_MS)}`, + ttlSeconds: (2 * RUNNING_COMMITS_CACHE_MS) / 1000, + schema: RunningCommitRows, + }, + warehouse.compiledQuery( + tenant, + CH.compile( + CH.runningCommitsQuery(), + { + orgId: tenant.orgId, + startTime: formatWarehouseDateTime(window.startMs), + endTime: formatWarehouseDateTime(window.endMs), + }, + { rowSchema: CH.runningCommitsRowSchema }, + ), + { profile: "list", context: "dependencyRunningCommits" }, ), - { profile: "list", context: "dependencyRunningCommits" }, ) .pipe( + Effect.map((result) => result.value), Effect.mapError( (cause) => new RunningCommitsReadError({ From 71e9c93d87ce84456ce31f1a2682851263227e0b Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 01:36:55 +0100 Subject: [PATCH 05/15] fix(security): production scans are queued once per page; failed scans say so - The tab reads repositories and findings together and both queued the same unscanned commits; only the repositories read queues now. - A running commit whose scan failed read as Scanning until the hourly retry; it now reads Scan failed. --- .../dependencies/dependencies.scene.test.ts | 30 +++++++++++++++ .../security/dependencies/view-findings.ts | 37 +++++++++++-------- .../DependencyScanService.test.ts | 6 ++- .../dependency-scan/DependencyScanService.ts | 3 +- 4 files changed, 58 insertions(+), 18 deletions(-) diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts index 70ad75cbf4..4a6379c8d1 100644 --- a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -182,6 +182,36 @@ describe("security dependencies tab", () => { ) }) + test("says a running commit's scan failed instead of claiming it is still scanning", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.id === PAYMENTS_ID + ? { + ...repository, + runningCommits: repository.runningCommits.map((commit) => ({ + ...commit, + scanStatus: "failed" as const, + })), + } + : repository, + ), + }, + { + ...findingsOf(NOW), + findings: findingsOf(NOW).findings.map((finding) => ({ ...finding, production: [] })), + }, + ), + ), + expectView(text("Scan failed")).toExist(), + expectView(text("Scanning")).toBeAbsent(), + ) + }) + test("caps a triage reason as the API does, untrimmed", () => { expect(reasonErrorOf(" ".repeat(10) + "x".repeat(495))).not.toBeNull() expect(reasonErrorOf("x".repeat(500))).toBeNull() diff --git a/apps/web/src/page/security/dependencies/view-findings.ts b/apps/web/src/page/security/dependencies/view-findings.ts index 945bdbab9a..ed1db78af8 100644 --- a/apps/web/src/page/security/dependencies/view-findings.ts +++ b/apps/web/src/page/security/dependencies/view-findings.ts @@ -149,14 +149,19 @@ const toolbar = ( const muted = (h: H, text: string, className?: string): Html => h.span([h.Class(cn("text-xs text-muted-foreground", className))], [text]) -/** Repositories with a running commit whose scan has not completed. */ -const pendingRepositories = (repositories: ReadonlyArray): ReadonlySet => - new Set( - repositories - .filter((repository) => - repository.runningCommits.some((commit) => commit.scanStatus !== "completed"), - ) - .map((repository) => repository.id), +type ProductionScanState = "pending" | "failed" + +/** Repositories whose running commits are not all scanned: one still pending, or one whose scan failed. */ +const productionScanStates = ( + repositories: ReadonlyArray, +): ReadonlyMap => + new Map( + repositories.flatMap((repository): ReadonlyArray => { + const statuses = repository.runningCommits.map((commit) => commit.scanStatus) + if (statuses.some((status) => status === null || status === "running")) + return [[repository.id, "pending"]] + return statuses.includes("failed") ? [[repository.id, "failed"]] : [] + }), ) /** Spans a finding's running services served over the last day. */ @@ -168,11 +173,13 @@ const productionCell = ( h: H, finding: DependencyFindingView, production: DependencyProductionCoverage, - pending: ReadonlySet, + scans: ReadonlyMap, ): Html => { const [busiest] = finding.production // A running commit not scanned yet may carry the finding: no answer until its scan lands. - if (busiest === undefined && pending.has(finding.repositoryId)) return muted(h, "Scanning") + const scan = scans.get(finding.repositoryId) + if (busiest === undefined && scan !== undefined) + return muted(h, scan === "pending" ? "Scanning" : "Scan failed") if (busiest === undefined) return muted(h, production.available && production.linkedServices > 0 ? "Not running" : "Unknown") const others = new Set(finding.production.map((service) => service.serviceName)).size - 1 @@ -211,7 +218,7 @@ const findingRow = ( isSelected: boolean, clock: Clock, production: DependencyProductionCoverage, - pending: ReadonlySet, + scans: ReadonlyMap, ): Html => { const open = Message.ClickedFinding({ findingId: finding.id }) return tableRow( @@ -241,7 +248,7 @@ const findingRow = ( ), ]), tableCell(h, [advisoryLink(h, finding)]), - tableCell(h, [productionCell(h, finding, production, pending)]), + tableCell(h, [productionCell(h, finding, production, scans)]), tableCell(h, [ finding.fixedIn === null ? muted(h, "No fix yet") @@ -305,7 +312,7 @@ const findingsTable = ( selectedId: string | null, clock: Clock, production: DependencyProductionCoverage, - pending: ReadonlySet, + scans: ReadonlyMap, ): Html => table(h, { ariaLabel: "Vulnerable packages" }, [ tableHeader(h, [ @@ -323,7 +330,7 @@ const findingsTable = ( tableBody( h, rows.map((finding) => - findingRow(h, finding, finding.id === selectedId, clock, production, pending), + findingRow(h, finding, finding.id === selectedId, clock, production, scans), ), ), ]) @@ -363,7 +370,7 @@ export const findingsSection = ( const rows = filteredFindings(findings, status, repositoryId) const content = rows.length > 0 - ? findingsTable(h, rows, model.selectedId, clock, production, pendingRepositories(repositories)) + ? findingsTable(h, rows, model.selectedId, clock, production, productionScanStates(repositories)) : status === "open" && repositoryId === null ? cleanState(h, repositories) : h.div( diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts index 0d2e7cc050..4a0a37c21d 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts @@ -264,14 +264,16 @@ describe("DependencyScanService production", () => { ["api-server", "staging", "a", 100], ], ) + // Only the repositories read queues: the tab fires both reads together. + assert.deepStrictEqual(sent, []) + + const { repositories } = yield* service.listRepositories(tenant) assert.deepStrictEqual( sent.map((job) => job.kind === "dependency-scan" ? [job.trigger, job.commitSha] : [job.kind], ), [["production", sha("d")]], ) - - const { repositories } = yield* service.listRepositories(tenant) assert.deepStrictEqual( repositories[0]?.runningCommits?.map((commit) => [ commit.commitSha.slice(0, 1), diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index 9d9828b1e3..018b7f2a85 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -373,7 +373,6 @@ export class DependencyScanService extends Context.Service [scan.repositoryId, scan])) const production = yield* productionOrNone(tenant, connected) + // The tab reads repositories and findings together; only this read queues, so each commit is queued once. + yield* queueProductionScans(connected, production) const runningCommitsOf = ( repositoryId: VcsRepositoryId, ): ReadonlyArray => { From 5430392cec6cb5d12de7218121dbf83b88327102 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 07:36:35 +0100 Subject: [PATCH 06/15] chore(security): restore the cause summary import after the merge --- .../backend/src/services/dependency-scan/DependencyScanner.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts index a3faa7719a..725b336e45 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -19,6 +19,7 @@ import type { } from "@maple/backend/services/integrations/vcs/VcsProviderClient" import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" import { DependencyScanSink } from "@maple/backend/services/integrations/vcs/PullRequestEventSink" +import { summarizeCause } from "@maple/backend/platform/describe-cause" import { MAX_VULNERABILITIES, PackageAdvisoryCatalog, From b2cc1f76119946cb0de2dd0c803214aafc9ca6de Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 08:56:17 +0100 Subject: [PATCH 07/15] fix(security): honest production claims and ranking on the Dependencies tab - Running commits rank by sample-weighted spans, not stored ones. - When the running-commits read hits its cap, coverage says incomplete and findings read Unknown instead of Not running. - 2000 findings are ranked before the 500 shown, so a deployed older finding is not cut before its traffic counts. - A failed first scan no longer shows the clean state. - The tree-too-large shortcut applies to branch-head scans only, so a production scan records its own failure and retries later. --- .../src/lab/security-dependencies-fixture.ts | 4 +- .../dependencies/dependencies.scene.test.ts | 39 ++++++++++++++++++- .../security/dependencies/view-findings.ts | 12 +++++- .../DependencyScanService.test.ts | 14 ++++++- .../dependency-scan/DependencyScanService.ts | 10 ++++- .../dependency-scan/DependencyScanner.ts | 5 ++- packages/domain/src/http/dependency-scan.ts | 2 + .../src/__sql_baseline__/catalog.sql | 4 +- .../query-engine/src/ch/queries/releases.ts | 3 +- 9 files changed, 80 insertions(+), 13 deletions(-) diff --git a/apps/web/src/lab/security-dependencies-fixture.ts b/apps/web/src/lab/security-dependencies-fixture.ts index a1c0d09969..7dfbc6c602 100644 --- a/apps/web/src/lab/security-dependencies-fixture.ts +++ b/apps/web/src/lab/security-dependencies-fixture.ts @@ -268,7 +268,7 @@ const CLI = { repositoryId: REPOSITORY_IDS.cli, repositoryFullName: "acme/cli" } /** Worst first, as the server sorts them: one per severity, a dismissed and a fixed finding. */ export const findingsOf = (nowMs: number): DependencyFindingsResponse => ({ - production: { available: true, runningServices: 4, linkedServices: 3 }, + production: { available: true, runningServices: 4, linkedServices: 3, complete: true }, findings: [ finding( nowMs, @@ -415,5 +415,5 @@ export const findingsOf = (nowMs: number): DependencyFindingsResponse => ({ export const noFindings: DependencyFindingsResponse = { findings: [], - production: { available: true, runningServices: 3, linkedServices: 3 }, + production: { available: true, runningServices: 3, linkedServices: 3, complete: true }, } diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts index 4a6379c8d1..6029301381 100644 --- a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -212,6 +212,43 @@ describe("security dependencies tab", () => { ) }) + test("does not call a repository clean when its only scan failed", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.latestScan === null + ? repository + : { + ...repository, + latestScan: { ...repository.latestScan, status: "failed" as const }, + }, + ), + }, + noFindings, + ), + ), + expectView(text("No known vulnerable packages")).toBeAbsent(), + ) + }) + + test("says Unknown, not Not running, when more commits ran than were read", () => { + scene( + { update: pageUpdate, view }, + given( + loaded(repositoriesOf(NOW), { + ...findingsOf(NOW), + production: { ...findingsOf(NOW).production, complete: false }, + }), + ), + expectView(text("Not running")).toBeAbsent(), + expectView(text("Unknown")).toExist(), + ) + }) + test("caps a triage reason as the API does, untrimmed", () => { expect(reasonErrorOf(" ".repeat(10) + "x".repeat(495))).not.toBeNull() expect(reasonErrorOf("x".repeat(500))).toBeNull() @@ -225,7 +262,7 @@ describe("security dependencies tab", () => { loaded(repositoriesOf(NOW), { ...findingsOf(NOW), findings: findingsOf(NOW).findings.map((finding) => ({ ...finding, production: [] })), - production: { available: true, runningServices: 0, linkedServices: 0 }, + production: { available: true, runningServices: 0, linkedServices: 0, complete: true }, }), ), expectView(text(/vcs\.ref\.head\.revision/)).toExist(), diff --git a/apps/web/src/page/security/dependencies/view-findings.ts b/apps/web/src/page/security/dependencies/view-findings.ts index ed1db78af8..a560432ca1 100644 --- a/apps/web/src/page/security/dependencies/view-findings.ts +++ b/apps/web/src/page/security/dependencies/view-findings.ts @@ -181,7 +181,12 @@ const productionCell = ( if (busiest === undefined && scan !== undefined) return muted(h, scan === "pending" ? "Scanning" : "Scan failed") if (busiest === undefined) - return muted(h, production.available && production.linkedServices > 0 ? "Not running" : "Unknown") + return muted( + h, + production.available && production.complete && production.linkedServices > 0 + ? "Not running" + : "Unknown", + ) const others = new Set(finding.production.map((service) => service.serviceName)).size - 1 return h.span( [h.Class("flex min-w-0 max-w-48 flex-col")], @@ -371,7 +376,10 @@ export const findingsSection = ( const content = rows.length > 0 ? findingsTable(h, rows, model.selectedId, clock, production, productionScanStates(repositories)) - : status === "open" && repositoryId === null + : // Clean only when a scan has read something; a failed first scan is not a clean bill. + status === "open" && + repositoryId === null && + repositories.some((repository) => repository.latestScan?.status === "completed") ? cleanState(h, repositories) : h.div( [h.Class("p-4")], diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts index ae6bf8af16..b73501baa8 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts @@ -270,7 +270,12 @@ describe("DependencyScanService production", () => { const { findings, production } = yield* service.listFindings(tenant, {}) // Services, not rows: api-server runs two commits and counts once. - assert.deepStrictEqual(production, { available: true, runningServices: 4, linkedServices: 2 }) + assert.deepStrictEqual(production, { + available: true, + runningServices: 4, + linkedServices: 2, + complete: true, + }) assert.deepStrictEqual( findings[0]?.production?.map((s) => [ s.serviceName, @@ -373,7 +378,12 @@ describe("DependencyScanService production", () => { const api = yield* seed yield* recordScan(api.id, "a", true) const { findings, production } = yield* service.listFindings(tenant, {}) - assert.deepStrictEqual(production, { available: false, runningServices: 0, linkedServices: 0 }) + assert.deepStrictEqual(production, { + available: false, + runningServices: 0, + linkedServices: 0, + complete: false, + }) assert.deepStrictEqual( findings.map((finding) => [finding.packageName, finding.production]), [["lodash", []]], diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index 07868fcf17..a1e0d844ca 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -44,6 +44,8 @@ import { /** Findings returned per read; the table sorts worst first, so the cut drops the least severe. */ export const FINDINGS_LIMIT = 500 +/** Findings read before ranking, so a deployed older finding is not cut before its traffic counts. */ +const FINDINGS_CANDIDATES = 2000 /** Production is what services served over the last day. */ const RUNNING_WINDOW_MS = 24 * 60 * 60 * 1000 /** Unscanned running commits queued per read, busiest first; the next read queues more. */ @@ -126,6 +128,8 @@ interface Production { readonly running: ReadonlyArray /** Distinct services reporting a commit. */ readonly runningServices: number + /** False when the read hit its row cap, so commits past it are unknown. */ + readonly complete: boolean /** Latest scan per running commit, any status: what the repository list shows. */ readonly latest: ReadonlyMap /** The matches of each running commit's latest completed scan. */ @@ -135,6 +139,7 @@ interface Production { const NO_PRODUCTION: Production = { available: false, + complete: false, running: [], runningServices: 0, latest: new Map(), @@ -365,6 +370,7 @@ export class DependencyScanService extends Context.Service b.score - a.score || a.order - b.order) + .slice(0, FINDINGS_LIMIT) yield* Effect.annotateCurrentSpan({ "result.rowCount": ranked.length, "maple.depscan.findings_in_production": ranked.filter( @@ -504,6 +511,7 @@ export class DependencyScanService extends Context.Service [CH.notInList($.bCommitSha, PLACEHOLDER_COMMIT_SHAS)]) .groupBy("serviceName", "environment", "commitSha") From b74fcb124ceff07d7962d490b159e421f6a124fd Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 09:17:52 +0100 Subject: [PATCH 08/15] test(web): sample the Dependencies tab in the lazy page slot test Every registered page needs a sample URL there. --- apps/web/src/view.lazy.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/web/src/view.lazy.test.ts b/apps/web/src/view.lazy.test.ts index 4b3c535316..83fa895b22 100644 --- a/apps/web/src/view.lazy.test.ts +++ b/apps/web/src/view.lazy.test.ts @@ -81,6 +81,7 @@ const SAMPLE_PATHS = [ "/security/runs", "/security/exposure", "/security/runtimes", + "/security/dependencies", "/security/runs/run_1", "/security/findings/00000000-0000-4000-8000-000000000005/fp_1", "/agent-sessions", From 5f3d029d3368f2aba5dc096731ddd97e5fd85e2f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 09:47:29 +0100 Subject: [PATCH 09/15] fix(security): Scan now always records a result; truthful filter counts and clean state - The tree-too-large shortcut applies to pushes only and compares the branch's own last scan, so Scan now records a result the page can see. - Status filter counts follow the selected repository. - The filter triggers keep their visible text as their accessible name. - The clean state needs every repository's latest scan completed. --- .../src/page/security/dependencies/view-findings.ts | 8 +++----- .../dependency-scan/DependencyScanner.test.ts | 3 +++ .../src/services/dependency-scan/DependencyScanner.ts | 11 +++++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apps/web/src/page/security/dependencies/view-findings.ts b/apps/web/src/page/security/dependencies/view-findings.ts index a560432ca1..b90ff6e7e3 100644 --- a/apps/web/src/page/security/dependencies/view-findings.ts +++ b/apps/web/src/page/security/dependencies/view-findings.ts @@ -21,7 +21,6 @@ import { relativeTime } from "../../kit/relative-time" import { securitySeverityBadge } from "../kit/parts" import { filteredFindings, - matchesStatus, RELATION_LABEL, repositoryFilterOf, SEVERITY_LABEL, @@ -88,7 +87,6 @@ const filterSelect = ( viewInputs: { entries, size: "sm", - ariaLabel: label, className: "h-7 w-auto min-w-28 text-xs", renderValue: (value) => [ h.span([h.Class("text-muted-foreground")], [`${label} `]), @@ -119,7 +117,7 @@ const toolbar = ( STATUS_FILTERS.map((status) => item( status, - `${STATUS_FILTER_LABEL[status]} (${findings.filter((finding) => matchesStatus(finding, status)).length})`, + `${STATUS_FILTER_LABEL[status]} (${filteredFindings(findings, status, repositoryFilterOf(model)).length})`, ), ), "Status", @@ -376,10 +374,10 @@ export const findingsSection = ( const content = rows.length > 0 ? findingsTable(h, rows, model.selectedId, clock, production, productionScanStates(repositories)) - : // Clean only when a scan has read something; a failed first scan is not a clean bill. + : // Clean only when every repository's latest scan read its lockfiles; a failed or missing one is not a clean bill. status === "open" && repositoryId === null && - repositories.some((repository) => repository.latestScan?.status === "completed") + repositories.every((repository) => repository.latestScan?.status === "completed") ? cleanState(h, repositories) : h.div( [h.Class("p-4")], diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts index 3af2c3963c..2d01eea0f4 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.test.ts @@ -362,6 +362,9 @@ describe("DependencyScanner", () => { const again = yield* scanner.scan(installation, repository, { trigger: "push" }) assert.strictEqual(again._tag, "Unchanged") assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 1) + // Scan now records its own result, so the page sees the scan it asked for. + yield* Effect.flip(scanner.scan(installation, repository, { trigger: "manual" })) + assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 2) }).pipe(Effect.provide(scannerFor(state))) }) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts index 43d5128b11..e236304b86 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -310,10 +310,13 @@ export class DependencyScanner extends Context.Service scan.repositoryId === repository.id, + ) if (last !== undefined && last.status === "failed" && last.error === TREE_TOO_LARGE) return { _tag: "Unchanged" } satisfies DependencyScanOutcome } From b948df360465b78df83ea4b56f06ef050d678665 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 10:28:46 +0100 Subject: [PATCH 10/15] fix(security): a commit two connected repositories share is not credited to either A fork or mirror holding the same commit made the running service's repository an arbitrary pick; such a commit now stays unlinked. --- .../DependencyScanService.test.ts | 20 +++++++++++++++++++ .../dependency-scan/DependencyScanService.ts | 13 +++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts index b73501baa8..36f55b605a 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts @@ -354,6 +354,26 @@ describe("DependencyScanService production", () => { }, ) + it.effect("leaves a commit two connected repositories share unlinked", () => { + const sent: Array = [] + const telemetry: Telemetry = { rows: [running("api-server", "production", "e", 500)] } + return Effect.gen(function* () { + const service = yield* DependencyScanService + const vcs = yield* VcsRepository + const api = yield* seed + const repositories = yield* vcs.listRepositoriesByInstallation(api.installationId, "active") + yield* Effect.forEach(repositories, (repository) => + vcs.upsertCommits(repository, [commitOf("e")]), + ) + const { repositories: listed } = yield* service.listRepositories(tenant) + assert.deepStrictEqual( + listed.map((repository) => repository.runningCommits.length), + [0, 0], + ) + assert.deepStrictEqual(sent, []) + }).pipe(Effect.provide(serviceFor(sent, telemetry))) + }) + it.effect("queues at most a handful of unscanned commits per read", () => { const sent: Array = [] const digits = "0123456789abcdef".split("") diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index a1e0d844ca..e105c095ab 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -337,10 +337,17 @@ export class DependencyScanService extends Context.Service row.sha), ) + // A commit in two connected repositories (a fork, a mirror) cannot say which one a service + // was built from, so it stays unlinked rather than credited to either. + const linked = commits.filter((commit) => connectedIds.has(commit.repositoryId)) + const owners = linked.reduce((acc, commit) => { + acc.set(commit.sha, new Set([...(acc.get(commit.sha) ?? []), commit.repositoryId])) + return acc + }, new Map>()) const repositoryOf = new Map( - commits - .filter((commit) => connectedIds.has(commit.repositoryId)) - .map((commit) => [commit.sha, commit.repositoryId] as const), + [...owners].flatMap(([sha, ids]) => + ids.size === 1 ? [[sha, [...ids][0]] as const] : [], + ), ) const running = withSha.flatMap((row): ReadonlyArray => { const repositoryId = repositoryOf.get(row.sha) From d2f1ae48e399aad8ae22d2a90ee3c8d29200a8e7 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 10:35:03 +0100 Subject: [PATCH 11/15] fix(web): a failed scan reads Last attempt, not Scanned --- .../src/page/security/dependencies/view-repositories.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/web/src/page/security/dependencies/view-repositories.ts b/apps/web/src/page/security/dependencies/view-repositories.ts index b181ff3ac4..bf08d2c94c 100644 --- a/apps/web/src/page/security/dependencies/view-repositories.ts +++ b/apps/web/src/page/security/dependencies/view-repositories.ts @@ -49,13 +49,20 @@ const lockfileIssue = (h: H, lockfile: DependencyScanLockfile): Html => ], ) +/** What the scan's time stands for: a failed scan read nothing, so it is not "Scanned". */ +const SCAN_SUMMARY_LABEL = { + running: "Scan started", + completed: "Scanned", + failed: "Last attempt", +} satisfies Record + const dot = (h: H): Html => h.span([h.AriaHidden(true), h.Class("text-foreground-faint")], ["ยท"]) const scanSummary = (h: H, scan: DependencyScanSummary, clock: Clock): Html => h.span( [h.Class("flex min-w-0 flex-wrap items-center gap-x-1.5 text-xs text-muted-foreground")], [ - scan.status === "running" ? "Scan started" : "Scanned", + SCAN_SUMMARY_LABEL[scan.status], relativeTime(h, { value: scan.finishedAt ?? scan.startedAt, nowMs: clock.nowMs, From b5c7c862b5face195bdf725a06b82cd9c3aab96c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 10:36:10 +0100 Subject: [PATCH 12/15] fix(web): the clean state needs every lockfile read A scan completes even when its lockfiles were too large, skipped or unsupported, so a repository nothing was read from could read as clean. The lab's clean scenario reads every lockfile. --- .../src/lab/security-dependencies-fixture.ts | 19 +++++++ apps/web/src/lab/security-lab/scenarios.ts | 3 +- .../dependencies/dependencies.scene.test.ts | 49 ++++++++++++++++++- .../security/dependencies/view-findings.ts | 9 ++-- 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/apps/web/src/lab/security-dependencies-fixture.ts b/apps/web/src/lab/security-dependencies-fixture.ts index 7dfbc6c602..ae8981a215 100644 --- a/apps/web/src/lab/security-dependencies-fixture.ts +++ b/apps/web/src/lab/security-dependencies-fixture.ts @@ -178,6 +178,25 @@ export const repositoriesOf = (nowMs: number): DependencyScanRepositoriesRespons }) /** Connected, before the repository's first scan. */ +/** The same repositories with every lockfile read, as the clean state requires. */ +export const cleanRepositoriesOf = (nowMs: number): DependencyScanRepositoriesResponse => ({ + repositories: repositoriesOf(nowMs).repositories.map((repository) => + repository.latestScan === null + ? repository + : { + ...repository, + latestScan: { + ...repository.latestScan, + lockfiles: repository.latestScan.lockfiles.map((lockfile) => ({ + ...lockfile, + status: "parsed" as const, + message: null, + })), + }, + }, + ), +}) + export const neverScannedOf = (nowMs: number): DependencyScanRepositoriesResponse => ({ repositories: [{ ...payments(nowMs), latestScan: null }], }) diff --git a/apps/web/src/lab/security-lab/scenarios.ts b/apps/web/src/lab/security-lab/scenarios.ts index 2c8782d742..75c0015757 100644 --- a/apps/web/src/lab/security-lab/scenarios.ts +++ b/apps/web/src/lab/security-lab/scenarios.ts @@ -45,6 +45,7 @@ import { unitsOf, } from "../security-fixture" import { + cleanRepositoriesOf, FINDING_IDS, findingsOf as dependencyFindingsOf, neverScannedOf, @@ -534,7 +535,7 @@ export const build = (scenario: Scenario, nowMs: number): Built => { return { _tag: "Dependencies", step: dependencies(nowMs, { - repositories: repositoriesOf(nowMs), + repositories: cleanRepositoriesOf(nowMs), findings: { ...dependencyFindingsOf(nowMs), findings: dependencyFindingsOf(nowMs).findings.filter( diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts index 6029301381..885af57ea7 100644 --- a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -235,6 +235,36 @@ describe("security dependencies tab", () => { ) }) + test("does not call a repository clean when its scan could not read a lockfile", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.latestScan === null + ? repository + : { + ...repository, + latestScan: { + ...repository.latestScan, + status: "completed" as const, + lockfiles: repository.latestScan.lockfiles.map((lockfile) => ({ + ...lockfile, + status: "too_large" as const, + })), + }, + }, + ), + }, + noFindings, + ), + ), + expectView(text("No known vulnerable packages")).toBeAbsent(), + ) + }) + test("says Unknown, not Not running, when more commits ran than were read", () => { scene( { update: pageUpdate, view }, @@ -272,10 +302,27 @@ describe("security dependencies tab", () => { }) test("scanned with nothing open shows the calm clean state", () => { + // Clean needs every lockfile read; the fixture's unsupported and failed ones are not. + const allParsed = { + repositories: repositoriesOf(NOW).repositories.map((repository) => + repository.latestScan === null + ? repository + : { + ...repository, + latestScan: { + ...repository.latestScan, + lockfiles: repository.latestScan.lockfiles.map((lockfile) => ({ + ...lockfile, + status: "parsed" as const, + })), + }, + }, + ), + } const settled = findingsOf(NOW).findings.filter((finding) => finding.status !== "open") scene( { update: pageUpdate, view }, - given(loaded(repositoriesOf(NOW), { ...findingsOf(NOW), findings: settled })), + given(loaded(allParsed, { ...findingsOf(NOW), findings: settled })), expectView(text("No known vulnerable packages")).toExist(), expectView(text("axios")).toBeAbsent(), ) diff --git a/apps/web/src/page/security/dependencies/view-findings.ts b/apps/web/src/page/security/dependencies/view-findings.ts index b90ff6e7e3..e64777fe82 100644 --- a/apps/web/src/page/security/dependencies/view-findings.ts +++ b/apps/web/src/page/security/dependencies/view-findings.ts @@ -341,6 +341,11 @@ const findingsTable = ( const packagesChecked = (repositories: ReadonlyArray): number => repositories.reduce((sum, repository) => sum + (repository.latestScan?.packageCount ?? 0), 0) +/** A repository is only clean when its latest scan completed and read every lockfile it found. */ +const readEverything = (repository: DependencyScanRepository): boolean => + repository.latestScan?.status === "completed" && + repository.latestScan.lockfiles.every((lockfile) => lockfile.status === "parsed") + /** Scanned and nothing open: a calm line, not an alarm. */ const cleanState = (h: H, repositories: ReadonlyArray): Html => empty( @@ -375,9 +380,7 @@ export const findingsSection = ( rows.length > 0 ? findingsTable(h, rows, model.selectedId, clock, production, productionScanStates(repositories)) : // Clean only when every repository's latest scan read its lockfiles; a failed or missing one is not a clean bill. - status === "open" && - repositoryId === null && - repositories.every((repository) => repository.latestScan?.status === "completed") + status === "open" && repositoryId === null && repositories.every(readEverything) ? cleanState(h, repositories) : h.div( [h.Class("p-4")], From dd042e1329b0e781769ded168818dd00b139dce7 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 11:58:13 +0100 Subject: [PATCH 13/15] fix(web): no clean state past unread advisories, no Scanning for unqueued commits A completed scan that recorded an error (advisories left unread, branch moved) no longer counts as clean, and a running commit with no scan row reads Not scanned yet. --- .../dependencies/dependencies.scene.test.ts | 64 ++++++++++++++++++- .../security/dependencies/view-findings.ts | 17 +++-- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts index 885af57ea7..6a840969df 100644 --- a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -152,7 +152,7 @@ describe("security dependencies tab", () => { ) }) - test("says a finding is being scanned, not absent, while a running commit's scan is pending", () => { + test("says a running commit is not scanned yet, not absent and not scanning, before any scan of it", () => { const repositories = repositoriesOf(NOW) scene( { update: pageUpdate, view }, @@ -177,6 +177,37 @@ describe("security dependencies tab", () => { }, ), ), + expectView(text("Not scanned yet")).toExist(), + expectView(text("Scanning")).toBeAbsent(), + expectView(text("api-gateway +1")).toBeAbsent(), + ) + }) + + test("says a finding is being scanned, not absent, while a running commit's scan runs", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.id === PAYMENTS_ID + ? { + ...repository, + runningCommits: repository.runningCommits.map((commit) => ({ + ...commit, + scanStatus: "running" as const, + })), + } + : repository, + ), + }, + { + ...findingsOf(NOW), + findings: findingsOf(NOW).findings.map((finding) => ({ ...finding, production: [] })), + }, + ), + ), expectView(text("Scanning")).toExist(), expectView(text("api-gateway +1")).toBeAbsent(), ) @@ -265,6 +296,37 @@ describe("security dependencies tab", () => { ) }) + test("does not call a repository clean when its scan left advisories unread", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.latestScan === null + ? repository + : { + ...repository, + latestScan: { + ...repository.latestScan, + status: "completed" as const, + error: "612 known vulnerabilities past the 500 a scan reads were left out", + lockfiles: repository.latestScan.lockfiles.map((lockfile) => ({ + ...lockfile, + status: "parsed" as const, + })), + }, + }, + ), + }, + noFindings, + ), + ), + expectView(text("No known vulnerable packages")).toBeAbsent(), + ) + }) + test("says Unknown, not Not running, when more commits ran than were read", () => { scene( { update: pageUpdate, view }, diff --git a/apps/web/src/page/security/dependencies/view-findings.ts b/apps/web/src/page/security/dependencies/view-findings.ts index e64777fe82..9b23cb9dac 100644 --- a/apps/web/src/page/security/dependencies/view-findings.ts +++ b/apps/web/src/page/security/dependencies/view-findings.ts @@ -147,17 +147,18 @@ const toolbar = ( const muted = (h: H, text: string, className?: string): Html => h.span([h.Class(cn("text-xs text-muted-foreground", className))], [text]) -type ProductionScanState = "pending" | "failed" +type ProductionScanState = "running" | "unscanned" | "failed" -/** Repositories whose running commits are not all scanned: one still pending, or one whose scan failed. */ +/** Repositories whose running commits are not all scanned: one scanning, one with no scan yet, or one failed. */ const productionScanStates = ( repositories: ReadonlyArray, ): ReadonlyMap => new Map( repositories.flatMap((repository): ReadonlyArray => { const statuses = repository.runningCommits.map((commit) => commit.scanStatus) - if (statuses.some((status) => status === null || status === "running")) - return [[repository.id, "pending"]] + if (statuses.includes("running")) return [[repository.id, "running"]] + // No scan row: queued from this read, or past the per-read cap, so nothing claims it is running. + if (statuses.includes(null)) return [[repository.id, "unscanned"]] return statuses.includes("failed") ? [[repository.id, "failed"]] : [] }), ) @@ -177,7 +178,10 @@ const productionCell = ( // A running commit not scanned yet may carry the finding: no answer until its scan lands. const scan = scans.get(finding.repositoryId) if (busiest === undefined && scan !== undefined) - return muted(h, scan === "pending" ? "Scanning" : "Scan failed") + return muted( + h, + scan === "running" ? "Scanning" : scan === "unscanned" ? "Not scanned yet" : "Scan failed", + ) if (busiest === undefined) return muted( h, @@ -341,9 +345,10 @@ const findingsTable = ( const packagesChecked = (repositories: ReadonlyArray): number => repositories.reduce((sum, repository) => sum + (repository.latestScan?.packageCount ?? 0), 0) -/** A repository is only clean when its latest scan completed and read every lockfile it found. */ +/** A repository is only clean when its latest scan completed whole and read every lockfile it found. */ const readEverything = (repository: DependencyScanRepository): boolean => repository.latestScan?.status === "completed" && + repository.latestScan.error === null && repository.latestScan.lockfiles.every((lockfile) => lockfile.status === "parsed") /** Scanned and nothing open: a calm line, not an alarm. */ From 13160c45168a92dacf089fe68e761af35547ee36 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 12:08:15 +0100 Subject: [PATCH 14/15] fix(web): Last seen shows the commit, not the last triage time updatedAt moves on every dismiss, accept or reopen, so pairing it with lastSeenSha read as a fresh detection. No stored time records when a scan last matched. --- .../dependencies/dependencies.scene.test.ts | 22 +++++++++++++++++++ .../page/security/dependencies/view-sheet.ts | 15 +++++-------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts index 6a840969df..943e767008 100644 --- a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -1,4 +1,5 @@ import { OrgId } from "@maple/domain" +import { formatRelativeTimeOrDate } from "@maple/ui/lib/time-format" import { DependencyScanForbiddenError, DependencyScanPersistenceError } from "@maple/domain/http" import type { DependencyFindingsResponse, DependencyScanRepositoriesResponse } from "@maple/domain/http" import { Result, Schema } from "effect" @@ -525,6 +526,27 @@ describe("security dependencies tab", () => { ) }) + test("the sheet does not date Last seen by the finding's last triage", () => { + const triagedAt = NOW - 3 * 60 * 1000 + const findings = findingsOf(NOW) + scene( + { update: pageUpdate, view }, + given( + opened( + loaded(repositoriesOf(NOW), { + ...findings, + findings: findings.findings.map((finding) => + finding.id === FINDING_IDS.axios ? { ...finding, updatedAt: triagedAt } : finding, + ), + }), + ), + ), + mounted, + expectView(text("Last seen")).toExist(), + expectView(text(formatRelativeTimeOrDate(triagedAt, NOW, "UTC"))).toBeAbsent(), + ) + }) + test("Dismiss needs a reason, then writes the status with it", () => { const write = UpdateFinding({ findingId: FINDING_IDS.axios, diff --git a/apps/web/src/page/security/dependencies/view-sheet.ts b/apps/web/src/page/security/dependencies/view-sheet.ts index 4d175f6ec2..89b220a228 100644 --- a/apps/web/src/page/security/dependencies/view-sheet.ts +++ b/apps/web/src/page/security/dependencies/view-sheet.ts @@ -94,19 +94,16 @@ const details = (h: H, finding: DependencyFindingView, clock: Clock): Html => detailRailRow(h, { label: "Repository" }, [value(h, finding.repositoryFullName)]), detailRailRow(h, { label: "Lockfile" }, [value(h, finding.lockfilePath, true)]), detailRailRow(h, { label: "First seen" }, [at(h, finding.firstSeenSha, finding.createdAt, clock)]), - detailRailRow(h, { label: "Last seen" }, [at(h, finding.lastSeenSha, finding.updatedAt, clock)]), + // Commit only: triage rewrites updatedAt, so no stored time says when a scan last matched. + detailRailRow(h, { label: "Last seen" }, [commit(h, finding.lastSeenSha)]), ...(finding.fixedSha === null ? [] - : [ - detailRailRow(h, { label: "Fixed at" }, [ - h.span( - [h.Class("font-mono text-xs"), h.Title(finding.fixedSha)], - [shortSha(finding.fixedSha)], - ), - ]), - ]), + : [detailRailRow(h, { label: "Fixed at" }, [commit(h, finding.fixedSha)])]), ]) +const commit = (h: H, sha: string): Html => + h.span([h.Class("font-mono text-xs"), h.Title(sha)], [shortSha(sha)]) + const REASON_IDS: Field.FieldIds = { control: "dependency-triage-reason", label: "dependency-triage-reason-label", From d38dbf9b184033eb675318d851db5dc68b679dd0 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 1 Nov 2026 12:11:50 +0100 Subject: [PATCH 15/15] fix(web): the reason field checks the reason as triage sends it, trimmed --- .../dependencies/dependencies.scene.test.ts | 14 ++++++++++++++ .../src/page/security/dependencies/view-sheet.ts | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts index 943e767008..c0f94e3888 100644 --- a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -575,6 +575,20 @@ describe("security dependencies tab", () => { ) }) + test("a reason at the cap with a trailing space is written without a length error", () => { + const reason = "x".repeat(500) + scene( + { update: pageUpdate, view }, + given(opened(loaded())), + mounted, + type(role("textbox", { name: "Reason" }), `${reason} `), + click(role("button", { name: "Dismiss" })), + Command.expectHas(UpdateFinding({ findingId: FINDING_IDS.axios, status: "dismissed", reason })), + expectView(text(/Keep the reason under/)).toBeAbsent(), + Command.resolve(UpdateFinding, Message.CompletedTriage({ result: forbidden })), + ) + }) + test("a dismissed finding is reopened without a reason", () => { const dismissed = apply(loaded(), [ () => diff --git a/apps/web/src/page/security/dependencies/view-sheet.ts b/apps/web/src/page/security/dependencies/view-sheet.ts index 89b220a228..cadbdff388 100644 --- a/apps/web/src/page/security/dependencies/view-sheet.ts +++ b/apps/web/src/page/security/dependencies/view-sheet.ts @@ -112,7 +112,8 @@ const REASON_IDS: Field.FieldIds = { } const reasonField = (h: H, model: Model): Html => { - const error = model.isReasonSubmitted ? reasonErrorOf(model.reason.value) : null + // Trimmed as triage sends it, so the field never errors on a reason that was written. + const error = model.isReasonSubmitted ? reasonErrorOf(model.reason.value.trim()) : null const state = Field.stateOf(model.reason, error !== null) return Field.field(h, { state }, [ Field.fieldLabel(h, { ids: REASON_IDS, state }, "Reason"),