diff --git a/apps/api/src/routes/internal/dependency-scan.http.ts b/apps/api/src/routes/internal/dependency-scan.http.ts index b587dc8ef0..0cc364ddfd 100644 --- a/apps/api/src/routes/internal/dependency-scan.http.ts +++ b/apps/api/src/routes/internal/dependency-scan.http.ts @@ -10,7 +10,7 @@ export const HttpDependencyScanLive = HttpApiBuilder.group(MapleInternalApi, "de return handlers .handle("listRepositories", () => - Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listRepositories(tenant.orgId)), + Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listRepositories(tenant)), ) .handle("scanRepository", ({ params }) => Effect.flatMap(CurrentTenant.Context, (tenant) => @@ -18,7 +18,7 @@ export const HttpDependencyScanLive = HttpApiBuilder.group(MapleInternalApi, "de ), ) .handle("listFindings", ({ query }) => - Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listFindings(tenant.orgId, query)), + Effect.flatMap(CurrentTenant.Context, (tenant) => scans.listFindings(tenant, query)), ) .handle("updateFinding", ({ params, payload }) => Effect.gen(function* () { diff --git a/apps/web/src/lab/entries/security.ts b/apps/web/src/lab/entries/security.ts index 5a001f52e9..15b5ffa95a 100644 --- a/apps/web/src/lab/entries/security.ts +++ b/apps/web/src/lab/entries/security.ts @@ -2,6 +2,9 @@ import { Clock, Effect, Option, Schema } from "effect" import { Command } from "foldkit" import type { Html, HtmlBuilder } from "foldkit/html" import { defineMessageUnion } from "foldkit/message" +import * as DependenciesPage from "../../page/security/dependencies/model" +import * as DependenciesUpdate from "../../page/security/dependencies/update" +import { view as dependenciesView } from "../../page/security/dependencies/view" import * as FindingPage from "../../page/security/finding/model" import * as FindingUpdate from "../../page/security/finding/update" import { breadcrumbsOf as findingCrumbs, view as findingView } from "../../page/security/finding/view" @@ -27,6 +30,7 @@ const Model = Schema.Struct({ run: Schema.NullOr(RunPage.Model), finding: Schema.NullOr(FindingPage.Model), telemetry: Schema.NullOr(TelemetryPage.Model), + dependencies: Schema.NullOr(DependenciesPage.Model), }) type Model = typeof Model.Type @@ -35,6 +39,7 @@ const Message = defineMessageUnion({ GotRunMessage: { message: RunPage.Message }, GotFindingMessage: { message: FindingPage.Message }, GotTelemetryMessage: { message: TelemetryPage.Message }, + GotDependenciesMessage: { message: DependenciesPage.Message }, }) type Message = typeof Message.Type @@ -54,7 +59,14 @@ const flags = Effect.all({ const init = (input: typeof Flags.Type) => { const built = build(input.scenario, input.nowMs) - const empty: Model = { scenario: input.scenario, tabs: null, run: null, finding: null, telemetry: null } + const empty: Model = { + scenario: input.scenario, + tabs: null, + run: null, + finding: null, + telemetry: null, + dependencies: null, + } switch (built._tag) { case "Tabs": return { @@ -84,6 +96,13 @@ const init = (input: typeof Flags.Type) => { Message.GotTelemetryMessage({ message }), ), } + case "Dependencies": + return { + model: { ...empty, dependencies: built.step.model }, + commands: Command.mapMessages(built.step.commands, (message) => + Message.GotDependenciesMessage({ message }), + ), + } } } @@ -129,6 +148,16 @@ const update = (model: Model, message: Message) => ), } }, + GotDependenciesMessage: ({ message: inner }) => { + if (model.dependencies === null) return { model } + const next = DependenciesUpdate.update(model.dependencies, inner, context) + return { + model: { ...model, dependencies: next.model }, + commands: Command.mapMessages((next.commands ?? []).filter(isLocalCommand), (child) => + Message.GotDependenciesMessage({ message: child }), + ), + } + }, }) const view = (model: Model, h: HtmlBuilder): Html => @@ -164,7 +193,15 @@ const view = (model: Model, h: HtmlBuilder): Html => viewInputs: { context }, toParentMessage: (message) => Message.GotTelemetryMessage({ message }), }) - : h.empty + : model.dependencies !== null + ? h.submodel({ + slotId: "security-lab-dependencies", + model: model.dependencies, + view: dependenciesView, + viewInputs: { context }, + toParentMessage: (message) => Message.GotDependenciesMessage({ message }), + }) + : h.empty export const lab = defineShellLab("Security Center", { Model, @@ -184,5 +221,7 @@ export const lab = defineShellLab("Security Center", { ? findingCrumbs(model.finding) : model.telemetry !== null ? tabsCrumbs(model.telemetry.tab) - : [], + : model.dependencies !== null + ? tabsCrumbs("dependencies") + : [], }) diff --git a/apps/web/src/lab/security-dependencies-fixture.ts b/apps/web/src/lab/security-dependencies-fixture.ts new file mode 100644 index 0000000000..7dfbc6c602 --- /dev/null +++ b/apps/web/src/lab/security-dependencies-fixture.ts @@ -0,0 +1,419 @@ +import { DependencyFindingId, DependencyScanId } from "@maple/domain/dependency-scan" +import { + type DependencyFindingsResponse, + type DependencyFindingView, + type DependencyScanRepositoriesResponse, + type DependencyScanRepository, + GitCommitSha, + VcsRepositoryId, +} from "@maple/domain/http" +import { Schema } from "effect" + +// Fixtures for the Security Center's Dependencies tab, for `/lab/security` and the page's scene +// tests. Built relative to `now` so relative times read the same. + +const HOUR = 3_600_000 +const DAY = 24 * HOUR + +const repositoryId = Schema.decodeSync(VcsRepositoryId) +const scanId = Schema.decodeSync(DependencyScanId) +const findingId = Schema.decodeSync(DependencyFindingId) +const sha = Schema.decodeSync(GitCommitSha) + +export const REPOSITORY_IDS = { + payments: repositoryId("6f1c2d3e-4a5b-4c6d-8e7f-901a2b3c4d5e"), + web: repositoryId("7a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d"), + cli: repositoryId("8b3c4d5e-6f7a-4b8c-9d0e-1f2a3b4c5d6e"), +} as const + +const SCAN_IDS = { + payments: scanId("1a2b3c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d"), + web: scanId("2b3c4d5e-6f7a-4b8c-9d0e-1f2a3b4c5d6e"), + cli: scanId("3c4d5e6f-7a8b-4c9d-8e0f-2a3b4c5d6e7f"), + older: scanId("4d5e6f7a-8b9c-4d0e-9f1a-3b4c5d6e7f8a"), +} as const + +const SHAS = { + payments: sha("4e1d9c2a7b3f8e6d5c4b3a2918f7e6d5c4b3a291"), + web: sha("9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0b"), + cli: sha("c0ffee1234567890abcdef1234567890abcdef12"), + older: sha("0123456789abcdef0123456789abcdef01234567"), +} as const + +export const PAYMENTS_ID = REPOSITORY_IDS.payments + +const service = (serviceName: string, commitSha: GitCommitSha, spanCount: number) => ({ + serviceName, + environment: "production", + commitSha, + spanCount, +}) + +/** What production ran over the last day: payments-api and web are deployed, cli is not. */ +const RUNNING = { + payments: [ + service("api-gateway", SHAS.payments, 2_400_000), + service("payments-worker", SHAS.payments, 180_000), + ], + web: [service("web-frontend", SHAS.web, 640_000)], +} as const + +const payments = (nowMs: number): DependencyScanRepository => ({ + id: REPOSITORY_IDS.payments, + fullName: "acme/payments-api", + htmlUrl: "https://github.com/acme/payments-api", + runningCommits: [{ commitSha: SHAS.payments, scanStatus: "completed", services: RUNNING.payments }], + trackedBranch: "main", + latestScan: { + id: SCAN_IDS.payments, + commitSha: SHAS.payments, + ref: "refs/heads/main", + trigger: "push", + status: "completed", + lockfiles: [ + { + path: "pnpm-lock.yaml", + kind: "pnpm-lock", + blobSha: "a1", + packageCount: 1184, + status: "parsed", + message: null, + }, + { + path: "services/ledger/go.sum", + kind: "go-sum", + blobSha: "a2", + packageCount: 96, + status: "parsed", + message: null, + }, + { + path: "tools/migrate/Cargo.lock", + kind: "cargo-lock", + blobSha: "a3", + packageCount: 0, + status: "failed", + message: "Line 212: expected a [[package]] table.", + }, + ], + packageCount: 1280, + vulnerableCount: 5, + error: null, + startedAt: nowMs - 2 * HOUR - 40_000, + finishedAt: nowMs - 2 * HOUR, + }, +}) + +const web = (nowMs: number): DependencyScanRepository => ({ + id: REPOSITORY_IDS.web, + fullName: "acme/web", + htmlUrl: "https://github.com/acme/web", + runningCommits: [{ commitSha: SHAS.web, scanStatus: "completed", services: RUNNING.web }], + trackedBranch: "main", + latestScan: { + id: SCAN_IDS.web, + commitSha: SHAS.web, + ref: "refs/heads/main", + trigger: "push", + status: "completed", + lockfiles: [ + { + path: "bun.lockb", + kind: "bun-lockb", + blobSha: "b1", + packageCount: 0, + status: "unsupported", + message: + "Binary bun.lockb is not readable. Run `bun install --save-text-lockfile` and commit bun.lock.", + }, + { + path: "docs/package-lock.json", + kind: "package-lock", + blobSha: "b2", + packageCount: 412, + status: "parsed", + message: null, + }, + ], + packageCount: 412, + vulnerableCount: 2, + error: null, + startedAt: nowMs - DAY - 30_000, + finishedAt: nowMs - DAY, + }, +}) + +const cli = (nowMs: number): DependencyScanRepository => ({ + id: REPOSITORY_IDS.cli, + fullName: "acme/cli", + htmlUrl: "https://github.com/acme/cli", + runningCommits: [], + trackedBranch: "release", + latestScan: { + id: SCAN_IDS.cli, + commitSha: SHAS.cli, + ref: "refs/heads/release", + trigger: "schedule", + status: "completed", + lockfiles: [ + { + path: "Cargo.lock", + kind: "cargo-lock", + blobSha: "c1", + packageCount: 214, + status: "parsed", + message: null, + }, + ], + packageCount: 214, + vulnerableCount: 1, + error: null, + startedAt: nowMs - 3 * DAY - 20_000, + finishedAt: nowMs - 3 * DAY, + }, +}) + +export const repositoriesOf = (nowMs: number): DependencyScanRepositoriesResponse => ({ + repositories: [payments(nowMs), web(nowMs), cli(nowMs)], +}) + +/** Connected, before the repository's first scan. */ +export const neverScannedOf = (nowMs: number): DependencyScanRepositoriesResponse => ({ + repositories: [{ ...payments(nowMs), latestScan: null }], +}) + +export const noRepositories: DependencyScanRepositoriesResponse = { repositories: [] } + +/** The payments repository scanned once more, later, as a completed Scan now would leave it. */ +export const rescannedOf = (nowMs: number): DependencyScanRepositoriesResponse => { + const [first, ...rest] = repositoriesOf(nowMs).repositories + return first === undefined || first.latestScan === null + ? repositoriesOf(nowMs) + : { + repositories: [ + { + ...first, + latestScan: { + ...first.latestScan, + id: SCAN_IDS.older, + commitSha: SHAS.older, + finishedAt: nowMs, + }, + }, + ...rest, + ], + } +} + +export const FINDING_IDS = { + axios: findingId("0a1b2c3d4e5f60718293a4b5c6d7e8f9"), + xnet: findingId("1b2c3d4e5f60718293a4b5c6d7e8f90a"), + vite: findingId("2c3d4e5f60718293a4b5c6d7e8f90a1b"), + tokio: findingId("3d4e5f60718293a4b5c6d7e8f90a1b2c"), + marked: findingId("4e5f60718293a4b5c6d7e8f90a1b2c3d"), + lodash: findingId("5f60718293a4b5c6d7e8f90a1b2c3d4e"), + jwt: findingId("60718293a4b5c6d7e8f90a1b2c3d4e5f"), +} as const + +type FindingSeed = Pick< + DependencyFindingView, + | "id" + | "repositoryId" + | "repositoryFullName" + | "lockfilePath" + | "ecosystem" + | "packageName" + | "advisoryId" + | "aliases" + | "severity" + | "summary" + | "versions" + | "fixedIn" + | "relation" +> + +const finding = ( + nowMs: number, + seed: FindingSeed, + firstSeenAgoMs: number, + production: DependencyFindingView["production"] = [], +): DependencyFindingView => { + const scan = + seed.repositoryId === REPOSITORY_IDS.payments + ? "payments" + : seed.repositoryId === REPOSITORY_IDS.web + ? "web" + : "cli" + return { + ...seed, + status: "open", + firstSeenScanId: SCAN_IDS.older, + lastSeenScanId: SCAN_IDS[scan], + firstSeenSha: SHAS.older, + lastSeenSha: SHAS[scan], + fixedScanId: null, + fixedSha: null, + statusReason: null, + statusChangedBy: null, + statusChangedAt: null, + createdAt: nowMs - firstSeenAgoMs, + updatedAt: nowMs - 2 * HOUR, + production, + } +} + +const PAYMENTS = { repositoryId: REPOSITORY_IDS.payments, repositoryFullName: "acme/payments-api" } as const +const WEB = { repositoryId: REPOSITORY_IDS.web, repositoryFullName: "acme/web" } as const +const CLI = { repositoryId: REPOSITORY_IDS.cli, repositoryFullName: "acme/cli" } as const + +/** Worst first, as the server sorts them: one per severity, a dismissed and a fixed finding. */ +export const findingsOf = (nowMs: number): DependencyFindingsResponse => ({ + production: { available: true, runningServices: 4, linkedServices: 3, complete: true }, + findings: [ + finding( + nowMs, + { + ...PAYMENTS, + id: FINDING_IDS.axios, + lockfilePath: "pnpm-lock.yaml", + ecosystem: "npm", + packageName: "axios", + advisoryId: "GHSA-jr5f-v2jv-69x6", + aliases: ["CVE-2025-27152"], + severity: "critical", + summary: + "axios requests to absolute URLs ignore baseURL, so a crafted URL can leak credentials to another host.", + versions: ["1.7.4"], + fixedIn: "1.8.2", + relation: "direct", + }, + 9 * DAY, + RUNNING.payments, + ), + finding( + nowMs, + { + ...PAYMENTS, + id: FINDING_IDS.xnet, + lockfilePath: "services/ledger/go.sum", + ecosystem: "Go", + packageName: "golang.org/x/net", + advisoryId: "GO-2025-3503", + aliases: ["CVE-2025-22870", "GHSA-qxp5-gwg8-xv66"], + severity: "high", + summary: "HTTP proxy bypass using IPv6 zone IDs.", + versions: ["0.33.0"], + fixedIn: "0.36.0", + relation: "transitive", + }, + 4 * DAY, + [RUNNING.payments[1]], + ), + finding( + nowMs, + { + ...PAYMENTS, + id: FINDING_IDS.vite, + lockfilePath: "pnpm-lock.yaml", + ecosystem: "npm", + packageName: "vite", + advisoryId: "GHSA-vg6x-rcgg-rjx6", + aliases: ["CVE-2025-24010"], + severity: "medium", + summary: + "Websites were able to send any requests to the development server and read the response.", + versions: ["5.4.10", "6.0.7"], + fixedIn: "6.0.9", + relation: "dev", + }, + 2 * DAY, + ), + finding( + nowMs, + { + ...CLI, + id: FINDING_IDS.tokio, + lockfilePath: "Cargo.lock", + ecosystem: "crates.io", + packageName: "tokio", + advisoryId: "RUSTSEC-2025-0023", + aliases: ["GHSA-rr8g-9fpq-6wmg"], + severity: "low", + summary: "Broadcast channel calls clone in parallel, but does not require Sync.", + versions: ["1.38.0"], + fixedIn: "1.38.2", + relation: "transitive", + }, + 20 * DAY, + ), + finding( + nowMs, + { + ...WEB, + id: FINDING_IDS.marked, + lockfilePath: "docs/package-lock.json", + ecosystem: "npm", + packageName: "marked", + advisoryId: "GHSA-4r62-v4vq-hr96", + aliases: [], + severity: "unknown", + summary: null, + versions: ["4.0.9"], + fixedIn: null, + relation: "unknown", + }, + DAY, + ), + { + ...finding( + nowMs, + { + ...WEB, + id: FINDING_IDS.lodash, + lockfilePath: "docs/package-lock.json", + ecosystem: "npm", + packageName: "lodash.template", + advisoryId: "GHSA-35jh-r3h4-6jhm", + aliases: ["CVE-2021-23337"], + severity: "high", + summary: "Command injection in lodash template.", + versions: ["4.5.0"], + fixedIn: null, + relation: "dev", + }, + 30 * DAY, + ), + status: "dismissed", + statusReason: "Only used by the docs build, never with user input.", + statusChangedAt: nowMs - 6 * DAY, + }, + { + ...finding( + nowMs, + { + ...PAYMENTS, + id: FINDING_IDS.jwt, + lockfilePath: "pnpm-lock.yaml", + ecosystem: "npm", + packageName: "jsonwebtoken", + advisoryId: "GHSA-8cf7-32gw-wr33", + aliases: ["CVE-2022-23539"], + severity: "high", + summary: "Insecure key types can be used for signature verification.", + versions: ["8.5.1"], + fixedIn: "9.0.0", + relation: "direct", + }, + 40 * DAY, + ), + status: "fixed", + fixedScanId: SCAN_IDS.payments, + fixedSha: SHAS.payments, + }, + ], +}) + +export const noFindings: DependencyFindingsResponse = { + findings: [], + production: { available: true, runningServices: 3, linkedServices: 3, complete: true }, +} diff --git a/apps/web/src/lab/security-lab/scenarios.ts b/apps/web/src/lab/security-lab/scenarios.ts index 018db34d5a..2c8782d742 100644 --- a/apps/web/src/lab/security-lab/scenarios.ts +++ b/apps/web/src/lab/security-lab/scenarios.ts @@ -1,5 +1,8 @@ import { OrgId } from "@maple/domain" import { + DependencyScanPersistenceError, + type DependencyFindingsResponse, + type DependencyScanRepositoriesResponse, SecurityAuditDisabledError, SecurityAuditForbiddenError, SecurityAuditTargetExistsError, @@ -10,6 +13,8 @@ import type { Command } from "foldkit" import { toLoadFailure } from "../../data/load-failure" import type { PageContext } from "../../page/contract" import * as AddTarget from "../../page/security/kit/add-target" +import * as DependenciesPage from "../../page/security/dependencies/model" +import * as DependenciesUpdate from "../../page/security/dependencies/update" import * as FindingPage from "../../page/security/finding/model" import * as FindingUpdate from "../../page/security/finding/update" import * as RunPage from "../../page/security/run/model" @@ -39,6 +44,14 @@ import { targetsOf, unitsOf, } from "../security-fixture" +import { + FINDING_IDS, + findingsOf as dependencyFindingsOf, + neverScannedOf, + noFindings, + noRepositories, + repositoriesOf, +} from "../security-dependencies-fixture" import { emptyExposureOf, exposureOf, runtimesOf } from "../security-telemetry-fixture" // Every Security Center state the lab renders, built by sending each page the fixture answers its @@ -68,6 +81,13 @@ export const SCENARIOS = [ "exposure", "exposure-empty", "runtimes", + "dependencies", + "dependencies-finding", + "dependencies-clean", + "dependencies-never-scanned", + "dependencies-no-github", + "dependencies-error", + "dependencies-loading", ] as const export const Scenario = Schema.Literals(SCENARIOS) export type Scenario = typeof Scenario.Type @@ -319,6 +339,56 @@ const exposure = (nowMs: number, data: TelemetryPage.Exposure): TelemetryStep => T.CompletedFetchExposure({ key: model.exposure.key ?? "", result: Result.succeed(data) }), ) +// DEPENDENCIES + +type DependenciesStep = Step +const D = DependenciesPage.Message + +type DependencyAnswers = Readonly<{ + repositories: DependencyScanRepositoriesResponse | "failed" + findings: DependencyFindingsResponse +}> + +const dependencies = ( + nowMs: number, + answers: DependencyAnswers | null, + then: ReadonlyArray<(model: DependenciesPage.Model) => DependenciesPage.Message> = [], +): DependenciesStep => { + const entered = DependenciesUpdate.init(AppRoute.SecurityDependencies(), context) + return fold( + { model: entered.model, commands: entered.commands ?? [] }, + (model, message) => DependenciesUpdate.update(model, message, context), + [ + () => D.ReadClock({ nowMs }), + () => D.CompletedReadIsAdmin({ isAdmin: true }), + ...(answers === null + ? [] + : [ + (model: DependenciesPage.Model) => + D.CompletedFetchRepositories({ + key: model.repositories.key ?? "", + result: + answers.repositories === "failed" + ? Result.fail( + toLoadFailure( + new DependencyScanPersistenceError({ + message: "Dependency scans are unavailable", + }), + ), + ) + : Result.succeed(answers.repositories), + }), + (model: DependenciesPage.Model) => + D.CompletedFetchFindings({ + key: model.findings.key ?? "", + result: Result.succeed(answers.findings), + }), + ]), + ...then, + ], + ) +} + // SCENARIOS export type Built = @@ -326,6 +396,7 @@ export type Built = | Readonly<{ _tag: "Run"; step: RunStep }> | Readonly<{ _tag: "Finding"; step: FindingStep }> | Readonly<{ _tag: "Telemetry"; step: TelemetryStep }> + | Readonly<{ _tag: "Dependencies"; step: DependenciesStep }> const overviewRoute = AppRoute.Security() @@ -450,5 +521,50 @@ export const build = (scenario: Scenario, nowMs: number): Built => { }), ), } + case "dependencies": + return { _tag: "Dependencies", step: populatedDependencies(nowMs) } + case "dependencies-finding": + return { + _tag: "Dependencies", + step: populatedDependencies(nowMs, [ + () => D.ClickedFinding({ findingId: FINDING_IDS.axios }), + ]), + } + case "dependencies-clean": + return { + _tag: "Dependencies", + step: dependencies(nowMs, { + repositories: repositoriesOf(nowMs), + findings: { + ...dependencyFindingsOf(nowMs), + findings: dependencyFindingsOf(nowMs).findings.filter( + (finding) => finding.status !== "open", + ), + }, + }), + } + case "dependencies-never-scanned": + return { + _tag: "Dependencies", + step: dependencies(nowMs, { repositories: neverScannedOf(nowMs), findings: noFindings }), + } + case "dependencies-no-github": + return { + _tag: "Dependencies", + step: dependencies(nowMs, { repositories: noRepositories, findings: noFindings }), + } + case "dependencies-error": + return { + _tag: "Dependencies", + step: dependencies(nowMs, { repositories: "failed", findings: noFindings }), + } + case "dependencies-loading": + return { _tag: "Dependencies", step: dependencies(nowMs, null) } } } + +const populatedDependencies = ( + nowMs: number, + then: ReadonlyArray<(model: DependenciesPage.Model) => DependenciesPage.Message> = [], +): DependenciesStep => + dependencies(nowMs, { repositories: repositoriesOf(nowMs), findings: dependencyFindingsOf(nowMs) }, then) diff --git a/apps/web/src/page/registry.ts b/apps/web/src/page/registry.ts index ba0868b657..b400b87a99 100644 --- a/apps/web/src/page/registry.ts +++ b/apps/web/src/page/registry.ts @@ -33,6 +33,7 @@ import * as DashboardTemplates from "./dashboards/templates" import * as DashboardView from "./dashboards/view" import * as DashboardWidget from "./dashboards/widget" import * as CodeReview from "./code-review" +import * as SecurityDependencies from "./security/dependencies" import * as SecurityFinding from "./security/finding" import * as SecurityRun from "./security/run" import * as SecurityTabs from "./security/tabs" @@ -152,6 +153,7 @@ export const pages = [ CodeReview.page, SecurityTabs.page, SecurityTelemetry.page, + SecurityDependencies.page, SecurityRun.page, SecurityFinding.page, AgentSessionsList.page, diff --git a/apps/web/src/page/security/dependencies/command.ts b/apps/web/src/page/security/dependencies/command.ts new file mode 100644 index 0000000000..b4a551ab36 --- /dev/null +++ b/apps/web/src/page/security/dependencies/command.ts @@ -0,0 +1,74 @@ +import { DependencyFindingId, DependencyFindingTriageStatus } from "@maple/domain/dependency-scan" +import { VcsRepositoryId } from "@maple/domain/http" +import { Clock, Effect, Schema } from "effect" +import { Command } from "foldkit" +import { toLoadFailure } from "../../../data/load-failure" +import { MapleInternalApiClient } from "../../../platform/api-clients" +import { readIsAdmin } from "../../alerts/shared/commands" +import { Message } from "./model" + +// The Dependencies tab's reads and writes on the api Worker. Names start with `SecurityApi` (and +// the clock's and session's with `ReadSecurity`) so the lab drops them. + +type Api = typeof MapleInternalApiClient.Service + +const call = (run: (api: Api) => Effect.Effect) => + Effect.gen(function* () { + const api = yield* MapleInternalApiClient + return yield* run(api) + }).pipe(Effect.mapError(toLoadFailure), Effect.result) + +export const ReadDependenciesClock = Command.define("ReadSecurityDependenciesClock", { + messages: [Message.ReadClock], + execute: Effect.map(Clock.currentTimeMillis, (nowMs) => Message.ReadClock({ nowMs })), +}) + +export const ReadDependenciesIsAdmin = Command.define("ReadSecurityDependenciesIsAdmin", { + messages: [Message.CompletedReadIsAdmin], + execute: Effect.map(readIsAdmin, (isAdmin) => Message.CompletedReadIsAdmin({ isAdmin })), +}) + +export const FetchRepositories = Command.define("SecurityApiDependencyRepositories", { + args: { key: Schema.String }, + messages: [Message.CompletedFetchRepositories], + execute: ({ key }) => + Effect.map( + call((api) => api.dependencyScan.listRepositories()), + (result) => Message.CompletedFetchRepositories({ key, result }), + ), +}) + +export const FetchFindings = Command.define("SecurityApiDependencyFindings", { + args: { key: Schema.String }, + messages: [Message.CompletedFetchFindings], + execute: ({ key }) => + Effect.map( + call((api) => api.dependencyScan.listFindings({ query: {} })), + (result) => Message.CompletedFetchFindings({ key, result }), + ), +}) + +export const ScanRepository = Command.define("SecurityApiScanRepository", { + args: { repositoryId: VcsRepositoryId }, + messages: [Message.CompletedScanRequest], + execute: ({ repositoryId }) => + Effect.map( + call((api) => api.dependencyScan.scanRepository({ params: { repositoryId } })), + (result) => Message.CompletedScanRequest({ repositoryId, result }), + ), +}) + +export const UpdateFinding = Command.define("SecurityApiUpdateDependencyFinding", { + args: { findingId: DependencyFindingId, status: DependencyFindingTriageStatus, reason: Schema.String }, + messages: [Message.CompletedTriage], + execute: ({ findingId, status, reason }) => + Effect.map( + call((api) => + api.dependencyScan.updateFinding({ + params: { findingId }, + payload: reason === "" ? { status } : { status, reason }, + }), + ), + (result) => Message.CompletedTriage({ result }), + ), +}) diff --git a/apps/web/src/page/security/dependencies/dependencies.scene.test.ts b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts new file mode 100644 index 0000000000..6029301381 --- /dev/null +++ b/apps/web/src/page/security/dependencies/dependencies.scene.test.ts @@ -0,0 +1,491 @@ +import { OrgId } from "@maple/domain" +import { DependencyScanForbiddenError, DependencyScanPersistenceError } from "@maple/domain/http" +import type { DependencyFindingsResponse, DependencyScanRepositoriesResponse } from "@maple/domain/http" +import { Result, Schema } from "effect" +import { + click, + Command, + expect as expectView, + expectOutMessage, + given, + role, + scene, + text, + type, + withViewInputs, + Mount, +} from "foldkit/scene" +import { describe, expect, test } from "vitest" +import { toLoadFailure } from "../../../data/load-failure" +import { + FINDING_IDS, + findingsOf, + neverScannedOf, + noFindings, + noRepositories, + PAYMENTS_ID, + repositoriesOf, + rescannedOf, +} from "../../../lab/security-dependencies-fixture" +import { AppRoute, hrefOf } from "../../../route" +import * as Dialog from "../../../ui/dialog" +import * as Field from "../../../ui/field" +import * as Select from "../../../ui/select" +import { type PageContext, PageOutMessage } from "../../contract" +import { UpdateFinding } from "./command" +import { reasonErrorOf } from "./derive" +import { Message, type Model } from "./model" +import { init, update } from "./update" +import { view as pageView } from "./view" + +const context: PageContext = { + orgId: Schema.decodeSync(OrgId)("org_acme"), + pinnedNamespace: null, + timeZone: "UTC", +} +const NOW = Date.UTC(2026, 10, 1, 9, 0) +const route = AppRoute.SecurityDependencies() +const pageUpdate = (model: Model, message: Message) => update(model, message, context) +const view = withViewInputs(pageView, { context })() +const names = (commands: ReadonlyArray<{ readonly name: string }> | undefined) => + (commands ?? []).map(({ name }) => name) +const apply = (model: Model, messages: ReadonlyArray<(model: Model) => Message>): Model => + messages.reduce((current, toMessage) => pageUpdate(current, toMessage(current)).model, model) + +const loaded = ( + repositories: DependencyScanRepositoriesResponse = repositoriesOf(NOW), + findings: DependencyFindingsResponse = findingsOf(NOW), + isAdmin = true, +): Model => + apply(init(route, context).model, [ + () => Message.ReadClock({ nowMs: NOW }), + () => Message.CompletedReadIsAdmin({ isAdmin }), + (model) => + Message.CompletedFetchRepositories({ + key: model.repositories.key ?? "", + result: Result.succeed(repositories), + }), + (model) => + Message.CompletedFetchFindings({ + key: model.findings.key ?? "", + result: Result.succeed(findings), + }), + ]) + +const forbiddenFailure = toLoadFailure( + new DependencyScanForbiddenError({ message: "Only org admins can triage findings" }), +) +const forbidden = Result.fail(forbiddenFailure) + +const mounted = Mount.resolve(Dialog.MountDialog, Dialog.Message.CompletedMountDialog()) + +const opened = (model: Model, findingId = FINDING_IDS.axios): Model => + pageUpdate(model, Message.ClickedFinding({ findingId })).model + +describe("security dependencies tab", () => { + test("entering reads the repositories and the findings", () => { + expect(names(init(route, context).commands)).toEqual( + expect.arrayContaining(["SecurityApiDependencyRepositories", "SecurityApiDependencyFindings"]), + ) + }) + + test("while loading, the tab strip shows and the content is busy", () => { + scene( + { update: pageUpdate, view }, + given(init(route, context).model), + expectView(role("link", { name: /Dependencies/ })).toExist(), + expectView(text("Repositories")).toBeAbsent(), + ) + }) + + test("a failed read shows the error with a retry that reads again", () => { + const failed = apply(init(route, context).model, [ + (model) => + Message.CompletedFetchRepositories({ + key: model.repositories.key ?? "", + result: Result.fail(toLoadFailure(new Error("unexpected body"))), + }), + ]) + scene( + { update: pageUpdate, view }, + given(failed), + expectView(text("Couldn't load dependency scans")).toExist(), + expectView(role("button", { name: "Try again" })).toExist(), + ) + expect(names(pageUpdate(failed, Message.ClickedRetry()).commands)).toContain( + "SecurityApiDependencyRepositories", + ) + }) + + test("with no connected repository, the tab is the GitHub on-ramp", () => { + scene( + { update: pageUpdate, view }, + given(loaded(noRepositories, noFindings)), + expectView(text("Connect a repository to scan its dependencies")).toExist(), + expectView(role("link", { name: "Connect GitHub" })).toHaveAttr( + "href", + hrefOf(AppRoute.Integrations({ search: {} })), + ), + expectView(text("Vulnerable packages")).toBeAbsent(), + ) + }) + + test("connected but never scanned says the first scan is coming and offers Scan now", () => { + scene( + { update: pageUpdate, view }, + given(loaded(neverScannedOf(NOW), noFindings)), + expectView(text("First scan is on its way")).toExist(), + expectView(text("Not scanned yet")).toExist(), + expectView(text("Vulnerable packages")).toBeAbsent(), + ) + }) + + test("names the busiest service running each finding, and says when one is not deployed", () => { + scene( + { update: pageUpdate, view }, + given(loaded(repositoriesOf(NOW), findingsOf(NOW))), + expectView(text("api-gateway +1")).toExist(), + expectView(text("2.6M spans/day")).toExist(), + expectView(text("payments-worker")).toExist(), + expectView(text("Not running")).toExist(), + expectView(text(/vcs\.ref\.head\.revision/)).toBeAbsent(), + ) + }) + + test("says a finding is being scanned, not absent, while a running commit's scan is pending", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.id === PAYMENTS_ID + ? { + ...repository, + runningCommits: repository.runningCommits.map((commit) => ({ + ...commit, + scanStatus: null, + })), + } + : repository, + ), + }, + { + ...findingsOf(NOW), + findings: findingsOf(NOW).findings.map((finding) => ({ ...finding, production: [] })), + }, + ), + ), + expectView(text("Scanning")).toExist(), + expectView(text("api-gateway +1")).toBeAbsent(), + ) + }) + + test("says a running commit's scan failed instead of claiming it is still scanning", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.id === PAYMENTS_ID + ? { + ...repository, + runningCommits: repository.runningCommits.map((commit) => ({ + ...commit, + scanStatus: "failed" as const, + })), + } + : repository, + ), + }, + { + ...findingsOf(NOW), + findings: findingsOf(NOW).findings.map((finding) => ({ ...finding, production: [] })), + }, + ), + ), + expectView(text("Scan failed")).toExist(), + expectView(text("Scanning")).toBeAbsent(), + ) + }) + + test("does not call a repository clean when its only scan failed", () => { + const repositories = repositoriesOf(NOW) + scene( + { update: pageUpdate, view }, + given( + loaded( + { + repositories: repositories.repositories.map((repository) => + repository.latestScan === null + ? repository + : { + ...repository, + latestScan: { ...repository.latestScan, status: "failed" as const }, + }, + ), + }, + noFindings, + ), + ), + expectView(text("No known vulnerable packages")).toBeAbsent(), + ) + }) + + test("says Unknown, not Not running, when more commits ran than were read", () => { + scene( + { update: pageUpdate, view }, + given( + loaded(repositoriesOf(NOW), { + ...findingsOf(NOW), + production: { ...findingsOf(NOW).production, complete: false }, + }), + ), + expectView(text("Not running")).toBeAbsent(), + expectView(text("Unknown")).toExist(), + ) + }) + + test("caps a triage reason as the API does, untrimmed", () => { + expect(reasonErrorOf(" ".repeat(10) + "x".repeat(495))).not.toBeNull() + expect(reasonErrorOf("x".repeat(500))).toBeNull() + expect(reasonErrorOf(" ")).not.toBeNull() + }) + + test("explains why findings cannot be tied to production when no service reports its commit", () => { + scene( + { update: pageUpdate, view }, + given( + loaded(repositoriesOf(NOW), { + ...findingsOf(NOW), + findings: findingsOf(NOW).findings.map((finding) => ({ ...finding, production: [] })), + production: { available: true, runningServices: 0, linkedServices: 0, complete: true }, + }), + ), + expectView(text(/vcs\.ref\.head\.revision/)).toExist(), + expectView(text("Unknown")).toExist(), + expectView(text("api-gateway +1")).toBeAbsent(), + ) + }) + + test("scanned with nothing open shows the calm clean state", () => { + const settled = findingsOf(NOW).findings.filter((finding) => finding.status !== "open") + scene( + { update: pageUpdate, view }, + given(loaded(repositoriesOf(NOW), { ...findingsOf(NOW), findings: settled })), + expectView(text("No known vulnerable packages")).toExist(), + expectView(text("axios")).toBeAbsent(), + ) + }) + + test("the populated tab shows the rail, repository problems and open findings worst first", () => { + scene( + { update: pageUpdate, view }, + given(loaded()), + expectView(text("3 / 3")).toExist(), + expectView(text("open, and 1 unrated")).toExist(), + expectView(text(/commit bun\.lock/)).toExist(), + expectView(text("Line 212: expected a [[package]] table.")).toExist(), + expectView(role("link", { name: "CVE-2025-27152" })).toHaveAttr( + "href", + "https://osv.dev/vulnerability/GHSA-jr5f-v2jv-69x6", + ), + // An advisory with no CVE alias shows its own id. + expectView(role("link", { name: "RUSTSEC-2025-0023" })).toExist(), + expectView(text("No fix yet")).toExist(), + expectView(text("lodash.template")).toBeAbsent(), + expectView(text("jsonwebtoken")).toBeAbsent(), + ) + }) + + test("the status filter shows dismissed and fixed findings, and All shows every one", () => { + const status = (value: string) => () => + Message.GotStatusSelectMessage({ message: Select.Message.SelectedItem({ index: 0, value }) }) + scene( + { update: pageUpdate, view }, + given(apply(loaded(), [status("dismissed")])), + expectView(text("lodash.template")).toExist(), + expectView(text("axios")).toBeAbsent(), + ) + scene( + { update: pageUpdate, view }, + given(apply(loaded(), [status("all")])), + expectView(text("lodash.template")).toExist(), + expectView(text("jsonwebtoken")).toExist(), + expectView(text("axios")).toExist(), + ) + }) + + test("the repository filter keeps one repository's findings", () => { + const filtered = apply(loaded(), [ + () => + Message.GotRepositorySelectMessage({ + message: Select.Message.SelectedItem({ index: 1, value: PAYMENTS_ID }), + }), + ]) + scene( + { update: pageUpdate, view }, + given(filtered), + expectView(text("axios")).toExist(), + expectView(text("tokio")).toBeAbsent(), + ) + }) + + test("Scan now requests a scan, then shows it queued and polls until a newer scan finishes", () => { + const clicked = pageUpdate(loaded(), Message.ClickedScanNow({ repositoryId: PAYMENTS_ID })) + expect(names(clicked.commands)).toEqual(["SecurityApiScanRepository"]) + expect( + names(pageUpdate(clicked.model, Message.ClickedScanNow({ repositoryId: PAYMENTS_ID })).commands), + ).toEqual([]) + const queued = pageUpdate( + clicked.model, + Message.CompletedScanRequest({ + repositoryId: PAYMENTS_ID, + result: Result.succeed({ queued: true }), + }), + ) + expect(queued.outMessage).toMatchObject({ title: "Scan queued" }) + scene( + { update: pageUpdate, view }, + given(queued.model), + expectView(role("button", { name: "Scan queued" })).toHaveAttr("disabled", "true"), + ) + const ticked = pageUpdate(queued.model, Message.TickedScanPoll({ nowMs: NOW + 5_000 })) + expect(names(ticked.commands)).toEqual(["SecurityApiDependencyRepositories"]) + const unchanged = pageUpdate( + ticked.model, + Message.CompletedFetchRepositories({ + key: ticked.model.repositories.key ?? "", + result: Result.succeed(repositoriesOf(NOW)), + }), + ) + expect(unchanged.model.pendingScans).toHaveLength(1) + const again = pageUpdate(unchanged.model, Message.TickedScanPoll({ nowMs: NOW + 10_000 })) + const finished = pageUpdate( + again.model, + Message.CompletedFetchRepositories({ + key: again.model.repositories.key ?? "", + result: Result.succeed(rescannedOf(NOW)), + }), + ) + expect(finished.model.pendingScans).toEqual([]) + expect(names(finished.commands)).toEqual(["SecurityApiDependencyFindings"]) + }) + + test("a pending scan stops polling after two minutes", () => { + const queued = apply(loaded(), [ + () => Message.ClickedScanNow({ repositoryId: PAYMENTS_ID }), + () => + Message.CompletedScanRequest({ + repositoryId: PAYMENTS_ID, + result: Result.succeed({ queued: true }), + }), + ]) + const late = pageUpdate(queued, Message.TickedScanPoll({ nowMs: NOW + 121_000 })) + expect(late.model.pendingScans).toEqual([]) + expect(names(late.commands)).toEqual([]) + }) + + test("a refused scan says why and lets Scan now be pressed again", () => { + const refused = pageUpdate( + pageUpdate(loaded(), Message.ClickedScanNow({ repositoryId: PAYMENTS_ID })).model, + Message.CompletedScanRequest({ + repositoryId: PAYMENTS_ID, + result: Result.fail(toLoadFailure(new DependencyScanPersistenceError({ message: "down" }))), + }), + ) + expect(refused.outMessage).toMatchObject({ type: "error" }) + expect( + names(pageUpdate(refused.model, Message.ClickedScanNow({ repositoryId: PAYMENTS_ID })).commands), + ).toEqual(["SecurityApiScanRepository"]) + }) + + test("a row opens the finding's sheet with the advisory and where the package is locked", () => { + scene( + { update: pageUpdate, view }, + given(opened(loaded())), + mounted, + expectView(role("dialog")).toExist(), + expectView(text(/ignore baseURL/)).toExist(), + expectView(text("1.8.2")).toExist(), + expectView(text("pnpm-lock.yaml")).toExist(), + expectView(role("button", { name: "Accept risk" })).toExist(), + ) + }) + + test("Dismiss needs a reason, then writes the status with it", () => { + const write = UpdateFinding({ + findingId: FINDING_IDS.axios, + status: "dismissed", + reason: "Not reachable", + }) + scene( + { update: pageUpdate, view }, + given(opened(loaded())), + mounted, + click(role("button", { name: "Dismiss" })), + Command.expectNone(), + expectView(text("Add a reason. It is shown on the finding with your name.")).toExist(), + type(role("textbox", { name: "Reason" }), "Not reachable"), + click(role("button", { name: "Dismiss" })), + Command.expectHas(write), + // The server refuses a member's write with a 403; it comes back as an error toast. + Command.resolve(UpdateFinding, Message.CompletedTriage({ result: forbidden })), + expectOutMessage( + PageOutMessage.RequestedToast({ + type: "error", + title: forbiddenFailure.title, + description: forbiddenFailure.message, + }), + ), + ) + }) + + test("a dismissed finding is reopened without a reason", () => { + const dismissed = apply(loaded(), [ + () => + Message.GotStatusSelectMessage({ + message: Select.Message.SelectedItem({ index: 1, value: "dismissed" }), + }), + () => Message.ClickedFinding({ findingId: FINDING_IDS.lodash }), + ]) + scene( + { update: pageUpdate, view }, + given(dismissed), + mounted, + expectView(text("Only used by the docs build, never with user input.")).toExist(), + click(role("button", { name: "Reopen" })), + Command.expectHas(UpdateFinding({ findingId: FINDING_IDS.lodash, status: "open", reason: "" })), + Command.resolve(UpdateFinding, Message.CompletedTriage({ result: forbidden })), + ) + }) + + test("a member sees the finding but not the triage actions", () => { + scene( + { update: pageUpdate, view }, + given(opened(loaded(repositoriesOf(NOW), findingsOf(NOW), false))), + mounted, + expectView(text("Only org admins can change a finding's status.")).toExist(), + expectView(role("button", { name: "Dismiss" })).toBeAbsent(), + ) + }) + + test("a saved triage reads the findings again and says what changed", () => { + const finding = findingsOf(NOW).findings[0]! + const saving = apply(opened(loaded()), [ + () => + Message.GotReasonMessage({ message: Field.Message.ChangedValue({ value: "Not reachable" }) }), + () => Message.ClickedTriage({ status: "accepted_risk" }), + ]) + const saved = pageUpdate( + saving, + Message.CompletedTriage({ result: Result.succeed({ ...finding, status: "accepted_risk" }) }), + ) + expect(names(saved.commands)).toEqual(["SecurityApiDependencyFindings"]) + expect(saved.outMessage).toMatchObject({ title: "Marked accepted risk" }) + expect(saved.model.reason.value).toBe("") + }) +}) diff --git a/apps/web/src/page/security/dependencies/derive.ts b/apps/web/src/page/security/dependencies/derive.ts new file mode 100644 index 0000000000..5cffef62a6 --- /dev/null +++ b/apps/web/src/page/security/dependencies/derive.ts @@ -0,0 +1,159 @@ +import type { AdvisorySeverity } from "@maple/domain/package-advisories" +import type { DependencyRelation } from "@maple/domain/dependency-scan" +import { + DEPENDENCY_FINDING_REASON_MAX_LENGTH, + type DependencyFindingView, + type DependencyProductionCoverage, + type DependencyScanRepository, + type VcsRepositoryId, +} from "@maple/domain/http" +import { Duration, Option } from "effect" +import { AsyncData } from "foldkit" +import type { LoadFailure } from "../../../data/load-failure" +import { type Model, type PendingScan, type StatusFilter, StatusFilter as StatusFilterSchema } from "./model" + +// What the Dependencies tab shows, read off its two slots. + +export const STATUS_FILTERS: ReadonlyArray = StatusFilterSchema.literals + +export const STATUS_FILTER_LABEL = { + open: "Open", + dismissed: "Dismissed", + accepted_risk: "Accepted risk", + fixed: "Fixed", + all: "All", +} satisfies Record + +export const SEVERITY_LABEL = { + critical: "Critical", + high: "High", + medium: "Medium", + low: "Low", + unknown: "Unrated", +} satisfies Record + +export const RELATION_LABEL = { + direct: "Direct", + dev: "Dev", + transitive: "Transitive", + unknown: "Unknown", +} satisfies Record + +/** A pending scan stops polling after this long, whether or not a newer scan showed up. */ +export const SCAN_POLL_LIMIT = Duration.minutes(2) + +export const repositoriesOf = (model: Model): ReadonlyArray => + Option.getOrNull(AsyncData.getData(model.repositories.data))?.repositories ?? [] + +export const allFindingsOf = (model: Model): ReadonlyArray => + Option.getOrNull(AsyncData.getData(model.findings.data))?.findings ?? [] + +export type PageState = + | Readonly<{ _tag: "Loading" }> + | Readonly<{ _tag: "Failed"; failure: LoadFailure }> + | Readonly<{ _tag: "NotConnected" }> + | Readonly<{ + _tag: "Ready" + repositories: ReadonlyArray + findings: ReadonlyArray + production: DependencyProductionCoverage + }> + +/** A failed read with nothing to show; a stale one keeps its data on screen. */ +const failureOf = (data: AsyncData.AsyncData): LoadFailure | null => + AsyncData.isFailure(data) ? Option.getOrNull(AsyncData.getError(data)) : null + +/** Either read failing fails the tab; it shows once both are in. */ +export const pageStateOf = (model: Model): PageState => { + const failure = failureOf(model.repositories.data) + const findingsFailure = failureOf(model.findings.data) + if (failure !== null) return { _tag: "Failed", failure } + if (findingsFailure !== null) return { _tag: "Failed", failure: findingsFailure } + const repositories = Option.getOrNull(AsyncData.getData(model.repositories.data)) + const findings = Option.getOrNull(AsyncData.getData(model.findings.data)) + if (repositories === null || findings === null) return { _tag: "Loading" } + if (repositories.repositories.length === 0) return { _tag: "NotConnected" } + return { + _tag: "Ready", + repositories: repositories.repositories, + findings: findings.findings, + production: findings.production, + } +} + +export const matchesStatus = (finding: DependencyFindingView, status: StatusFilter): boolean => + status === "all" || finding.status === status + +/** The table's rows: the server's order (worst first) under the two filters. */ +export const filteredFindings = ( + findings: ReadonlyArray, + status: StatusFilter, + repositoryId: VcsRepositoryId | null, +): ReadonlyArray => + findings.filter( + (finding) => + matchesStatus(finding, status) && + (repositoryId === null || finding.repositoryId === repositoryId), + ) + +export const statusFilterOf = (model: Model): StatusFilter => + STATUS_FILTERS.find((status) => status === model.statusSelect.value) ?? "open" + +export const repositoryFilterOf = (model: Model): VcsRepositoryId | null => + repositoriesOf(model).find((repository) => repository.id === model.repositorySelect.value)?.id ?? null + +export type SeverityCounts = Readonly> + +const NO_SEVERITIES = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 } satisfies SeverityCounts + +export const openSeverityCounts = (findings: ReadonlyArray): SeverityCounts => + findings + .filter((finding) => finding.status === "open") + .reduce( + (counts, finding) => ({ ...counts, [finding.severity]: counts[finding.severity] + 1 }), + NO_SEVERITIES, + ) + +export const scannedCount = (repositories: ReadonlyArray): number => + repositories.filter((repository) => repository.latestScan !== null).length + +export const isScanPending = (model: Model, repositoryId: VcsRepositoryId): boolean => + model.requestingScans.includes(repositoryId) || + model.pendingScans.some((pending) => pending.repositoryId === repositoryId) + +/** A pending scan is done once its repository shows a newer scan that has finished. */ +const isSettled = (pending: PendingScan, repositories: ReadonlyArray): boolean => { + const scan = repositories.find((repository) => repository.id === pending.repositoryId)?.latestScan ?? null + return scan !== null && scan.id !== pending.previousScanId && scan.status !== "running" +} + +/** The scans still to poll for, and whether any finished (so the findings are read again). */ +export const settlePendingScans = ( + pendingScans: ReadonlyArray, + repositories: ReadonlyArray, +): Readonly<{ pending: ReadonlyArray; hasSettled: boolean }> => { + const pending = pendingScans.filter((scan) => !isSettled(scan, repositories)) + return { pending, hasSettled: pending.length < pendingScans.length } +} + +export const unexpiredScans = ( + pendingScans: ReadonlyArray, + nowMs: number, +): ReadonlyArray => + pendingScans.filter((scan) => nowMs - scan.requestedAtMs < Duration.toMillis(SCAN_POLL_LIMIT)) + +export const selectedFindingOf = (model: Model): DependencyFindingView | null => + model.selectedId === null + ? null + : (allFindingsOf(model).find((finding) => finding.id === model.selectedId) ?? null) + +export const reasonErrorOf = (reason: string): string | null => + reason.trim() === "" + ? "Add a reason. It is shown on the finding with your name." + : // The API checks the reason as sent, so the cap applies to it untrimmed. + reason.length > DEPENDENCY_FINDING_REASON_MAX_LENGTH + ? `Keep the reason under ${DEPENDENCY_FINDING_REASON_MAX_LENGTH} characters.` + : null + +/** A commit as its first seven characters, as GitHub abbreviates it. */ +export const shortSha = (sha: string): string => sha.slice(0, 7) diff --git a/apps/web/src/page/security/dependencies/index.ts b/apps/web/src/page/security/dependencies/index.ts new file mode 100644 index 0000000000..a67e683560 --- /dev/null +++ b/apps/web/src/page/security/dependencies/index.ts @@ -0,0 +1,24 @@ +import { definePage } from "../../contract" +import { breadcrumbsOf } from "../kit/layout" +import { Message, Model } from "./model" +import { subscriptions } from "./subscription" +import { init, update } from "./update" +import { view } from "./view" + +/** + * `/security/dependencies`, in the Security frame: lockfile scans of the connected repositories, + * so no code-audit plan gates it. + */ +export const page = definePage( + "SecurityDependencies", + { Model, Message }, + { + routes: ["SecurityDependencies"], + key: () => "security-dependencies", + init, + update, + view, + subscriptions, + breadcrumbs: () => breadcrumbsOf("dependencies"), + }, +) diff --git a/apps/web/src/page/security/dependencies/model.ts b/apps/web/src/page/security/dependencies/model.ts new file mode 100644 index 0000000000..dec9585968 --- /dev/null +++ b/apps/web/src/page/security/dependencies/model.ts @@ -0,0 +1,81 @@ +import { + DependencyFindingsResponse, + DependencyFindingView, + DependencyScanRepositoriesResponse, + VcsRepositoryId, +} from "@maple/domain/http" +import { + DependencyFindingId, + DependencyFindingTriageStatus, + DependencyScanId, +} from "@maple/domain/dependency-scan" +import { Schema } from "effect" +import { defineMessageUnion } from "foldkit/message" +import { KeyedSchema } from "../../../data/keyed" +import { LoadFailure } from "../../../data/load-failure" +import * as Dialog from "../../../ui/dialog" +import * as Field from "../../../ui/field" +import * as Select from "../../../ui/select" + +// `/security/dependencies`: the connected repositories' latest lockfile scans and the vulnerable +// packages they found. Every finding is read once; the filters narrow it on the client. + +export const StatusFilter = Schema.Literals(["open", "dismissed", "accepted_risk", "fixed", "all"]) +export type StatusFilter = typeof StatusFilter.Type + +/** A scan someone asked for: polled until the repository's latest scan is a newer, finished one. */ +export const PendingScan = Schema.Struct({ + repositoryId: VcsRepositoryId, + requestedAtMs: Schema.Finite, + /** The latest scan when it was asked for; null before the first. */ + previousScanId: Schema.NullOr(DependencyScanId), +}) +export type PendingScan = typeof PendingScan.Type + +export const Model = Schema.Struct({ + nowMs: Schema.Finite, + isAdmin: Schema.Boolean, + repositories: KeyedSchema(DependencyScanRepositoriesResponse, LoadFailure), + findings: KeyedSchema(DependencyFindingsResponse, LoadFailure), + statusSelect: Select.Model, + repositorySelect: Select.Model, + /** Repositories whose Scan now request is in flight. */ + requestingScans: Schema.Array(VcsRepositoryId), + pendingScans: Schema.Array(PendingScan), + /** The finding the detail sheet shows. */ + selectedId: Schema.NullOr(DependencyFindingId), + sheet: Dialog.Model, + reason: Field.Model, + isReasonSubmitted: Schema.Boolean, + savingStatus: Schema.NullOr(DependencyFindingTriageStatus), +}) +export type Model = typeof Model.Type + +const result = (schema: S) => ({ + key: Schema.String, + result: Schema.Result(schema, LoadFailure), +}) + +export const Message = defineMessageUnion({ + ReadClock: { nowMs: Schema.Finite }, + CompletedReadIsAdmin: { isAdmin: Schema.Boolean }, + CompletedFetchRepositories: result(DependencyScanRepositoriesResponse), + CompletedFetchFindings: result(DependencyFindingsResponse), + ClickedRetry: {}, + ClickedScanNow: { repositoryId: VcsRepositoryId }, + CompletedScanRequest: { + repositoryId: VcsRepositoryId, + result: Schema.Result(Schema.Struct({ queued: Schema.Boolean }), LoadFailure), + }, + TickedScanPoll: { nowMs: Schema.Finite }, + GotStatusSelectMessage: { message: Select.Message }, + GotRepositorySelectMessage: { message: Select.Message }, + ClickedFinding: { findingId: DependencyFindingId }, + /** An advisory link inside a row: it opens OSV, not the row's sheet. */ + ClickedAdvisoryLink: {}, + GotSheetMessage: { message: Dialog.Message }, + GotReasonMessage: { message: Field.Message }, + ClickedTriage: { status: DependencyFindingTriageStatus }, + CompletedTriage: { result: Schema.Result(DependencyFindingView, LoadFailure) }, +}) +export type Message = typeof Message.Type diff --git a/apps/web/src/page/security/dependencies/subscription.ts b/apps/web/src/page/security/dependencies/subscription.ts new file mode 100644 index 0000000000..57e7005921 --- /dev/null +++ b/apps/web/src/page/security/dependencies/subscription.ts @@ -0,0 +1,33 @@ +import { Clock, Duration, Effect, Schema, Stream } from "effect" +import { Subscription } from "foldkit" +import { pollTicks } from "../../../data/poll" +import type { PageSubscriptionInput } from "../../contract" +import { Message, type Model } from "./model" + +/** While a scan someone asked for is pending, the repositories are read every 5 s. */ +const SCAN_POLL = Duration.seconds(5) +/** The clock is read every minute, so the last-scan times move. */ +const CLOCK = Duration.minutes(1) + +const clockTicks = (every: Duration.Input, toMessage: (nowMs: number) => A) => + pollTicks(every).pipe(Stream.mapEffect(() => Effect.map(Clock.currentTimeMillis, toMessage))) + +export const subscriptions = Subscription.make, Message>()((entry) => ({ + scanPoll: entry( + { isPending: Schema.Boolean }, + { + modelToDependencies: ({ model }) => ({ isPending: model.pendingScans.length > 0 }), + dependenciesToStream: ({ isPending }) => + isPending + ? clockTicks(SCAN_POLL, (nowMs) => Message.TickedScanPoll({ nowMs })) + : Stream.empty, + }, + ), + clock: entry( + {}, + { + modelToDependencies: () => ({}), + dependenciesToStream: () => clockTicks(CLOCK, (nowMs) => Message.ReadClock({ nowMs })), + }, + ), +})) diff --git a/apps/web/src/page/security/dependencies/update.ts b/apps/web/src/page/security/dependencies/update.ts new file mode 100644 index 0000000000..9d7fd6d009 --- /dev/null +++ b/apps/web/src/page/security/dependencies/update.ts @@ -0,0 +1,245 @@ +import type { DependencyFindingStatus, DependencyFindingTriageStatus } from "@maple/domain/dependency-scan" +import { Result } from "effect" +import { Command } from "foldkit" +import * as Keyed from "../../../data/keyed" +import { requestKey } from "../../../data/request-key" +import { isClerkAuthEnabled } from "../../../lib/services/common/auth-mode" +import type { AppRoute } from "../../../route" +import * as Dialog from "../../../ui/dialog" +import * as Field from "../../../ui/field" +import * as Select from "../../../ui/select" +import { type PageContext, PageOutMessage, type PageReturn } from "../../contract" +import { chainAll, slotStep } from "../../kit/keyed-step" +import { + FetchFindings, + FetchRepositories, + ReadDependenciesClock, + ReadDependenciesIsAdmin, + ScanRepository, + UpdateFinding, +} from "./command" +import { + isScanPending, + reasonErrorOf, + repositoriesOf, + selectedFindingOf, + settlePendingScans, + STATUS_FILTER_LABEL, + unexpiredScans, +} from "./derive" +import { Message, type Model } from "./model" + +type Return = PageReturn + +export type SecurityDependenciesRoute = Extract + +export const SHEET_ID = "security-dependency-finding" + +const toast = (type: "success" | "error", title: string, description = "") => + PageOutMessage.RequestedToast({ type, title, description }) + +const readRepositories = (model: Model, context: PageContext, mode: Keyed.ReadMode): Return => + slotStep( + model, + model.repositories, + (repositories) => ({ repositories }), + requestKey(context.orgId, "dependencyScan.listRepositories", {}), + mode, + (key) => FetchRepositories({ key }), + ) + +const readFindings = (model: Model, context: PageContext, mode: Keyed.ReadMode): Return => + slotStep( + model, + model.findings, + (findings) => ({ findings }), + requestKey(context.orgId, "dependencyScan.listFindings", {}), + mode, + (key) => FetchFindings({ key }), + ) + +const readAll = (model: Model, context: PageContext, mode: Keyed.ReadMode): Return => + chainAll(model, [ + (next) => readRepositories(next, context, mode), + (next) => readFindings(next, context, mode), + ]) + +export const initialModel = (): Model => ({ + nowMs: 0, + isAdmin: !isClerkAuthEnabled, + repositories: Keyed.idle(), + findings: Keyed.idle(), + statusSelect: Select.init("security-dependency-status", { value: "open", selectedIndex: 0 }), + repositorySelect: Select.init("security-dependency-repository", { value: "all", selectedIndex: 0 }), + requestingScans: [], + pendingScans: [], + selectedId: null, + sheet: Dialog.init(SHEET_ID, "sheet"), + reason: Field.init(), + isReasonSubmitted: false, + savingStatus: null, +}) + +export const init = (_route: SecurityDependenciesRoute, context: PageContext): Return => { + const loaded = readAll(initialModel(), context, "Load") + return { + ...loaded, + commands: [ + ReadDependenciesClock(), + ...(isClerkAuthEnabled ? [ReadDependenciesIsAdmin()] : []), + ...(loaded.commands ?? []), + ], + } +} + +const selectStep = ( + model: Model, + select: Select.Model, + message: Select.Message, + write: (select: Select.Model) => Partial, + wrap: (message: Select.Message) => Message, +): Return => { + const next = Select.update(select, message) + return { + model: { ...model, ...write(next.model) }, + commands: Command.mapMessages(next.commands ?? [], wrap), + } +} + +const sheetStep = (model: Model, result: ReturnType): Return => ({ + model: { ...model, sheet: result.model }, + commands: Command.mapMessages(result.commands ?? [], (message) => Message.GotSheetMessage({ message })), +}) + +/** A finished scan changes the findings, so they are read again. */ +const afterRepositories = (model: Model, context: PageContext): Return => { + if (model.pendingScans.length === 0) return { model } + const { pending, hasSettled } = settlePendingScans(model.pendingScans, repositoriesOf(model)) + const settled = { ...model, pendingScans: pending } + return hasSettled ? readFindings(settled, context, "Supersede") : { model: settled } +} + +const TRIAGE_TOAST = { + open: "Finding reopened", + dismissed: `Marked ${STATUS_FILTER_LABEL.dismissed.toLowerCase()}`, + accepted_risk: `Marked ${STATUS_FILTER_LABEL.accepted_risk.toLowerCase()}`, +} satisfies Record + +/** The answer's status as a triage one; a write never answers `fixed`, so it reads as open. */ +const triageStatusOf = (status: DependencyFindingStatus): DependencyFindingTriageStatus => + status === "fixed" ? "open" : status + +const triage = (model: Model, status: DependencyFindingTriageStatus): Return => { + const finding = selectedFindingOf(model) + if (finding === null || !model.isAdmin || model.savingStatus !== null) return { model } + const reason = model.reason.value.trim() + // Reopening needs no reason; setting a finding aside does. + if (status !== "open" && reasonErrorOf(reason) !== null) + return { model: { ...model, isReasonSubmitted: true } } + return { + model: { ...model, isReasonSubmitted: true, savingStatus: status }, + commands: [UpdateFinding({ findingId: finding.id, status, reason })], + } +} + +export const update = (model: Model, message: Message, context: PageContext): Return => + Message.match(message, { + ReadClock: ({ nowMs }) => ({ model: { ...model, nowMs } }), + CompletedReadIsAdmin: ({ isAdmin }) => ({ model: { ...model, isAdmin } }), + CompletedFetchRepositories: ({ key, result }) => + afterRepositories( + { ...model, repositories: Keyed.accept(model.repositories, key, result) }, + context, + ), + CompletedFetchFindings: ({ key, result }) => ({ + model: { ...model, findings: Keyed.accept(model.findings, key, result) }, + }), + ClickedRetry: () => readAll(model, context, "Refresh"), + ClickedScanNow: ({ repositoryId }) => + isScanPending(model, repositoryId) + ? { model } + : { + model: { ...model, requestingScans: [...model.requestingScans, repositoryId] }, + commands: [ScanRepository({ repositoryId })], + }, + CompletedScanRequest: ({ repositoryId, result }) => { + const requested = { + ...model, + requestingScans: model.requestingScans.filter((id) => id !== repositoryId), + } + return Result.match(result, { + onSuccess: () => { + const repository = repositoriesOf(model).find((entry) => entry.id === repositoryId) + const pending = { + repositoryId, + requestedAtMs: model.nowMs, + previousScanId: repository?.latestScan?.id ?? null, + } + return { + model: { ...requested, pendingScans: [...requested.pendingScans, pending] }, + outMessage: toast("success", "Scan queued", repository?.fullName ?? ""), + } + }, + onFailure: (failure) => ({ + model: requested, + outMessage: toast("error", failure.title, failure.message), + }), + }) + }, + TickedScanPoll: ({ nowMs }) => { + const pendingScans = unexpiredScans(model.pendingScans, nowMs) + const next = { ...model, nowMs, pendingScans } + return pendingScans.length === 0 ? { model: next } : readRepositories(next, context, "Supersede") + }, + GotStatusSelectMessage: ({ message: inner }) => + selectStep( + model, + model.statusSelect, + inner, + (statusSelect) => ({ statusSelect }), + (next) => Message.GotStatusSelectMessage({ message: next }), + ), + GotRepositorySelectMessage: ({ message: inner }) => + selectStep( + model, + model.repositorySelect, + inner, + (repositorySelect) => ({ repositorySelect }), + (next) => Message.GotRepositorySelectMessage({ message: next }), + ), + ClickedFinding: ({ findingId }) => + sheetStep( + { ...model, selectedId: findingId, reason: Field.init(), isReasonSubmitted: false }, + Dialog.open(model.sheet), + ), + ClickedAdvisoryLink: () => ({ model }), + GotSheetMessage: ({ message: inner }) => sheetStep(model, Dialog.update(model.sheet, inner)), + GotReasonMessage: ({ message: inner }) => ({ + model: { ...model, reason: Field.update(model.reason, inner).model }, + }), + ClickedTriage: ({ status }) => triage(model, status), + CompletedTriage: ({ result }) => + Result.match(result, { + onSuccess: (finding) => ({ + ...readFindings( + { + ...model, + savingStatus: null, + isReasonSubmitted: false, + reason: Field.init(), + }, + context, + "Supersede", + ), + outMessage: toast( + "success", + TRIAGE_TOAST[triageStatusOf(finding.status)], + finding.packageName, + ), + }), + onFailure: (failure) => ({ + model: { ...model, savingStatus: null }, + outMessage: toast("error", failure.title, failure.message), + }), + }), + }) diff --git a/apps/web/src/page/security/dependencies/view-findings.ts b/apps/web/src/page/security/dependencies/view-findings.ts new file mode 100644 index 0000000000..b90ff6e7e3 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view-findings.ts @@ -0,0 +1,398 @@ +import type { + DependencyFindingView, + DependencyProductionCoverage, + DependencyScanRepository, +} from "@maple/domain/http" +import { type AdvisorySeverity, displayAdvisoryId } from "@maple/domain/package-advisories" +import { countLabel, formatRate } from "@maple/ui/lib/format" +import { cn } from "@maple/ui/lib/utils" +import type { Html, HtmlBuilder } from "foldkit/html" +import { badge } from "../../../ui/display" +import { empty, emptyDescription, emptyHeader, emptyMedia, emptyTitle } from "../../../ui/empty" +import { icon } from "../../../ui/icon" +import { middleTruncate } from "../../../ui/middle-truncate" +import * as Select from "../../../ui/select" +import * as SelectView from "../../../ui/select-view" +import { table, tableBody, tableCell, tableHead, tableHeader, tableRow } from "../../../ui/table" +import { textLink } from "../../../ui/text-link" +import { listToolbar } from "../../kit/list-toolbar" +import { nothingInWindow } from "../../kit/nothing-in-window" +import { relativeTime } from "../../kit/relative-time" +import { securitySeverityBadge } from "../kit/parts" +import { + filteredFindings, + RELATION_LABEL, + repositoryFilterOf, + SEVERITY_LABEL, + STATUS_FILTER_LABEL, + STATUS_FILTERS, + statusFilterOf, +} from "./derive" +import { Message, type Model } from "./model" + +// The findings table: one row per advisory on a package in a lockfile, worst first as the server +// sorts them, under a status and a repository filter. + +type H = HtmlBuilder +type Clock = Readonly<{ nowMs: number; timeZone: string }> + +/** The code audit's severity badge; an advisory with no score reads as "Unrated". */ +export const dependencySeverityBadge = (h: H, severity: AdvisorySeverity, className?: string): Html => + severity === "unknown" + ? badge(h, { variant: "muted", className: cn("text-muted-foreground", className) }, [ + SEVERITY_LABEL.unknown, + ]) + : securitySeverityBadge(h, severity, className) + +export const advisoryHref = (advisoryId: string): string => + `https://osv.dev/vulnerability/${encodeURIComponent(advisoryId)}` + +export const advisoryLink = (h: H, finding: DependencyFindingView, className?: string): Html => + textLink( + h, + { + className: cn("font-mono text-xs", className), + href: advisoryHref(finding.advisoryId), + attributes: [ + h.Target("_blank"), + h.Rel("noopener noreferrer"), + h.OnClick(Message.ClickedAdvisoryLink(), { propagation: "Stop" }), + ], + }, + [displayAdvisoryId({ id: finding.advisoryId, aliases: finding.aliases })], + ) + +export const versionsLabel = (finding: DependencyFindingView): string => + finding.versions.length === 0 + ? finding.packageName + : `${finding.packageName}@${finding.versions.join(", ")}` + +const item = (value: string, label: string): SelectView.SelectEntry => ({ + _tag: "Item", + item: { value, label }, +}) + +const filterSelect = ( + h: H, + select: Select.Model, + entries: ReadonlyArray, + label: string, + renderValue: (value: string) => string, + wrap: (message: Select.Message) => Message, +): Html => + h.submodel({ + slotId: select.id, + model: select, + view: SelectView.view, + viewInputs: { + entries, + size: "sm", + className: "h-7 w-auto min-w-28 text-xs", + renderValue: (value) => [ + h.span([h.Class("text-muted-foreground")], [`${label} `]), + renderValue(value), + ], + toView: (parts) => h.div([h.Class("contents")], parts), + }, + toParentMessage: wrap, + }) + +const toolbar = ( + h: H, + model: Model, + findings: ReadonlyArray, + repositories: ReadonlyArray, +): Html => { + const statusLabel = (value: string) => + STATUS_FILTER_LABEL[STATUS_FILTERS.find((status) => status === value) ?? "open"] + const repositoryLabel = (value: string) => + repositories.find((repository) => repository.id === value)?.fullName ?? "All repositories" + return listToolbar(h, { + framed: true, + leading: h.h2([h.Class("px-1 text-sm font-medium")], ["Vulnerable packages"]), + trailing: [ + filterSelect( + h, + model.statusSelect, + STATUS_FILTERS.map((status) => + item( + status, + `${STATUS_FILTER_LABEL[status]} (${filteredFindings(findings, status, repositoryFilterOf(model)).length})`, + ), + ), + "Status", + statusLabel, + (message) => Message.GotStatusSelectMessage({ message }), + ), + // One repository needs no repository filter. + ...(repositories.length < 2 + ? [] + : [ + filterSelect( + h, + model.repositorySelect, + [ + item("all", "All repositories"), + ...repositories.map((repository) => item(repository.id, repository.fullName)), + ], + "Repository", + repositoryLabel, + (message) => Message.GotRepositorySelectMessage({ message }), + ), + ]), + ], + }) +} + +const muted = (h: H, text: string, className?: string): Html => + h.span([h.Class(cn("text-xs text-muted-foreground", className))], [text]) + +type ProductionScanState = "pending" | "failed" + +/** Repositories whose running commits are not all scanned: one still pending, or one whose scan failed. */ +const productionScanStates = ( + repositories: ReadonlyArray, +): ReadonlyMap => + new Map( + repositories.flatMap((repository): ReadonlyArray => { + const statuses = repository.runningCommits.map((commit) => commit.scanStatus) + if (statuses.some((status) => status === null || status === "running")) + return [[repository.id, "pending"]] + return statuses.includes("failed") ? [[repository.id, "failed"]] : [] + }), + ) + +/** Spans a finding's running services served over the last day. */ +export const productionSpans = (finding: DependencyFindingView): number => + finding.production.reduce((sum, service) => sum + service.spanCount, 0) + +/** The busiest service running the finding, and how many more; unknown without commit telemetry. */ +const productionCell = ( + h: H, + finding: DependencyFindingView, + production: DependencyProductionCoverage, + scans: ReadonlyMap, +): Html => { + const [busiest] = finding.production + // A running commit not scanned yet may carry the finding: no answer until its scan lands. + const scan = scans.get(finding.repositoryId) + if (busiest === undefined && scan !== undefined) + return muted(h, scan === "pending" ? "Scanning" : "Scan failed") + if (busiest === undefined) + return muted( + h, + production.available && production.complete && production.linkedServices > 0 + ? "Not running" + : "Unknown", + ) + const others = new Set(finding.production.map((service) => service.serviceName)).size - 1 + return h.span( + [h.Class("flex min-w-0 max-w-48 flex-col")], + [ + h.span( + [h.Class("truncate text-sm"), h.Title(busiest.serviceName)], + [others > 0 ? `${busiest.serviceName} +${others}` : busiest.serviceName], + ), + muted(h, `${formatRate(productionSpans(finding))} spans/day`, "tabular-nums"), + ], + ) +} + +/** + * Said when findings cannot be tied to what runs: no service reports the commit it was built from, + * or none of those commits belongs to a connected repository. + */ +const productionHint = (h: H, production: DependencyProductionCoverage): ReadonlyArray => { + const text = !production.available + ? "Production traffic could not be read, so findings are ranked by severity alone." + : production.runningServices === 0 + ? "No service reports the commit it runs (vcs.ref.head.revision), so findings cannot be tied to production. Set it to rank them by traffic." + : production.linkedServices === 0 + ? "The commits your services run are not in a connected repository's tracked branch, so findings cannot be tied to production." + : null + return text === null + ? [] + : [h.p([h.Class("border-b bg-surface-sunken px-4 py-2 text-xs text-muted-foreground")], [text])] +} + +const findingRow = ( + h: H, + finding: DependencyFindingView, + isSelected: boolean, + clock: Clock, + production: DependencyProductionCoverage, + scans: ReadonlyMap, +): Html => { + const open = Message.ClickedFinding({ findingId: finding.id }) + return tableRow( + h, + [ + tableCell(h, [dependencySeverityBadge(h, finding.severity)], "w-24"), + tableCell(h, [ + h.button( + [ + h.Type("button"), + h.OnClick(open, { propagation: "Stop" }), + h.Class( + "flex min-w-0 max-w-72 flex-col items-start text-left focus-visible:underline focus-visible:outline-none", + ), + ], + [ + h.span( + [h.Class("truncate font-mono text-sm"), h.Title(versionsLabel(finding))], + [finding.packageName], + ), + muted( + h, + `${finding.ecosystem} · ${finding.versions.join(", ")}`, + "truncate font-mono", + ), + ], + ), + ]), + tableCell(h, [advisoryLink(h, finding)]), + tableCell(h, [productionCell(h, finding, production, scans)]), + tableCell(h, [ + finding.fixedIn === null + ? muted(h, "No fix yet") + : h.span([h.Class("font-mono text-xs tabular-nums")], [finding.fixedIn]), + ]), + tableCell(h, [muted(h, RELATION_LABEL[finding.relation])], "hidden md:table-cell"), + tableCell( + h, + [ + h.span( + [h.Class("flex min-w-0 max-w-64 flex-col")], + [ + h.span( + [h.Class("truncate text-sm"), h.Title(finding.repositoryFullName)], + [finding.repositoryFullName], + ), + middleTruncate(h, { + text: finding.lockfilePath, + tail: Math.min(24, finding.lockfilePath.length), + mono: true, + className: "text-xs text-muted-foreground", + }), + ], + ), + ], + "hidden lg:table-cell", + ), + tableCell( + h, + [ + ...(finding.status === "open" + ? [] + : [ + badge(h, { variant: "meta", className: "mr-2" }, [ + STATUS_FILTER_LABEL[finding.status], + ]), + ]), + relativeTime(h, { + value: finding.createdAt, + nowMs: clock.nowMs, + timeZone: clock.timeZone, + variant: "orDate", + tooltip: "title", + className: "whitespace-nowrap text-xs text-muted-foreground", + }), + ], + "whitespace-nowrap text-right", + ), + ], + cn( + "cursor-pointer hover:bg-surface-sunken active:bg-surface-sunken", + isSelected && "bg-surface-strong", + ), + [h.Key(finding.id), h.OnClick(open)], + ) +} + +const findingsTable = ( + h: H, + rows: ReadonlyArray, + selectedId: string | null, + clock: Clock, + production: DependencyProductionCoverage, + scans: ReadonlyMap, +): Html => + table(h, { ariaLabel: "Vulnerable packages" }, [ + tableHeader(h, [ + tableRow(h, [ + tableHead(h, ["Severity"]), + tableHead(h, ["Package"]), + tableHead(h, ["Advisory"]), + tableHead(h, ["In production"]), + tableHead(h, ["Fixed in"]), + tableHead(h, ["Relation"], "hidden md:table-cell"), + tableHead(h, ["Location"], "hidden lg:table-cell"), + tableHead(h, ["First seen"], "text-right"), + ]), + ]), + tableBody( + h, + rows.map((finding) => + findingRow(h, finding, finding.id === selectedId, clock, production, scans), + ), + ), + ]) + +const packagesChecked = (repositories: ReadonlyArray): number => + repositories.reduce((sum, repository) => sum + (repository.latestScan?.packageCount ?? 0), 0) + +/** Scanned and nothing open: a calm line, not an alarm. */ +const cleanState = (h: H, repositories: ReadonlyArray): Html => + empty( + h, + [ + emptyHeader(h, [ + emptyMedia(h, { variant: "icon" }, [icon(h, "ShieldCheckIcon")]), + emptyTitle(h, ["No known vulnerable packages"]), + emptyDescription(h, [ + `The latest scans checked ${countLabel(packagesChecked(repositories), "package")} against OSV.dev. New advisories are matched on every scan.`, + ]), + ]), + ], + "md:py-14", + ) + +export const findingsSection = ( + h: H, + model: Model, + state: Readonly<{ + findings: ReadonlyArray + repositories: ReadonlyArray + production: DependencyProductionCoverage + }>, + clock: Clock, +): Html => { + const { findings, repositories, production } = state + const status = statusFilterOf(model) + const repositoryId = repositoryFilterOf(model) + const rows = filteredFindings(findings, status, repositoryId) + const content = + rows.length > 0 + ? findingsTable(h, rows, model.selectedId, clock, production, productionScanStates(repositories)) + : // Clean only when every repository's latest scan read its lockfiles; a failed or missing one is not a clean bill. + status === "open" && + repositoryId === null && + repositories.every((repository) => repository.latestScan?.status === "completed") + ? cleanState(h, repositories) + : h.div( + [h.Class("p-4")], + [ + nothingInWindow(h, { + title: "No findings match", + description: + status === "all" + ? "No findings for these filters." + : `No ${STATUS_FILTER_LABEL[status].toLowerCase()} findings for these filters.`, + }), + ], + ) + return h.section( + [h.AriaLabel("Vulnerable packages"), h.Class("overflow-hidden rounded-md border bg-card")], + [toolbar(h, model, findings, repositories), ...productionHint(h, production), content], + ) +} diff --git a/apps/web/src/page/security/dependencies/view-repositories.ts b/apps/web/src/page/security/dependencies/view-repositories.ts new file mode 100644 index 0000000000..b181ff3ac4 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view-repositories.ts @@ -0,0 +1,182 @@ +import type { DependencyLockfileStatus, DependencyScanLockfile } from "@maple/domain/dependency-scan" +import type { DependencyScanRepository, DependencyScanSummary } from "@maple/domain/http" +import { countLabel } from "@maple/ui/lib/format" +import type { Html, HtmlBuilder } from "foldkit/html" +import { button } from "../../../ui/button" +import { badge } from "../../../ui/display" +import { icon } from "../../../ui/icon" +import { panel, panelHeader } from "../../../ui/panel" +import { relativeTime } from "../../kit/relative-time" +import { repositoryName } from "../kit/parts" +import { isScanPending, shortSha } from "./derive" +import { Message, type Model } from "./model" + +// Each connected repository: its tracked branch, the latest scan, the lockfiles it could not read +// and a Scan now button. + +type H = HtmlBuilder +type Clock = Readonly<{ nowMs: number; timeZone: string }> + +/** Said when the scanner left no message of its own. */ +const LOCKFILE_FALLBACK = { + parsed: "", + unsupported: "This lockfile format is not supported yet.", + too_large: "Too large to scan.", + failed: "Could not be read.", + skipped: "Past the lockfiles a scan reads.", +} satisfies Record + +const LOCKFILE_STATUS_LABEL = { + parsed: "Parsed", + unsupported: "Unsupported", + too_large: "Too large", + failed: "Failed", + skipped: "Skipped", +} satisfies Record + +const lockfileIssue = (h: H, lockfile: DependencyScanLockfile): Html => + h.li( + [h.Key(lockfile.path), h.Class("flex min-w-0 flex-wrap items-baseline gap-x-2 gap-y-0.5 text-xs")], + [ + h.span([h.Class("font-mono text-foreground")], [lockfile.path]), + badge(h, { variant: lockfile.status === "failed" ? "warn" : "muted" }, [ + LOCKFILE_STATUS_LABEL[lockfile.status], + ]), + h.span( + [h.Class("min-w-0 text-muted-foreground")], + [lockfile.message ?? LOCKFILE_FALLBACK[lockfile.status]], + ), + ], + ) + +const dot = (h: H): Html => h.span([h.AriaHidden(true), h.Class("text-foreground-faint")], ["·"]) + +const scanSummary = (h: H, scan: DependencyScanSummary, clock: Clock): Html => + h.span( + [h.Class("flex min-w-0 flex-wrap items-center gap-x-1.5 text-xs text-muted-foreground")], + [ + scan.status === "running" ? "Scan started" : "Scanned", + relativeTime(h, { + value: scan.finishedAt ?? scan.startedAt, + nowMs: clock.nowMs, + timeZone: clock.timeZone, + variant: "orDate", + tooltip: "title", + }), + dot(h), + h.span([h.Class("font-mono"), h.Title(scan.commitSha)], [shortSha(scan.commitSha)]), + dot(h), + countLabel(scan.lockfiles.length, "lockfile"), + dot(h), + countLabel(scan.packageCount, "package"), + ], + ) + +export const scanButton = (h: H, model: Model, repository: DependencyScanRepository): Html => { + const isRequesting = model.requestingScans.includes(repository.id) + const isQueued = !isRequesting && isScanPending(model, repository.id) + const isRunning = repository.latestScan?.status === "running" + return button( + h, + { + variant: "outline", + size: "sm", + isLoading: isRequesting, + isDisabled: isQueued || isRunning, + attributes: [h.OnClick(Message.ClickedScanNow({ repositoryId: repository.id }))], + }, + [isQueued ? "Scan queued" : isRunning ? "Scanning" : "Scan now"], + ) +} + +const scanState = (h: H, scan: DependencyScanSummary | null): ReadonlyArray => + scan === null + ? [badge(h, { variant: "muted" }, ["Not scanned yet"])] + : scan.status === "failed" + ? [badge(h, { variant: "crit" }, ["Scan failed"])] + : [] + +const repositoryRow = (h: H, model: Model, repository: DependencyScanRepository, clock: Clock): Html => { + const scan = repository.latestScan + const issues = (scan?.lockfiles ?? []).filter((lockfile) => lockfile.status !== "parsed") + return h.li( + [h.Key(repository.id), h.Class("flex flex-col gap-2 px-4 py-3")], + [ + h.div( + [h.Class("flex flex-wrap items-center justify-between gap-x-4 gap-y-2")], + [ + h.div( + [h.Class("flex min-w-0 flex-col gap-0.5")], + [ + h.span( + [h.Class("flex min-w-0 items-center gap-2")], + [ + icon(h, "GithubIcon", { + size: 14, + className: "shrink-0 text-muted-foreground", + }), + h.a( + [ + h.Href(repository.htmlUrl), + h.Target("_blank"), + h.Rel("noopener noreferrer"), + h.Class( + "min-w-0 text-sm font-medium hover:underline active:underline", + ), + ], + [repositoryName(h, repository.fullName)], + ), + h.span( + [h.Class("shrink-0 font-mono text-xs text-muted-foreground")], + [repository.trackedBranch], + ), + ], + ), + scan === null + ? h.span( + [h.Class("text-xs text-muted-foreground")], + ["Scanned on the next push, or now."], + ) + : scanSummary(h, scan, clock), + ], + ), + h.div( + [h.Class("flex shrink-0 items-center gap-2")], + [...scanState(h, scan), scanButton(h, model, repository)], + ), + ], + ), + ...(scan?.error === null || scan?.error === undefined + ? [] + : [h.p([h.Class("text-xs text-severity-error")], [scan.error])]), + ...(issues.length === 0 + ? [] + : [ + h.ul( + [h.Class("flex flex-col gap-1 pl-5.5")], + issues.map((lockfile) => lockfileIssue(h, lockfile)), + ), + ]), + ], + ) +} + +export const repositoriesPanel = ( + h: H, + model: Model, + repositories: ReadonlyArray, + clock: Clock, +): Html => + panel(h, { className: "overflow-hidden" }, [ + panelHeader(h, { + title: "Repositories", + action: h.span( + [h.Class("text-xs text-muted-foreground")], + ["Tracked branch, rescanned on every push"], + ), + }), + h.ul( + [h.Class("divide-y divide-border")], + repositories.map((repository) => repositoryRow(h, model, repository, clock)), + ), + ]) diff --git a/apps/web/src/page/security/dependencies/view-sheet.ts b/apps/web/src/page/security/dependencies/view-sheet.ts new file mode 100644 index 0000000000..4d175f6ec2 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view-sheet.ts @@ -0,0 +1,279 @@ +import type { DependencyFindingView } from "@maple/domain/http" +import { displayAdvisoryId } from "@maple/domain/package-advisories" +import { formatRate } from "@maple/ui/lib/format" +import type { Html, HtmlBuilder } from "foldkit/html" +import { button } from "../../../ui/button" +import { detailRailGroup, detailRailRow } from "../../../ui/detail-rail" +import { badge } from "../../../ui/display" +import * as DialogView from "../../../ui/dialog-view" +import * as Field from "../../../ui/field" +import { textarea } from "../../../ui/textarea" +import { relativeTime } from "../../kit/relative-time" +import { sheetDetailHeader } from "../../kit/sheet-detail-header" +import { RELATION_LABEL, reasonErrorOf, selectedFindingOf, shortSha, STATUS_FILTER_LABEL } from "./derive" +import { Message, type Model } from "./model" +import { advisoryLink, dependencySeverityBadge } from "./view-findings" + +// The finding's sheet: what the advisory says, where the package is locked, and its triage. + +type H = HtmlBuilder +type Clock = Readonly<{ nowMs: number; timeZone: string }> + +const value = (h: H, text: string, mono = false): Html => + h.span([h.Class(mono ? "truncate font-mono text-xs" : "truncate text-sm"), h.Title(text)], [text]) + +const at = (h: H, sha: string, epochMs: number, clock: Clock): Html => + h.span( + [h.Class("flex min-w-0 items-center gap-1.5 text-sm")], + [ + h.span([h.Class("font-mono text-xs"), h.Title(sha)], [shortSha(sha)]), + relativeTime(h, { + value: epochMs, + nowMs: clock.nowMs, + timeZone: clock.timeZone, + variant: "orDate", + tooltip: "title", + className: "text-xs text-muted-foreground", + }), + ], + ) + +/** The services running a commit that carries the finding, busiest first. */ +const production = (h: H, finding: DependencyFindingView): ReadonlyArray => + finding.production.length === 0 + ? [] + : [ + detailRailGroup( + h, + "Running in production", + // The environment labels the row, so a long service name keeps the value's width. + finding.production.map((service) => + detailRailRow(h, { label: service.environment }, [ + h.span( + [h.Class("flex min-w-0 items-center justify-end gap-1.5 text-xs")], + [ + h.span( + [h.Class("truncate text-sm"), h.Title(service.serviceName)], + [service.serviceName], + ), + h.span( + [ + h.Class("shrink-0 font-mono text-muted-foreground"), + h.Title(service.commitSha), + ], + [shortSha(service.commitSha)], + ), + h.span( + [h.Class("shrink-0 tabular-nums text-muted-foreground")], + [`${formatRate(service.spanCount)} spans/day`], + ), + ], + ), + ]), + ), + ), + ] + +/** The ids besides the one the advisory link already shows. */ +const aliasesBesides = (finding: DependencyFindingView): ReadonlyArray => { + const shown = displayAdvisoryId({ id: finding.advisoryId, aliases: finding.aliases }) + return [finding.advisoryId, ...finding.aliases].filter((name) => name !== shown) +} + +const details = (h: H, finding: DependencyFindingView, clock: Clock): Html => + detailRailGroup(h, "Details", [ + detailRailRow(h, { label: "Advisory" }, [advisoryLink(h, finding)]), + ...(aliasesBesides(finding).length === 0 + ? [] + : [detailRailRow(h, { label: "Aliases" }, [value(h, aliasesBesides(finding).join(", "), true)])]), + detailRailRow(h, { label: "Versions" }, [value(h, finding.versions.join(", "), true)]), + detailRailRow(h, { label: "Fixed in" }, [ + finding.fixedIn === null ? value(h, "No fix yet") : value(h, finding.fixedIn, true), + ]), + detailRailRow(h, { label: "Relation" }, [value(h, RELATION_LABEL[finding.relation])]), + detailRailRow(h, { label: "Repository" }, [value(h, finding.repositoryFullName)]), + detailRailRow(h, { label: "Lockfile" }, [value(h, finding.lockfilePath, true)]), + detailRailRow(h, { label: "First seen" }, [at(h, finding.firstSeenSha, finding.createdAt, clock)]), + detailRailRow(h, { label: "Last seen" }, [at(h, finding.lastSeenSha, finding.updatedAt, clock)]), + ...(finding.fixedSha === null + ? [] + : [ + detailRailRow(h, { label: "Fixed at" }, [ + h.span( + [h.Class("font-mono text-xs"), h.Title(finding.fixedSha)], + [shortSha(finding.fixedSha)], + ), + ]), + ]), + ]) + +const REASON_IDS: Field.FieldIds = { + control: "dependency-triage-reason", + label: "dependency-triage-reason-label", + error: "dependency-triage-reason-error", + description: "dependency-triage-reason-description", +} + +const reasonField = (h: H, model: Model): Html => { + const error = model.isReasonSubmitted ? reasonErrorOf(model.reason.value) : null + const state = Field.stateOf(model.reason, error !== null) + return Field.field(h, { state }, [ + Field.fieldLabel(h, { ids: REASON_IDS, state }, "Reason"), + textarea(h, { + id: REASON_IDS.control, + attributes: [ + h.Placeholder("Why is this not a risk here?"), + h.AriaLabelledBy(REASON_IDS.label), + h.AriaDescribedBy(error === null ? (REASON_IDS.description ?? "") : REASON_IDS.error), + h.AriaInvalid(error !== null), + h.Rows(3), + h.Value(model.reason.value), + h.OnInput((next) => + Message.GotReasonMessage({ message: Field.Message.ChangedValue({ value: next }) }), + ), + h.OnBlur(Message.GotReasonMessage({ message: Field.Message.BlurredControl() })), + ], + }), + error === null + ? Field.fieldDescription(h, { ids: REASON_IDS, state }, "Required to dismiss or accept the risk.") + : Field.fieldError(h, { ids: REASON_IDS, state, attributes: [h.Role("alert")] }, error), + ]) +} + +const note = (h: H, text: string): Html => h.p([h.Class("text-xs text-muted-foreground")], [text]) + +const triage = (h: H, model: Model, finding: DependencyFindingView, clock: Clock): Html => { + if (!model.isAdmin) + return detailRailGroup(h, "Triage", [note(h, "Only org admins can change a finding's status.")]) + if (finding.status === "fixed") + return detailRailGroup(h, "Triage", [ + note(h, "A later scan no longer finds this package version. It reopens if it comes back."), + ]) + if (finding.status !== "open") + return detailRailGroup(h, "Triage", [ + h.div( + [h.Class("flex flex-col gap-3")], + [ + h.p( + [h.Class("text-sm")], + [ + `${STATUS_FILTER_LABEL[finding.status]}`, + ...(finding.statusChangedAt === null + ? [] + : [ + " ", + relativeTime(h, { + value: finding.statusChangedAt, + nowMs: clock.nowMs, + timeZone: clock.timeZone, + variant: "orDate", + tooltip: "title", + className: "text-muted-foreground", + }), + ]), + ], + ), + ...(finding.statusReason === null + ? [] + : [ + h.p( + [h.Class("font-prose text-sm text-muted-foreground")], + [finding.statusReason], + ), + ]), + button( + h, + { + variant: "outline", + isLoading: model.savingStatus === "open", + className: "self-start", + attributes: [h.OnClick(Message.ClickedTriage({ status: "open" }))], + }, + ["Reopen"], + ), + ], + ), + ]) + return detailRailGroup(h, "Triage", [ + h.div( + [h.Class("flex flex-col gap-3")], + [ + reasonField(h, model), + h.div( + [h.Class("flex gap-2")], + [ + button( + h, + { + variant: "outline", + isLoading: model.savingStatus === "dismissed", + isDisabled: model.savingStatus !== null, + attributes: [h.OnClick(Message.ClickedTriage({ status: "dismissed" }))], + }, + ["Dismiss"], + ), + button( + h, + { + variant: "outline", + isLoading: model.savingStatus === "accepted_risk", + isDisabled: model.savingStatus !== null, + attributes: [h.OnClick(Message.ClickedTriage({ status: "accepted_risk" }))], + }, + ["Accept risk"], + ), + ], + ), + ], + ), + ]) +} + +const content = (h: H, model: Model, finding: DependencyFindingView, clock: Clock): ReadonlyArray => [ + sheetDetailHeader(h, { + dialogId: model.sheet.id, + kind: [finding.ecosystem, finding.repositoryFullName], + title: [finding.packageName], + adornment: + finding.status === "open" + ? null + : badge(h, { variant: "meta" }, [STATUS_FILTER_LABEL[finding.status]]), + description: finding.summary ?? `A known vulnerability in ${finding.packageName}`, + meta: [dependencySeverityBadge(h, finding.severity), advisoryLink(h, finding)], + }), + DialogView.dialogPanel( + h, + "sheet", + model.sheet.id, + [ + h.p( + [h.Class("font-prose text-sm text-pretty")], + [finding.summary ?? "The advisory publishes no summary. Read it on OSV.dev."], + ), + ...production(h, finding), + details(h, finding, clock), + triage(h, model, finding, clock), + ], + "flex flex-col gap-6", + ), +] + +/** The sheet; its content renders only while a finding is picked and still in the list. */ +export const findingSheet = (h: H, model: Model, clock: Clock): Html => + h.submodel({ + slotId: `${model.sheet.id}-sheet`, + model: model.sheet, + view: DialogView.view, + viewInputs: { + className: "p-0 sm:max-w-[480px]", + side: "right", + hasTrigger: false, + isPopupFocused: true, + toView: ({ portal }) => portal, + toContent: () => { + const finding = selectedFindingOf(model) + return finding === null ? [] : content(h, model, finding, clock) + }, + }, + toParentMessage: (message) => Message.GotSheetMessage({ message }), + }) diff --git a/apps/web/src/page/security/dependencies/view-states.ts b/apps/web/src/page/security/dependencies/view-states.ts new file mode 100644 index 0000000000..dc56ff1616 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view-states.ts @@ -0,0 +1,130 @@ +import type { DependencyFindingView, DependencyScanRepository } from "@maple/domain/http" +import type { Html, HtmlBuilder } from "foldkit/html" +import { AppRoute, hrefOf } from "../../../route" +import { linkButton } from "../../../ui/button" +import { empty, emptyContent, emptyDescription, emptyHeader, emptyMedia, emptyTitle } from "../../../ui/empty" +import { icon } from "../../../ui/icon" +import { textLink } from "../../../ui/text-link" +import { statRail, statRailItem } from "../../kit/stat-rail" +import { placeholder } from "../kit/parts" +import { openSeverityCounts, scannedCount } from "./derive" +import type { Message, Model } from "./model" +import { scanButton } from "./view-repositories" + +// The Dependencies tab's rail and its whole-tab states: loading, no GitHub, first scan pending. + +type H = HtmlBuilder + +export const dependenciesLoading = (h: H): ReadonlyArray => [ + h.div( + [h.AriaBusy(true), h.AriaLabel("Loading"), h.Class("grid grid-cols-2 gap-px md:grid-cols-5")], + Array.from({ length: 5 }, () => placeholder(h, "h-24")), + ), + placeholder(h, "h-32 w-full"), + placeholder(h, "h-64 w-full"), +] + +/** No connected repository: the GitHub on-ramp the code audits use. */ +export const notConnected = (h: H): Html => + empty( + h, + [ + emptyHeader(h, [ + emptyMedia(h, { variant: "icon" }, [icon(h, "GithubIcon")]), + emptyTitle(h, ["Connect a repository to scan its dependencies"]), + emptyDescription(h, [ + "Maple reads the lockfiles on each repository's tracked branch, matches every package against OSV.dev and rescans on every push.", + ]), + ]), + emptyContent(h, [ + linkButton(h, { href: hrefOf(AppRoute.Integrations({ search: {} })) }, [ + icon(h, "GithubIcon"), + "Connect GitHub", + ]), + h.p( + [h.Class("text-xs text-muted-foreground")], + ["npm, pnpm, Yarn, Bun, Go and Cargo lockfiles. Read-only, code is never executed."], + ), + ]), + ], + "rounded-md border md:py-16", + ) + +/** Connected, never scanned: the first scan follows the connection or the next push. */ +export const firstScan = ( + h: H, + model: Model, + repositories: ReadonlyArray, +): Html => { + const only = repositories.length === 1 ? (repositories[0] ?? null) : null + return empty( + h, + [ + emptyHeader(h, [ + emptyMedia(h, { variant: "icon" }, [icon(h, "CubeIcon")]), + emptyTitle(h, ["First scan is on its way"]), + emptyDescription(h, [ + "Each connected repository is scanned shortly after it is connected and again on every push to its tracked branch.", + ]), + ]), + ...(only === null ? [] : [emptyContent(h, [scanButton(h, model, only)])]), + ], + "rounded-md border md:py-14", + ) +} + +export const severityRail = ( + h: H, + findings: ReadonlyArray, + repositories: ReadonlyArray, +): Html => { + const counts = openSeverityCounts(findings) + return statRail(h, { columns: 5 }, [ + statRailItem(h, { + eyebrow: "Critical", + value: String(counts.critical), + tone: counts.critical > 0 ? "crit" : undefined, + compact: true, + subline: "open, fix first", + }), + statRailItem(h, { + eyebrow: "High", + value: String(counts.high), + tone: counts.high > 0 ? "warn" : undefined, + compact: true, + subline: "open", + }), + statRailItem(h, { eyebrow: "Medium", value: String(counts.medium), compact: true, subline: "open" }), + statRailItem(h, { + eyebrow: "Low", + value: String(counts.low), + compact: true, + subline: counts.unknown > 0 ? `open, and ${counts.unknown} unrated` : "open", + }), + statRailItem(h, { + eyebrow: "Repositories", + value: `${scannedCount(repositories)} / ${repositories.length}`, + compact: true, + subline: "scanned / connected", + }), + ]) +} + +export const footnote = (h: H): Html => + h.p( + [h.Class("text-xs text-muted-foreground")], + [ + "Advisories from ", + textLink( + h, + { + tone: "muted", + className: "underline", + href: "https://osv.dev", + attributes: [h.Target("_blank"), h.Rel("noopener noreferrer")], + }, + ["OSV.dev"], + ), + ". Findings are kept per lockfile and package, and marked fixed when a scan no longer finds them.", + ], + ) diff --git a/apps/web/src/page/security/dependencies/view.ts b/apps/web/src/page/security/dependencies/view.ts new file mode 100644 index 0000000000..bf721c0b86 --- /dev/null +++ b/apps/web/src/page/security/dependencies/view.ts @@ -0,0 +1,57 @@ +import { Submodel } from "foldkit" +import type { Html, HtmlBuilder } from "foldkit/html" +import { errorState } from "../../../ui/error-state" +import type { PageViewInputs } from "../../contract" +import { securityLayout } from "../kit/layout" +import { pageStateOf, scannedCount } from "./derive" +import { Message, type Model } from "./model" +import { findingsSection } from "./view-findings" +import { repositoriesPanel } from "./view-repositories" +import { findingSheet } from "./view-sheet" +import { dependenciesLoading, firstScan, footnote, notConnected, severityRail } from "./view-states" + +// The Dependencies tab inside the Security frame. No plan gate: it reads lockfiles. + +type H = HtmlBuilder +type Clock = Readonly<{ nowMs: number; timeZone: string }> + +const body = (h: H, model: Model, clock: Clock): ReadonlyArray => { + const state = pageStateOf(model) + switch (state._tag) { + case "Loading": + return dependenciesLoading(h) + case "Failed": + return [ + errorState(h, { + failure: state.failure, + title: "Couldn't load dependency scans", + onRetry: Message.ClickedRetry(), + }), + ] + case "NotConnected": + return [notConnected(h)] + case "Ready": + return scannedCount(state.repositories) === 0 + ? [ + firstScan(h, model, state.repositories), + repositoriesPanel(h, model, state.repositories, clock), + ] + : [ + severityRail(h, state.findings, state.repositories), + repositoriesPanel(h, model, state.repositories, clock), + h.div( + [h.Class("flex flex-col gap-2")], + [findingsSection(h, model, state, clock), footnote(h)], + ), + ] + } +} + +export const view = Submodel.defineView((model, inputs, h) => { + const clock: Clock = { nowMs: model.nowMs, timeZone: inputs.context.timeZone } + return securityLayout( + h, + { active: "dependencies", overlay: [findingSheet(h, model, clock)] }, + body(h, model, clock), + ) +}) diff --git a/apps/web/src/page/security/kit/layout.ts b/apps/web/src/page/security/kit/layout.ts index ff243f9f8d..9d5d1d24fc 100644 --- a/apps/web/src/page/security/kit/layout.ts +++ b/apps/web/src/page/security/kit/layout.ts @@ -6,9 +6,10 @@ import { pageTitle } from "../../../ui/page-layout" import { dashboardPage } from "../../kit/dashboard-page" import { linkTabStrip } from "../../kit/underline-tabs" // The frame of the Security tabs: title block, the tab strip, content. Overview, Findings and Runs -// are the code audits' page; Exposure and Runtimes read the org's telemetry. +// are the code audits' page; Exposure and Runtimes read the org's telemetry; Dependencies reads +// the connected repositories' lockfiles. -export type SecurityTab = "overview" | "findings" | "runs" | "exposure" | "runtimes" +export type SecurityTab = "overview" | "findings" | "runs" | "exposure" | "runtimes" | "dependencies" const TABS: ReadonlyArray<{ tab: SecurityTab; label: string; icon: IconName; href: string }> = [ { tab: "overview", label: "Overview", icon: "ShieldCheckIcon", href: hrefOf(AppRoute.Security()) }, @@ -26,6 +27,12 @@ const TABS: ReadonlyArray<{ tab: SecurityTab; label: string; icon: IconName; hre }, { tab: "exposure", label: "Exposure", icon: "GlobeIcon", href: hrefOf(AppRoute.SecurityExposure()) }, { tab: "runtimes", label: "Runtimes", icon: "MicrochipIcon", href: hrefOf(AppRoute.SecurityRuntimes()) }, + { + tab: "dependencies", + label: "Dependencies", + icon: "CubeIcon", + href: hrefOf(AppRoute.SecurityDependencies()), + }, ] export const breadcrumbsOf = (tab: SecurityTab): ReadonlyArray => { @@ -59,6 +66,8 @@ const DESCRIPTION = { exposure: "Scanners probing your HTTP services over the last 7 days, read from server spans.", runtimes: "The language runtime each service reports, checked for known vulnerabilities and end-of-life dates.", + dependencies: + "Known vulnerable packages in your connected repositories' lockfiles, rescanned on every push.", } satisfies Record /** The title block: the page `

` and its one line of context, like every other page header. */ diff --git a/apps/web/src/route/app-route.ts b/apps/web/src/route/app-route.ts index aec4e460a8..39438035d0 100644 --- a/apps/web/src/route/app-route.ts +++ b/apps/web/src/route/app-route.ts @@ -49,6 +49,7 @@ export const AppRoute = defineRouteUnion({ SecurityRuns: { search: Search.SecurityRunsSearch }, SecurityExposure: {}, SecurityRuntimes: {}, + SecurityDependencies: {}, SecurityRun: { runId: SecurityAuditRunId }, SecurityFinding: { targetId: SecurityAuditTargetId, fingerprint: Schema.String }, Dashboard: { dashboardId: DashboardId, search: Search.DashboardSearch }, diff --git a/apps/web/src/route/route.test.ts b/apps/web/src/route/route.test.ts index ba7509c7ec..d178d646e2 100644 --- a/apps/web/src/route/route.test.ts +++ b/apps/web/src/route/route.test.ts @@ -144,6 +144,7 @@ describe("routes", () => { expect(parse("/security/runs")._tag).toBe("SecurityRuns") expect(parse("/security/exposure")._tag).toBe("SecurityExposure") expect(parse("/security/runtimes")._tag).toBe("SecurityRuntimes") + expect(parse("/security/dependencies")._tag).toBe("SecurityDependencies") expect(parse("/security/runs/run_9f3c2a1e7b")).toMatchObject({ _tag: "SecurityRun", runId: "run_9f3c2a1e7b", diff --git a/apps/web/src/route/routers.ts b/apps/web/src/route/routers.ts index 77cf931252..e449ec87bc 100644 --- a/apps/web/src/route/routers.ts +++ b/apps/web/src/route/routers.ts @@ -150,6 +150,7 @@ const routers = { ), SecurityExposure: pipe(path("security", "exposure"), Route.mapTo(AppRoute.SecurityExposure)), SecurityRuntimes: pipe(path("security", "runtimes"), Route.mapTo(AppRoute.SecurityRuntimes)), + SecurityDependencies: pipe(path("security", "dependencies"), Route.mapTo(AppRoute.SecurityDependencies)), SecurityRun: pipe( path("security", "runs"), slash(schemaSegment("runId", SecurityAuditRunId)), diff --git a/apps/web/src/view.lazy.test.ts b/apps/web/src/view.lazy.test.ts index 4b3c535316..83fa895b22 100644 --- a/apps/web/src/view.lazy.test.ts +++ b/apps/web/src/view.lazy.test.ts @@ -81,6 +81,7 @@ const SAMPLE_PATHS = [ "/security/runs", "/security/exposure", "/security/runtimes", + "/security/dependencies", "/security/runs/run_1", "/security/findings/00000000-0000-4000-8000-000000000005/fp_1", "/agent-sessions", diff --git a/packages/backend/package.json b/packages/backend/package.json index 5c97f5249f..ff997e4490 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -36,10 +36,10 @@ "@maple/chat-platform": "workspace:*", "@maple/db": "workspace:*", "@maple/domain": "workspace:*", - "@maple/lockfiles": "workspace:*", "@maple/email": "workspace:*", "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", + "@maple/lockfiles": "workspace:*", "@maple/prometheus-otlp": "workspace:*", "@maple/query-engine": "workspace:*", "@maple/query-engine-integrations": "workspace:*", diff --git a/packages/backend/scripts/depscan-vs-osv-scanner.ts b/packages/backend/scripts/depscan-vs-osv-scanner.ts index fa6e807658..25ac01c690 100644 --- a/packages/backend/scripts/depscan-vs-osv-scanner.ts +++ b/packages/backend/scripts/depscan-vs-osv-scanner.ts @@ -40,7 +40,7 @@ const compare = (path: string) => const content = yield* Effect.promise(() => Bun.file(path).text()) const parsed = yield* parseLockfile({ path: basename(path), content, manifests }) const catalog = yield* PackageAdvisoryCatalog - const ours = yield* catalog.advisories(parsed.packages) + const { byPackage: ours } = yield* catalog.advisories(parsed.packages) const theirs = new Map( (osvScanner(path).results ?? []) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts index defbecfb11..b73501baa8 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.test.ts @@ -10,7 +10,14 @@ import { recordingQueueLayer, testRepoLayer, } from "@maple/backend/services/integrations/vcs/__tests__/harness" -import { DependencyScanService } from "./DependencyScanService" +import { systemTenant } from "@maple/backend/services/alerts/system-tenant" +import { + DependencyScanService, + PRODUCTION_SCANS_PER_READ, + type RunningCommitRow, + RunningCommits, + RunningCommitsReadError, +} from "./DependencyScanService" import { DependencyScanStore } from "./DependencyScanStore" const trackedDbs: TestDb[] = [] @@ -20,12 +27,25 @@ const org = Schema.decodeSync(OrgId)("org_deps") const admin = Schema.decodeSync(UserId)("user_admin") const sha = (digit: string) => Schema.decodeSync(GitCommitSha)(digit.repeat(40)) -const serviceFor = (sent: Array) => { +const tenant = systemTenant(org) + +/** What telemetry says services ran; `undefined` makes the read fail. */ +interface Telemetry { + rows: ReadonlyArray | undefined +} + +const serviceFor = (sent: Array, telemetry: Telemetry = { rows: [] }) => { const db = createTestDb(trackedDbs) const store = DependencyScanStore.layer.pipe(Layer.provide(db.layer)) const vcs = testRepoLayer(db) + const running = Layer.succeed(RunningCommits, { + read: () => + telemetry.rows === undefined + ? Effect.fail(new RunningCommitsReadError({ message: "warehouse down", cause: undefined })) + : Effect.succeed(telemetry.rows), + }) return Layer.effect(DependencyScanService, DependencyScanService.make).pipe( - Layer.provideMerge(Layer.mergeAll(store, vcs, recordingQueueLayer(sent))), + Layer.provideMerge(Layer.mergeAll(store, vcs, recordingQueueLayer(sent), running)), ) } @@ -110,7 +130,7 @@ describe("DependencyScanService", () => { return Effect.gen(function* () { const service = yield* DependencyScanService const api = yield* seed - const before = yield* service.listRepositories(org) + const before = yield* service.listRepositories(tenant) assert.deepStrictEqual( before.repositories.map((repository) => [repository.fullName, repository.latestScan]), [ @@ -122,7 +142,7 @@ describe("DependencyScanService", () => { const latest = yield* recordScan(api.id, "b", true) // An older commit scanned because it runs in production is not the branch's latest scan. yield* recordScan(api.id, "c", false) - const after = yield* service.listRepositories(org) + const after = yield* service.listRepositories(tenant) assert.strictEqual(after.repositories[0]?.latestScan?.id, latest.id) assert.strictEqual(after.repositories[0]?.latestScan?.vulnerableCount, 1) }).pipe(Effect.provide(serviceFor(sent))) @@ -159,10 +179,10 @@ describe("DependencyScanService", () => { const vcs = yield* VcsRepository const api = yield* seed yield* recordScan(api.id, "a", true) - assert.strictEqual((yield* service.listFindings(org, {})).findings.length, 1) - const [finding] = (yield* service.listFindings(org, {})).findings + assert.strictEqual((yield* service.listFindings(tenant, {})).findings.length, 1) + const [finding] = (yield* service.listFindings(tenant, {})).findings yield* vcs.markRepositoryRemoved(api.id) - assert.deepStrictEqual((yield* service.listFindings(org, {})).findings, []) + assert.deepStrictEqual((yield* service.listFindings(tenant, {})).findings, []) if (finding === undefined) return const error = yield* Effect.flip( service.updateFinding(org, admin, finding.id, { status: "dismissed", reason: "gone" }), @@ -177,7 +197,7 @@ describe("DependencyScanService", () => { const service = yield* DependencyScanService const api = yield* seed yield* recordScan(api.id, "a", true) - const { findings } = yield* service.listFindings(org, {}) + const { findings } = yield* service.listFindings(tenant, {}) assert.deepStrictEqual( findings.map((finding) => [finding.repositoryFullName, finding.packageName, finding.status]), [["octo/api", "lodash", "open"]], @@ -192,7 +212,7 @@ describe("DependencyScanService", () => { [dismissed.status, dismissed.statusReason, dismissed.statusChangedBy], ["dismissed", "Only used in a build script", admin], ) - assert.deepStrictEqual((yield* service.listFindings(org, { status: "open" })).findings, []) + assert.deepStrictEqual((yield* service.listFindings(tenant, { status: "open" })).findings, []) const missing = Schema.decodeSync(DependencyFindingId)("0".repeat(32)) const error = yield* Effect.flip(service.updateFinding(org, admin, missing, { status: "open" })) @@ -200,3 +220,174 @@ describe("DependencyScanService", () => { }).pipe(Effect.provide(serviceFor(sent))) }) }) + +const commitOf = (digit: string) => ({ + sha: digit.repeat(40), + message: `commit ${digit}`, + authorName: null, + authorEmail: null, + authorLogin: null, + authorAvatarUrl: null, + authoredAt: null, + committedAt: 1, + htmlUrl: `https://github.com/octo/api/commit/${digit.repeat(40)}`, +}) + +const running = ( + serviceName: string, + environment: string, + commit: string, + spanCount: number, +): RunningCommitRow => ({ + serviceName, + environment, + commitSha: commit.length === 1 ? commit.repeat(40) : commit, + spanCount, +}) + +describe("DependencyScanService production", () => { + it.effect( + "joins findings to the services running a commit whose scan matched them, and queues unscanned commits", + () => { + const sent: Array = [] + const telemetry: Telemetry = { + rows: [ + running("api-server", "production", "c", 9000), + running("worker", "production", "d", 50), + running("api-server", "staging", "a", 100), + // Not a commit of a connected repository, and a placeholder revision. + running("billing", "production", "f", 400), + running("legacy", "production", "unknown", 10), + ], + } + return Effect.gen(function* () { + const service = yield* DependencyScanService + const vcs = yield* VcsRepository + const api = yield* seed + yield* vcs.upsertCommits(api, [commitOf("a"), commitOf("c"), commitOf("d")]) + yield* recordScan(api.id, "c", false) + yield* recordScan(api.id, "a", true) + + const { findings, production } = yield* service.listFindings(tenant, {}) + // Services, not rows: api-server runs two commits and counts once. + assert.deepStrictEqual(production, { + available: true, + runningServices: 4, + linkedServices: 2, + complete: true, + }) + assert.deepStrictEqual( + findings[0]?.production?.map((s) => [ + s.serviceName, + s.environment, + s.commitSha.slice(0, 1), + s.spanCount, + ]), + [ + ["api-server", "production", "c", 9000], + ["api-server", "staging", "a", 100], + ], + ) + // Only the repositories read queues: the tab fires both reads together. + assert.deepStrictEqual(sent, []) + + const { repositories } = yield* service.listRepositories(tenant) + assert.deepStrictEqual( + sent.map((job) => + job.kind === "dependency-scan" ? [job.trigger, job.commitSha] : [job.kind], + ), + [["production", sha("d")]], + ) + assert.deepStrictEqual( + repositories[0]?.runningCommits?.map((commit) => [ + commit.commitSha.slice(0, 1), + commit.scanStatus, + ]), + [ + ["c", "completed"], + ["d", null], + ["a", "completed"], + ], + ) + }).pipe(Effect.provide(serviceFor(sent, telemetry))) + }, + ) + + it.effect( + "retries a running commit whose scan failed an hour ago, and keeps its older completed scan's matches", + () => { + const sent: Array = [] + const telemetry: Telemetry = { rows: [running("api-server", "production", "c", 9000)] } + return Effect.gen(function* () { + const service = yield* DependencyScanService + const store = yield* DependencyScanStore + const vcs = yield* VcsRepository + const api = yield* seed + yield* vcs.upsertCommits(api, [commitOf("c")]) + yield* recordScan(api.id, "a", true) + yield* recordScan(api.id, "c", false) + yield* TestClock.adjust("1 second") + const failed = yield* store.startScan({ + orgId: org, + repositoryId: api.id, + commitSha: sha("c"), + ref: null, + trigger: "production", + defaultBranchHead: false, + }) + yield* store.failScan({ orgId: org, scanId: failed.id, error: "GitHub answered 502" }) + + const { findings } = yield* service.listFindings(tenant, {}) + assert.deepStrictEqual( + findings.map((finding) => finding.production?.map((s) => s.serviceName)), + [["api-server"]], + ) + assert.deepStrictEqual(sent, []) + + yield* TestClock.adjust("61 minutes") + yield* service.listRepositories(tenant) + assert.deepStrictEqual( + sent.map((job) => (job.kind === "dependency-scan" ? job.commitSha : undefined)), + [sha("c")], + ) + }).pipe(Effect.provide(serviceFor(sent, telemetry))) + }, + ) + + it.effect("queues at most a handful of unscanned commits per read", () => { + const sent: Array = [] + const digits = "0123456789abcdef".split("") + const telemetry: Telemetry = { + rows: digits.map((digit, i) => running(`svc-${i}`, "production", digit, 100 - i)), + } + return Effect.gen(function* () { + const service = yield* DependencyScanService + const vcs = yield* VcsRepository + const api = yield* seed + yield* vcs.upsertCommits(api, digits.map(commitOf)) + yield* service.listRepositories(tenant) + assert.strictEqual(sent.length, PRODUCTION_SCANS_PER_READ) + assert.strictEqual(sent[0]?.kind === "dependency-scan" ? sent[0].commitSha : undefined, sha("0")) + }).pipe(Effect.provide(serviceFor(sent, telemetry))) + }) + + it.effect("still lists findings, unranked by traffic, when telemetry cannot be read", () => { + const sent: Array = [] + return Effect.gen(function* () { + const service = yield* DependencyScanService + const api = yield* seed + yield* recordScan(api.id, "a", true) + const { findings, production } = yield* service.listFindings(tenant, {}) + assert.deepStrictEqual(production, { + available: false, + runningServices: 0, + linkedServices: 0, + complete: false, + }) + assert.deepStrictEqual( + findings.map((finding) => [finding.packageName, finding.production]), + [["lodash", []]], + ) + }).pipe(Effect.provide(serviceFor(sent, { rows: undefined }))) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanService.ts b/packages/backend/src/services/dependency-scan/DependencyScanService.ts index b06655ca55..a1e0d844ca 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanService.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanService.ts @@ -1,4 +1,4 @@ -import type { DependencyFindingRow, DependencyScanRow } from "@maple/db/tables" +import type { DependencyFindingRow, DependencyScanMatchRow, DependencyScanRow } from "@maple/db/tables" import type { DependencyFindingId } from "@maple/domain/dependency-scan" import { DependencyFindingNotFoundError, @@ -9,26 +9,160 @@ import { type DependencyScanRepositoriesResponse, DependencyScanRepositoryNotFoundError, type DependencyScanSummary, + type DependencyRunningCommit, + type DependencyRunningService, + GitCommitSha, isInstallationProcessable, + type VcsInstallation, type VcsRepo, type VcsRepositoryId, + type VcsSyncJob, } from "@maple/domain/http" import type { OrgId, UserId } from "@maple/domain/primitives" import type { DependencyFindingStatus } from "@maple/domain/dependency-scan" -import { Context, Effect, Layer, Option } from "effect" +import { EdgeCacheService } from "@maple/cache" +import { Clock, Context, Effect, Layer, Option, Schema } from "effect" +import { CH, formatWarehouseDateTime } from "@maple/query-engine" +import type { TenantContext } from "@maple/backend/services/auth/AuthService" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" import { VcsSyncQueue } from "@maple/backend/services/integrations/vcs/VcsSyncQueue" import { DependencyScanStore } from "./DependencyScanStore" +import { + findingKey, + latestCompletedPerCommit, + latestScanPerCommit, + needsScan, + productionByFinding, + type RunningService, + riskScore, + scanAt, +} from "./production" // The Dependencies tab's reads and writes: repositories with their latest scan, findings with // their repository named, triage, and "scan now", which goes through the VCS sync queue. /** Findings returned per read; the table sorts worst first, so the cut drops the least severe. */ export const FINDINGS_LIMIT = 500 +/** Findings read before ranking, so a deployed older finding is not cut before its traffic counts. */ +const FINDINGS_CANDIDATES = 2000 +/** Production is what services served over the last day. */ +const RUNNING_WINDOW_MS = 24 * 60 * 60 * 1000 +/** Unscanned running commits queued per read, busiest first; the next read queues more. */ +export const PRODUCTION_SCANS_PER_READ = 10 + +export type RunningCommitRow = typeof CH.runningCommitsRowSchema.Type +const RunningCommitRows = Schema.Array(CH.runningCommitsRowSchema) +const RUNNING_COMMITS_CACHE_BUCKET = "dependency-running-commits" +/** Running commits move with deploys, not by the second: one warehouse read per org a minute. */ +const RUNNING_COMMITS_CACHE_MS = 60_000 + +export class RunningCommitsReadError extends Schema.TaggedError()( + "@maple/dependency-scan/RunningCommitsReadError", + { message: Schema.String, cause: Schema.Defect() }, +) {} + +/** Which commit each service served traffic from in a window, busiest first. */ +export class RunningCommits extends Context.Service< + RunningCommits, + { + readonly read: ( + tenant: TenantContext, + window: { readonly startMs: number; readonly endMs: number }, + ) => Effect.Effect, RunningCommitsReadError> + } +>()("@maple/backend/services/dependency-scan/RunningCommits") { + static readonly layer = Layer.effect( + this, + Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + const edgeCache = yield* EdgeCacheService + return { + // The tab's two reads land together: the first reads the warehouse, the second the cache. + read: (tenant, window) => + edgeCache + .getOrCompute( + { + bucket: RUNNING_COMMITS_CACHE_BUCKET, + key: `${tenant.orgId}:${Math.floor(window.endMs / RUNNING_COMMITS_CACHE_MS)}`, + ttlSeconds: (2 * RUNNING_COMMITS_CACHE_MS) / 1000, + schema: RunningCommitRows, + }, + warehouse.compiledQuery( + tenant, + CH.compile( + CH.runningCommitsQuery(), + { + orgId: tenant.orgId, + startTime: formatWarehouseDateTime(window.startMs), + endTime: formatWarehouseDateTime(window.endMs), + }, + { rowSchema: CH.runningCommitsRowSchema }, + ), + { profile: "list", context: "dependencyRunningCommits" }, + ), + ) + .pipe( + Effect.map((result) => result.value), + Effect.mapError( + (cause) => + new RunningCommitsReadError({ + message: "Could not read running commits", + cause, + }), + ), + ), + } + }), + ).pipe(Layer.provide(WarehouseQueryService.layer)) +} + +interface Connected { + readonly installation: VcsInstallation + readonly repository: VcsRepo +} + +interface Production { + readonly available: boolean + /** Running (service, environment, commit) rows whose commit belongs to a connected repository. */ + readonly running: ReadonlyArray + /** Distinct services reporting a commit. */ + readonly runningServices: number + /** False when the read hit its row cap, so commits past it are unknown. */ + readonly complete: boolean + /** Latest scan per running commit, any status: what the repository list shows. */ + readonly latest: ReadonlyMap + /** The matches of each running commit's latest completed scan. */ + readonly matches: ReadonlyArray + readonly completed: ReadonlyMap +} + +const NO_PRODUCTION: Production = { + available: false, + complete: false, + running: [], + runningServices: 0, + latest: new Map(), + matches: [], + completed: new Map(), +} + +const distinctServices = (services: ReadonlyArray<{ readonly serviceName: string }>): number => + new Set(services.map((service) => service.serviceName)).size + +const decodeSha = Schema.decodeUnknownOption(GitCommitSha) + +const toRunningService = (service: RunningService): DependencyRunningService => ({ + serviceName: service.serviceName, + environment: service.environment, + commitSha: service.commitSha, + spanCount: service.spanCount, +}) export interface DependencyScanServiceApi { + /** Each repository with its latest scan and the commits services run from it. */ readonly listRepositories: ( - orgId: OrgId, + tenant: TenantContext, ) => Effect.Effect readonly requestScan: ( orgId: OrgId, @@ -37,8 +171,9 @@ export interface DependencyScanServiceApi { { readonly queued: boolean }, DependencyScanPersistenceError | DependencyScanRepositoryNotFoundError > + /** Most urgent first; production comes from telemetry and is left out when it cannot be read. */ readonly listFindings: ( - orgId: OrgId, + tenant: TenantContext, query: { readonly status?: DependencyFindingStatus | undefined readonly repositoryId?: VcsRepositoryId | undefined @@ -54,8 +189,8 @@ export interface DependencyScanServiceApi { const persistence = (message: string) => - (error: { readonly message: string }): DependencyScanPersistenceError => - new DependencyScanPersistenceError({ message: `${message}: ${error.message}` }) + (error: { readonly _tag: string; readonly message: string }): DependencyScanPersistenceError => + new DependencyScanPersistenceError({ message: `${message}: ${error.message}`, cause: error._tag }) export const scanSummaryOf = (row: DependencyScanRow): DependencyScanSummary => ({ id: row.id, @@ -74,7 +209,9 @@ export const scanSummaryOf = (row: DependencyScanRow): DependencyScanSummary => export const findingViewOf = ( row: DependencyFindingRow, repositoryFullName: string, + production: ReadonlyArray, ): DependencyFindingView => ({ + production, id: row.id, repositoryId: row.repositoryId, repositoryFullName, @@ -109,6 +246,7 @@ export class DependencyScanService extends Context.Service - vcs.listRepositoriesByInstallation(installation.id, "active"), + vcs + .listRepositoriesByInstallation(installation.id, "active") + .pipe( + Effect.map((list) => + list.map((repository): Connected => ({ installation, repository })), + ), + ), ) return repositories.flat() }) + /** + * Queues a scan of each running commit that needs one, busiest first and capped. A queue + * failure is logged here: the read it runs beside still answers. + */ + const queueProductionScans = Effect.fn("DependencyScanService.queueProductionScans")(function* ( + connected: ReadonlyArray, + production: Production, + ) { + const now = yield* Clock.currentTimeMillis + // Rows come busiest first, so a commit's first row is its busiest. + const unscanned = production.running + .filter((service) => needsScan(production.latest, service, now)) + .filter( + (service, index, all) => + all.findIndex( + (other) => + other.repositoryId === service.repositoryId && + other.commitSha === service.commitSha, + ) === index, + ) + .slice(0, PRODUCTION_SCANS_PER_READ) + const jobs = unscanned.flatMap((service): ReadonlyArray => { + const owner = connected.find((entry) => entry.repository.id === service.repositoryId) + return owner === undefined + ? [] + : [ + { + kind: "dependency-scan", + provider: owner.repository.provider, + externalInstallationId: owner.installation.externalInstallationId, + externalRepoId: owner.repository.externalRepoId, + trigger: "production", + commitSha: service.commitSha, + }, + ] + }) + yield* Effect.annotateCurrentSpan({ "maple.depscan.production_queued": jobs.length }) + if (jobs.length === 0) return + yield* queue + .sendBatch(jobs) + .pipe( + Effect.catch((error) => + Effect.logWarning("[DependencyScan] could not queue production scans").pipe( + Effect.annotateLogs({ error: error.message }), + ), + ), + ) + }) + + /** The commits services ran over the last day, joined to connected repositories and their scans. */ + const readProduction = Effect.fn("DependencyScanService.readProduction")(function* ( + tenant: TenantContext, + connected: ReadonlyArray, + ) { + const now = yield* Clock.currentTimeMillis + const rows = yield* runningCommits.read(tenant, { + startMs: now - RUNNING_WINDOW_MS, + endMs: now, + }) + const withSha = rows.flatMap((row) => + Option.match(decodeSha(row.commitSha), { + onNone: () => [], + onSome: (sha) => [{ ...row, sha }], + }), + ) + const connectedIds = new Set(connected.map((entry) => entry.repository.id)) + const commits = yield* vcs.findCommitsByShas( + tenant.orgId, + withSha.map((row) => row.sha), + ) + const repositoryOf = new Map( + commits + .filter((commit) => connectedIds.has(commit.repositoryId)) + .map((commit) => [commit.sha, commit.repositoryId] as const), + ) + const running = withSha.flatMap((row): ReadonlyArray => { + const repositoryId = repositoryOf.get(row.sha) + return repositoryId === undefined + ? [] + : [ + { + repositoryId, + commitSha: row.sha, + serviceName: row.serviceName, + environment: row.environment, + spanCount: row.spanCount, + }, + ] + }) + const scans = yield* store.scansAtCommits({ + orgId: tenant.orgId, + repositoryIds: running.map((service) => service.repositoryId), + commitShas: running.map((service) => service.commitSha), + }) + const completed = latestCompletedPerCommit(scans) + const matches = yield* store.matchesOfScans({ + orgId: tenant.orgId, + scanIds: [...completed.values()].map((scan) => scan.id), + }) + const production: Production = { + available: true, + running, + runningServices: distinctServices(rows), + complete: rows.length < CH.RUNNING_COMMITS_CAP, + latest: latestScanPerCommit(scans), + matches, + completed, + } + yield* Effect.annotateCurrentSpan({ + "maple.depscan.running_services": production.runningServices, + "maple.depscan.linked_services": distinctServices(running), + }) + return production + }) + + /** Production when telemetry and the scans can be read; the tab still lists findings without it. */ + const productionOrNone = (tenant: TenantContext, connected: ReadonlyArray) => + readProduction(tenant, connected).pipe( + Effect.catch((error) => + Effect.logWarning("[DependencyScan] production read failed").pipe( + Effect.annotateLogs({ orgId: tenant.orgId, error: error.message }), + Effect.as(NO_PRODUCTION), + ), + ), + ) + const listRepositories: DependencyScanServiceApi["listRepositories"] = Effect.fn( "DependencyScanService.listRepositories", - )(function* (orgId) { - const repositories = yield* connectedRepositories(orgId).pipe( + )(function* (tenant) { + const orgId = tenant.orgId + const connected = yield* connectedRepositories(orgId).pipe( Effect.mapError(persistence("Could not list repositories")), ) const branchScans = yield* store .latestBranchScans({ orgId }) .pipe(Effect.mapError(persistence("Could not list scans"))) const latest = new Map(branchScans.map((scan) => [scan.repositoryId, scan])) + const production = yield* productionOrNone(tenant, connected) + // The tab reads repositories and findings together; only this read queues, so each commit is queued once. + yield* queueProductionScans(connected, production) + const runningCommitsOf = ( + repositoryId: VcsRepositoryId, + ): ReadonlyArray => { + const services = production.running.filter( + (service) => service.repositoryId === repositoryId, + ) + const shas = [...new Set(services.map((service) => service.commitSha))] + return shas.map((commitSha) => ({ + commitSha, + scanStatus: scanAt(production.latest, { repositoryId, commitSha })?.status ?? null, + services: services + .filter((service) => service.commitSha === commitSha) + .map(toRunningService), + })) + } return { - repositories: repositories - .map((repository: VcsRepo) => { + repositories: connected + .map(({ repository }) => { const scan = latest.get(repository.id) return { id: repository.id, @@ -143,6 +432,7 @@ export class DependencyScanService extends Context.Service a.fullName.localeCompare(b.fullName)), @@ -179,27 +469,53 @@ export class DependencyScanService extends Context.Service [repository.id, repository.fullName])) + const names = new Map(connected.map(({ repository }) => [repository.id, repository.fullName])) const rows = yield* store .listFindings({ orgId, - limit: FINDINGS_LIMIT, + limit: FINDINGS_CANDIDATES, status: query.status, repositoryId: query.repositoryId, repositoryIds: [...names.keys()], }) .pipe(Effect.mapError(persistence("Could not list findings"))) - const findings = rows.flatMap((row) => { - const name = names.get(row.repositoryId) - return name === undefined ? [] : [findingViewOf(row, name)] + const production = yield* productionOrNone(tenant, connected) + const byFinding = productionByFinding( + production.running, + production.completed, + production.matches, + ) + const ranked = rows + .map((row, order) => { + const services = byFinding.get(findingKey(row)) ?? [] + const spans = services.reduce((sum, service) => sum + service.spanCount, 0) + return { row, order, services, score: riskScore(row, spans) } + }) + .sort((a, b) => b.score - a.score || a.order - b.order) + .slice(0, FINDINGS_LIMIT) + yield* Effect.annotateCurrentSpan({ + "result.rowCount": ranked.length, + "maple.depscan.findings_in_production": ranked.filter( + (entry) => entry.services.length > 0, + ).length, }) - yield* Effect.annotateCurrentSpan({ "result.rowCount": findings.length }) - return { findings } + return { + findings: ranked.map(({ row, services }) => + findingViewOf(row, names.get(row.repositoryId) ?? "", services.map(toRunningService)), + ), + production: { + available: production.available, + complete: production.complete, + runningServices: production.runningServices, + linkedServices: distinctServices(production.running), + }, + } }) const updateFinding: DependencyScanServiceApi["updateFinding"] = Effect.fn( @@ -214,7 +530,7 @@ export class DependencyScanService extends Context.Service repository.id === existing.value.repositoryId)) + if (!connected.some((entry) => entry.repository.id === existing.value.repositoryId)) return yield* notFound const row = yield* store .setFindingStatus({ @@ -240,12 +556,14 @@ export class DependencyScanService extends Context.Service "unknown repository", onSome: (r) => r.fullName, }), + [], ) }) @@ -259,6 +577,13 @@ export class DependencyScanService extends Context.Service { const scanner = yield* DependencyScanner yield* scanner.scan(installation, repository, { trigger: "schedule" }) + // A production scan of an older commit in between does not hide the branch's last scan. + yield* scanner.scan(installation, repository, { trigger: "production", commitSha: sha("9") }) state.head = sha("b") state.files.set("README.md", { sha: "readme-2", content: "# shop 2" }) const unchanged = yield* scanner.scan(installation, repository, { trigger: "push" }) @@ -349,6 +351,23 @@ describe("DependencyScanner", () => { }, ) + it.effect("records a tree too large to list once, not on every push", () => { + const state = world({ "package-lock.json": packageLock("4.17.15") }) + return Effect.gen(function* () { + const scanner = yield* DependencyScanner + const store = yield* DependencyScanStore + state.truncated = true + yield* Effect.flip(scanner.scan(installation, repository, { trigger: "push" })) + state.head = sha("b") + const again = yield* scanner.scan(installation, repository, { trigger: "push" }) + assert.strictEqual(again._tag, "Unchanged") + assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 1) + // Scan now records its own result, so the page sees the scan it asked for. + yield* Effect.flip(scanner.scan(installation, repository, { trigger: "manual" })) + assert.strictEqual((yield* store.listScans({ orgId: repository.orgId, limit: 10 })).length, 2) + }).pipe(Effect.provide(scannerFor(state))) + }) + it.effect("records an oversized lockfile without reading it and keeps its findings open", () => { const state = world({ "package-lock.json": packageLock("4.17.15") }) return Effect.gen(function* () { @@ -383,6 +402,14 @@ describe("DependencyScanner", () => { 7, ) assert.deepStrictEqual(yield* findingsOf, []) + // A commit's content never changes, so asking again reads nothing. + const reads = state.blobReads.length + const again = yield* scanner.scan(installation, repository, { + trigger: "production", + commitSha: sha("e"), + }) + assert.strictEqual(again._tag, "Unchanged") + assert.strictEqual(state.blobReads.length, reads) }).pipe(Effect.provide(scannerFor(state))) }) }) diff --git a/packages/backend/src/services/dependency-scan/DependencyScanner.ts b/packages/backend/src/services/dependency-scan/DependencyScanner.ts index 90cfb21fed..e236304b86 100644 --- a/packages/backend/src/services/dependency-scan/DependencyScanner.ts +++ b/packages/backend/src/services/dependency-scan/DependencyScanner.ts @@ -12,7 +12,7 @@ import { } from "@maple/domain/http" import { type PackageAdvisory, PackageEcosystem, packageKey } from "@maple/domain/package-advisories" import { parseLockfile } from "@maple/lockfiles" -import { Cause, Context, Effect, Layer, Option, Schema } from "effect" +import { Cause, Clock, Context, Effect, Layer, Option, Schema } from "effect" import type { VcsProviderClient, VcsTreeFile, @@ -20,7 +20,10 @@ import type { import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" import { DependencyScanSink } from "@maple/backend/services/integrations/vcs/PullRequestEventSink" import { summarizeCause } from "@maple/backend/platform/describe-cause" -import { PackageAdvisoryCatalog } from "@maple/backend/services/security/PackageAdvisoryCatalog" +import { + MAX_VULNERABILITIES, + PackageAdvisoryCatalog, +} from "@maple/backend/services/security/PackageAdvisoryCatalog" import { type DependencyScanMatchInput, DependencyScanStore } from "./DependencyScanStore" import { MAX_LOCKFILES, @@ -29,16 +32,21 @@ import { sameLockfiles, selectLockfiles, } from "./lockfile-selection" +import { RESCAN_AFTER_MS } from "./production" // Reads a repository's lockfiles at one commit through the VCS provider, looks their packages up // in OSV and stores the scan. A lockfile's parse is cached by its blob sha (and its manifests'), // so a rescan for new advisories reads nothing from the provider but the tree. const PARSE_CACHE_BUCKET = "lockfile-packages" +/** Part of the parse cache key: bump it when a parser fix should reach lockfiles already cached. */ +const PARSER_VERSION = 1 const PARSE_CACHE_TTL_SECONDS = 7 * 24 * 60 * 60 /** Larger lockfiles are recorded as too large rather than read into the Worker's memory. */ -export const MAX_LOCKFILE_BYTES = 15 * 1024 * 1024 -const READ_CONCURRENCY = 4 +export const MAX_LOCKFILE_BYTES = 8 * 1024 * 1024 +/** Two lockfiles at a time keeps a monorepo's largest pair well inside a Worker's memory. */ +const READ_CONCURRENCY = 2 +const MANIFEST_CONCURRENCY = 4 const LockedPackageRow = Schema.Struct({ ecosystem: PackageEcosystem, @@ -66,7 +74,7 @@ export interface DependencyScanRequest { export type DependencyScanOutcome = | { readonly _tag: "Scanned"; readonly scan: DependencyScanRow } - /** A push that left every lockfile as the last scan read it. */ + /** A push that left every lockfile as the last scan read it, or a commit already scanned. */ | { readonly _tag: "Unchanged" } /** The tracked branch has no commit yet. */ | { readonly _tag: "EmptyRepository" } @@ -95,6 +103,8 @@ export class DependencySources extends Context.Service< ).pipe(Layer.provide(VcsProviderRegistry.layer)) } +const TREE_TOO_LARGE = "The repository tree is too large to list in one read" + interface ReadLockfile { readonly lockfile: DependencyScanLockfile readonly packages: ReadonlyArray @@ -150,7 +160,7 @@ export class DependencyScanner extends Context.Service [manifest.relativePath, text] as const)), - { concurrency: READ_CONCURRENCY }, + { concurrency: MANIFEST_CONCURRENCY }, ) const parsed = yield* parseLockfile({ path: selected.file.path, @@ -163,9 +173,11 @@ export class DependencyScanner extends Context.Service manifest.file.sha)].join( - ":", - ), + key: [ + `v${PARSER_VERSION}`, + selected.file.sha, + ...manifests.map((manifest) => manifest.file.sha), + ].join(":"), ttlSeconds: PARSE_CACHE_TTL_SECONDS, schema: CachedLockfile, }, @@ -252,6 +264,24 @@ export class DependencyScanner extends Context.Service + scan.status === "completed" || + (scan.status === "running" && now - scan.startedAt <= RESCAN_AFTER_MS), + ) + if (covered) return { _tag: "Unchanged" } satisfies DependencyScanOutcome + } + const tree = yield* provider .fetchTreeFiles(installation, repo, commitSha) .pipe(Effect.mapError(scanError("Could not list the repository tree"))) @@ -265,11 +295,10 @@ export class DependencyScanner extends Context.Service scan.repositoryId === repository.id, + ) + if (last !== undefined && last.status === "failed" && last.error === TREE_TOO_LARGE) + return { _tag: "Unchanged" } satisfies DependencyScanOutcome + } + const started = yield* store .startScan({ orgId, @@ -295,7 +337,7 @@ export class DependencyScanner extends Context.Service read.lockfile), ...skipped.map((lockfile) => @@ -333,6 +374,10 @@ export class DependencyScanner extends Context.Service + // The scan row records the failure; a defect still fails the job so it shows. + Effect.catch((error) => Effect.logWarning("[DependencyScan] scan failed").pipe( Effect.annotateLogs({ orgId: repository.orgId, repository: repository.fullName, trigger: job.trigger, - cause: summarizeCause(cause), + error: error.message, }), ), ), diff --git a/packages/backend/src/services/dependency-scan/production.test.ts b/packages/backend/src/services/dependency-scan/production.test.ts new file mode 100644 index 0000000000..a3546b01fc --- /dev/null +++ b/packages/backend/src/services/dependency-scan/production.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest" +import { riskScore } from "./production" + +const rank = ( + findings: ReadonlyArray<{ + readonly name: string + readonly severity: "critical" | "high" | "medium" | "low" | "unknown" + readonly relation: "direct" | "dev" | "transitive" | "unknown" + readonly spans: number + }>, +) => + [...findings].sort((a, b) => riskScore(b, b.spans) - riskScore(a, a.spans)).map((finding) => finding.name) + +describe("riskScore", () => { + it("puts a medium advisory a busy service runs above a high one nothing deploys", () => { + expect( + rank([ + { name: "high, not deployed", severity: "high", relation: "direct", spans: 0 }, + { name: "medium, in production", severity: "medium", relation: "transitive", spans: 50_000 }, + ]), + ).toEqual(["medium, in production", "high, not deployed"]) + }) + + it("ranks the same advisory by the traffic that runs it", () => { + expect( + rank([ + { name: "quiet", severity: "high", relation: "transitive", spans: 10 }, + { name: "busy", severity: "high", relation: "transitive", spans: 1_000_000 }, + { name: "undeployed", severity: "high", relation: "transitive", spans: 0 }, + ]), + ).toEqual(["busy", "quiet", "undeployed"]) + }) + + it("keeps a critical advisory first even when it is only a direct dependency nothing deploys", () => { + expect( + rank([ + { name: "low, busy", severity: "low", relation: "transitive", spans: 1_000_000 }, + { name: "critical, undeployed", severity: "critical", relation: "direct", spans: 0 }, + ]), + ).toEqual(["critical, undeployed", "low, busy"]) + }) + + it("ranks a dev-only dependency below the same advisory shipped to users", () => { + expect( + rank([ + { name: "dev", severity: "high", relation: "dev", spans: 0 }, + { name: "transitive", severity: "high", relation: "transitive", spans: 0 }, + ]), + ).toEqual(["transitive", "dev"]) + }) +}) diff --git a/packages/backend/src/services/dependency-scan/production.ts b/packages/backend/src/services/dependency-scan/production.ts new file mode 100644 index 0000000000..ddaaaeef35 --- /dev/null +++ b/packages/backend/src/services/dependency-scan/production.ts @@ -0,0 +1,134 @@ +import type { DependencyScanMatchRow, DependencyScanRow } from "@maple/db/tables" +import type { DependencyRelation } from "@maple/domain/dependency-scan" +import type { GitCommitSha, VcsRepositoryId } from "@maple/domain/http" +import type { AdvisorySeverity } from "@maple/domain/package-advisories" + +// Which findings the code running in production carries, and how the findings rank by it. Pure: +// the service reads the running commits, their scans and matches, and hands them in. + +/** One service serving traffic from a commit of a connected repository. */ +export interface RunningService { + readonly repositoryId: VcsRepositoryId + readonly commitSha: GitCommitSha + readonly serviceName: string + readonly environment: string + readonly spanCount: number +} + +export interface FindingKeyFields { + readonly repositoryId: VcsRepositoryId + readonly lockfilePath: string + readonly ecosystem: string + readonly packageName: string + readonly advisoryId: string +} + +export const findingKey = (finding: FindingKeyFields): string => + [ + finding.repositoryId, + finding.lockfilePath, + finding.ecosystem, + finding.packageName, + finding.advisoryId, + ].join("\u0000") + +const commitKey = (repositoryId: string, commitSha: string) => `${repositoryId}@${commitSha}` + +/** The latest scan per (repository, commit), from scans listed newest first. */ +export const latestScanPerCommit = ( + scans: ReadonlyArray, +): ReadonlyMap => + scans.reduce((latest, scan) => { + const key = commitKey(scan.repositoryId, scan.commitSha) + if (!latest.has(key)) latest.set(key, scan) + return latest + }, new Map()) + +/** The latest completed scan per (repository, commit): what the matches come from. */ +export const latestCompletedPerCommit = ( + scans: ReadonlyArray, +): ReadonlyMap => + latestScanPerCommit(scans.filter((scan) => scan.status === "completed")) + +/** A failed or stuck scan is retried once it is this old, so one bad read does not stick. */ +export const RESCAN_AFTER_MS = 60 * 60 * 1000 + +/** Whether a running commit should be scanned: never scanned, or its last scan failed or stalled. */ +export const needsScan = ( + latest: ReadonlyMap, + running: Pick, + nowMs: number, +): boolean => { + const scan = scanAt(latest, running) + if (scan === undefined) return true + return scan.status !== "completed" && nowMs - scan.startedAt > RESCAN_AFTER_MS +} + +export const scanAt = ( + latest: ReadonlyMap, + running: Pick, +): DependencyScanRow | undefined => latest.get(commitKey(running.repositoryId, running.commitSha)) + +/** + * The running services per finding key: a service counts when the latest completed scan of the + * commit it runs (`completed`, from `latestCompletedPerCommit`) matched the finding. Busiest first. + */ +export function productionByFinding( + running: ReadonlyArray, + completed: ReadonlyMap, + matches: ReadonlyArray, +): ReadonlyMap> { + const scanById = new Map([...completed.values()].map((scan) => [scan.id, scan])) + const servicesByCommit = running.reduce((byCommit, service) => { + const key = commitKey(service.repositoryId, service.commitSha) + byCommit.set(key, [...(byCommit.get(key) ?? []), service]) + return byCommit + }, new Map>()) + const byFinding = matches.reduce((acc, match) => { + const scan = scanById.get(match.scanId) + if (scan === undefined) return acc + const key = findingKey({ ...match, repositoryId: scan.repositoryId }) + const services = servicesByCommit.get(commitKey(scan.repositoryId, scan.commitSha)) ?? [] + const known = acc.get(key) ?? [] + const added = services.filter( + (service) => + !known.some( + (other) => + other.serviceName === service.serviceName && + other.environment === service.environment && + other.commitSha === service.commitSha, + ), + ) + acc.set(key, [...known, ...added]) + return acc + }, new Map>()) + return new Map( + [...byFinding].map(([key, services]) => [ + key, + [...services].sort((a, b) => b.spanCount - a.spanCount), + ]), + ) +} + +const SEVERITY_WEIGHT = { critical: 8, high: 4, medium: 2, low: 1, unknown: 1 } satisfies Record< + AdvisorySeverity, + number +> + +/** A dev-only dependency rarely ships; a direct one is the team's own choice to upgrade. */ +const RELATION_WEIGHT = { direct: 1.5, transitive: 1, unknown: 1, dev: 0.25 } satisfies Record< + DependencyRelation, + number +> + +/** + * How urgent a finding is: its severity, how the code depends on the package, and the traffic of + * the services running it. A finding in production outranks the same one that is not deployed. + */ +export const riskScore = ( + finding: { readonly severity: AdvisorySeverity; readonly relation: DependencyRelation }, + productionSpans: number, +): number => + SEVERITY_WEIGHT[finding.severity] * + RELATION_WEIGHT[finding.relation] * + (productionSpans > 0 ? 2 + Math.log10(1 + productionSpans) : 1) diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts index 4cbc58f278..9c3ead13d1 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts @@ -373,14 +373,6 @@ const GithubApiRecursiveTree = Schema.Struct({ }) export type GithubRecursiveTree = typeof GithubApiRecursiveTree.Type -const GithubApiBlob = Schema.Struct({ - sha: Schema.String, - size: Schema.Number, - encoding: Schema.String, - content: Schema.String, -}) -export type GithubBlob = typeof GithubApiBlob.Type - const GithubApiReviewCommentList = Schema.Array( Schema.Struct({ id: Schema.Number, @@ -586,7 +578,6 @@ const decodeCollaboratorPermission = Schema.decodeUnknownEffect(GithubApiCollabo const decodeGitCommit = Schema.decodeUnknownEffect(GithubApiGitCommit) const decodeGitTreeListing = Schema.decodeUnknownEffect(GithubApiGitTreeListing) const decodeRecursiveTree = Schema.decodeUnknownEffect(GithubApiRecursiveTree) -const decodeBlob = Schema.decodeUnknownEffect(GithubApiBlob) const decodeComparison = Schema.decodeUnknownEffect(GithubApiComparison) const decodeReviewThreads = Schema.decodeUnknownEffect(GithubReviewThreadsResponse) const decodeMutation = Schema.decodeUnknownEffect(GithubMutationResponse) @@ -935,14 +926,19 @@ export class GithubAppClient extends Context.Service()( return grantsActions(yield* installationToken(externalInstallationId)) }) - const authedGet = (_config: ResolvedAppConfig, token: string, url: string) => + const authedGet = ( + _config: ResolvedAppConfig, + token: string, + url: string, + accept = "application/vnd.github+json", + ) => rateLimitedFetch( tracedFetch( url, { headers: { authorization: `token ${token}`, - accept: "application/vnd.github+json", + accept, "x-github-api-version": GITHUB_API_VERSION, "user-agent": USER_AGENT, }, @@ -2183,8 +2179,11 @@ export class GithubAppClient extends Context.Service()( ) }) - /** One blob by its sha; unlike the contents API it serves files up to 100 MB. */ - const getBlob = Effect.fn("GithubAppClient.getBlob")(function* ( + /** + * One blob's bytes as text, by its sha; unlike the contents API it serves files up to 100 MB. + * Read raw, so a large lockfile is never held as base64 JSON as well. + */ + const getBlobText = Effect.fn("GithubAppClient.getBlobText")(function* ( externalInstallationId: string, owner: string, repo: string, @@ -2196,11 +2195,12 @@ export class GithubAppClient extends Context.Service()( config, token, `${config.apiBaseUrl}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/git/blobs/${encodeURIComponent(sha)}`, + "application/vnd.github.raw+json", ) if (!isOk(response)) return yield* failure(response, "Read blob", "repository") - return yield* decodeBlob(yield* parseJson(response, "Read blob")).pipe( + return yield* response.text.pipe( Effect.mapError( - (cause) => new GithubAppError({ message: "Unexpected blob payload", cause }), + (cause) => new GithubAppError({ message: "Could not read the blob", cause }), ), ) }) @@ -2477,7 +2477,7 @@ export class GithubAppClient extends Context.Service()( searchCode, getSourceFile, getRecursiveTree, - getBlob, + getBlobText, listCheckRunAnnotations, streamJobLog, canReadActions, diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts index 216c225bcb..89e8fbf92e 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts @@ -1320,14 +1320,9 @@ export class GithubProvider extends Context.Service - client.getBlob(installation.externalInstallationId, repo.owner, repo.name, blobSha).pipe( - Effect.map((blob) => - blob.encoding === "base64" - ? Buffer.from(blob.content.replace(/\s/g, ""), "base64").toString("utf8") - : blob.content, - ), - Effect.mapError(toVcsCommitError), - ) + client + .getBlobText(installation.externalInstallationId, repo.owner, repo.name, blobSha) + .pipe(Effect.mapError(toVcsCommitError)) const resolveRef: VcsProviderClient["resolveRef"] = (installation, repo, ref) => client.getCommit(installation.externalInstallationId, repo.owner, repo.name, ref).pipe( diff --git a/packages/backend/src/services/security/PackageAdvisoryCatalog.test.ts b/packages/backend/src/services/security/PackageAdvisoryCatalog.test.ts index 2e39a894f1..51196d45eb 100644 --- a/packages/backend/src/services/security/PackageAdvisoryCatalog.test.ts +++ b/packages/backend/src/services/security/PackageAdvisoryCatalog.test.ts @@ -53,7 +53,11 @@ const osv = (fake: FakeOsv) => const id = decodeURIComponent(url.pathname.replace("/v1/vulns/", "")) fake.requests.push(`vuln:${id}`) const vuln = fixture.vulns.find((candidate) => candidate.id === id) - return vuln === undefined ? json(request, { message: "not found" }, 404) : json(request, vuln) + if (vuln !== undefined) return json(request, vuln) + // Synthetic ids a test invents answer with a bare record. + return id.startsWith("GHSA-many-") + ? json(request, { id, aliases: [] }) + : json(request, { message: "not found" }, 404) }), ) @@ -88,7 +92,7 @@ describe("PackageAdvisoryCatalog", () => { const fake: FakeOsv = { requests: [] } return Effect.gen(function* () { const catalog = yield* PackageAdvisoryCatalog - const first = yield* catalog.advisories([...refs, ...refs]) + const { byPackage: first } = yield* catalog.advisories([...refs, ...refs]) assert.deepStrictEqual( first .get(keyOf("minimist")) @@ -105,7 +109,7 @@ describe("PackageAdvisoryCatalog", () => { assert.strictEqual(reads.length, fixture.vulns.length) fake.requests.length = 0 - const second = yield* catalog.advisories(refs) + const { byPackage: second } = yield* catalog.advisories(refs) assert.deepStrictEqual(fake.requests, [`querybatch:${refs.length}`]) assert.deepStrictEqual([...second.keys()], [...first.keys()]) }).pipe(Effect.provide(catalogWith(fake))) @@ -123,7 +127,7 @@ describe("PackageAdvisoryCatalog", () => { } return Effect.gen(function* () { const catalog = yield* PackageAdvisoryCatalog - const advisories = yield* catalog.advisories(refs) + const { byPackage: advisories } = yield* catalog.advisories(refs) assert.deepStrictEqual( advisories .get(keyOf("lodash")) @@ -148,9 +152,9 @@ describe("PackageAdvisoryCatalog", () => { }).pipe(Effect.provide(catalogWith(fake))) }) - it.effect("refuses more distinct vulnerabilities than a scan reads", () => { - const many = Array.from({ length: MAX_VULNERABILITIES + 1 }, (_, i) => ({ - id: `GHSA-many-${i}`, + it.effect("reads at most the capped number of vulnerabilities and counts the rest as unread", () => { + const many = Array.from({ length: MAX_VULNERABILITIES + 3 }, (_, i) => ({ + id: `GHSA-many-${String(i).padStart(4, "0")}`, modified: "1", })) const fake: FakeOsv = { @@ -159,9 +163,12 @@ describe("PackageAdvisoryCatalog", () => { } return Effect.gen(function* () { const catalog = yield* PackageAdvisoryCatalog - const error = yield* Effect.flip(catalog.advisories(refs)) - assert.strictEqual(error._tag, "@maple/security/PackageAdvisoryCatalogError") - assert.isFalse(fake.requests.some((request) => request.startsWith("vuln:"))) + const { unread } = yield* catalog.advisories(refs) + assert.strictEqual(unread, 3) + assert.strictEqual( + fake.requests.filter((request) => request.startsWith("vuln:")).length, + MAX_VULNERABILITIES, + ) }).pipe(Effect.provide(catalogWith(fake))) }) }) diff --git a/packages/backend/src/services/security/PackageAdvisoryCatalog.ts b/packages/backend/src/services/security/PackageAdvisoryCatalog.ts index 26ba1ea6f1..7cbcc61e74 100644 --- a/packages/backend/src/services/security/PackageAdvisoryCatalog.ts +++ b/packages/backend/src/services/security/PackageAdvisoryCatalog.ts @@ -15,9 +15,11 @@ const CACHE_BUCKET = "osv-vulns" const CACHE_TTL_SECONDS = 7 * 24 * 60 * 60 const CACHE_READ_TIMEOUT_MS = 150 const REQUEST_TIMEOUT = "15 seconds" -/** Distinct vulnerabilities read per call; a scan past it fails rather than store a partial answer. */ +/** Distinct vulnerabilities read per call; past it the answer says how many it left unread. */ export const MAX_VULNERABILITIES = 500 const CONCURRENCY = 16 +/** Pages followed per query past the first; a package with more than a page of advisories is rare. */ +const MAX_EXTRA_PAGES = 5 const RETRY = { times: 2, schedule: Schedule.exponential("250 millis") } as const const QueryBatchResponse = Schema.Struct({ @@ -44,11 +46,16 @@ export class PackageAdvisoryCatalogError extends Schema.TaggedError, - ) => Effect.Effect>, PackageAdvisoryCatalogError> + readonly advisories: (packages: ReadonlyArray) => Effect.Effect< + { + readonly byPackage: ReadonlyMap> + readonly unread: number + }, + PackageAdvisoryCatalogError + > } interface Pending { @@ -113,27 +120,36 @@ export class PackageAdvisoryCatalog extends Context.Service< const queryAll = ( pending: ReadonlyArray, found: ReadonlyMap>, + depth = 0, ): Effect.Effect>, PackageAdvisoryCatalogError> => pending.length === 0 ? Effect.succeed(found) - : Effect.forEach(Arr.chunksOf(pending, BATCH_SIZE), queryBatch, { concurrency: 2 }).pipe( - Effect.flatMap((batches) => { - const pages = batches.flat() - const next = new Map(found) - pages.forEach((page) => { - const key = packageKey(page.ref) - next.set(key, [...(next.get(key) ?? []), ...page.vulns]) - }) - return queryAll( - pages.flatMap((page) => - page.nextPageToken === undefined - ? [] - : [{ ref: page.ref, pageToken: page.nextPageToken }], - ), - next, - ) - }), - ) + : depth > MAX_EXTRA_PAGES + ? Effect.fail( + new PackageAdvisoryCatalogError({ + message: `OSV kept paging past ${MAX_EXTRA_PAGES} pages`, + cause: undefined, + }), + ) + : Effect.forEach(Arr.chunksOf(pending, BATCH_SIZE), queryBatch, { concurrency: 2 }).pipe( + Effect.flatMap((batches) => { + const pages = batches.flat() + const next = new Map(found) + pages.forEach((page) => { + const key = packageKey(page.ref) + next.set(key, [...(next.get(key) ?? []), ...page.vulns]) + }) + return queryAll( + pages.flatMap((page) => + page.nextPageToken === undefined + ? [] + : [{ ref: page.ref, pageToken: page.nextPageToken }], + ), + next, + depth + 1, + ) + }), + ) const fetchRecord = (vuln: VulnRef) => edgeCache @@ -172,19 +188,15 @@ export class PackageAdvisoryCatalog extends Context.Service< "maple.depscan.package_count": distinct.length, "maple.depscan.vulnerability_count": vulns.length, }) - if (vulns.length > MAX_VULNERABILITIES) { - return yield* new PackageAdvisoryCatalogError({ - message: `${vulns.length} known vulnerabilities, more than the ${MAX_VULNERABILITIES} a scan reads`, - cause: undefined, - }) - } + // Sorted, so the same vulnerabilities are read each time the cap bites. + const read = [...vulns].sort((a, b) => a.id.localeCompare(b.id)).slice(0, MAX_VULNERABILITIES) const records = new Map( - (yield* Effect.forEach(vulns, fetchRecord, { concurrency: CONCURRENCY })).map((record) => [ + (yield* Effect.forEach(read, fetchRecord, { concurrency: CONCURRENCY })).map((record) => [ record.id, record, ]), ) - return new Map( + const byPackage = new Map( distinct.flatMap((ref) => { const matched = (found.get(packageKey(ref)) ?? []).flatMap((vuln) => { const record = records.get(vuln.id) @@ -194,6 +206,7 @@ export class PackageAdvisoryCatalog extends Context.Service< return list.length === 0 ? [] : [[packageKey(ref), list] as const] }), ) + return { byPackage, unread: vulns.length - read.length } }) return { advisories } satisfies PackageAdvisoryCatalogApi diff --git a/packages/domain/src/http/dependency-scan.ts b/packages/domain/src/http/dependency-scan.ts index 1944b9ab68..08ebdbb089 100644 --- a/packages/domain/src/http/dependency-scan.ts +++ b/packages/domain/src/http/dependency-scan.ts @@ -34,6 +34,23 @@ export const DependencyScanSummary = Schema.Struct({ }) export type DependencyScanSummary = typeof DependencyScanSummary.Type +/** A service serving traffic from one commit over the last day; spans are what it served. */ +export const DependencyRunningService = Schema.Struct({ + serviceName: Schema.String, + environment: Schema.String, + commitSha: GitCommitSha, + spanCount: Schema.Finite, +}) +export type DependencyRunningService = typeof DependencyRunningService.Type + +/** A commit of the repository that services run, and the scan that read it (null until one does). */ +export const DependencyRunningCommit = Schema.Struct({ + commitSha: GitCommitSha, + scanStatus: Schema.NullOr(DependencyScanStatus), + services: Schema.Array(DependencyRunningService), +}) +export type DependencyRunningCommit = typeof DependencyRunningCommit.Type + /** A connected repository and its latest scan of the tracked branch, null before the first. */ export const DependencyScanRepository = Schema.Struct({ id: VcsRepositoryId, @@ -41,6 +58,8 @@ export const DependencyScanRepository = Schema.Struct({ htmlUrl: Schema.String, trackedBranch: Schema.String, latestScan: Schema.NullOr(DependencyScanSummary), + /** Commits services ran over the last day, busiest first. */ + runningCommits: Schema.Array(DependencyRunningCommit), }) export type DependencyScanRepository = typeof DependencyScanRepository.Type @@ -53,10 +72,29 @@ export type DependencyScanRepositoriesResponse = typeof DependencyScanRepositori export const DependencyFindingView = Schema.Struct({ ...DependencyFinding.fields, repositoryFullName: Schema.String, + /** Services whose running commit carries the finding, busiest first; empty when none does. */ + production: Schema.Array(DependencyRunningService), }) export type DependencyFindingView = typeof DependencyFindingView.Type -export const DependencyFindingsResponse = Schema.Struct({ findings: Schema.Array(DependencyFindingView) }) +/** How much of production the findings could be joined to. */ +export const DependencyProductionCoverage = Schema.Struct({ + /** False when telemetry could not be read; the findings are then ranked without it. */ + available: Schema.Boolean, + /** Services reporting a commit (`vcs.ref.head.revision`) over the last day. */ + runningServices: Schema.Int, + /** Of those, the ones whose commit belongs to a connected repository. */ + linkedServices: Schema.Int, + /** False when more commits ran than were read: a finding then is not known to be absent. */ + complete: Schema.Boolean, +}) +export type DependencyProductionCoverage = typeof DependencyProductionCoverage.Type + +/** Findings most urgent first: severity, how the code depends on the package, production traffic. */ +export const DependencyFindingsResponse = Schema.Struct({ + findings: Schema.Array(DependencyFindingView), + production: DependencyProductionCoverage, +}) export type DependencyFindingsResponse = typeof DependencyFindingsResponse.Type export const DependencyFindingsQuery = Schema.Struct({ @@ -74,7 +112,8 @@ export const DependencyScanQueued = Schema.Struct({ queued: Schema.Boolean }) export class DependencyScanPersistenceError extends Schema.TaggedError()( "@maple/http/dependency-scan/DependencyScanPersistenceError", - { message: Schema.String }, + /** `cause` names the failure behind it (its tag), never its details. */ + { message: Schema.String, cause: Schema.optionalKey(Schema.String) }, { httpApiStatus: 503 }, ) {} diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index c9e17a3cba..151d07d76f 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -4086,6 +4086,62 @@ SELECT LIMIT 5000 FORMAT JSON +-- builder:releases:runningCommitsQuery:default [aa2f361b] +SELECT + service_windows.bServiceName AS serviceName, + service_windows.bEnvironment AS environment, + service_windows.bCommitSha AS commitSha, + sum(service_windows.bEstimatedSpanCount) AS spanCount + FROM ( +SELECT + toStartOfHour(service_overview_spans.Timestamp) AS bBucket, + service_overview_spans.ServiceName AS bServiceName, + service_overview_spans.ServiceNamespace AS bServiceNamespace, + service_overview_spans.DeploymentEnv AS bEnvironment, + service_overview_spans.CommitSha AS bCommitSha, + count() AS bSpanCount, + sum(service_overview_spans.SampleRate) AS bEstimatedSpanCount, + countIf(service_overview_spans.StatusCode = 'Error') AS bErrorCount, + sumIf(service_overview_spans.SampleRate, service_overview_spans.StatusCode = 'Error') AS bEstimatedErrorCount, + sum(toFloat64(Duration)) AS bDurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS bDurationQuantiles, + min(service_overview_spans.Timestamp) AS bFirstSeen, + countIf((service_overview_spans.StatusCode != 'Error' AND service_overview_spans.Duration < 500000000)) AS bApdexSatisfiedCount, + countIf(((service_overview_spans.StatusCode != 'Error' AND service_overview_spans.Duration >= 500000000) AND service_overview_spans.Duration < 2000000000)) AS bApdexToleratingCount + FROM service_overview_spans + WHERE service_overview_spans.OrgId = 'org_sql_catalog' + AND service_overview_spans.Timestamp >= '2026-01-01 10:30:00' + AND service_overview_spans.Timestamp <= '2026-01-03 14:15:00' + AND (Timestamp < if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) OR Timestamp >= toStartOfHour(toDateTime('2026-01-03 14:15:00'))) + GROUP BY bBucket, bServiceName, bServiceNamespace, bEnvironment, bCommitSha +UNION ALL +SELECT + service_overview_hourly.Hour AS bBucket, + service_overview_hourly.ServiceName AS bServiceName, + service_overview_hourly.ServiceNamespace AS bServiceNamespace, + service_overview_hourly.DeploymentEnv AS bEnvironment, + service_overview_hourly.CommitSha AS bCommitSha, + sum(service_overview_hourly.SpanCount) AS bSpanCount, + sum(service_overview_hourly.EstimatedSpanCount) AS bEstimatedSpanCount, + sum(service_overview_hourly.ErrorCount) AS bErrorCount, + sum(service_overview_hourly.EstimatedErrorCount) AS bEstimatedErrorCount, + sum(service_overview_hourly.DurationSum) AS bDurationSum, + quantilesTDigestMergeState(0.5, 0.95, 0.99)(DurationQuantiles) AS bDurationQuantiles, + min(service_overview_hourly.FirstSeen) AS bFirstSeen, + sum(service_overview_hourly.ApdexSatisfiedCount) AS bApdexSatisfiedCount, + sum(service_overview_hourly.ApdexToleratingCount) AS bApdexToleratingCount + FROM service_overview_hourly + WHERE service_overview_hourly.OrgId = 'org_sql_catalog' + AND service_overview_hourly.Hour >= if(toDateTime('2026-01-01 10:30:00') = toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')), toStartOfHour(toDateTime('2026-01-01 10:30:00')) + INTERVAL 1 HOUR) + AND service_overview_hourly.Hour < toStartOfHour(toDateTime('2026-01-03 14:15:00')) + GROUP BY bBucket, bServiceName, bServiceNamespace, bEnvironment, bCommitSha +) AS service_windows + WHERE service_windows.bCommitSha NOT IN ('', 'unknown', 'N/A') + GROUP BY serviceName, environment, commitSha + ORDER BY spanCount DESC + LIMIT 200 + FORMAT JSON + -- builder:releases:serviceDeploymentsQuery:hourInterior [ae47e5db] SELECT ranked.serviceName AS serviceName, diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index 0a23a579a6..c49412a128 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -1759,6 +1759,13 @@ export const builderFixtures: ReadonlyArray = [ window, ), }, + // Dependency scanner: which commit each service runs, ranked by traffic. + { + module: "releases", + name: "runningCommitsQuery", + label: "default", + compile: () => CH.compileUnsafe(CH.runningCommitsQuery(), window), + }, { module: "releases", name: "releasesTimelineQuery", diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 7ac613e241..e74afc78e6 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -340,6 +340,9 @@ export { // Queries — Releases export { releasesListQuery, + runningCommitsQuery, + runningCommitsRowSchema, + RUNNING_COMMITS_CAP, releasesListRowSchema, releasesTimelineQuery, releaseErrorFingerprintsQuery, diff --git a/packages/query-engine/src/ch/queries/releases.ts b/packages/query-engine/src/ch/queries/releases.ts index e7bbc305a2..de5e820173 100644 --- a/packages/query-engine/src/ch/queries/releases.ts +++ b/packages/query-engine/src/ch/queries/releases.ts @@ -126,6 +126,45 @@ export function releasesListQuery(opts: ReleasesListOpts = {}) { .format("JSON") } +// Running commits +// +// Which commit each service served traffic from in the window, for the dependency scanner: a +// vulnerable package matters most in the commit a busy service is running. + +/** At most this many (service, environment, commit) rows, busiest first. */ +export const RUNNING_COMMITS_CAP = 200 + +export interface RunningCommitsOutput { + readonly serviceName: string + readonly environment: string + readonly commitSha: string + readonly spanCount: number +} + +export const runningCommitsRowSchema = Schema.Struct({ + serviceName: Schema.String, + environment: Schema.String, + commitSha: Schema.String, + spanCount: CHNumber, +}) satisfies CompiledQueryRowSchema + +/** The releases list without its latency and error columns, ordered by traffic. */ +export function runningCommitsQuery() { + return serviceOverviewWindows({}) + .select(($) => ({ + serviceName: $.bServiceName, + environment: $.bEnvironment, + commitSha: $.bCommitSha, + // Sample-weighted, so a sampled service ranks by the traffic it served. + spanCount: CH.sum($.bEstimatedSpanCount), + })) + .where(($) => [CH.notInList($.bCommitSha, PLACEHOLDER_COMMIT_SHAS)]) + .groupBy("serviceName", "environment", "commitSha") + .orderBy(["spanCount", "desc"]) + .limit(RUNNING_COMMITS_CAP) + .format("JSON") +} + // Releases timeline // // The org-wide sibling of `serviceReleasesTimelineQuery`: per bucket, per