diff --git a/apps/api/scripts/quality/check-coverage.ts b/apps/api/scripts/quality/check-coverage.ts index 335cebf8..775442e6 100644 --- a/apps/api/scripts/quality/check-coverage.ts +++ b/apps/api/scripts/quality/check-coverage.ts @@ -5,14 +5,12 @@ * `coverageThreshold` is documented but not enforced, so we parse the * text report ourselves and exit non-zero on regression. * - * The threshold is a ratchet, not a wishlist: it sits a few points - * below the current measured rate so a small slip triggers the alarm. - * Raise it as coverage climbs; never lower it to silence a regression. + * The floor lives in coverage-thresholds.ts, shared with the sharded + * verification runner. */ import { spawnSync } from "node:child_process"; +import { MIN_FUNCTION, MIN_LINE } from "./coverage-thresholds"; -const MIN_LINE = 0.65; -const MIN_FUNCTION = 0.7; const MAX_TEST_OUTPUT_BUFFER_BYTES = 64 * 1024 * 1024; const FORBIDDEN_OUTPUT = [ @@ -49,16 +47,26 @@ const runCoverage = (): { * letting them into the coverage gate turns the ordinary merge gate red * for reasons unrelated to the change under review. */ - const result = spawnSync("bun", ["test", "tests", "--coverage"], { - encoding: "utf8", - maxBuffer: MAX_TEST_OUTPUT_BUFFER_BYTES, - env: { - ...process.env, - NODE_ENV: "test", - LOG_LEVEL: "error", - NODE_NO_WARNINGS: "1", - }, - }); + const result = spawnSync( + "bun", + [ + ...(process.env.AGENT_SANDBOX === "1" ? ["--no-env-file"] : []), + "test", + "tests", + "--coverage", + ...process.argv.slice(2), + ], + { + encoding: "utf8", + maxBuffer: MAX_TEST_OUTPUT_BUFFER_BYTES, + env: { + ...process.env, + NODE_ENV: "test", + LOG_LEVEL: "error", + NODE_NO_WARNINGS: "1", + }, + } + ); return { combined: result.stdout + result.stderr, @@ -80,7 +88,14 @@ const parseAllFilesRow = (output: string): ICoverageResult | null => { const functionPct = parseFloat(parts[1] ?? ""); const linePct = parseFloat(parts[2] ?? ""); - if (Number.isNaN(linePct) || Number.isNaN(functionPct)) { + if ( + !Number.isFinite(linePct) || + !Number.isFinite(functionPct) || + linePct < 0 || + linePct > 100 || + functionPct < 0 || + functionPct > 100 + ) { return null; } @@ -118,7 +133,7 @@ if (warningLines.length > 0) { console.error(` ${line}`); } - process.exit(1); + process.exit(86); } if (exitCode !== 0) { @@ -145,7 +160,7 @@ if (!lineOk || !functionOk) { `\n\nTo raise: add tests for under-covered surfaces (queues / SSE / web push / setup).` + `\nTo lower the threshold: do not. Treat the gate as a ratchet.` ); - process.exit(1); + process.exit(86); } console.log( diff --git a/apps/api/scripts/quality/coverage-thresholds.ts b/apps/api/scripts/quality/coverage-thresholds.ts new file mode 100644 index 00000000..e61a0fd1 --- /dev/null +++ b/apps/api/scripts/quality/coverage-thresholds.ts @@ -0,0 +1,9 @@ +/* + * The coverage floor is a ratchet, not a wishlist: it sits a few points + * below the current measured rate so a small slip triggers the alarm. + * Raise it as coverage climbs; never lower it to silence a regression. + * Shared by the single-process gate (check-coverage.ts) and the sharded + * verification runner, so both enforce the same numbers. + */ +export const MIN_LINE = 0.65; +export const MIN_FUNCTION = 0.7; diff --git a/apps/api/security-spec/f14-sse-stream-lifetime.test.ts b/apps/api/security-spec/f14-sse-stream-lifetime.test.ts index d685b2a5..19e0271b 100644 --- a/apps/api/security-spec/f14-sse-stream-lifetime.test.ts +++ b/apps/api/security-spec/f14-sse-stream-lifetime.test.ts @@ -129,8 +129,32 @@ const openStream = (userId: string, jti: string): IStreamFixture => { }), }); + /* + * The stream opens with a ping so Elysia can flush the response headers + * before the first notification. The fixture consumes it, so every test + * reads the real payloads exactly as it would have without the handshake. + */ + let opened = false; + + const next = async (): Promise> => { + if (!opened) { + opened = true; + + const handshake = await generator.next(); + + if ( + handshake.done === true || + handshake.value !== JSON.stringify({ type: "ping" }) + ) { + throw new Error("f14: the stream did not open with a ping"); + } + } + + return generator.next(); + }; + return { - next: () => generator.next(), + next, publish: (message) => valkeyPubSub.publish(userNotificationChannel(userId), message), credential, diff --git a/apps/api/src/api/notifications/notifications.sse.ts b/apps/api/src/api/notifications/notifications.sse.ts index 2819d107..e0ad0d28 100644 --- a/apps/api/src/api/notifications/notifications.sse.ts +++ b/apps/api/src/api/notifications/notifications.sse.ts @@ -157,6 +157,15 @@ export const notificationsStreamHandler = async function* ( let lastPingAtMs = nowMs(); try { + /* + * Elysia turns a generator into a response only after its first + * `yield`. A stream that stays silent until a notification arrives, or + * until the keepalive below, would keep the browser's `EventSource` + * from opening and any proxy from seeing bytes for up to 25 seconds. + * A ping on open flushes the headers at once; the client ignores it. + */ + yield JSON.stringify({ type: "ping" }); + while (!isAborted()) { /* * The credential is re-checked before every payload, not once per diff --git a/apps/api/src/templates/email/build.ts b/apps/api/src/templates/email/build.ts index 2b907377..6203bf0e 100644 --- a/apps/api/src/templates/email/build.ts +++ b/apps/api/src/templates/email/build.ts @@ -103,6 +103,18 @@ const precompilePartials = (): Record => { return partials; }; +/** Avoid replacing artifacts while parallel test and build consumers read them. */ +const writeArtifact = (outputPath: string, content: string): void => { + if ( + fs.existsSync(outputPath) && + fs.readFileSync(outputPath, "utf8") === content + ) { + return; + } + + fs.writeFileSync(outputPath, content, "utf8"); +}; + const buildTemplate = (templatePath: string): void => { const source = fs.readFileSync(templatePath, "utf8"); const baseTemplate = precompileToString(source); @@ -123,10 +135,9 @@ const buildTemplate = (templatePath: string): void => { contentTemplate = precompileToString(contentSource); } - fs.writeFileSync( + writeArtifact( outputPath, - JSON.stringify({ baseTemplate, contentTemplate }, null, 2), - "utf8" + JSON.stringify({ baseTemplate, contentTemplate }, null, 2) ); console.log(`✓ Built: ${path.relative(__dirname, outputPath)}`); }; @@ -137,7 +148,7 @@ const buildPartialsManifest = (): void => { fs.ensureDirSync(DIST_DIR); const manifestPath = path.join(DIST_DIR, "partials.json"); - fs.writeFileSync(manifestPath, JSON.stringify(partials, null, 2), "utf8"); + writeArtifact(manifestPath, JSON.stringify(partials, null, 2)); console.log( `✓ Built partials manifest: ${path.relative(__dirname, manifestPath)}` ); diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json index aa58b5db..f126a939 100644 --- a/apps/api/tsconfig.json +++ b/apps/api/tsconfig.json @@ -1,5 +1,7 @@ { "compilerOptions": { + "incremental": true, + "tsBuildInfoFile": "./node_modules/.cache/tsc/api.tsbuildinfo", "target": "ES2022", "module": "ES2022", "moduleResolution": "bundler", @@ -29,6 +31,11 @@ "@/*": ["./src/*"] } }, - "include": ["src/**/*.ts", "tests/**/*.ts", "security-spec/**/*.ts", "scripts/**/*.ts"], + "include": [ + "src/**/*.ts", + "tests/**/*.ts", + "security-spec/**/*.ts", + "scripts/**/*.ts" + ], "exclude": ["node_modules", "dist", "drizzle", "src/templates/email/dist"] } diff --git a/apps/ui/.size-limit.json b/apps/ui/.size-limit.json index d2d8fb03..fe2a1202 100644 --- a/apps/ui/.size-limit.json +++ b/apps/ui/.size-limit.json @@ -9,7 +9,8 @@ "dist/assets/query-*.js" ], "limit": "255 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "Modulepreload runtime + shared shell", @@ -24,120 +25,140 @@ "dist/assets/dist-*.js" ], "limit": "165 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "CSS (Tailwind compiled)", "path": "dist/assets/*.css", "limit": "12 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "LoginPage chunk", "path": "dist/assets/LoginPage-*.js", "limit": "20 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "DashboardPage chunk", "path": "dist/assets/DashboardPage-*.js", "limit": "5 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "SettingsPage chunk", "path": "dist/assets/SettingsPage-*.js", "limit": "20 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "BillingPage chunk", "path": "dist/assets/BillingPage-*.js", "limit": "4 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "NotificationsPage chunk", "path": "dist/assets/NotificationsPage-*.js", "limit": "4 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "NotificationsPreferencesPage chunk", "path": "dist/assets/NotificationsPreferencesPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "InvitationsPage chunk", "path": "dist/assets/InvitationsPage-*.js", "limit": "4 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "InvitationAcceptPage chunk", "path": "dist/assets/InvitationAcceptPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "OwnershipTransferAcceptPage chunk", "path": "dist/assets/OwnershipTransferAcceptPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "JoinRequestsPage chunk", "path": "dist/assets/JoinRequestsPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "AuditLogPage chunk", "path": "dist/assets/AuditLogPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "ProfilePage chunk", "path": "dist/assets/ProfilePage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "SignUpPage chunk", "path": "dist/assets/SignUpPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "ForgotPasswordPage chunk", "path": "dist/assets/ForgotPasswordPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "ResetPasswordPage chunk", "path": "dist/assets/ResetPasswordPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "VerifyEmailPage chunk", "path": "dist/assets/VerifyEmailPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "OAuthCallbackPage chunk", "path": "dist/assets/OAuthCallbackPage-*.js", "limit": "3 KB", - "gzip": true + "gzip": true, + "running": false }, { "name": "NotFoundPage chunk", "path": "dist/assets/NotFoundPage-*.js", "limit": "2 KB", - "gzip": true + "gzip": true, + "running": false } ] diff --git a/apps/ui/scripts/codegen/new-feature.ts b/apps/ui/scripts/codegen/new-feature.ts index e7f2711e..92578edd 100644 --- a/apps/ui/scripts/codegen/new-feature.ts +++ b/apps/ui/scripts/codegen/new-feature.ts @@ -322,7 +322,8 @@ if (namespaceEnabled) { name: `${Name} translations (all locales)`, path: `dist/assets/${lower}-*.js`, limit: "10 KB", - gzip: true + gzip: true, + running: false }); writeFileSync( budgetPath, diff --git a/apps/ui/tests/lint-meta/feature-namespace.test.ts b/apps/ui/tests/lint-meta/feature-namespace.test.ts index 34cd9e88..dcd56e0a 100644 --- a/apps/ui/tests/lint-meta/feature-namespace.test.ts +++ b/apps/ui/tests/lint-meta/feature-namespace.test.ts @@ -73,7 +73,8 @@ test("namespace scaffolding wires dictionaries, lint scope, and a separate bundl name: "Posts translations (all locales)", path: "dist/assets/posts-*.js", limit: "10 KB", - gzip: true + gzip: true, + running: false } ]); diff --git a/apps/ui/tsconfig.json b/apps/ui/tsconfig.json index dd37df1a..13e886fc 100644 --- a/apps/ui/tsconfig.json +++ b/apps/ui/tsconfig.json @@ -1,5 +1,7 @@ { "compilerOptions": { + "incremental": true, + "tsBuildInfoFile": "./node_modules/.cache/tsc/ui.tsbuildinfo", "target": "ES2023", "lib": ["ES2023", "DOM", "DOM.Iterable"], "module": "ESNext", diff --git a/tools/agent/README.md b/tools/agent/README.md index b745e66b..83140c09 100644 --- a/tools/agent/README.md +++ b/tools/agent/README.md @@ -48,7 +48,27 @@ Local release evidence excludes GitHub-only CodeQL, dependency review, secret sc Each `sandbox:up` creates pinned Postgres and Valkey containers with random loopback ports, random Postgres and Valkey credentials and checkout/run ownership labels. Credentials live only in a mode-0600 ignored descriptor under `.agent-state/sandboxes`; JSON output omits them. Verification ignores caller database URLs and uses the descriptor after checking ownership and live bindings. Database tests are destructive **inside that sandbox**. -Within one run, verification lanes execute in parallel. Stateless lanes (tooling quality, API and UI checks, UI unit tests, builds, size gates) share nothing; each stateful lane (API tests, security spec, browser run, coverage) owns its own Postgres database and Valkey index inside the sandbox (`app_tests`, `app_security`, `app_e2e`, `app_coverage`, created and migrated by the runner), so they never truncate each other's tables or share rate-limit counters. The budget defaults to roughly one lane per three cores, between two and six; `AGENT_VERIFY_PARALLEL=1` restores a strictly sequential run and `AGENT_VERIFY_PARALLEL=8` widens it. Reports list checks in a fixed order whatever the completion order. +Within one run, verification schedules ready checks against a shared CPU-slot budget. API/UI/tooling checks run their constituent scripts independently; contract tests keep them aligned with each app's complete `check` command. Builds start early, size checks wait for a successful UI build without occupying slots, and static checks overlap sandbox preparation. Failed prerequisites block their dependents while independent checks finish. + +Local defaults reserve two host cores and budget roughly 2 GiB per slot, capped at 24 slots. A 16-core, 64-GiB host gets 14 slots, with four workers each for Playwright and Vitest; each pool consumes four slots from that same budget. Real CI defaults to at most four slots and one worker per test pool. The runner captures the caller's CI setting before assigning `CI=true` to isolated child processes, so local browser tests can run in parallel while retaining CI safety checks and zero retries. + +| Override | Meaning | +| ----------------------------- | ------------------------------------------------------------------- | +| `AGENT_VERIFY_PARALLEL=8` | Set the shared CPU-slot budget. | +| `AGENT_VERIFY_TEST_WORKERS=2` | Set workers per Playwright/Vitest pool, bounded by the slot budget. | +| `AGENT_VERIFY_PARALLEL=1` | Run one task and one test worker at a time. | + +The two API suites are the longest single processes of a run, so locally both are sharded the same way. Coverage in `release-local` comes from the shards' LCOV reports: line coverage merges exactly (a line hit in any shard is covered), function coverage can only be combined as the best shard per file because Bun's LCOV carries no per-function records, so when that lower bound alone misses the floor the whole suite runs once through `check-coverage.ts` and its verdict decides. The security spec in detail: its files are packed largest-first into as many shards as there are browser workers (four on a large host), each shard runs on its own database and Valkey index (`app_security_1` to `app_security_4`), and an aggregate task merges the JUnit reports so the evidence and the findings manifest read one run. With one worker (CI) the spec runs whole on `app_security` exactly as before. The Vitest pool for UI tests grows to half the slot budget, capped at eight, since it becomes the long pole once the spec is sharded; `AGENT_VERIFY_TEST_WORKERS` pins both pools. + +Stateful lanes own separate Postgres databases and Valkey indexes inside the sandbox: `app_tests`, `app_security` (or its shards), and `app_e2e`. Only the selected profile's databases are prepared. `release-local` runs the API suite once with coverage; the same execution provides JUnit inventory evidence and enforces the coverage and warning gates. Email templates are prepared before readers start, and subsequent builds leave identical artifacts untouched. + +Agent verification emits a coverage summary and `lcov.info`, without generating HTML pages; Vitest still enforces every configured coverage threshold. Standalone `test:ci` retains its configured reporters. Bundle budgets retain their exact gzip byte limits and set `running: false` to avoid unbudgeted headless-Chrome execution measurements; generated translation budgets use the same setting. + +Typechecking stores incremental project information in ignored `node_modules/.cache/tsc` files, with separate files for API, UI, and tooling. TypeScript still checks affected dependents after edits. Generated-code validation uses a fresh in-memory program. Deleting the cache forces a cold typecheck; no verification pass is reused or inferred from a cache. + +JSON results retain a fixed check order and add `execution` with the selected budget and each task's `startedAfterMs`, `durationMs`, and status. Start offsets include dependency and capacity waiting, making it possible to distinguish a late start from slow execution. UI test, browser, and runtime startup durations are included in check results. Sandbox startup (`sandbox:up`) is outside verification timing; reuse an owned sandbox across iterations and remove it when the task is finished. + +The automatic budget assumes this verification is the main workload. On a busy host or when several checkouts verify simultaneously, lower `AGENT_VERIFY_PARALLEL` to share resources; memory pressure can slow down every lane and trigger test deadlines. Failed commands retain private diagnostic logs under ignored `.agent-state/verification`, and failed browser runs retain a private `.agent-state/playwright-*.xml.log`. Separate checkouts and sandboxes can run concurrently. Operations that generate/build/verify in the same checkout are serialized by a checkout lock, even with different sandbox IDs. A lease prevents overlapping verification suites on the same ID. `sandbox:down` refuses live leases, checks labels again, and removes only that run's containers and volumes. After an interrupted process exits, `sandbox:down -- --id=` can reclaim its stale lease. It never globally prunes Docker or flushes a shared cache. If Docker cleanup fails, preserve the descriptor and retry the same ID. diff --git a/tools/agent/checks.ts b/tools/agent/checks.ts index 50eabb69..af15df3b 100644 --- a/tools/agent/checks.ts +++ b/tools/agent/checks.ts @@ -9,21 +9,61 @@ export interface ICommandCheck { lane?: SandboxLaneName; /** Runs only after the named check has finished (it reads that check's output). */ after?: string; + priority?: number; } +/** Constituent scripts must exactly cover each aggregate gate (checked by contracts.test.ts). */ +export const CHECK_GROUPS = [ + { + id: "tooling.quality", + app: "root", + script: "agent:quality", + parts: ["agent:typecheck", "agent:lint", "agent:format:check"], + }, + { + id: "api.check", + app: "api", + script: "check", + parts: ["typecheck", "lint", "lint:meta", "check:lint-meta-docs", "knip"], + }, + { + id: "ui.check", + app: "ui", + script: "check", + parts: [ + "lint", + "lint:meta", + "check:lint-meta-docs", + "format:check", + "typecheck", + "knip", + ], + }, +] as const; /** References existing package scripts; parity tests reject drift. No arbitrary shell fragments. */ export const STATIC_CHECKS: readonly ICommandCheck[] = [ - { id: "tooling.quality", app: "root", script: "agent:quality" }, - { id: "api.check", app: "api", script: "check" }, - { id: "ui.check", app: "ui", script: "check" }, + ...CHECK_GROUPS.flatMap((group) => + group.parts.map((script) => ({ + id: `${group.id}.${script}`, + app: group.app, + script, + priority: + script.includes("typecheck") || script.endsWith("lint") ? 30 : 0, + })) + ), { id: "acl.drift", app: "api", script: "generate:acl-types:check" }, { id: "api.scripts", app: "api", script: "check:scripts-docs" }, { id: "ui.scripts", app: "ui", script: "check:scripts-docs" }, { id: "docs.data", app: "docs", script: "check:docs-data" }, ]; export const RELEASE_CHECKS: readonly ICommandCheck[] = [ - { id: "api.coverage", app: "api", script: "test:coverage", lane: "coverage" }, { id: "api.build", app: "api", script: "build" }, - { id: "ui.build", app: "ui", script: "build", nodeEnv: "production" }, + { + id: "ui.build", + app: "ui", + script: "build", + nodeEnv: "production", + priority: 40, + }, { id: "ui.bundle", app: "ui", script: "size:check", after: "ui.build" }, { id: "ui.modulepreload", @@ -31,7 +71,13 @@ export const RELEASE_CHECKS: readonly ICommandCheck[] = [ script: "size:check:modulepreload", after: "ui.build", }, - { id: "docs.build", app: "docs", script: "build:ci", nodeEnv: "production" }, + { + id: "docs.build", + app: "docs", + script: "build:ci", + nodeEnv: "production", + priority: 40, + }, ]; export const PROFILES = [ "openapi", diff --git a/tools/agent/contracts.test.ts b/tools/agent/contracts.test.ts index d4888fae..d8f3ee0e 100644 --- a/tools/agent/contracts.test.ts +++ b/tools/agent/contracts.test.ts @@ -2,7 +2,7 @@ import { expect, test } from "bun:test"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { RELEASE_CHECKS, STATIC_CHECKS } from "./checks"; +import { CHECK_GROUPS, RELEASE_CHECKS, STATIC_CHECKS } from "./checks"; import { runProcess } from "./process"; import { testEvidence } from "./reports"; import { inspectTask } from "./tasks"; @@ -10,6 +10,29 @@ import { isRecord, parseRecord } from "./validation"; const root = fileURLToPath(new URL("../../", import.meta.url)); +test("parallel static tasks preserve every constituent of the aggregate package gates", () => { + for (const group of CHECK_GROUPS) { + const path = + group.app === "root" + ? join(root, "package.json") + : join(root, "apps", group.app, "package.json"); + const scripts = parseRecord(readFileSync(path, "utf8")).scripts; + + if (!isRecord(scripts)) { + throw new Error("Package scripts missing"); + } + + expect(scripts[group.script]).toBe( + group.parts.map((script) => `bun run ${script}`).join(" && ") + ); + expect( + STATIC_CHECKS.filter((check) => check.id.startsWith(`${group.id}.`)).map( + (check) => check.script + ) + ).toEqual([...group.parts]); + } +}); + test("every check adapter references an existing package script", () => { for (const check of [...STATIC_CHECKS, ...RELEASE_CHECKS]) { if (check.app === "docs" && !existsSync(join(root, "apps/docs"))) { diff --git a/tools/agent/coverage.test.ts b/tools/agent/coverage.test.ts new file mode 100644 index 00000000..96ea94c8 --- /dev/null +++ b/tools/agent/coverage.test.ts @@ -0,0 +1,83 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runProcess } from "./process"; +import { testEvidence } from "./reports"; + +const ROOT = fileURLToPath(new URL("../../", import.meta.url)); + +test("one coverage execution provides JUnit evidence without weakening quality gates", async () => { + const directory = mkdtempSync(join(tmpdir(), "bs-coverage-gate-")); + const scenarios = [ + { output: "All files | 80.00 | 75.00 |", code: 0, expected: "passed" }, + { output: "All files | 60.00 | 75.00 |", code: 0, expected: "failed" }, + { + output: "All files | 80.00 | 75.00 |\nwarn: forbidden", + code: 0, + expected: "failed", + }, + { output: "No coverage table", code: 0, expected: "blocked" }, + { output: "All files | Infinity | 75.00 |", code: 0, expected: "blocked" }, + { output: "All files | 80.00 | 75.00 |", code: 1, expected: "blocked" }, + ] as const; + + try { + for (const scenario of scenarios) { + const report = join(directory, "tests.xml"); + const calls = join(directory, "calls.json"); + const fakeBun = `#!${process.execPath} +import { appendFileSync, writeFileSync } from "node:fs"; +const args = process.argv.slice(2); +appendFileSync(${JSON.stringify(calls)}, JSON.stringify(args) + "\\n"); +const report = args.find(arg => arg.startsWith("--reporter-outfile=")); +if (!report) throw new Error("JUnit report argument was not forwarded"); +writeFileSync(report.slice("--reporter-outfile=".length), ''); +console.log(${JSON.stringify(scenario.output)}); +process.exit(${String(scenario.code)}); +`; + + rmSync(calls, { force: true }); + rmSync(report, { force: true }); + writeFileSync(join(directory, "bun"), fakeBun, { mode: 0o700 }); + const result = await runProcess( + [ + process.execPath, + "--no-env-file", + "scripts/quality/check-coverage.ts", + "--reporter=junit", + `--reporter-outfile=${report}`, + ], + { + cwd: join(ROOT, "apps/api"), + env: { PATH: directory, AGENT_SANDBOX: "1" }, + timeoutMs: 10_000, + } + ); + + expect(result.status).toBe("completed"); + const evidence = testEvidence( + "api.tests", + readFileSync(report, "utf8"), + result.code, + "bun", + result.code === 86 + ); + + expect(evidence.status).toBe(scenario.expected); + expect(readFileSync(calls, "utf8")).toBe( + JSON.stringify([ + "--no-env-file", + "test", + "tests", + "--coverage", + "--reporter=junit", + `--reporter-outfile=${report}`, + ]) + "\n" + ); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}, 90_000); diff --git a/tools/agent/environment.ts b/tools/agent/environment.ts index 89f261db..9afdc46c 100644 --- a/tools/agent/environment.ts +++ b/tools/agent/environment.ts @@ -18,3 +18,12 @@ export function openApiUrl(): string { export function dockerTestsEnabled(): boolean { return process.env.AGENT_DOCKER_TESTS === "true"; } + +/** Capture real caller policy before isolated child processes receive CI=true. */ +export function verificationEnvironment(): Record { + return { + CI: process.env.CI, + AGENT_VERIFY_PARALLEL: process.env.AGENT_VERIFY_PARALLEL, + AGENT_VERIFY_TEST_WORKERS: process.env.AGENT_VERIFY_TEST_WORKERS, + }; +} diff --git a/tools/agent/generate/typecheck.ts b/tools/agent/generate/typecheck.ts index 496766b7..1d3ee8fd 100644 --- a/tools/agent/generate/typecheck.ts +++ b/tools/agent/generate/typecheck.ts @@ -67,7 +67,12 @@ export function validateGeneratedTypes( ); const program = ts.createProgram({ rootNames: [...new Set([...parsed.fileNames, ...addedFiles])], - options: { ...parsed.options, noEmit: true, incremental: false }, + options: { + ...parsed.options, + noEmit: true, + incremental: false, + tsBuildInfoFile: undefined, + }, host, }); const errors = [ diff --git a/tools/agent/junit.test.ts b/tools/agent/junit.test.ts new file mode 100644 index 00000000..f24ba0c5 --- /dev/null +++ b/tools/agent/junit.test.ts @@ -0,0 +1,91 @@ +import { expect, test } from "bun:test"; +import { mergeJunitReports, shardFiles, suiteDurations } from "./junit"; +import { testEvidence } from "./reports"; +import { shardLane, SANDBOX_LANES } from "./sandbox/lifecycle"; + +const report = (name: string, failures: number): string => + ` + + + + ${ + failures > 0 + ? '' + : "" + } + + +`; + +test("merged shard reports read as one passing run", () => { + const merged = mergeJunitReports([report("f01", 0), report("f02", 0)]); + const evidence = testEvidence("security.tests", merged, 0); + + expect((merged.match(/ { + const merged = mergeJunitReports([report("f01", 0), report("f02", 1)]); + + expect(testEvidence("security.tests", merged, 1).status).toBe("failed"); + expect(mergeJunitReports([report("f01", 0), "not xml"])).toBe(""); + expect(testEvidence("security.tests", "", 0).status).toBe("blocked"); +}); + +test("files pack largest-first into balanced, deterministic shards", () => { + const files = [ + { path: "a", size: 10 }, + { path: "b", size: 9 }, + { path: "c", size: 8 }, + { path: "d", size: 1 }, + { path: "e", size: 1 }, + ]; + const shards = shardFiles(files, 3); + + expect(shards).toEqual([["a"], ["b", "e"], ["c", "d"]]); + expect(shardFiles(files, 1)).toEqual([["a", "b", "c", "d", "e"]]); + expect(shardFiles([], 3)).toEqual([]); + expect(shardFiles(files, 3)).toEqual(shards); +}); + +test("shard lanes are distinct from each other and from the fixed lanes", () => { + const lanes = [ + ...[1, 2, 3, 4].map((index) => shardLane("security", index, 4)), + ...[1, 2, 3, 4].map((index) => shardLane("tests", index, 4)), + ]; + const fixed = Object.values(SANDBOX_LANES); + const keys = [...lanes, ...fixed].map( + (lane) => `${lane.database}/${String(lane.valkeyDb)}` + ); + + expect(new Set(keys).size).toBe(keys.length); + expect(lanes[0]).toEqual({ + name: "security-1", + database: "app_security_1", + valkeyDb: 5, + }); + expect(lanes[4]).toEqual({ + name: "tests-1", + database: "app_tests_1", + valkeyDb: 9, + }); + expect(shardLane("security", 1, 1)).toBe(SANDBOX_LANES.security); + expect(shardLane("tests", 1, 1)).toBe(SANDBOX_LANES.tests); + expect(() => shardLane("security", 5, 4)).toThrow("Invalid shard"); + expect(() => shardLane("tests", 1, 5)).toThrow("Invalid shard"); +}); + +test("suite durations come from the per-file testsuite time attributes", () => { + const merged = mergeJunitReports([ + report("f01", 0), + report("f02", 0), + ]).replace( + '', + '' + ); + + expect(suiteDurations(merged)).toEqual({ "security-spec/f01.test.ts": 41.5 }); + expect(suiteDurations("")).toEqual({}); +}); diff --git a/tools/agent/junit.ts b/tools/agent/junit.ts new file mode 100644 index 00000000..b2963568 --- /dev/null +++ b/tools/agent/junit.ts @@ -0,0 +1,76 @@ +/** + * Joins the JUnit reports of test shards into one document. Each report is a + * `` root; the suites inside keep their attributes, so evidence + * and manifest checks read the merged file exactly as they read a single run. + */ +export function mergeJunitReports(reports: readonly string[]): string { + const bodies: string[] = []; + + for (const report of reports) { + const open = report.indexOf(""); + + if (open === -1 || close === -1) { + return ""; + } + + const start = report.indexOf(">", open); + + if (start === -1 || start > close) { + return ""; + } + + bodies.push(report.slice(start + 1, close)); + } + + return `\n${bodies.join("")}\n`; +} + +/** + * Greedy longest-first packing of files into `count` shards by size, a + * stand-in for duration. Deterministic for a given file list. + */ +export function shardFiles( + files: readonly { readonly path: string; readonly size: number }[], + count: number +): string[][] { + const shards: { size: number; paths: string[] }[] = Array.from( + { length: Math.max(1, count) }, + () => ({ size: 0, paths: [] }) + ); + const ordered = [...files].sort((left, right) => + right.size === left.size + ? left.path.localeCompare(right.path) + : right.size - left.size + ); + + for (const file of ordered) { + const target = shards.reduce((best, shard) => + shard.size < best.size ? shard : best + ); + + target.size += file.size; + target.paths.push(file.path); + } + + return shards + .map((shard) => shard.paths.sort()) + .filter((paths) => paths.length > 0); +} + +/** Seconds per file from the `testsuite` elements bun writes, one per file. */ +export function suiteDurations(xml: string): Record { + const durations: Record = {}; + + for (const match of xml.matchAll(/]*)>/g)) { + const attributes = match[1] ?? ""; + const file = /\bfile="([^"]+)"/.exec(attributes)?.[1]; + const time = Number(/\btime="([^"]+)"/.exec(attributes)?.[1]); + + if (file !== undefined && Number.isFinite(time) && time >= 0) { + durations[file] = (durations[file] ?? 0) + time; + } + } + + return durations; +} diff --git a/tools/agent/lanes.test.ts b/tools/agent/lanes.test.ts index 4c5abd4a..f9d253f3 100644 --- a/tools/agent/lanes.test.ts +++ b/tools/agent/lanes.test.ts @@ -82,8 +82,8 @@ test("concurrency honours AGENT_VERIFY_PARALLEL and otherwise stays in band", () expect(defaultConcurrency({ AGENT_VERIFY_PARALLEL: "9" })).toBe(9); expect( defaultConcurrency({ AGENT_VERIFY_PARALLEL: "zero" }) - ).toBeGreaterThanOrEqual(2); - expect(defaultConcurrency({})).toBeLessThanOrEqual(6); + ).toBeGreaterThanOrEqual(1); + expect(defaultConcurrency({})).toBeLessThanOrEqual(24); }); test("checks report in the declared order regardless of completion order", () => { diff --git a/tools/agent/lanes.ts b/tools/agent/lanes.ts index 8d3d4744..48c0cc9a 100644 --- a/tools/agent/lanes.ts +++ b/tools/agent/lanes.ts @@ -1,32 +1,13 @@ -import { availableParallelism } from "node:os"; +import { executionBudget } from "./scheduling"; import type { ICheckResult } from "./result"; import { isAborted } from "./validation"; export type Lane = () => Promise; -const MIN_PARALLEL = 2; -const MAX_PARALLEL = 6; -const CORES_PER_LANE = 3; - -/** - * Each lane is a whole toolchain process (type-aware ESLint, vitest workers, - * a Playwright run), not a single thread, so the default leaves roughly - * three cores per lane and stays inside a modest band. `AGENT_VERIFY_PARALLEL` - * overrides it; `1` restores strictly sequential runs. - */ export function defaultConcurrency( - env: Record = process.env + env?: Record ): number { - const requested = Number(env.AGENT_VERIFY_PARALLEL ?? ""); - - if (Number.isInteger(requested) && requested >= 1) { - return requested; - } - - return Math.max( - MIN_PARALLEL, - Math.min(MAX_PARALLEL, Math.floor(availableParallelism() / CORES_PER_LANE)) - ); + return executionBudget(env).slots; } /** diff --git a/tools/agent/lcov.test.ts b/tools/agent/lcov.test.ts new file mode 100644 index 00000000..191e9f1c --- /dev/null +++ b/tools/agent/lcov.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from "bun:test"; +import { coverageVerdict, formatCoverage, mergeLcov } from "./lcov"; + +const record = ( + file: string, + lines: [number, number][], + fnf: number, + fnh: number +): string => + [ + "TN:", + `SF:${file}`, + `FNF:${String(fnf)}`, + `FNH:${String(fnh)}`, + ...lines.map(([line, hits]) => `DA:${String(line)},${String(hits)}`), + `LF:${String(lines.length)}`, + `LH:${String(lines.filter(([, hits]) => hits > 0).length)}`, + "end_of_record", + ].join("\n"); + +test("lines merge as a union across shards, functions as the best shard per file", () => { + const shardA = record( + "src/a.ts", + [ + [1, 1], + [2, 0], + [3, 0], + ], + 4, + 2 + ); + const shardB = + record( + "src/a.ts", + [ + [1, 0], + [2, 1], + [3, 0], + ], + 4, + 3 + ) + + "\n" + + record("src/b.ts", [[1, 1]], 1, 1); + const summary = mergeLcov([shardA, shardB]); + + // a.ts: 2 of 3 lines after the union, best shard 3 of 4 functions; b.ts: 1/1 and 1/1. + expect(summary.files).toBe(2); + expect(summary.linePct).toBeCloseTo((2 / 3 + 1) / 2, 6); + expect(summary.functionPct).toBeCloseTo((3 / 4 + 1) / 2, 6); + expect(formatCoverage(summary)).toContain("lines 83.33%"); +}); + +test("the verdict fails on lines, only asks for confirmation on the function bound", () => { + const fine = { linePct: 0.9, functionPct: 0.9, files: 1 }; + const lowLines = { linePct: 0.1, functionPct: 0.9, files: 1 }; + const lowFunctionBound = { linePct: 0.9, functionPct: 0.2, files: 1 }; + + expect(coverageVerdict(fine)).toBe("passed"); + expect(coverageVerdict(lowLines)).toBe("lines_below_floor"); + expect(coverageVerdict(lowFunctionBound)).toBe("functions_unconfirmed"); + expect(coverageVerdict(mergeLcov([]))).toBe("lines_below_floor"); +}); + +test("a file without functions counts as fully covered for functions, as Bun does", () => { + const summary = mergeLcov([record("src/constants.ts", [[1, 1]], 0, 0)]); + + expect(summary.functionPct).toBe(1); +}); diff --git a/tools/agent/lcov.ts b/tools/agent/lcov.ts new file mode 100644 index 00000000..f713ac45 --- /dev/null +++ b/tools/agent/lcov.ts @@ -0,0 +1,119 @@ +import { + MIN_FUNCTION, + MIN_LINE, +} from "../../apps/api/scripts/quality/coverage-thresholds"; + +export interface ICoverageSummary { + /** Mean of per-file line percentages, which is how Bun's "All files" row is computed. */ + linePct: number; + /** Same mean for functions; across shards a lower bound (see mergeLcov). */ + functionPct: number; + files: number; +} + +interface IFileCoverage { + lines: Map; + functionsTotal: number; + functionsCovered: number; +} + +/** + * Merges the LCOV reports of test shards into the figure Bun's own text + * reporter prints: the unweighted mean of per-file percentages (verified + * against `bun test --coverage` on the same run). Lines are exact: a line + * counts as covered when any shard hit it, exactly as a single run would. + * Bun's LCOV carries only per-file function counts (FNF/FNH, no FN/FNDA), + * so functions can only be combined as the best single shard per file: a + * lower bound, never an overstatement. Callers that see the bound miss the + * floor must confirm with a whole-suite run before failing. + */ +export function mergeLcov(reports: readonly string[]): ICoverageSummary { + const files = new Map(); + + for (const report of reports) { + let current: IFileCoverage | undefined; + let functionsCovered = 0; + let functionsTotal = 0; + + for (const rawLine of report.split("\n")) { + const line = rawLine.trim(); + + if (line.startsWith("SF:")) { + const path = line.slice(3); + const existing = files.get(path) ?? { + lines: new Map(), + functionsTotal: 0, + functionsCovered: 0, + }; + + files.set(path, existing); + current = existing; + functionsCovered = 0; + functionsTotal = 0; + } else if (line.startsWith("DA:") && current !== undefined) { + const [lineNumber, hits] = line.slice(3).split(","); + const number = Number(lineNumber); + const hit = Number(hits) > 0; + + if (Number.isSafeInteger(number)) { + current.lines.set( + number, + (current.lines.get(number) ?? false) || hit + ); + } + } else if (line.startsWith("FNF:")) { + functionsTotal = Number(line.slice(4)); + } else if (line.startsWith("FNH:")) { + functionsCovered = Number(line.slice(4)); + } else if (line === "end_of_record" && current !== undefined) { + current.functionsTotal = Math.max( + current.functionsTotal, + functionsTotal + ); + current.functionsCovered = Math.max( + current.functionsCovered, + functionsCovered + ); + current = undefined; + } + } + } + + const withLines = [...files.values()].filter((file) => file.lines.size > 0); + const linePcts = withLines.map( + (file) => [...file.lines.values()].filter(Boolean).length / file.lines.size + ); + const functionPcts = withLines.map((file) => + file.functionsTotal === 0 ? 1 : file.functionsCovered / file.functionsTotal + ); + const mean = (values: readonly number[]): number => + values.length === 0 + ? 0 + : values.reduce((total, value) => total + value, 0) / values.length; + + return { + linePct: mean(linePcts), + functionPct: mean(functionPcts), + files: withLines.length, + }; +} + +export type CoverageVerdict = + "passed" | "lines_below_floor" | "functions_unconfirmed"; + +/** Lines decide directly; a function bound under the floor only asks for confirmation. */ +export function coverageVerdict(summary: ICoverageSummary): CoverageVerdict { + if (summary.files === 0 || summary.linePct < MIN_LINE) { + return "lines_below_floor"; + } + + return summary.functionPct < MIN_FUNCTION + ? "functions_unconfirmed" + : "passed"; +} + +export function formatCoverage(summary: ICoverageSummary): string { + const pct = (value: number): string => `${(value * 100).toFixed(2)}%`; + + return `lines ${pct(summary.linePct)}, functions >= ${pct(summary.functionPct)} across ${String(summary.files)} files (mean of per-file rates, as Bun reports)`; +} diff --git a/tools/agent/profile-runner.ts b/tools/agent/profile-runner.ts index 7285fc03..631049fc 100644 --- a/tools/agent/profile-runner.ts +++ b/tools/agent/profile-runner.ts @@ -1,34 +1,45 @@ -import { existsSync, readFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + statSync, + writeFileSync, +} from "node:fs"; import { join } from "node:path"; import { type ICommandCheck } from "./checks"; -import type { Lane } from "./lanes"; +import type { ITask } from "./scheduler"; +import type { IExecutionBudget } from "./scheduling"; +import { mergeJunitReports, shardFiles, suiteDurations } from "./junit"; +import { coverageVerdict, formatCoverage, mergeLcov } from "./lcov"; import { hostEnvironment } from "./environment"; import { inventoryEvidence } from "./inventory"; -import { runProcess } from "./process"; +import { runProcess, type IProcessResult } from "./process"; import { testEvidence } from "./reports"; import type { ICheckResult, IVerificationResult } from "./result"; import { acquireLease } from "./sandbox/lease"; -import { runLanes } from "./lanes"; import { ensureLaneDatabases, inspectSandbox, SANDBOX_LANES, sandboxEnv, - EXTRA_LANES, + shardLane, type ISandbox, - type SandboxLaneName, + type ISandboxLane, + type ShardedSuite, } from "./sandbox/lifecycle"; import { securityManifestEvidence } from "./security-evidence"; -import { isAborted } from "./validation"; +import { isAborted, isRecord, parseRecord } from "./validation"; const NO_ENV_FILE = "--no-env-file"; +const API_COVERAGE = "api.coverage"; export class ProfileRunner { state: ISandbox | undefined; releaseLease: (() => void) | undefined; - private env: Record = { + private readonly env: Record = { ...hostEnvironment(), CI: "true", DOTENV_CONFIG_PATH: "/dev/null", @@ -40,16 +51,28 @@ export class ProfileRunner { private readonly result: IVerificationResult, private readonly signal?: AbortSignal, private readonly sandboxId?: string, - readonly concurrency = 1 + readonly budget: IExecutionBudget = { + slots: 1, + testWorkers: 1, + uiTestWorkers: 1, + securityShards: 1, + apiShards: 1, + } ) {} + /** Shard reports collected for each suite's aggregate evidence. */ + private readonly shardReports: Record< + ShardedSuite, + Map + > = { security: new Map(), tests: new Map() }; + /** Environment for a stateful lane: its own database and Valkey index. */ - laneEnv(lane: SandboxLaneName): Record { + laneEnv(lane: ISandboxLane): Record { if (this.state === undefined) { throw new Error("Sandbox absent"); } - return sandboxEnv(this.state, SANDBOX_LANES[lane]); + return sandboxEnv(this.state, lane); } add(check: ICheckResult): void { @@ -57,6 +80,24 @@ export class ProfileRunner { process.stderr.write(`${check.checkId}: ${check.status}\n`); } + private logFailure(id: string, run: IProcessResult): void { + const directory = join( + this.root, + ".agent-state", + "verification", + this.result.runId + ); + + mkdirSync(directory, { recursive: true, mode: 0o700 }); + const path = join( + directory, + `${id.replaceAll(/[^a-zA-Z0-9.-]/g, "_")}.log` + ); + + writeFileSync(path, run.stdout + run.stderr, { mode: 0o600 }); + process.stderr.write(`${id}: output saved to ${path}\n`); + } + async command( id: string, app: string, @@ -87,6 +128,10 @@ export class ProfileRunner { this.add(check); + if (check.status !== "passed") { + this.logFailure(id, run); + } + return check; } @@ -101,7 +146,10 @@ export class ProfileRunner { return; } - const base = check.lane === undefined ? this.env : this.laneEnv(check.lane); + const base = + check.lane === undefined + ? this.env + : this.laneEnv(SANDBOX_LANES[check.lane]); await this.command( check.id, @@ -111,53 +159,235 @@ export class ProfileRunner { ); } - async tests(security: boolean): Promise { - const report = join(this.temp, security ? "security.xml" : "api.xml"); - const laneEnv = this.laneEnv(security ? "security" : "tests"); + private suiteEnv( + suite: ShardedSuite, + lane: ISandboxLane + ): Record { + return suite === "security" + ? { + ...this.laneEnv(lane), + SECURITY_SPEC: "true", + ACCOUNT_DOMAIN_CLAIMING: "true", + GOOGLE_OAUTH_CLIENT_ID: "spec-google-client-id", + GOOGLE_OAUTH_CLIENT_SECRET: "spec-google-client-secret", + } + : { ...this.laneEnv(lane), SECURITY_SPEC: "false" }; + } + + private async bunTests( + argv: readonly string[], + report: string, + env: Record + ): Promise<{ run: IProcessResult; xml: string }> { const run = await runProcess( [ process.execPath, NO_ENV_FILE, "run", - "scripts/quality/run-tests-clean.ts", - security ? "security-spec" : "tests", + ...argv, "--reporter=junit", `--reporter-outfile=${report}`, ], - { - cwd: join(this.root, "apps/api"), - env: { - ...laneEnv, - SECURITY_SPEC: security ? "true" : "false", - ...(security - ? { - ACCOUNT_DOMAIN_CLAIMING: "true", - GOOGLE_OAUTH_CLIENT_ID: "spec-google-client-id", - GOOGLE_OAUTH_CLIENT_SECRET: "spec-google-client-secret", - } - : {}), - }, - signal: this.signal, - } + { cwd: join(this.root, "apps/api"), env, signal: this.signal } + ); + + return { + run, + xml: existsSync(report) ? readFileSync(report, "utf8") : "", + }; + } + + private timingsPath(suite: ShardedSuite): string { + return join( + this.root, + ".agent-state", + "verification", + "timings", + `${suite}.json` + ); + } + + /** Per-file seconds recorded by the previous run; empty on a fresh checkout. */ + recordedDurations(suite: ShardedSuite): Record { + const path = this.timingsPath(suite); + + if (!existsSync(path)) { + return {}; + } + + try { + const value = parseRecord(readFileSync(path, "utf8")).files; + + return isRecord(value) + ? Object.fromEntries( + Object.entries(value).filter( + (entry): entry is [string, number] => + typeof entry[1] === "number" && Number.isFinite(entry[1]) + ) + ) + : {}; + } catch { + return {}; + } + } + + private recordDurations(suite: ShardedSuite, xml: string): void { + const files = suiteDurations(xml); + + if (Object.keys(files).length === 0) { + return; + } + + mkdirSync(join(this.timingsPath(suite), ".."), { + recursive: true, + mode: 0o700, + }); + writeFileSync( + this.timingsPath(suite), + JSON.stringify({ schemaVersion: 1, files }, null, 2), + { mode: 0o600 } + ); + } + + private listTestFiles(directory: string, prefix: string): string[] { + return readdirSync(join(this.root, "apps/api", directory), { + withFileTypes: true, + }).flatMap((entry) => + entry.isDirectory() + ? this.listTestFiles( + `${directory}/${entry.name}`, + `${prefix}/${entry.name}` + ) + : entry.name.endsWith(".test.ts") + ? [`${prefix}/${entry.name}`] + : [] ); - const xml = existsSync(report) ? readFileSync(report, "utf8") : ""; + } + + /** + * Suite files packed into `count` shards, longest first. Duration comes + * from the previous run's JUnit report; a file without a record is assumed + * to be as long as the median recorded file, and size decides ties, so a + * fresh checkout still gets a sensible split. + */ + shardedFiles(suite: ShardedSuite, count: number): string[][] { + const directory = suite === "security" ? "security-spec" : "tests"; + const recorded = this.recordedDurations(suite); + const known = Object.values(recorded).sort((left, right) => left - right); + const median = known[Math.floor(known.length / 2)] ?? 1; + const files = this.listTestFiles(directory, directory).map((path) => ({ + path, + // Seconds dominate; bytes only order files of equal duration. + size: + (recorded[path] ?? median) * 1_000_000 + + statSync(join(this.root, "apps/api", path)).size, + })); + + return shardFiles(files, count); + } + + /** + * One shard of a suite on its own lane. With a single shard this is the + * whole suite and records its evidence directly. + */ + async shard( + suite: ShardedSuite, + index: number, + count: number, + files: readonly string[], + coverage = false + ): Promise { + const lane = shardLane(suite, index, count); + const coverageDir = join(this.temp, `coverage-${suite}-${String(index)}`); + const { run, xml } = await this.bunTests( + [ + "scripts/quality/run-tests-clean.ts", + ...files, + ...(coverage + ? [ + "--coverage", + "--coverage-reporter=lcov", + `--coverage-dir=${coverageDir}`, + ] + : []), + ], + join(this.temp, `${suite}-${String(index)}.xml`), + this.suiteEnv(suite, lane) + ); + const lcovPath = join(coverageDir, "lcov.info"); + + this.shardReports[suite].set(index, { + xml, + run, + lcov: + coverage && existsSync(lcovPath) ? readFileSync(lcovPath, "utf8") : "", + }); + + if (count === 1) { + await this.aggregate(suite, 1, coverage); + + return; + } + + const checkId = `${suite === "security" ? "security.tests" : "api.tests"}.${String(index)}`; const check = run.status === "blocked" - ? { - checkId: security ? "security.tests" : "api.tests", - status: "blocked" as const, - reason: run.reason, - } + ? { checkId, status: "blocked" as const, reason: run.reason } + : testEvidence(checkId, xml, run.code, "bun", run.code === 86); + + this.add({ ...check, durationMs: run.durationMs }); + + if (check.status !== "passed") { + this.logFailure(checkId, run); + } + } + + /** Merges the shard reports so evidence, inventory, manifest and coverage see one run. */ + async aggregate( + suite: ShardedSuite, + count: number, + coverage = false + ): Promise { + const checkId = suite === "security" ? "security.tests" : "api.tests"; + const shards = Array.from({ length: count }, (_, offset) => + this.shardReports[suite].get(offset + 1) + ); + const present = shards.flatMap((shard) => + shard === undefined ? [] : [shard] + ); + + if (present.length !== count) { + for (const id of [ + checkId, + ...(suite === "security" ? ["security.manifest"] : []), + ...(coverage ? [API_COVERAGE] : []), + ]) { + this.add({ checkId: id, status: "blocked", reason: "shard_missing" }); + } + + return; + } + + const runs = present.map((shard) => shard.run); + const xml = mergeJunitReports(present.map((shard) => shard.xml)); + const blocked = runs.find((run) => run.status === "blocked"); + const exit = runs.reduce( + (worst, run) => + worst === null || run.code === null ? null : Math.max(worst, run.code), + 0 + ); + const check = + blocked !== undefined + ? { checkId, status: "blocked" as const, reason: blocked.reason } : testEvidence( - security ? "security.tests" : "api.tests", + checkId, xml, - run.code, + exit, "bun", - run.code === 86 + runs.some((run) => run.code === 86) ); - - this.add({ - ...(security + const evidence = + suite === "security" ? check : inventoryEvidence( this.root, @@ -165,15 +395,96 @@ export class ProfileRunner { xml, check, this.result.checkout?.fingerprint - )), - durationMs: run.durationMs, - }); + ); + const durationMs = runs.reduce((total, run) => total + run.durationMs, 0); + + this.add({ ...evidence, durationMs }); + this.recordDurations(suite, xml); + + if (evidence.status !== "passed") { + for (const run of runs) { + this.logFailure(checkId, run); + } + } - if (security) { + if (suite === "security") { this.add( securityManifestEvidence(this.root, xml, check.status !== "blocked") ); } + + if (coverage) { + await this.coverageEvidence( + evidence.status, + present.map((shard) => shard.lcov), + durationMs + ); + } + } + + /** + * Line coverage merges exactly across shards. Function coverage merges as + * a lower bound; when that bound alone misses the floor, the whole suite + * runs once through the single-process gate so the verdict is the real one. + */ + private async coverageEvidence( + tests: ICheckResult["status"], + reports: readonly string[], + durationMs: number + ): Promise { + if (tests !== "passed") { + this.add({ + checkId: API_COVERAGE, + status: tests, + reason: "tests_did_not_pass", + durationMs, + }); + + return; + } + + const summary = mergeLcov(reports); + const verdict = coverageVerdict(summary); + + process.stderr.write(`api.coverage: ${formatCoverage(summary)}\n`); + + if (verdict === "passed") { + this.add({ + checkId: API_COVERAGE, + status: "passed", + reason: "coverage_gate_passed", + durationMs, + }); + + return; + } + + if (verdict === "lines_below_floor") { + this.add({ + checkId: API_COVERAGE, + status: "failed", + reason: "coverage_below_floor", + durationMs, + }); + + return; + } + + const confirmation = await this.command( + API_COVERAGE, + "api", + [ + process.execPath, + NO_ENV_FILE, + "run", + "scripts/quality/check-coverage.ts", + ], + this.suiteEnv("tests", SANDBOX_LANES.tests) + ); + + if (confirmation.status === "passed") { + confirmation.reason = "coverage_gate_passed_after_full_run"; + } } async uiTests(fingerprint: string): Promise { @@ -191,14 +502,18 @@ export class ProfileRunner { "scripts/quality/run-tests-clean.ts", "run", "--coverage", + "--coverage.reporter=text-summary", + "--coverage.reporter=lcovonly", + `--maxWorkers=${String(this.budget.uiTestWorkers)}`, "--reporter=junit", `--outputFile=${report}`, ], { cwd: join(this.root, "apps/ui"), env: this.env, signal: this.signal } ); - this.add( - run.status === "blocked" + this.add({ + durationMs: run.durationMs, + ...(run.status === "blocked" ? { checkId: "ui.tests", status: "blocked", reason: run.reason } : inventoryEvidence( this.root, @@ -212,8 +527,15 @@ export class ProfileRunner { run.code === 86 ), fingerprint - ) - ); + )), + }); + + if ( + this.result.checks.find((check) => check.checkId === "ui.tests") + ?.status !== "passed" + ) { + this.logFailure("ui.tests", run); + } } /** Full-stack adapter starts the selected checkout rather than trusting an ambient server. */ @@ -229,101 +551,93 @@ export class ProfileRunner { featureSandbox, this.signal, fingerprint, - SANDBOX_LANES.e2e + SANDBOX_LANES.e2e, + this.budget.testWorkers )) { this.add(check); } } - /** The feature lanes: API tests, UI tests and the browser run, each on its own state. */ - featureLanes(featureSandbox: ISandbox, fingerprint: string): Lane[] { - return [ - () => this.tests(false), - () => this.uiTests(fingerprint), - () => this.e2e(featureSandbox, fingerprint), - ]; - } - - /** - * One lane per script. A check with `after` waits for that check to finish - * (a size gate reads the build it names) but everything else runs as the - * concurrency budget allows. - */ - scriptLanes(checks: readonly ICommandCheck[]): Lane[] { - const done = new Map>(); - const resolvers = new Map void>(); - - for (const check of checks) { - done.set( - check.id, - new Promise((resolve) => { - resolvers.set(check.id, resolve); - }) - ); - } - - return checks.map((check) => async () => { - try { - if (check.after !== undefined) { - await done.get(check.after); - } - - if (!isAborted(this.signal)) { - await this.script(check); + /** Tasks declare their evidence IDs so concurrent completions cannot cross-contaminate status. */ + task( + id: string, + run: () => Promise, + after: readonly string[] = [], + slots = 1, + priority = 0, + evidenceIds: readonly string[] = [id] + ): ITask { + return { + id, + after, + slots, + priority, + run: async () => { + await run(); + + return evidenceIds.every((checkId) => + this.result.checks.some( + (check) => + check.checkId === checkId && + (check.status === "passed" || check.status === "not_applicable") + ) + ); + }, + blocked: (reason) => { + for (const checkId of evidenceIds) { + if (!this.result.checks.some((check) => check.checkId === checkId)) { + this.add({ checkId, status: "blocked", reason }); + } } - } finally { - resolvers.get(check.id)?.(); - } - }); - } - - async scripts(checks: readonly ICommandCheck[]): Promise { - await this.lanes(this.scriptLanes(checks)); + }, + }; } - async lanes(lanes: readonly Lane[]): Promise { - await runLanes(lanes, this.concurrency, this.signal); + scriptTasks(checks: readonly ICommandCheck[]): ITask[] { + return checks.map((check) => + this.task( + check.id, + () => this.script(check), + [ + ...(check.after === undefined ? [] : [check.after]), + ...(check.id === "api.build" ? ["api.templates"] : []), + ], + 1, + check.priority + ) + ); } - async prepareSandbox(): Promise { + async prepareSandbox(lanes: readonly ISandboxLane[]): Promise { if (this.sandboxId === undefined || this.sandboxId === "") { throw new Error("Owned sandbox required"); } this.releaseLease = acquireLease(this.root, this.sandboxId); this.state = await inspectSandbox(this.root, this.sandboxId); - this.env = sandboxEnv(this.state); + await ensureLaneDatabases(this.root, this.state, lanes); this.add({ checkId: "sandbox.ready", status: "passed", reason: "owned_services_ready", }); - await ensureLaneDatabases(this.root, this.state); - - const migrations = await Promise.all( - [undefined, ...EXTRA_LANES].map((lane) => - this.command( - lane === undefined ? "api.migrate" : `api.migrate.${lane}`, - "api", - [process.execPath, NO_ENV_FILE, "run", "db:prepare"], - lane === undefined ? this.env : this.laneEnv(lane) - ) - ) - ); + } - if (migrations.some((migration) => migration.status !== "passed")) { - throw new Error("Migration prerequisite failed"); - } + async migrate(lane: ISandboxLane): Promise { + await this.command( + `api.migrate.${lane.name}`, + "api", + [process.execPath, NO_ENV_FILE, "run", "db:prepare"], + this.laneEnv(lane) + ); + } - const templates = await this.command("api.templates", "api", [ + async templates(): Promise { + await this.command("api.templates", "api", [ process.execPath, NO_ENV_FILE, "run", "build:templates", ]); - - if (templates.status !== "passed") { - throw new Error("Template prerequisite failed"); - } } } diff --git a/tools/agent/profile-tasks.test.ts b/tools/agent/profile-tasks.test.ts new file mode 100644 index 00000000..198bdb43 --- /dev/null +++ b/tools/agent/profile-tasks.test.ts @@ -0,0 +1,168 @@ +import { expect, test } from "bun:test"; +import { fileURLToPath } from "node:url"; +import { ProfileRunner } from "./profile-runner"; +import { aggregateChecks, profileTasks } from "./profile-tasks"; +import type { IVerificationResult } from "./result"; + +const result: IVerificationResult = { + schemaVersion: 1, + runId: "fixture", + profile: "release-local", + startedAt: "", + finishedAt: "", + checkout: null, + source: "owned-sandbox", + status: "blocked", + checks: [], +}; +const ROOT = fileURLToPath(new URL("../../", import.meta.url)); +const runner = new ProfileRunner( + ROOT, + "/unused", + result, + undefined, + "fixture", + { + slots: 14, + testWorkers: 4, + uiTestWorkers: 7, + securityShards: 4, + apiShards: 4, + } +); +const serial = new ProfileRunner( + ROOT, + "/unused", + result, + undefined, + "fixture", + { + slots: 1, + testWorkers: 1, + uiTestWorkers: 1, + securityShards: 1, + apiShards: 1, + } +); + +test("profiles prepare only used lanes and release tests supply coverage once", () => { + const release = profileTasks(runner, "release-local", "fixture"); + const feature = profileTasks(runner, "feature", "fixture"); + const security = profileTasks(runner, "security", "fixture"); + const migrations = (tasks: typeof release): string[] => + tasks + .filter((task) => task.id.startsWith("api.migrate.")) + .map((task) => task.id); + + expect(migrations(release)).toEqual([ + "api.migrate.security-1", + "api.migrate.security-2", + "api.migrate.security-3", + "api.migrate.security-4", + "api.migrate.tests-1", + "api.migrate.tests-2", + "api.migrate.tests-3", + "api.migrate.tests-4", + "api.migrate.e2e", + ]); + expect(migrations(feature)).toEqual([ + "api.migrate.tests-1", + "api.migrate.tests-2", + "api.migrate.tests-3", + "api.migrate.tests-4", + "api.migrate.e2e", + ]); + expect(migrations(security)).toEqual([ + "api.migrate.security-1", + "api.migrate.security-2", + "api.migrate.security-3", + "api.migrate.security-4", + ]); + expect(release.filter((task) => task.id === "api.tests")).toHaveLength(1); + expect( + release.filter((task) => /^api\.tests\.\d$/.test(task.id)) + ).toHaveLength(4); + expect(release.find((task) => task.id === "api.tests")?.after).toEqual([ + "api.tests.1", + "api.tests.2", + "api.tests.3", + "api.tests.4", + ]); + expect(release.some((task) => task.id === "api.coverage")).toBe(false); + expect( + profileTasks(serial, "release-local", "fixture").filter((task) => + task.id.startsWith("api.tests") + ) + ).toHaveLength(1); + expect(release.find((task) => task.id === "ui.e2e")?.slots).toBe(4); + expect(release.find((task) => task.id === "ui.tests")?.slots).toBe(7); + expect(release.find((task) => task.id === "ui.tests")?.after).toEqual([]); + expect(release.find((task) => task.id === "ui.build")?.after).toEqual([]); + expect(release.find((task) => task.id === "api.build")?.after).toContain( + "api.templates" + ); + expect(release.find((task) => task.id === "ui.bundle")?.after).toEqual([ + "ui.build", + ]); +}); + +test("missing aggregate constituents block evidence instead of implying a complete check", () => { + expect(aggregateChecks([]).every((check) => check.status === "blocked")).toBe( + true + ); +}); + +test("the security spec shards across lanes and aggregates, or runs whole with one worker", () => { + const sharded = profileTasks(runner, "security", "fixture"); + const shardTasks = sharded.filter((task) => + /^security\.tests\.\d$/.test(task.id) + ); + const aggregate = sharded.find((task) => task.id === "security.tests"); + + expect(shardTasks.map((task) => task.id)).toEqual([ + "security.tests.1", + "security.tests.2", + "security.tests.3", + "security.tests.4", + ]); + expect(shardTasks[2]?.after).toEqual([ + "api.migrate.security-3", + "api.templates", + ]); + expect(aggregate?.after).toEqual(shardTasks.map((task) => task.id)); + + const whole = profileTasks(serial, "security", "fixture"); + + expect( + whole.filter((task) => task.id.startsWith("security.tests")) + ).toHaveLength(1); + expect(whole.find((task) => task.id === "security.tests")?.after).toEqual([ + "api.migrate.security", + "api.templates", + ]); + expect(whole.map((task) => task.id)).toContain("api.migrate.security"); +}); + +test("shard files cover every spec file exactly once", () => { + const shards = runner.shardedFiles("security", 4); + const whole = runner.shardedFiles("security", 1); + const flat = shards.flat().sort(); + + expect(shards).toHaveLength(4); + expect(flat).toEqual([...(whole[0] ?? [])].sort()); + expect(new Set(flat).size).toBe(flat.length); + expect(flat.length).toBeGreaterThan(10); +}); + +test("API test files are found recursively and cover every file once", () => { + const shards = runner.shardedFiles("tests", 4); + const flat = shards.flat(); + + expect(shards).toHaveLength(4); + expect(new Set(flat).size).toBe(flat.length); + expect(flat.length).toBeGreaterThan(100); + expect( + flat.every((path) => path.startsWith("tests/") && path.endsWith(".test.ts")) + ).toBe(true); + expect(flat.some((path) => path.split("/").length > 2)).toBe(true); +}); diff --git a/tools/agent/profile-tasks.ts b/tools/agent/profile-tasks.ts new file mode 100644 index 00000000..ce818e42 --- /dev/null +++ b/tools/agent/profile-tasks.ts @@ -0,0 +1,184 @@ +import { + CHECK_GROUPS, + RELEASE_CHECKS, + STATIC_CHECKS, + type Profile, +} from "./checks"; +import type { ProfileRunner } from "./profile-runner"; +import type { ICheckResult } from "./result"; +import type { ITask } from "./scheduler"; +import { + SANDBOX_LANES, + shardLane, + type ISandboxLane, + type ShardedSuite, +} from "./sandbox/lifecycle"; +import { requireValue } from "./validation"; + +export function profileTasks( + runner: ProfileRunner, + profile: Profile, + fingerprint: string +): ITask[] { + const tasks = profile === "security" ? [] : runner.scriptTasks(STATIC_CHECKS); + const release = profile === "release-local"; + const feature = profile === "feature" || release; + const security = profile === "security" || release; + const shardCount: Record = { + security: security ? runner.budget.securityShards : 0, + tests: feature ? runner.budget.apiShards : 0, + }; + const laneList = (suite: ShardedSuite): ISandboxLane[] => + Array.from({ length: shardCount[suite] }, (_, offset) => + shardLane(suite, offset + 1, shardCount[suite]) + ); + const lanes: ISandboxLane[] = [ + ...laneList("security"), + ...laneList("tests"), + ...(feature ? [SANDBOX_LANES.e2e] : []), + ]; + + if (lanes.length > 0) { + tasks.push( + runner.task( + "sandbox.ready", + () => runner.prepareSandbox(lanes), + [], + 1, + 100 + ) + ); + tasks.push( + runner.task("api.templates", () => runner.templates(), [], 1, 100) + ); + + for (const lane of lanes) { + tasks.push( + runner.task( + `api.migrate.${lane.name}`, + () => runner.migrate(lane), + ["sandbox.ready"], + 1, + 100 + ) + ); + } + } + + /* + * A sharded suite is one task per shard plus an aggregate that merges the + * reports. With one shard the single task carries the suite's evidence. + */ + const shardedSuite = ( + suite: ShardedSuite, + aggregateId: string, + evidence: readonly string[], + priority: number, + coverage: boolean + ): void => { + const files = runner.shardedFiles(suite, shardCount[suite]); + const count = files.length; + const shardIds = files.map( + (_, offset) => `${aggregateId}.${String(offset + 1)}` + ); + + files.forEach((shardFiles, offset) => { + const index = offset + 1; + const lane = shardLane(suite, index, count); + + tasks.push( + runner.task( + count === 1 ? aggregateId : (shardIds[offset] ?? ""), + () => runner.shard(suite, index, count, shardFiles, coverage), + [`api.migrate.${lane.name}`, "api.templates"], + 1, + priority, + count === 1 ? evidence : undefined + ) + ); + }); + + if (count > 1) { + tasks.push( + runner.task( + aggregateId, + () => runner.aggregate(suite, count, coverage), + shardIds, + 1, + priority, + evidence + ) + ); + } + }; + + if (feature) { + shardedSuite( + "tests", + "api.tests", + release ? ["api.tests", "api.coverage"] : ["api.tests"], + 70, + release + ); + tasks.push( + runner.task( + "ui.tests", + () => runner.uiTests(fingerprint), + [], + runner.budget.uiTestWorkers, + 70 + ) + ); + tasks.push( + runner.task( + "ui.e2e", + () => + runner.e2e(requireValue(runner.state, "Sandbox absent"), fingerprint), + ["api.migrate.e2e", "api.templates"], + runner.budget.testWorkers, + 80, + ["runtime.ready", "openapi.drift", "ui.e2e"] + ) + ); + } + + if (security) { + shardedSuite( + "security", + "security.tests", + ["security.tests", "security.manifest"], + 90, + false + ); + } + + if (release) { + tasks.push(...runner.scriptTasks(RELEASE_CHECKS)); + } + + return tasks; +} + +/** Keep aggregate check IDs for consumers while exposing each constituent's result. */ +export function aggregateChecks( + checks: readonly ICheckResult[] +): ICheckResult[] { + return CHECK_GROUPS.map((group) => { + const parts = group.parts.map((script) => + checks.find((check) => check.checkId === `${group.id}.${script}`) + ); + const status = parts.some( + (check) => check === undefined || check.status === "blocked" + ) + ? "blocked" + : parts.some((check) => check?.status === "failed") + ? "failed" + : "passed"; + + return { + checkId: group.id, + status, + reason: `constituent_checks_${status}`, + }; + }); +} diff --git a/tools/agent/profiles.ts b/tools/agent/profiles.ts index d23a0ea0..e7e826a6 100644 --- a/tools/agent/profiles.ts +++ b/tools/agent/profiles.ts @@ -5,10 +5,13 @@ import { now } from "../../apps/api/src/lib/time/now"; import { identifyCheckout } from "./checkout"; import { RELEASE_CHECKS, STATIC_CHECKS, type Profile } from "./checks"; import { openApiUrl } from "./environment"; -import { defaultConcurrency, orderChecks } from "./lanes"; +import { orderChecks } from "./lanes"; +import { executionBudget } from "./scheduling"; +import { runTasks } from "./scheduler"; +import { aggregateChecks, profileTasks } from "./profile-tasks"; import { ProfileRunner } from "./profile-runner"; import type { IVerificationResult } from "./result"; -import { isAborted, requireValue } from "./validation"; +import { isAborted } from "./validation"; import { verify } from "./verification"; import { acquireWorkspace } from "./workspace-lock"; @@ -17,18 +20,38 @@ const CHECK_ORDER: readonly string[] = [ "sandbox.ready", "api.migrate", "api.migrate.tests", + "api.migrate.tests-1", + "api.migrate.tests-2", + "api.migrate.tests-3", + "api.migrate.tests-4", "api.migrate.security", + "api.migrate.security-1", + "api.migrate.security-2", + "api.migrate.security-3", + "api.migrate.security-4", "api.migrate.e2e", "api.migrate.coverage", "api.templates", + "tooling.quality", + "api.check", + "ui.check", ...STATIC_CHECKS.map((check) => check.id), + "security.tests.1", + "security.tests.2", + "security.tests.3", + "security.tests.4", "security.tests", "security.manifest", + "api.tests.1", + "api.tests.2", + "api.tests.3", + "api.tests.4", "api.tests", "ui.tests", "openapi.drift", "runtime.ready", "ui.e2e", + "api.coverage", ...RELEASE_CHECKS.map((check) => check.id), "checkout.stable", "run.completed", @@ -59,53 +82,44 @@ export async function runProfile( const temp = mkdtempSync(join(tmpdir(), "bs-verification-")); let releaseWorkspace: (() => void) | undefined; + const budget = executionBudget(); const runner = new ProfileRunner( root, temp, result, signal, sandboxId, - defaultConcurrency() + budget ); try { releaseWorkspace = acquireWorkspace(root); result.checkout = identifyCheckout(root); - if (profile !== "static") { - await runner.prepareSandbox(); - } + const tasks = profileTasks(runner, profile, result.checkout.fingerprint); - /* - * Every lane below is independent: stateless checks share nothing, and - * each stateful lane (API tests, security spec, browser run, coverage) - * owns a database and a Valkey index inside the sandbox. They run - * together within the concurrency budget instead of one after another. - */ - const lanes = [ - ...(profile === "security" ? [] : runner.scriptLanes(STATIC_CHECKS)), - ...(profile === "security" || profile === "release-local" - ? [() => runner.tests(true)] - : []), - ...(profile === "feature" || profile === "release-local" - ? runner.featureLanes( - requireValue(runner.state, "Sandbox absent"), - result.checkout.fingerprint - ) - : []), - ...(profile === "release-local" - ? runner.scriptLanes(RELEASE_CHECKS) - : []), - ]; + process.stderr.write( + `Verification budget: ${String(budget.slots)} CPU slots; ${String(budget.testWorkers)} browser workers and shards per API suite; ${String(budget.uiTestWorkers)} vitest workers\n` + ); + result.execution = { + ...budget, + tasks: await runTasks(tasks, budget.slots, signal, (id, state) => { + if (state === "started") { + process.stderr.write(`${id}: running\n`); + } + }), + }; - await runner.lanes(lanes); + if (profile !== "security") { + for (const check of aggregateChecks(result.checks)) { + runner.add(check); + } + } if (isAborted(signal)) { throw new Error("interrupted"); } - result.checks = orderChecks(result.checks, CHECK_ORDER); - if (identifyCheckout(root).fingerprint !== result.checkout.fingerprint) { runner.add({ checkId: "checkout.stable", @@ -149,6 +163,7 @@ export async function runProfile( }); result.status = "blocked"; } finally { + result.checks = orderChecks(result.checks, CHECK_ORDER); runner.releaseLease?.(); releaseWorkspace?.(); result.finishedAt = now(); diff --git a/tools/agent/result.ts b/tools/agent/result.ts index 1fe26a1f..714282fb 100644 --- a/tools/agent/result.ts +++ b/tools/agent/result.ts @@ -1,3 +1,6 @@ +import type { ITaskTiming } from "./scheduler"; +import type { IExecutionBudget } from "./scheduling"; + const INVALID_EVIDENCE = "Invalid verification evidence"; /** Versioned evidence for a declared check, never a production certification. */ @@ -21,6 +24,7 @@ export interface IVerificationResult { source: "caller-selected-api" | "owned-sandbox" | "checkout"; status: Status; checks: ICheckResult[]; + execution?: IExecutionBudget & { tasks: ITaskTiming[] }; } export const exitCode = (status: Status): number => diff --git a/tools/agent/runtime.ts b/tools/agent/runtime.ts index 8fdaa4ee..38f47ce0 100644 --- a/tools/agent/runtime.ts +++ b/tools/agent/runtime.ts @@ -248,13 +248,17 @@ export async function fullStackChecks( state: ISandbox, signal?: AbortSignal, expectedFingerprint?: string, - lane: ISandboxLane = SANDBOX_LANES.default + lane: ISandboxLane = SANDBOX_LANES.default, + workers = 1 ): Promise { let runtime: IRuntime | undefined; const report = join(root, ".agent-state", `playwright-${randomUUID()}.xml`); try { + const startup = performance.now(); + runtime = await startRuntime(root, state, signal, lane); + const startupMs = Math.round(performance.now() - startup); const schema = await checkOpenapi( root, `${runtime.apiUrl}/swagger/json`, @@ -267,6 +271,7 @@ export async function fullStackChecks( "node_modules/@playwright/test/cli.js", "test", "--project=chromium", + `--workers=${String(workers)}`, "--retries=0", "--grep-invert=Visual regression", "--reporter=junit", @@ -288,15 +293,29 @@ export async function fullStackChecks( "playwright" ); + if (browser.status !== "passed") { + writeFileSync(`${report}.log`, run.stdout + run.stderr, { mode: 0o600 }); + process.stderr.write(`ui.e2e: output saved to ${report}.log\n`); + } + return [ + { + checkId: "runtime.ready", + status: "passed", + reason: "owned_runtime_ready", + durationMs: startupMs, + }, schema, - inventoryEvidence( - root, - "ui.e2e", - existsSync(report) ? readFileSync(report, "utf8") : "", - browser, - expectedFingerprint - ), + { + durationMs: run.durationMs, + ...inventoryEvidence( + root, + "ui.e2e", + existsSync(report) ? readFileSync(report, "utf8") : "", + browser, + expectedFingerprint + ), + }, ]; } catch { return [ diff --git a/tools/agent/sandbox/lifecycle.test.ts b/tools/agent/sandbox/lifecycle.test.ts index 7cfecbfb..661894b5 100644 --- a/tools/agent/sandbox/lifecycle.test.ts +++ b/tools/agent/sandbox/lifecycle.test.ts @@ -7,6 +7,8 @@ import { runProcess } from "../process"; import { acquireLease } from "./lease"; import { downSandbox, + ensureLaneDatabases, + SANDBOX_LANES, inspectSandbox, publicSandbox, readSandbox, @@ -125,6 +127,22 @@ if (dockerTestsEnabled()) { expect(secondRead.stdout.trim()).toBe(""); expect(secondWrite.code).toBe(0); + await ensureLaneDatabases(ROOT, first, [SANDBOX_LANES.security]); + await ensureLaneDatabases(ROOT, first, [SANDBOX_LANES.security]); + const selected = await query( + first, + "SELECT datname FROM pg_database WHERE datname LIKE 'app_%' ORDER BY datname;" + ); + + expect(selected.code).toBe(0); + expect(selected.stdout.trim()).toBe("app_security"); + const otherSandbox = await query( + second, + "SELECT datname FROM pg_database WHERE datname LIKE 'app_%';" + ); + + expect(otherSandbox.stdout.trim()).toBe(""); + const createMarker = await query( first, "CREATE TABLE isolation_marker (value text); INSERT INTO isolation_marker VALUES ('first');" diff --git a/tools/agent/sandbox/lifecycle.ts b/tools/agent/sandbox/lifecycle.ts index 7c8c8040..15a223a7 100644 --- a/tools/agent/sandbox/lifecycle.ts +++ b/tools/agent/sandbox/lifecycle.ts @@ -35,28 +35,69 @@ export interface ISandbox { * counters. Stateless lanes (lint, typecheck, unit tests, builds) need none. */ export interface ISandboxLane { + readonly name: string; readonly database: string; readonly valkeyDb: number; } export const SANDBOX_LANES = { - default: { database: "app", valkeyDb: 0 }, - tests: { database: "app_tests", valkeyDb: 1 }, - security: { database: "app_security", valkeyDb: 2 }, - e2e: { database: "app_e2e", valkeyDb: 3 }, - coverage: { database: "app_coverage", valkeyDb: 4 }, + default: { name: "default", database: "app", valkeyDb: 0 }, + tests: { name: "tests", database: "app_tests", valkeyDb: 1 }, + security: { name: "security", database: "app_security", valkeyDb: 2 }, + e2e: { name: "e2e", database: "app_e2e", valkeyDb: 3 }, + coverage: { name: "coverage", database: "app_coverage", valkeyDb: 4 }, } as const satisfies Record; export type SandboxLaneName = keyof typeof SANDBOX_LANES; /** Every lane database except the default that `sandbox:up` created. */ -export const EXTRA_LANES: readonly SandboxLaneName[] = [ - "tests", - "security", - "e2e", - "coverage", +export const EXTRA_LANES: readonly ISandboxLane[] = [ + SANDBOX_LANES.tests, + SANDBOX_LANES.security, + SANDBOX_LANES.e2e, + SANDBOX_LANES.coverage, ]; +export const MAX_SHARDS = 4; +export type ShardedSuite = "security" | "tests"; + +const SHARD_VALKEY_BASE: Record = { + security: 4, + tests: 8, +}; + +/** + * The two API suites are the longest single processes of a run. Sharding + * them across processes needs a database and a Valkey index per shard; with + * one shard the suite's ordinary lane is used so CI output does not change. + */ +export function shardLane( + suite: ShardedSuite, + index: number, + count: number +): ISandboxLane { + if ( + !Number.isSafeInteger(index) || + !Number.isSafeInteger(count) || + count < 1 || + count > MAX_SHARDS || + index < 1 || + index > count + ) { + throw new Error("Invalid shard"); + } + + if (count === 1) { + return SANDBOX_LANES[suite]; + } + + return { + name: `${suite}-${String(index)}`, + database: `app_${suite}_${String(index)}`, + valkeyDb: SHARD_VALKEY_BASE[suite] + index, + }; +} + const VALKEY_PONG = "PONG"; const ID = /^[a-f0-9]{32}$/; const ownerOf = (root: string): string => @@ -471,7 +512,8 @@ export function publicSandbox(state: ISandbox): object { */ export async function ensureLaneDatabases( root: string, - state: ISandbox + state: ISandbox, + lanes: readonly ISandboxLane[] = EXTRA_LANES ): Promise { await owned(root, state, state.postgres); @@ -488,9 +530,7 @@ export async function ensureLaneDatabases( ]); const existing = new Set(listed.split("\n").map((line) => line.trim())); - for (const name of EXTRA_LANES) { - const { database } = SANDBOX_LANES[name]; - + for (const database of new Set(lanes.map((lane) => lane.database))) { if (existing.has(database)) { continue; } diff --git a/tools/agent/scheduler.test.ts b/tools/agent/scheduler.test.ts new file mode 100644 index 00000000..9752f2d4 --- /dev/null +++ b/tools/agent/scheduler.test.ts @@ -0,0 +1,298 @@ +import { expect, test } from "bun:test"; +import { runTasks, type ITask } from "./scheduler"; +import { executionBudget } from "./scheduling"; + +test("local budget uses a 16-core host with headroom and conservative CI defaults", () => { + const memory = 64 * 1024 ** 3; + + expect(executionBudget({}, 16, memory)).toMatchObject({ + slots: 14, + testWorkers: 4, + }); + expect(executionBudget({ CI: "true" }, 16, memory)).toMatchObject({ + slots: 4, + testWorkers: 1, + }); + expect(executionBudget({ CI: "true" }, 2, memory)).toMatchObject({ + slots: 2, + testWorkers: 1, + }); + expect(executionBudget({}, 16, 4 * 1024 ** 3).slots).toBe(2); + expect( + executionBudget( + { AGENT_VERIFY_PARALLEL: "1", AGENT_VERIFY_TEST_WORKERS: "8" }, + 16, + memory + ) + ).toMatchObject({ slots: 1, testWorkers: 1 }); + expect( + executionBudget( + { AGENT_VERIFY_PARALLEL: "8", AGENT_VERIFY_TEST_WORKERS: "2" }, + 16, + memory + ) + ).toMatchObject({ slots: 8, testWorkers: 2 }); +}); + +test("waiting size gates do not occupy a slot needed by independent builds", async () => { + const seen: string[] = []; + let finishBuild = (): void => undefined; + const gate = new Promise((resolve) => { + finishBuild = resolve; + }); + + const blocked = (): void => { + throw new Error("Unexpected blocked task"); + }; + + const tasks: ITask[] = [ + { + id: "ui.build", + run: async () => { + await gate; + seen.push("ui.build"); + + return true; + }, + blocked, + }, + { + id: "ui.size", + after: ["ui.build"], + run: () => { + seen.push("ui.size"); + + return Promise.resolve(true); + }, + blocked, + }, + { + id: "docs.build", + run: () => { + seen.push("docs.build"); + + return Promise.resolve(true); + }, + blocked, + }, + ]; + const running = runTasks(tasks, 2); + + try { + await Bun.sleep(5); + expect(seen).toEqual(["docs.build"]); + } finally { + finishBuild(); + await running; + } + + expect(seen).toEqual(["docs.build", "ui.build", "ui.size"]); +}); + +test("worker pools share the slot budget and all independent tasks finish", async () => { + let used = 0; + let peak = 0; + const tasks: ITask[] = [3, 2, 1, 1].map((slots, index) => ({ + id: String(index), + slots, + blocked: () => undefined, + run: async () => { + used += slots; + peak = Math.max(peak, used); + await Bun.sleep(5); + used -= slots; + + return true; + }, + })); + const timings = await runTasks(tasks, 4); + + expect(peak).toBe(4); + expect(timings).toHaveLength(4); + expect(timings.every((timing) => timing.status === "passed")).toBe(true); +}); + +test("small checks cannot starve a ready higher-priority worker pool", async () => { + const seen: string[] = []; + let finish = (): void => undefined; + const gate = new Promise((resolve) => { + finish = resolve; + }); + const blocked = (): void => undefined; + const tasks: ITask[] = [ + { + id: "active", + slots: 2, + priority: 100, + blocked, + run: async () => { + await gate; + + return true; + }, + }, + { id: "prepare", priority: 100, blocked, run: () => Promise.resolve(true) }, + { + id: "browser", + slots: 3, + priority: 80, + after: ["prepare"], + blocked, + run: () => { + seen.push("browser"); + + return Promise.resolve(true); + }, + }, + { + id: "small", + after: ["prepare"], + blocked, + run: () => { + seen.push("small"); + + return Promise.resolve(true); + }, + }, + ]; + const running = runTasks(tasks, 4); + + try { + await Bun.sleep(5); + expect(seen).toEqual([]); + } finally { + finish(); + await running; + } + + expect(seen).toEqual(["browser", "small"]); +}); + +test("failed and crashed prerequisites block descendants but not independent work", async () => { + const blockedIds: string[] = []; + const seen: string[] = []; + const task = (id: string, after: string[] = []): ITask => ({ + id, + after, + blocked: () => { + blockedIds.push(id); + }, + run: () => { + seen.push(id); + + return Promise.resolve(id !== "failed"); + }, + }); + const tasks = [ + task("failed"), + task("dependent", ["failed"]), + task("descendant", ["dependent"]), + task("independent"), + { ...task("crashed"), run: () => Promise.reject(new Error("crash")) }, + task("crash-dependent", ["crashed"]), + ]; + + await runTasks(tasks, 2); + expect(seen).toEqual(["failed", "independent"]); + expect(blockedIds.sort()).toEqual([ + "crash-dependent", + "crashed", + "dependent", + "descendant", + ]); +}); + +test("abort records unstarted tasks as blocked and waits for active work", async () => { + const controller = new AbortController(); + const seen: string[] = []; + const tasks: ITask[] = ["first", "second"].map((id) => ({ + id, + blocked: () => { + seen.push(`${id}.blocked`); + }, + run: async () => { + controller.abort(); + await Bun.sleep(5); + seen.push(id); + + return true; + }, + })); + + await runTasks(tasks, 1, controller.signal); + expect(seen).toEqual(["first", "second.blocked"]); +}); + +test("invalid graphs fail before executing any task", async () => { + const task = (id: string, after: string[] = []): ITask => ({ + id, + after, + run: () => Promise.reject(new Error("Should not execute")), + blocked: () => undefined, + }); + + const invalid = [ + { + tasks: [task("first", ["missing"])], + capacity: 1, + message: "Invalid verification dependency", + }, + { + tasks: [task("first", ["second"]), task("second", ["first"])], + capacity: 1, + message: "Invalid verification dependency", + }, + { + tasks: [task("first"), task("first")], + capacity: 1, + message: "Duplicate verification task", + }, + { + tasks: [task("first")], + capacity: 0, + message: "Invalid verification capacity", + }, + ]; + + for (const fixture of invalid) { + let failure: unknown; + + try { + await runTasks(fixture.tasks, fixture.capacity); + } catch (error) { + failure = error; + } + + expect(failure instanceof Error ? failure.message : "").toContain( + fixture.message + ); + } +}); + +test("the UI pool grows to half the budget locally and stays with the browser pool in CI", () => { + const local = executionBudget({}, 20, 64 * 1024 ** 3); + const ci = executionBudget({ CI: "true" }, 20, 64 * 1024 ** 3); + const pinned = executionBudget( + { AGENT_VERIFY_TEST_WORKERS: "2" }, + 20, + 64 * 1024 ** 3 + ); + + expect(local).toMatchObject({ + slots: 18, + testWorkers: 4, + uiTestWorkers: 8, + securityShards: 4, + }); + expect(ci).toMatchObject({ + slots: 4, + testWorkers: 1, + uiTestWorkers: 1, + securityShards: 1, + }); + expect(pinned).toMatchObject({ + testWorkers: 2, + uiTestWorkers: 2, + securityShards: 2, + }); +}); diff --git a/tools/agent/scheduler.ts b/tools/agent/scheduler.ts new file mode 100644 index 00000000..98226288 --- /dev/null +++ b/tools/agent/scheduler.ts @@ -0,0 +1,164 @@ +import { isAborted } from "./validation"; + +export interface ITask { + id: string; + after?: readonly string[]; + slots?: number; + priority?: number; + run: () => Promise; + blocked: (reason: string) => void; +} + +export interface ITaskTiming { + id: string; + startedAfterMs: number; + durationMs: number; + status: "passed" | "failed" | "blocked"; +} + +function validateTasks(tasks: readonly ITask[]): void { + const byId = new Map(tasks.map((task) => [task.id, task])); + + if (byId.size !== tasks.length) { + throw new Error("Duplicate verification task"); + } + + const visited = new Set(); + const visiting = new Set(); + + const visit = (id: string): void => { + if (visited.has(id)) { + return; + } + + const task = byId.get(id); + + if (task === undefined || visiting.has(id)) { + throw new Error(`Invalid verification dependency: ${id}`); + } + + visiting.add(id); + + for (const dependency of task.after ?? []) { + visit(dependency); + } + + visiting.delete(id); + visited.add(id); + }; + + for (const task of tasks) { + if (!Number.isSafeInteger(task.slots ?? 1) || (task.slots ?? 1) < 1) { + throw new Error(`Invalid verification slots: ${task.id}`); + } + + visit(task.id); + } +} + +/** Only ready work occupies capacity; failed dependencies block their descendants. */ +export async function runTasks( + tasks: readonly ITask[], + capacity: number, + signal?: AbortSignal, + progress: (id: string, state: "started" | "finished") => void = () => + undefined +): Promise { + validateTasks(tasks); + + if (!Number.isSafeInteger(capacity) || capacity < 1) { + throw new Error("Invalid verification capacity"); + } + + if (tasks.some((task) => (task.slots ?? 1) > capacity)) { + throw new Error("Verification task exceeds capacity"); + } + + const pending = [...tasks].sort( + (left, right) => (right.priority ?? 0) - (left.priority ?? 0) + ); + const running = new Map>(); + const completed = new Map(); + const timings: ITaskTiming[] = []; + const start = performance.now(); + let used = 0; + + while (pending.length > 0 || running.size > 0) { + // Starting tasks removes them from pending; iterate a stable snapshot. + const batch = pending.slice(); + + for (const task of batch) { + const dependencies = task.after ?? []; + const interrupted = isAborted(signal); + const failed = dependencies.some((id) => completed.get(id) === false); + + if (interrupted || failed) { + pending.splice(pending.indexOf(task), 1); + completed.set(task.id, false); + task.blocked(interrupted ? "interrupted" : "prerequisite_failed"); + timings.push({ + id: task.id, + startedAfterMs: Math.round(performance.now() - start), + durationMs: 0, + status: "blocked", + }); + continue; + } + + const slots = task.slots ?? 1; + + if (dependencies.some((id) => !completed.has(id))) { + continue; + } + + // Reserve capacity for ready, higher-priority work rather than starving its worker pool. + if (used + slots > capacity) { + break; + } + + pending.splice(pending.indexOf(task), 1); + used += slots; + const started = performance.now(); + + progress(task.id, "started"); + const execution = Promise.resolve() + .then(() => task.run()) + .then( + (passed) => { + completed.set(task.id, passed); + + return passed ? ("passed" as const) : ("failed" as const); + }, + () => { + completed.set(task.id, false); + task.blocked(isAborted(signal) ? "interrupted" : "task_exception"); + + return "blocked" as const; + } + ) + .then((status) => { + timings.push({ + id: task.id, + startedAfterMs: Math.round(started - start), + durationMs: Math.round(performance.now() - started), + status, + }); + used -= slots; + running.delete(task.id); + progress(task.id, "finished"); + }); + + running.set(task.id, execution); + } + + if (running.size > 0) { + await Promise.race(running.values()); + } + } + + const order = new Map(tasks.map((task, index) => [task.id, index])); + + return timings.sort( + (left, right) => (order.get(left.id) ?? 0) - (order.get(right.id) ?? 0) + ); +} diff --git a/tools/agent/scheduling.ts b/tools/agent/scheduling.ts new file mode 100644 index 00000000..36ecad47 --- /dev/null +++ b/tools/agent/scheduling.ts @@ -0,0 +1,63 @@ +import { availableParallelism, totalmem } from "node:os"; +import { verificationEnvironment } from "./environment"; + +export interface IExecutionBudget { + slots: number; + /** Playwright workers; also the number of security-spec shards. */ + testWorkers: number; + /** Vitest workers: the UI suite is the long pole once the spec is sharded. */ + uiTestWorkers: number; + securityShards: number; + apiShards: number; +} + +const MAX_UI_WORKERS = 8; +const MAX_SHARDS = 4; + +const GIB = 1024 ** 3; + +/** Reserve host headroom; worker pools consume slots from the same budget as checks. */ +export function executionBudget( + env: Record = verificationEnvironment(), + cores = availableParallelism(), + memoryBytes = totalmem() +): IExecutionBudget { + const ci = env.CI !== undefined && env.CI !== "" && env.CI !== "false"; + const requested = Number(env.AGENT_VERIFY_PARALLEL ?? ""); + const automatic = Math.max( + 1, + Math.min( + ci ? 4 : 24, + ci ? cores : cores - 2, + Math.floor(memoryBytes / (2 * GIB)) + ) + ); + const slots = + Number.isSafeInteger(requested) && requested > 0 ? requested : automatic; + const requestedWorkers = Number(env.AGENT_VERIFY_TEST_WORKERS ?? ""); + const testWorkers = Math.min( + slots, + Number.isSafeInteger(requestedWorkers) && requestedWorkers > 0 + ? requestedWorkers + : ci + ? 1 + : Math.max(1, Math.min(4, Math.floor(slots / 3))) + ); + + const uiTestWorkers = Math.min( + slots, + Number.isSafeInteger(requestedWorkers) && requestedWorkers > 0 + ? requestedWorkers + : ci + ? testWorkers + : Math.max(testWorkers, Math.min(MAX_UI_WORKERS, Math.floor(slots / 2))) + ); + + return { + slots, + testWorkers, + uiTestWorkers, + securityShards: Math.min(MAX_SHARDS, testWorkers), + apiShards: Math.min(MAX_SHARDS, testWorkers), + }; +} diff --git a/tools/agent/typecheck-cache.test.ts b/tools/agent/typecheck-cache.test.ts new file mode 100644 index 00000000..ec0936a0 --- /dev/null +++ b/tools/agent/typecheck-cache.test.ts @@ -0,0 +1,61 @@ +import { expect, test } from "bun:test"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runProcess } from "./process"; + +const ROOT = fileURLToPath(new URL("../../", import.meta.url)); + +test("warm incremental typechecking catches changes to imported types", async () => { + const directory = mkdtempSync(join(tmpdir(), "bs-typecheck-cache-")); + const source = join(directory, "value.ts"); + const cache = join(directory, "cache.tsbuildinfo"); + + try { + writeFileSync( + join(directory, "tsconfig.json"), + JSON.stringify({ + compilerOptions: { + strict: true, + noEmit: true, + incremental: true, + tsBuildInfoFile: cache, + types: [], + skipLibCheck: true, + }, + include: ["*.ts"], + }) + ); + writeFileSync(source, "export interface IValue { value: string; }\n"); + writeFileSync( + join(directory, "consumer.ts"), + 'import type { IValue } from "./value";\nexport const value: IValue = { value: "valid" };\n' + ); + const run = () => + runProcess( + [ + process.execPath, + join(ROOT, "apps/api/node_modules/typescript/bin/tsc"), + "-p", + directory, + ], + { cwd: directory, timeoutMs: 10_000 } + ); + + const cold = await run(); + + expect(cold.code).toBe(0); + expect(existsSync(cache)).toBe(true); + const warm = await run(); + + expect(warm.code).toBe(0); + writeFileSync(source, "export interface IValue { value: number; }\n"); + const changed = await run(); + + expect(changed.code).not.toBe(0); + expect(changed.stdout).toContain("consumer.ts"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/tools/tsconfig.json b/tools/tsconfig.json index a96ddc81..2a20ea90 100644 --- a/tools/tsconfig.json +++ b/tools/tsconfig.json @@ -1,6 +1,8 @@ { "extends": "../apps/api/tsconfig.json", "compilerOptions": { + "incremental": true, + "tsBuildInfoFile": "../apps/api/node_modules/.cache/tsc/tools.tsbuildinfo", "types": [], "typeRoots": ["../apps/api/node_modules/@types"], "noUnusedLocals": true