From e7f3d145e3c01de892f384bec6be56066e0db296 Mon Sep 17 00:00:00 2001 From: Aleksandar Grbic Date: Mon, 14 Sep 2026 22:23:41 +0200 Subject: [PATCH] feat(agent): run verification lanes in parallel on per-lane databases Every stage of agent:verify ran one after another; a release-local pass took ten minutes or more while most cores idled. Lanes are now independent: stateless checks share nothing, and each stateful lane (API tests, security spec, browser run, coverage) owns a Postgres database and a Valkey index inside the sandbox, created and migrated by the runner. runLanes bounds concurrency (roughly a lane per three cores, AGENT_VERIFY_PARALLEL overrides, 1 restores sequential runs), a failing lane never cancels the others, size gates wait for the build they read, and reports keep a fixed order. Two things the lanes exposed: the F04 security spec opened a raw Valkey client without the database index (now the app's own client options), and the docs build:ci regenerated the tracked OG image, which differs by machine and tripped checkout.stable. Release-local on a 20-core machine: 103 s, three runs in a row. Inventories accepted for the tests #455 added. --- .gitignore | 3 + .../f04-mfa-attempt-accounting.test.ts | 12 +- apps/docs/package.json | 2 +- apps/ui/.prettierignore | 1 + docs/maintenance/product-feedback.md | 2 +- tools/agent/README.md | 2 + tools/agent/checks.ts | 17 ++- tools/agent/inventories/api.tests.json | 3 + tools/agent/inventories/ui.tests.json | 38 ++++-- tools/agent/lanes.test.ts | 129 ++++++++++++++++++ tools/agent/lanes.ts | 86 ++++++++++++ tools/agent/profile-runner.ts | 118 ++++++++++++---- tools/agent/profiles.ts | 77 ++++++++--- tools/agent/runtime.ts | 17 ++- tools/agent/sandbox/lifecycle.ts | 85 +++++++++++- 15 files changed, 513 insertions(+), 79 deletions(-) create mode 100644 tools/agent/lanes.test.ts create mode 100644 tools/agent/lanes.ts diff --git a/.gitignore b/.gitignore index f109aae2..06931528 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ ROADMAP.md .mcp.json # Owned disposable agent verification state (contains test credentials). .agent-state/ + +# tsforge review cache (local tooling by the same author; never part of the template) +.tsforge/ diff --git a/apps/api/security-spec/f04-mfa-attempt-accounting.test.ts b/apps/api/security-spec/f04-mfa-attempt-accounting.test.ts index 88c167e4..c0f084c0 100644 --- a/apps/api/security-spec/f04-mfa-attempt-accounting.test.ts +++ b/apps/api/security-spec/f04-mfa-attempt-accounting.test.ts @@ -30,6 +30,7 @@ import { beforeEach, describe, expect, test } from "bun:test"; import { Redis } from "ioredis"; +import { getValkeyAppClientOptions } from "../src/clients/valkey/valkey.utils"; import { now } from "../src/lib/time/now"; import { mfaService } from "../src/api/auth/services/mfa.service"; import { MFA_MAX_CHALLENGE_ATTEMPTS } from "../src/api/auth/mfa.constants"; @@ -45,16 +46,11 @@ import { import { seedVerifiedUser } from "../tests/helpers/auth"; import { raceAll, requireDbOrFail, requireValkeyOrFail } from "./harness"; -/** Opens a raw client against the same Valkey the cache provider uses. */ +/** Opens a raw client against the same Valkey (and database index) the app uses. */ const valkeyClient = async (): Promise => { - const client = new Redis({ - host: process.env.VALKEY_HOST ?? "127.0.0.1", - port: Number(process.env.VALKEY_PORT ?? 6379), - password: process.env.VALKEY_PASSWORD, - lazyConnect: true, - }); + const client = new Redis(getValkeyAppClientOptions({ connectTimeout: 500 })); - await client.connect(); + await client.ping(); return client; }; diff --git a/apps/docs/package.json b/apps/docs/package.json index 6977a2d6..2b35d105 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -26,7 +26,7 @@ "check:components": "bun run scripts/check-unused-components.mjs", "build:site": "bun run generate:scaffold-manifest && bun run generate:og-image && astro build && bun run sanitize:llms", "build": "bun run generate:scaffold-manifest && bun run generate:og-image && astro build && bun run sanitize:llms", - "build:ci": "bun run check:docs-data && bun run check:components && bun run generate:og-image && astro build && bun run sanitize:llms && bun run check:fragments && bun run check:rendered-markdown && bun run check:agent-surface", + "build:ci": "bun run check:docs-data && bun run check:components && astro build && bun run sanitize:llms && bun run check:fragments && bun run check:rendered-markdown && bun run check:agent-surface", "preview": "bun run build:site && wrangler dev", "astro": "astro", "deploy": "bun run build:ci && wrangler deploy", diff --git a/apps/ui/.prettierignore b/apps/ui/.prettierignore index 4c76cd9c..c32f789a 100644 --- a/apps/ui/.prettierignore +++ b/apps/ui/.prettierignore @@ -29,3 +29,4 @@ scripts/lint-meta/RULES.md # Untracked working docs (code-review swarms, scratchpads). Anything matching # this glob is operator-local and should not gate format:check. code-review-swarm-*.md +.tsforge diff --git a/docs/maintenance/product-feedback.md b/docs/maintenance/product-feedback.md index 1e3c3fc7..1d6518d4 100644 --- a/docs/maintenance/product-feedback.md +++ b/docs/maintenance/product-feedback.md @@ -27,7 +27,7 @@ agent-built product. Existing security and coverage gates remain enforced. | 29 | Feature-page size budgets are set for demo-sized pages. | Policy retained: route budgets remain reviewed per feature; no blanket increase to 25 KB. | | 30 | `bun test` at the api root runs the infra-gated security spec. | Addressed: API contract and validation guide specify bun run test, separate from security specs. | | 31 | The agent runtime migrates but never seeds. | Implemented: db:prepare runs migrations then the seed hook in owned runtimes and production; product reference seeds must be idempotent. | -| 32 | `agent:verify --profile=feature` takes ~5 minutes and is strictly serial. | Partial: human verification output includes durations. Stateful lanes remain serial; parallelization needs isolation evidence. | +| 32 | `agent:verify --profile=feature` takes ~5 minutes and is strictly serial. | Implemented: `agent:verify` runs its lanes in parallel within a per-core budget (`AGENT_VERIFY_PARALLEL` overrides); the stateful lanes each own a database and Valkey index inside the sandbox, which is the isolation evidence the serial design was waiting for. Durations stay in the human output. | | 33 | Local `bun run test` silently skips every database test. | Implemented: explicitly requested unreachable DB fails; unconfigured unit-only runs remain distinct. Regression rejects old silent return. | | 34 | `docker compose -p run …` outside `dev.sh` silently recreates the dev stack. | Documented: use infra/compose/compose/dev.sh for project/overlay consistency. | | 35 | Component anatomy vs. real pages: the `single-semantic-module` rule fights every non-trivial file. | Implemented upstream with 21 and 51: only exported declarations define a module's category, so page files keep their private helpers and constants; two exported categories still conflict, which is the boundary the rule exists for. | diff --git a/tools/agent/README.md b/tools/agent/README.md index d17797f7..b745e66b 100644 --- a/tools/agent/README.md +++ b/tools/agent/README.md @@ -48,6 +48,8 @@ Local release evidence excludes GitHub-only CodeQL, dependency review, secret sc Each `sandbox:up` creates pinned Postgres and Valkey containers with random loopback ports, random Postgres and Valkey credentials and checkout/run ownership labels. Credentials live only in a mode-0600 ignored descriptor under `.agent-state/sandboxes`; JSON output omits them. Verification ignores caller database URLs and uses the descriptor after checking ownership and live bindings. Database tests are destructive **inside that sandbox**. +Within one run, verification lanes execute in parallel. Stateless lanes (tooling quality, API and UI checks, UI unit tests, builds, size gates) share nothing; each stateful lane (API tests, security spec, browser run, coverage) owns its own Postgres database and Valkey index inside the sandbox (`app_tests`, `app_security`, `app_e2e`, `app_coverage`, created and migrated by the runner), so they never truncate each other's tables or share rate-limit counters. The budget defaults to roughly one lane per three cores, between two and six; `AGENT_VERIFY_PARALLEL=1` restores a strictly sequential run and `AGENT_VERIFY_PARALLEL=8` widens it. Reports list checks in a fixed order whatever the completion order. + Separate checkouts and sandboxes can run concurrently. Operations that generate/build/verify in the same checkout are serialized by a checkout lock, even with different sandbox IDs. A lease prevents overlapping verification suites on the same ID. `sandbox:down` refuses live leases, checks labels again, and removes only that run's containers and volumes. After an interrupted process exits, `sandbox:down -- --id=` can reclaim its stale lease. It never globally prunes Docker or flushes a shared cache. If Docker cleanup fails, preserve the descriptor and retry the same ID. The harness disables optional outbound providers and does not inherit developer secrets. This is development tooling for reviewed source, not a security sandbox for hostile code. API code still executes locally and can access local files/network. Do not execute an untrusted submission under credentials or mistake Docker data isolation for an OS sandbox. diff --git a/tools/agent/checks.ts b/tools/agent/checks.ts index dc47e00e..50eabb69 100644 --- a/tools/agent/checks.ts +++ b/tools/agent/checks.ts @@ -1,8 +1,14 @@ +import type { SandboxLaneName } from "./sandbox/lifecycle"; + export interface ICommandCheck { id: string; app: "api" | "ui" | "docs" | "root"; script: string; nodeEnv?: "production"; + /** Stateful scripts run against their own lane database. */ + lane?: SandboxLaneName; + /** Runs only after the named check has finished (it reads that check's output). */ + after?: string; } /** References existing package scripts; parity tests reject drift. No arbitrary shell fragments. */ export const STATIC_CHECKS: readonly ICommandCheck[] = [ @@ -15,11 +21,16 @@ export const STATIC_CHECKS: readonly ICommandCheck[] = [ { id: "docs.data", app: "docs", script: "check:docs-data" }, ]; export const RELEASE_CHECKS: readonly ICommandCheck[] = [ - { id: "api.coverage", app: "api", script: "test:coverage" }, + { id: "api.coverage", app: "api", script: "test:coverage", lane: "coverage" }, { id: "api.build", app: "api", script: "build" }, { id: "ui.build", app: "ui", script: "build", nodeEnv: "production" }, - { id: "ui.bundle", app: "ui", script: "size:check" }, - { id: "ui.modulepreload", app: "ui", script: "size:check:modulepreload" }, + { id: "ui.bundle", app: "ui", script: "size:check", after: "ui.build" }, + { + id: "ui.modulepreload", + app: "ui", + script: "size:check:modulepreload", + after: "ui.build", + }, { id: "docs.build", app: "docs", script: "build:ci", nodeEnv: "production" }, ]; export const PROFILES = [ diff --git a/tools/agent/inventories/api.tests.json b/tools/agent/inventories/api.tests.json index fefd3c8c..73be03a1 100644 --- a/tools/agent/inventories/api.tests.json +++ b/tools/agent/inventories/api.tests.json @@ -641,6 +641,9 @@ "[\"tests/lib/audit-log/audit-log.service.test.ts\",\"returns success=false without throwing when the DB insert fails\"]", "[\"tests/lib/audit-log/audit-log.service.test.ts\",\"still returns events matched by the account:{id} resource convention\"]", "[\"tests/lib/audit-log/audit-log.service.test.ts\",\"writes a row with the given action and metadata\"]", + "[\"tests/lib/cache/cache.generation.test.ts\",\"an untouched namespace is at generation 0 and keys carry it\"]", + "[\"tests/lib/cache/cache.generation.test.ts\",\"bumping moves every scoped key so cached reads become unreachable\"]", + "[\"tests/lib/cache/cache.generation.test.ts\",\"bumps are cumulative and return the new generation\"]", "[\"tests/lib/cache/cache.service.test.ts\",\"del removes a previously set key\"]", "[\"tests/lib/cache/cache.service.test.ts\",\"exposes the full ICacheService contract\"]", "[\"tests/lib/cache/cache.service.test.ts\",\"round-trips a value through the singleton (memory provider in test env)\"]", diff --git a/tools/agent/inventories/ui.tests.json b/tools/agent/inventories/ui.tests.json index bf773aca..4018f9a2 100644 --- a/tools/agent/inventories/ui.tests.json +++ b/tools/agent/inventories/ui.tests.json @@ -187,21 +187,9 @@ "[\"src/features/auth/Auth.queries.test.tsx\",\"useLogin > surfaces 401 as a mutation error\"]", "[\"src/features/auth/Auth.queries.test.tsx\",\"useLogin > throws on an empty response body (defensive guard)\"]", "[\"src/features/auth/Auth.queries.test.tsx\",\"useLogout > succeeds on 204\"]", - "[\"src/features/auth/Auth.queries.test.tsx\",\"useMe > propagates 401 as an ApiError (consumer distinguishes auth failure from anonymous)\"]", - "[\"src/features/auth/Auth.queries.test.tsx\",\"useMe > propagates 5xx server errors instead of silently logging the user out\"]", - "[\"src/features/auth/Auth.queries.test.tsx\",\"useMe > propagates network errors so the offline fallback can render\"]", - "[\"src/features/auth/Auth.queries.test.tsx\",\"useMe > returns null when the API responds 200 `{ user: null }` (anonymous probe)\"]", - "[\"src/features/auth/Auth.queries.test.tsx\",\"useMe > returns null when the response data is absent\"]", - "[\"src/features/auth/Auth.queries.test.tsx\",\"useMe > returns the full session payload when the API responds 200 with the authenticated shape\"]", "[\"src/features/auth/Auth.queries.test.tsx\",\"useMfaStatus > propagates 401 as an ApiError\"]", "[\"src/features/auth/Auth.queries.test.tsx\",\"useMfaStatus > propagates non-ApiError failures\"]", "[\"src/features/auth/Auth.queries.test.tsx\",\"useMfaStatus > returns the status payload from data.data\"]", - "[\"src/features/auth/Auth.queries.utils.test.ts\",\"isAuthenticatedMe > returns false for non-object primitives\"]", - "[\"src/features/auth/Auth.queries.utils.test.ts\",\"isAuthenticatedMe > returns false for null\"]", - "[\"src/features/auth/Auth.queries.utils.test.ts\",\"isAuthenticatedMe > returns false for the anonymous shape `{ user: null }`\"]", - "[\"src/features/auth/Auth.queries.utils.test.ts\",\"isAuthenticatedMe > returns false for undefined\"]", - "[\"src/features/auth/Auth.queries.utils.test.ts\",\"isAuthenticatedMe > returns false when `user` key is absent (openapi-fetch empty-content branch)\"]", - "[\"src/features/auth/Auth.queries.utils.test.ts\",\"isAuthenticatedMe > returns true when `user` is a non-null object (authenticated shape)\"]", "[\"src/features/auth/Auth.queries.utils.test.ts\",\"resolveAuthStatus > returns 'anonymous' for an explicit null data\"]", "[\"src/features/auth/Auth.queries.utils.test.ts\",\"resolveAuthStatus > returns 'authed' for an IMe payload\"]", "[\"src/features/auth/Auth.queries.utils.test.ts\",\"resolveAuthStatus > returns 'offline' for a 5xx ApiError (not a forced-logout)\"]", @@ -498,6 +486,13 @@ "[\"src/lib/guards/isRecord.test.ts\",\"isRecord > rejects functions\"]", "[\"src/lib/guards/isRecord.test.ts\",\"isRecord > rejects null and undefined\"]", "[\"src/lib/guards/isRecord.test.ts\",\"isRecord > rejects primitives\"]", + "[\"src/lib/i18n/config.test.ts\",\"application configuration does not preload secondary locale resources\"]", + "[\"src/lib/i18n/locale-backend.test.ts\",\"locale backend > keeps secondary dictionaries unloaded until the language is requested\"]", + "[\"src/lib/i18n/locale-backend.test.ts\",\"locale backend > reports an unknown namespace as a load failure\"]", + "[\"src/lib/i18n/locale-backend.test.ts\",\"locale backend > retains the bundled fallback when a dictionary does not exist\"]", + "[\"src/lib/i18n/useNamespace.test.tsx\",\"useNamespace > allows a loaded fallback when the selected language is missing\"]", + "[\"src/lib/i18n/useNamespace.test.tsx\",\"useNamespace > returns the loaded dictionary\"]", + "[\"src/lib/i18n/useNamespace.test.tsx\",\"useNamespace > throws when no usable dictionary exists\"]", "[\"src/lib/logger/logger.events.test.ts\",\"LOG_EVENTS > has no duplicate event names\"]", "[\"src/lib/logger/logger.events.test.ts\",\"LOG_EVENTS > is a non-empty list of event names\"]", "[\"src/lib/logger/logger.events.test.ts\",\"LOG_EVENTS > namespaces every event as dotted lowercase segments\"]", @@ -513,6 +508,18 @@ "[\"src/lib/logger/logger.utils.test.ts\",\"emit (logger.utils) > routes error level to console.error, not console.log\"]", "[\"src/lib/logger/logger.utils.test.ts\",\"emit (logger.utils) > writes an info entry to console.log with level + timestamp + app\"]", "[\"src/lib/logger/logger.utils.test.ts\",\"emit (logger.utils) in production mode > records a Sentry breadcrumb and never writes to the console\"]", + "[\"src/lib/session/session.utils.test.ts\",\"isAuthenticatedMe > returns false for non-object primitives\"]", + "[\"src/lib/session/session.utils.test.ts\",\"isAuthenticatedMe > returns false for null\"]", + "[\"src/lib/session/session.utils.test.ts\",\"isAuthenticatedMe > returns false for the anonymous shape `{ user: null }`\"]", + "[\"src/lib/session/session.utils.test.ts\",\"isAuthenticatedMe > returns false for undefined\"]", + "[\"src/lib/session/session.utils.test.ts\",\"isAuthenticatedMe > returns false when `user` key is absent (openapi-fetch empty-content branch)\"]", + "[\"src/lib/session/session.utils.test.ts\",\"isAuthenticatedMe > returns true when `user` is a non-null object (authenticated shape)\"]", + "[\"src/lib/session/useMe.test.tsx\",\"useMe > propagates 401 as an ApiError (consumer distinguishes auth failure from anonymous)\"]", + "[\"src/lib/session/useMe.test.tsx\",\"useMe > propagates 5xx server errors instead of silently logging the user out\"]", + "[\"src/lib/session/useMe.test.tsx\",\"useMe > propagates network errors so the offline fallback can render\"]", + "[\"src/lib/session/useMe.test.tsx\",\"useMe > returns null when the API responds 200 `{ user: null }` (anonymous probe)\"]", + "[\"src/lib/session/useMe.test.tsx\",\"useMe > returns null when the response data is absent\"]", + "[\"src/lib/session/useMe.test.tsx\",\"useMe > returns the full session payload when the API responds 200 with the authenticated shape\"]", "[\"src/lib/storage/localStorage.test.ts\",\"localStore > clear() removes only the namespaced keys, not foreign ones\"]", "[\"src/lib/storage/localStorage.test.ts\",\"localStore > namespaces keys under the configured prefix + version\"]", "[\"src/lib/storage/localStorage.test.ts\",\"localStore > returns null and does not throw when JSON is corrupted\"]", @@ -559,6 +566,12 @@ "[\"tests/factories/factories.test.ts\",\"makeUser > produces a payload that passes the userSchema\"]", "[\"tests/factories/factories.test.ts\",\"makeUser > resetUserFactory makes the next call's id predictable again\"]", "[\"tests/factories/factories.test.ts\",\"makeUser > respects overrides\"]", + "[\"tests/lint-meta/eslint-cache.test.ts\",\"the dictionary digest in settings invalidates the ESLint result cache\"]", + "[\"tests/lint-meta/eslint-cache.test.ts\",\"without the digest a deleted key hides behind the cached result\"]", + "[\"tests/lint-meta/feature-namespace.test.ts\",\"a namespace collision is rejected before the feature is written\"]", + "[\"tests/lint-meta/feature-namespace.test.ts\",\"namespace scaffolding wires dictionaries, lint scope, and a separate bundle budget\"]", + "[\"tests/lint-meta/i18n-locales.test.ts\",\"all shipped locale namespaces contain matching non-empty translations\"]", + "[\"tests/lint-meta/i18n-plugin.test.ts\",\"the installed translation rule accepts counted plurals without hiding missing keys\"]", "[\"tests/lint-meta/lint-meta.test.ts\",\"RULES.md catalog > matches generate-rules-md output\"]", "[\"tests/lint-meta/lint-meta.test.ts\",\"checkCanonicalHelpersSingleHome > returns no violations on a clean file when the registry is empty\"]", "[\"tests/lint-meta/lint-meta.test.ts\",\"checkDependencyPairs > exact-deps fixture has no overlap\"]", @@ -648,6 +661,7 @@ "[\"tests/lint-meta/lint-meta.test.ts\",\"lint-meta guardrails > checkScriptRawFetch flags fetch in scripts outside allowlist\"]", "[\"tests/lint-meta/lint-meta.test.ts\",\"lint-meta guardrails > checkUiEnvCascadeDrift flags vite-config-only keys missing from vite-env.d.ts\"]", "[\"tests/lint-meta/lint-meta.test.ts\",\"lint-meta guardrails > parseDotenvKeys ignores comments and blank lines\"]", + "[\"tests/render-with-providers.test.tsx\",\"each provider render has independent query and translation state\"]", "[\"tests/sw/sw-url-sanitize.test.ts\",\"clientPathMatches > matches exact same-origin path+search+hash\"]", "[\"tests/sw/sw-url-sanitize.test.ts\",\"clientPathMatches > rejects malformed client URLs\"]", "[\"tests/sw/sw-url-sanitize.test.ts\",\"clientPathMatches > rejects off-origin clients\"]", diff --git a/tools/agent/lanes.test.ts b/tools/agent/lanes.test.ts new file mode 100644 index 00000000..4c5abd4a --- /dev/null +++ b/tools/agent/lanes.test.ts @@ -0,0 +1,129 @@ +import { expect, test } from "bun:test"; +import { defaultConcurrency, orderChecks, runLanes } from "./lanes"; +import type { ICheckResult } from "./result"; +import { SANDBOX_LANES, sandboxEnv, type ISandbox } from "./sandbox/lifecycle"; + +const sandbox: ISandbox = { + version: 1, + id: "0123456789abcdef0123456789abcdef", + owner: "owner", + root: "/repo", + createdAt: "2026-09-14T00:00:00.000Z", + password: "pg-secret", + valkeyPassword: "valkey-secret", + postgres: "pg", + valkey: "valkey", + postgresPort: 45000, + valkeyPort: 45001, +}; + +test("lanes run together within the budget and every lane completes", async () => { + let inFlight = 0; + let peak = 0; + const finished: number[] = []; + const lanes = Array.from({ length: 6 }, (_, index) => async () => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await Bun.sleep(10); + inFlight -= 1; + finished.push(index); + }); + + await runLanes(lanes, 3); + + expect(finished.sort((left, right) => left - right)).toEqual([ + 0, 1, 2, 3, 4, 5, + ]); + expect(peak).toBe(3); +}); + +test("a failing lane does not cancel the others and is rethrown afterwards", async () => { + const seen: string[] = []; + const lanes = [ + async () => { + await Bun.sleep(5); + seen.push("first"); + }, + () => Promise.reject(new Error("lane exploded")), + async () => { + await Bun.sleep(15); + seen.push("third"); + }, + ]; + + let caught: unknown = undefined; + + try { + await runLanes(lanes, 2); + } catch (error) { + caught = error; + } + + expect(caught instanceof Error ? caught.message : "").toBe("lane exploded"); + expect(seen.sort()).toEqual(["first", "third"]); +}); + +test("an abort stops lanes that have not started", async () => { + const controller = new AbortController(); + const started: number[] = []; + const lanes = Array.from({ length: 4 }, (_, index) => async () => { + started.push(index); + controller.abort(); + await Bun.sleep(1); + }); + + await runLanes(lanes, 1, controller.signal); + + expect(started).toEqual([0]); +}); + +test("concurrency honours AGENT_VERIFY_PARALLEL and otherwise stays in band", () => { + expect(defaultConcurrency({ AGENT_VERIFY_PARALLEL: "1" })).toBe(1); + expect(defaultConcurrency({ AGENT_VERIFY_PARALLEL: "9" })).toBe(9); + expect( + defaultConcurrency({ AGENT_VERIFY_PARALLEL: "zero" }) + ).toBeGreaterThanOrEqual(2); + expect(defaultConcurrency({})).toBeLessThanOrEqual(6); +}); + +test("checks report in the declared order regardless of completion order", () => { + const checks: ICheckResult[] = [ + { checkId: "ui.e2e", status: "passed", reason: "x" }, + { checkId: "mystery", status: "passed", reason: "x" }, + { checkId: "api.tests", status: "passed", reason: "x" }, + { checkId: "sandbox.ready", status: "passed", reason: "x" }, + ]; + + expect( + orderChecks(checks, ["sandbox.ready", "api.tests", "ui.e2e"]).map( + (check) => check.checkId + ) + ).toEqual(["sandbox.ready", "api.tests", "ui.e2e", "mystery"]); +}); + +test("each stateful lane gets its own database and Valkey index in the same sandbox", () => { + const base = sandboxEnv(sandbox); + const tests = sandboxEnv(sandbox, SANDBOX_LANES.tests); + const e2e = sandboxEnv(sandbox, SANDBOX_LANES.e2e); + + expect(base.DATABASE_URL).toBe( + "postgresql://app:pg-secret@127.0.0.1:45000/app" + ); + expect(base.VALKEY_DB).toBe("0"); + expect(tests.DATABASE_URL).toBe( + "postgresql://app:pg-secret@127.0.0.1:45000/app_tests" + ); + expect(tests.TEST_DATABASE_URL).toBe(tests.DATABASE_URL); + expect(tests.VALKEY_DB).toBe("1"); + expect(e2e.DATABASE_URL).toBe( + "postgresql://app:pg-secret@127.0.0.1:45000/app_e2e" + ); + expect(e2e.VALKEY_DB).toBe("3"); + expect( + new Set( + Object.values(SANDBOX_LANES).map( + (lane) => `${lane.database}/${String(lane.valkeyDb)}` + ) + ).size + ).toBe(Object.keys(SANDBOX_LANES).length); +}); diff --git a/tools/agent/lanes.ts b/tools/agent/lanes.ts new file mode 100644 index 00000000..8d3d4744 --- /dev/null +++ b/tools/agent/lanes.ts @@ -0,0 +1,86 @@ +import { availableParallelism } from "node:os"; +import type { ICheckResult } from "./result"; +import { isAborted } from "./validation"; + +export type Lane = () => Promise; + +const MIN_PARALLEL = 2; +const MAX_PARALLEL = 6; +const CORES_PER_LANE = 3; + +/** + * Each lane is a whole toolchain process (type-aware ESLint, vitest workers, + * a Playwright run), not a single thread, so the default leaves roughly + * three cores per lane and stays inside a modest band. `AGENT_VERIFY_PARALLEL` + * overrides it; `1` restores strictly sequential runs. + */ +export function defaultConcurrency( + env: Record = process.env +): number { + const requested = Number(env.AGENT_VERIFY_PARALLEL ?? ""); + + if (Number.isInteger(requested) && requested >= 1) { + return requested; + } + + return Math.max( + MIN_PARALLEL, + Math.min(MAX_PARALLEL, Math.floor(availableParallelism() / CORES_PER_LANE)) + ); +} + +/** + * Runs lanes with at most `concurrency` in flight. A lane that throws does not + * stop the others; the first error is rethrown once every started lane has + * settled, so partial results are never lost. An abort stops new lanes from + * starting; running ones observe the signal themselves. + */ +export async function runLanes( + lanes: readonly Lane[], + concurrency: number, + signal?: AbortSignal +): Promise { + const queue = [...lanes]; + const failures: unknown[] = []; + + const worker = async (): Promise => { + for (;;) { + const lane = queue.shift(); + + if (lane === undefined || isAborted(signal)) { + return; + } + + try { + await lane(); + } catch (error) { + failures.push(error); + } + } + }; + + await Promise.all( + Array.from({ length: Math.max(1, concurrency) }, () => worker()) + ); + + if (failures.length > 0) { + throw failures[0]; + } +} + +/** + * Lanes finish in whatever order the machine allows; reports read better in + * the declared order. Unknown ids keep their completion order at the end. + */ +export function orderChecks( + checks: readonly ICheckResult[], + order: readonly string[] +): ICheckResult[] { + const rank = new Map(order.map((id, index) => [id, index])); + + return [...checks].sort( + (left, right) => + (rank.get(left.checkId) ?? order.length) - + (rank.get(right.checkId) ?? order.length) + ); +} diff --git a/tools/agent/profile-runner.ts b/tools/agent/profile-runner.ts index 2829d1f2..7285fc03 100644 --- a/tools/agent/profile-runner.ts +++ b/tools/agent/profile-runner.ts @@ -1,13 +1,23 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { type ICommandCheck } from "./checks"; +import type { Lane } from "./lanes"; import { hostEnvironment } from "./environment"; import { inventoryEvidence } from "./inventory"; import { runProcess } from "./process"; import { testEvidence } from "./reports"; import type { ICheckResult, IVerificationResult } from "./result"; import { acquireLease } from "./sandbox/lease"; -import { inspectSandbox, sandboxEnv, type ISandbox } from "./sandbox/lifecycle"; +import { runLanes } from "./lanes"; +import { + ensureLaneDatabases, + inspectSandbox, + SANDBOX_LANES, + sandboxEnv, + EXTRA_LANES, + type ISandbox, + type SandboxLaneName, +} from "./sandbox/lifecycle"; import { securityManifestEvidence } from "./security-evidence"; import { isAborted } from "./validation"; @@ -29,9 +39,19 @@ export class ProfileRunner { private readonly temp: string, private readonly result: IVerificationResult, private readonly signal?: AbortSignal, - private readonly sandboxId?: string + private readonly sandboxId?: string, + readonly concurrency = 1 ) {} + /** Environment for a stateful lane: its own database and Valkey index. */ + laneEnv(lane: SandboxLaneName): Record { + if (this.state === undefined) { + throw new Error("Sandbox absent"); + } + + return sandboxEnv(this.state, SANDBOX_LANES[lane]); + } + add(check: ICheckResult): void { this.result.checks.push(check); process.stderr.write(`${check.checkId}: ${check.status}\n`); @@ -81,18 +101,19 @@ export class ProfileRunner { return; } + const base = check.lane === undefined ? this.env : this.laneEnv(check.lane); + await this.command( check.id, check.app, [process.execPath, NO_ENV_FILE, "run", check.script], - check.nodeEnv !== undefined - ? { ...this.env, NODE_ENV: check.nodeEnv } - : this.env + check.nodeEnv !== undefined ? { ...base, NODE_ENV: check.nodeEnv } : base ); } async tests(security: boolean): Promise { const report = join(this.temp, security ? "security.xml" : "api.xml"); + const laneEnv = this.laneEnv(security ? "security" : "tests"); const run = await runProcess( [ process.execPath, @@ -106,7 +127,7 @@ export class ProfileRunner { { cwd: join(this.root, "apps/api"), env: { - ...this.env, + ...laneEnv, SECURITY_SPEC: security ? "true" : "false", ...(security ? { @@ -155,13 +176,7 @@ export class ProfileRunner { } } - async feature(featureSandbox: ISandbox, fingerprint: string): Promise { - if (isAborted(this.signal)) { - throw new Error("interrupted"); - } - - await this.tests(false); - + async uiTests(fingerprint: string): Promise { if (isAborted(this.signal)) { throw new Error("interrupted"); } @@ -199,8 +214,10 @@ export class ProfileRunner { fingerprint ) ); + } - // Full-stack adapter starts the selected checkout rather than trusting an ambient server. + /** Full-stack adapter starts the selected checkout rather than trusting an ambient server. */ + async e2e(featureSandbox: ISandbox, fingerprint: string): Promise { if (isAborted(this.signal)) { throw new Error("interrupted"); } @@ -211,20 +228,61 @@ export class ProfileRunner { this.root, featureSandbox, this.signal, - fingerprint + fingerprint, + SANDBOX_LANES.e2e )) { this.add(check); } } - async scripts(checks: readonly ICommandCheck[]): Promise { + /** The feature lanes: API tests, UI tests and the browser run, each on its own state. */ + featureLanes(featureSandbox: ISandbox, fingerprint: string): Lane[] { + return [ + () => this.tests(false), + () => this.uiTests(fingerprint), + () => this.e2e(featureSandbox, fingerprint), + ]; + } + + /** + * One lane per script. A check with `after` waits for that check to finish + * (a size gate reads the build it names) but everything else runs as the + * concurrency budget allows. + */ + scriptLanes(checks: readonly ICommandCheck[]): Lane[] { + const done = new Map>(); + const resolvers = new Map void>(); + for (const check of checks) { - if (isAborted(this.signal)) { - return; + done.set( + check.id, + new Promise((resolve) => { + resolvers.set(check.id, resolve); + }) + ); + } + + return checks.map((check) => async () => { + try { + if (check.after !== undefined) { + await done.get(check.after); + } + + if (!isAborted(this.signal)) { + await this.script(check); + } + } finally { + resolvers.get(check.id)?.(); } + }); + } - await this.script(check); - } + async scripts(checks: readonly ICommandCheck[]): Promise { + await this.lanes(this.scriptLanes(checks)); + } + + async lanes(lanes: readonly Lane[]): Promise { + await runLanes(lanes, this.concurrency, this.signal); } async prepareSandbox(): Promise { @@ -240,14 +298,20 @@ export class ProfileRunner { status: "passed", reason: "owned_services_ready", }); - const migration = await this.command("api.migrate", "api", [ - process.execPath, - NO_ENV_FILE, - "run", - "db:prepare", - ]); + await ensureLaneDatabases(this.root, this.state); + + const migrations = await Promise.all( + [undefined, ...EXTRA_LANES].map((lane) => + this.command( + lane === undefined ? "api.migrate" : `api.migrate.${lane}`, + "api", + [process.execPath, NO_ENV_FILE, "run", "db:prepare"], + lane === undefined ? this.env : this.laneEnv(lane) + ) + ) + ); - if (migration.status !== "passed") { + if (migrations.some((migration) => migration.status !== "passed")) { throw new Error("Migration prerequisite failed"); } diff --git a/tools/agent/profiles.ts b/tools/agent/profiles.ts index d970a23d..d23a0ea0 100644 --- a/tools/agent/profiles.ts +++ b/tools/agent/profiles.ts @@ -5,12 +5,36 @@ import { now } from "../../apps/api/src/lib/time/now"; import { identifyCheckout } from "./checkout"; import { RELEASE_CHECKS, STATIC_CHECKS, type Profile } from "./checks"; import { openApiUrl } from "./environment"; +import { defaultConcurrency, orderChecks } from "./lanes"; import { ProfileRunner } from "./profile-runner"; import type { IVerificationResult } from "./result"; -import { isAborted } from "./validation"; +import { isAborted, requireValue } from "./validation"; import { verify } from "./verification"; import { acquireWorkspace } from "./workspace-lock"; +/** Reporting order; lanes complete in machine order. */ +const CHECK_ORDER: readonly string[] = [ + "sandbox.ready", + "api.migrate", + "api.migrate.tests", + "api.migrate.security", + "api.migrate.e2e", + "api.migrate.coverage", + "api.templates", + ...STATIC_CHECKS.map((check) => check.id), + "security.tests", + "security.manifest", + "api.tests", + "ui.tests", + "openapi.drift", + "runtime.ready", + "ui.e2e", + ...RELEASE_CHECKS.map((check) => check.id), + "checkout.stable", + "run.completed", + "run.prerequisites", +]; + export async function runProfile( root: string, profile: Profile, @@ -35,7 +59,14 @@ export async function runProfile( const temp = mkdtempSync(join(tmpdir(), "bs-verification-")); let releaseWorkspace: (() => void) | undefined; - const runner = new ProfileRunner(root, temp, result, signal, sandboxId); + const runner = new ProfileRunner( + root, + temp, + result, + signal, + sandboxId, + defaultConcurrency() + ); try { releaseWorkspace = acquireWorkspace(root); @@ -45,29 +76,35 @@ export async function runProfile( await runner.prepareSandbox(); } - if (profile !== "security") { - await runner.scripts(STATIC_CHECKS); - } + /* + * Every lane below is independent: stateless checks share nothing, and + * each stateful lane (API tests, security spec, browser run, coverage) + * owns a database and a Valkey index inside the sandbox. They run + * together within the concurrency budget instead of one after another. + */ + const lanes = [ + ...(profile === "security" ? [] : runner.scriptLanes(STATIC_CHECKS)), + ...(profile === "security" || profile === "release-local" + ? [() => runner.tests(true)] + : []), + ...(profile === "feature" || profile === "release-local" + ? runner.featureLanes( + requireValue(runner.state, "Sandbox absent"), + result.checkout.fingerprint + ) + : []), + ...(profile === "release-local" + ? runner.scriptLanes(RELEASE_CHECKS) + : []), + ]; + + await runner.lanes(lanes); if (isAborted(signal)) { throw new Error("interrupted"); } - if (profile === "security" || profile === "release-local") { - await runner.tests(true); - } - - if (profile === "feature" || profile === "release-local") { - if (runner.state === undefined) { - throw new Error("Sandbox absent"); - } - - await runner.feature(runner.state, result.checkout.fingerprint); - } - - if (profile === "release-local") { - await runner.scripts(RELEASE_CHECKS); - } + result.checks = orderChecks(result.checks, CHECK_ORDER); if (identifyCheckout(root).fingerprint !== result.checkout.fingerprint) { runner.add({ diff --git a/tools/agent/runtime.ts b/tools/agent/runtime.ts index 749c7e53..8fdaa4ee 100644 --- a/tools/agent/runtime.ts +++ b/tools/agent/runtime.ts @@ -13,7 +13,12 @@ import { inventoryEvidence } from "./inventory"; import { runProcess } from "./process"; import { testEvidence } from "./reports"; import type { ICheckResult } from "./result"; -import { sandboxEnv, type ISandbox } from "./sandbox/lifecycle"; +import { + SANDBOX_LANES, + sandboxEnv, + type ISandbox, + type ISandboxLane, +} from "./sandbox/lifecycle"; import { isAborted, parseRecord, requireValue } from "./validation"; import { checkOpenapi } from "./verification"; @@ -27,7 +32,8 @@ export interface IRuntime { export async function startRuntime( root: string, state: ISandbox, - signal?: AbortSignal + signal?: AbortSignal, + lane: ISandboxLane = SANDBOX_LANES.default ): Promise { const dir = join(root, ".agent-state", `runtime-${randomUUID()}`); @@ -110,7 +116,7 @@ export async function startRuntime( await reservation.stop(true); const uiUrl = `http://localhost:${uiPort}`; const env = { - ...sandboxEnv(state), + ...sandboxEnv(state, lane), QUEUES_ENABLED: "false", ACCOUNT_DOMAIN_CLAIMING: "false", FRONTEND_URL: uiUrl, @@ -241,13 +247,14 @@ export async function fullStackChecks( root: string, state: ISandbox, signal?: AbortSignal, - expectedFingerprint?: string + expectedFingerprint?: string, + lane: ISandboxLane = SANDBOX_LANES.default ): Promise { let runtime: IRuntime | undefined; const report = join(root, ".agent-state", `playwright-${randomUUID()}.xml`); try { - runtime = await startRuntime(root, state, signal); + runtime = await startRuntime(root, state, signal, lane); const schema = await checkOpenapi( root, `${runtime.apiUrl}/swagger/json`, diff --git a/tools/agent/sandbox/lifecycle.ts b/tools/agent/sandbox/lifecycle.ts index b1e49a23..7c8c8040 100644 --- a/tools/agent/sandbox/lifecycle.ts +++ b/tools/agent/sandbox/lifecycle.ts @@ -27,6 +27,36 @@ export interface ISandbox { postgresPort: number; valkeyPort: number; } +/** + * Verification lanes that mutate state each get their own Postgres database + * and Valkey database index inside the one sandbox, so the API tests, the + * security spec, the browser run and the coverage run can execute at the + * same time without truncating each other's tables or sharing rate-limit + * counters. Stateless lanes (lint, typecheck, unit tests, builds) need none. + */ +export interface ISandboxLane { + readonly database: string; + readonly valkeyDb: number; +} + +export const SANDBOX_LANES = { + default: { database: "app", valkeyDb: 0 }, + tests: { database: "app_tests", valkeyDb: 1 }, + security: { database: "app_security", valkeyDb: 2 }, + e2e: { database: "app_e2e", valkeyDb: 3 }, + coverage: { database: "app_coverage", valkeyDb: 4 }, +} as const satisfies Record; + +export type SandboxLaneName = keyof typeof SANDBOX_LANES; + +/** Every lane database except the default that `sandbox:up` created. */ +export const EXTRA_LANES: readonly SandboxLaneName[] = [ + "tests", + "security", + "e2e", + "coverage", +]; + const VALKEY_PONG = "PONG"; const ID = /^[a-f0-9]{32}$/; const ownerOf = (root: string): string => @@ -367,8 +397,11 @@ export async function upSandbox(root: string): Promise { } /** Explicit test defaults override Bun's ambient dotenv. No developer service credentials are inherited. */ -export function sandboxEnv(state: ISandbox): Record { - const db = `postgresql://app:${state.password}@127.0.0.1:${state.postgresPort}/app`; +export function sandboxEnv( + state: ISandbox, + lane: ISandboxLane = SANDBOX_LANES.default +): Record { + const db = `postgresql://app:${state.password}@127.0.0.1:${state.postgresPort}/${lane.database}`; return { AGENT_SANDBOX: "1", @@ -389,6 +422,7 @@ export function sandboxEnv(state: ISandbox): Record { RUN_VALKEY_NETWORK_TESTS: "true", VALKEY_HOST: "127.0.0.1", VALKEY_PORT: String(state.valkeyPort), + VALKEY_DB: String(lane.valkeyDb), VALKEY_PASSWORD: state.valkeyPassword, CACHE_PROVIDER: "valkey", CACHE_ENABLED: "true", @@ -429,3 +463,50 @@ export function publicSandbox(state: ISandbox): object { ownership: "current-checkout", }; } + +/** + * Creates the extra lane databases in the sandbox's Postgres when they are + * missing. Idempotent: a second verification on the same sandbox finds them + * present. Uses the container's local socket, which the image trusts. + */ +export async function ensureLaneDatabases( + root: string, + state: ISandbox +): Promise { + await owned(root, state, state.postgres); + + const listed = await docker(root, [ + "exec", + state.postgres, + "psql", + "-U", + "app", + "-d", + "app", + "-tAc", + "SELECT datname FROM pg_database", + ]); + const existing = new Set(listed.split("\n").map((line) => line.trim())); + + for (const name of EXTRA_LANES) { + const { database } = SANDBOX_LANES[name]; + + if (existing.has(database)) { + continue; + } + + await docker(root, [ + "exec", + state.postgres, + "psql", + "-U", + "app", + "-d", + "app", + "-v", + "ON_ERROR_STOP=1", + "-c", + `CREATE DATABASE "${database}"`, + ]); + } +}