diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95dd3c9..7c4aeb4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,8 +33,53 @@ jobs: exit 1 } - build: + bench-live: needs: validate-tag + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - name: Require OpenRouter credential + env: + COMPLETION_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: test -n "$COMPLETION_API_KEY" + + # Plain release profile: the same build the release job ships, so the + # gate measures the binary users will actually get. + - run: cargo build --quiet --release + - name: Install benchmark dependencies + working-directory: bench + run: bun install --frozen-lockfile + + - name: Run diff-file live benchmark + working-directory: bench + env: + REVIEW_MODEL: z-ai/glm-5.2 + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: bun run bench:live -- --run-id "release-${{ github.ref_name }}" --json-out "${{ runner.temp }}/bench-live-report.json" + + - name: Compare against the recorded baseline + id: compare-baseline + working-directory: bench + run: bun run bench:compare -- --result "${{ runner.temp }}/bench-live-report.json" + + - name: Report benchmark regression + if: steps.compare-baseline.outcome == 'failure' + run: | + echo "::error::Release blocked by a benchmark regression. See the 'Compare against the recorded baseline' step above for the failing metric(s)." + echo "## Release blocked" >> "$GITHUB_STEP_SUMMARY" + echo "The live benchmark regressed past tolerance against \`bench/baseline.json\`. See the 'Compare against the recorded baseline' step log for the failing metric(s)." >> "$GITHUB_STEP_SUMMARY" + + build: + needs: [validate-tag, bench-live] strategy: matrix: include: diff --git a/bench/README.md b/bench/README.md index acf2215..a3b69ea 100644 --- a/bench/README.md +++ b/bench/README.md @@ -358,9 +358,11 @@ scorer call or scorer identity. A finding that is later suppressed does exercise the scorer and must retain its exact identity and usage record. It refuses to run without `POSTIL_API_KEY`, `OPENROUTER_API_KEY`, `MODEL_API_KEY`, -or `LLM_API_KEY` and never logs or prints the key value. Live mode is -**not run in CI**: it spends real tokens and depends on an -external provider. Every live run writes its JSON report under +or `LLM_API_KEY` and never logs or prints the key value. Live mode spends real +tokens and depends on an external provider, so it is **not run on ordinary +pushes or pull requests**; the release pipeline runs a full-corpus live pass +against `bench/baseline.json` before every tagged release (see "Release gate" +below). Every live run writes its JSON report under `.runs/live//` (gitignored), beside raw per-attempt stdout and stderr. Set `--run-id ` or `POSTIL_BENCH_SCREEN_RUN_ID` to name the immutable namespace; an omitted ID gets a unique generated value. Reusing an ID fails @@ -434,3 +436,40 @@ per case, diff-only with no repository context or policy docs. **Neither severity metric is a peer-comparison claim**: no competitor has been run on the same fixtures. Results vary across runs because model inference is nondeterministic. Treat them as internal evidence, not a published benchmark. + +## Release gate + +The `Release` workflow runs a full-corpus diff-file live pass +(`REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live`) against the plain release +binary before it builds any target, then checks the result with +`bun run bench:compare`. A material regression blocks the release: `build` +depends on the `bench-live` job. + +`compare-baseline.ts` computes five metrics from the live report and compares +each against the matching model entry in `bench/baseline.json`: authored-target +detection rate, false/unrelated finding count, gate-verdict correctness (does +the CLI's exit code agree with the authored must-block/advisory/clean +classification), mean provider cost per case, and p95 review latency. Each +metric has its own tolerance (see the exported `*_MAX_*` constants at the top +of `compare-baseline.ts`), wide enough to absorb ordinary run-to-run inference +variance but not a real behavioral, cost, or latency regression. The baseline +also records the fixture-corpus and evaluator-source SHA-256 digests the live +report already computes; a mismatch fails loudly instead of comparing metrics +across an unrelated fixture set. + +```sh +# Compare an existing report against the committed baseline. +bun run bench:compare -- --result +bun run bench:compare -- --run-id # resolves .runs/live//report.json + +# Re-baseline deliberately, after confirming the new numbers are an accepted +# tradeoff (a model change, a fixture change, an intentional pipeline change): +REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live -- --json-out live-report.json +bun run bench:compare -- --result live-report.json --record +``` + +`--record` is the only thing that writes `bench/baseline.json`; nothing else +updates it implicitly. When the gate fails in CI, the printed table shows +baseline vs. observed vs. verdict for every metric, so the failing metric and +by how much is visible directly in the job log without downloading the report +artifact. diff --git a/bench/baseline.json b/bench/baseline.json new file mode 100644 index 0000000..9abd690 --- /dev/null +++ b/bench/baseline.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": 1, + "corpus": { + "fixtureCorpusSha256": "3c4f952f9411bf2c4e5f0a61b5975e47d203f7681a8235d53d717c743ea9cb54", + "evaluatorSha256": "64dd1975bd5743e15b073b8c6dbad91e37cef3388f793073b216d002de80d78a" + }, + "profiles": { + "z-ai/glm-5.2": { + "populated": true, + "generatedAt": "2026-07-30T01:06:07.029Z", + "reviewMode": "exhaustive", + "sourceRunAt": "2026-07-30T01:05:48.700Z", + "totalCases": 70, + "scoredCases": 69, + "detectionRate": 0.9649122807017544, + "falsePositives": 4, + "gateVerdictCorrectness": 0.7681159420289855, + "meanCostUsdPerCase": 0.004135275999999999, + "latencyMs": { + "p50": 4528, + "p95": 48585 + } + } + } +} diff --git a/bench/package.json b/bench/package.json index fcaddf7..762ea39 100644 --- a/bench/package.json +++ b/bench/package.json @@ -8,6 +8,7 @@ "bench": "bun run src/run.ts", "bench:live": "bun run src/run.ts --live", "bench:live-models": "POSTIL_BENCH_MODE=live bun run src/run.ts", + "bench:compare": "bun run src/compare-baseline.ts", "verify-admission": "bun run src/verify-admission.ts", "scorer-eval": "bun run src/scorer-eval.ts" }, diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts new file mode 100644 index 0000000..3e5e1b4 --- /dev/null +++ b/bench/src/compare-baseline.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, test } from "bun:test"; +import { + DETECTION_RATE_MAX_DROP_PP, + FALSE_FINDINGS_MAX_INCREASE, + GATE_VERDICT_MAX_DROP_PP, + LATENCY_P95_MAX_INCREASE_RATIO, + MEAN_COST_MAX_INCREASE_RATIO, + compareMetrics, + extractObservedMetrics, + percentile, + type BaselineProfile, + type LiveReportForComparison, +} from "./compare-baseline"; + +function fakeReport(overrides: { + falsePositives?: number; + observedProviderCostUsdDecimal?: string; + results?: LiveReportForComparison["results"]; + detected?: number; +} = {}): LiveReportForComparison { + const results: LiveReportForComparison["results"] = overrides.results ?? [ + { id: "must-block-1", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 1 }, + { id: "must-block-2", type: "defect", scored: true, truthSeverity: "error", durationMs: 2000, exitCode: 1 }, + { id: "advisory-1", type: "defect", scored: true, truthSeverity: "warn", durationMs: 1500, exitCode: 0 }, + { id: "clean-1", type: "clean", scored: true, truthSeverity: null, durationMs: 500, exitCode: 0 }, + ]; + const defectCases = results.filter((r) => r.type === "defect").length; + return { + summary: { + model: "z-ai/glm-5.2", + reviewMode: "exhaustive", + fixtureCorpusSha256: "corpus-sha", + evaluatorSha256: "evaluator-sha", + totalCases: results.length, + scoredCases: results.filter((r) => r.scored).length, + defectCases, + detected: overrides.detected ?? defectCases, + falsePositives: overrides.falsePositives ?? 0, + observedProviderCostUsdDecimal: overrides.observedProviderCostUsdDecimal ?? "0.004", + ranAt: "2026-01-01T00:00:00.000Z", + }, + results, + }; +} + +const populatedBaseline: Extract = { + populated: true, + generatedAt: "2026-01-01T00:00:00.000Z", + reviewMode: "exhaustive", + sourceRunAt: "2026-01-01T00:00:00.000Z", + totalCases: 70, + scoredCases: 70, + detectionRate: 0.95, + falsePositives: 2, + gateVerdictCorrectness: 0.97, + meanCostUsdPerCase: 0.003, + latencyMs: { p50: 4000, p95: 9000 }, +}; + +describe("percentile", () => { + test("nearest-rank over a sorted sample", () => { + const sorted = [10, 20, 30, 40, 50]; + expect(percentile(sorted, 50)).toBe(30); + expect(percentile(sorted, 95)).toBe(50); + expect(percentile(sorted, 1)).toBe(10); + }); + + test("single-element sample returns that element at any percentile", () => { + expect(percentile([42], 50)).toBe(42); + expect(percentile([42], 95)).toBe(42); + }); + + test("rejects an empty sample", () => { + expect(() => percentile([], 50)).toThrow("empty sample"); + }); +}); + +describe("extractObservedMetrics", () => { + test("derives detection rate, gate-verdict correctness, cost, and latency from a live report", () => { + const metrics = extractObservedMetrics(fakeReport()); + // 2 defects, 2 detected. + expect(metrics.detectionRate).toBeCloseTo(1, 5); + // All four scored cases: two must-block exit 1 (correct), one advisory + // exit 0 (correct), one clean exit 0 (correct) => 4/4 correct. + expect(metrics.gateVerdictCorrectness).toBeCloseTo(1, 5); + expect(metrics.meanCostUsdPerCase).toBeCloseTo(0.004 / 4, 6); + expect(metrics.latencyMs.p50).toBeGreaterThan(0); + expect(metrics.latencyMs.p95).toBe(2000); + }); + + test("scores a wrongly-blocked advisory case as an incorrect gate verdict", () => { + const report = fakeReport({ + results: [ + { id: "must-block-1", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 1 }, + // An advisory (warn) case that the model over-blocked: exit 1 when it + // should have exited 0. + { id: "advisory-1", type: "defect", scored: true, truthSeverity: "warn", durationMs: 1500, exitCode: 1 }, + ], + }); + const metrics = extractObservedMetrics(report); + expect(metrics.gateVerdictCorrectness).toBeCloseTo(0.5, 5); + }); + + test("excludes unscored (errored) cases from gate-verdict and latency scoring", () => { + const report = fakeReport({ + results: [ + { id: "must-block-1", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 1 }, + { id: "errored-1", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined }, + ], + }); + const metrics = extractObservedMetrics(report); + expect(metrics.gateVerdictCorrectness).toBeCloseTo(1, 5); + expect(metrics.latencyMs.p50).toBe(1000); + }); + + test("throws when every case is unscored", () => { + const report = fakeReport({ + results: [ + { id: "errored-1", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined }, + ], + }); + expect(() => extractObservedMetrics(report)).toThrow("no scored cases"); + }); +}); + +describe("compareMetrics", () => { + test("passes when observed metrics match baseline exactly", () => { + const baseline = { ...populatedBaseline, detectionRate: 1, gateVerdictCorrectness: 1 }; + const observed = extractObservedMetrics(fakeReport({ + falsePositives: baseline.falsePositives, + observedProviderCostUsdDecimal: String(baseline.meanCostUsdPerCase * 4), + })); + const comparison = compareMetrics(baseline, observed); + expect(comparison.ok).toBe(true); + expect(comparison.rows.every((row) => row.verdict === "PASS")).toBe(true); + }); + + test("fails when detection rate drops more than the tolerance", () => { + const baseline = { ...populatedBaseline, detectionRate: 0.98 }; + const observed = extractObservedMetrics(fakeReport({ + detected: 1, + results: [ + { id: "must-block-1", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 1 }, + { id: "must-block-2", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 0 }, + ], + })); + expect(observed.detectionRate).toBeCloseTo(0.5, 5); + expect(0.98 - observed.detectionRate).toBeGreaterThan(DETECTION_RATE_MAX_DROP_PP / 100); + const comparison = compareMetrics(baseline, observed); + expect(comparison.ok).toBe(false); + expect(comparison.rows.find((row) => row.metric === "detection rate")?.verdict).toBe("FAIL"); + }); + + test("tolerates a detection-rate drop within the percentage-point budget", () => { + // Baseline 69/70, observed 68/70: a ~1.4pp drop, inside the 2pp budget. + const baseline = { ...populatedBaseline, detectionRate: 69 / 70 }; + const observed = extractObservedMetrics(fakeReport()); // detectionRate 1.0 here; force via override below + const nearlyEqual = { ...observed, detectionRate: 68 / 70 }; + const comparison = compareMetrics(baseline, nearlyEqual); + expect(comparison.rows.find((row) => row.metric === "detection rate")?.verdict).toBe("PASS"); + }); + + test("fails when false/unrelated findings rise past the absolute budget", () => { + const observed = extractObservedMetrics(fakeReport({ + falsePositives: populatedBaseline.falsePositives + FALSE_FINDINGS_MAX_INCREASE + 1, + })); + const comparison = compareMetrics(populatedBaseline, observed); + expect(comparison.rows.find((row) => row.metric === "false/unrelated findings")?.verdict).toBe("FAIL"); + expect(comparison.ok).toBe(false); + }); + + test("fails when gate-verdict correctness drops more than the tolerance", () => { + const baseline = { ...populatedBaseline, gateVerdictCorrectness: 1 }; + const observed = { ...extractObservedMetrics(fakeReport()), gateVerdictCorrectness: 1 - (GATE_VERDICT_MAX_DROP_PP / 100) - 0.01 }; + const comparison = compareMetrics(baseline, observed); + expect(comparison.rows.find((row) => row.metric === "gate verdict correctness")?.verdict).toBe("FAIL"); + }); + + test("fails when mean cost rises past the ratio budget", () => { + const observed = { ...extractObservedMetrics(fakeReport()), meanCostUsdPerCase: populatedBaseline.meanCostUsdPerCase * (1 + MEAN_COST_MAX_INCREASE_RATIO) + 0.001 }; + const comparison = compareMetrics(populatedBaseline, observed); + expect(comparison.rows.find((row) => row.metric === "mean cost per case")?.verdict).toBe("FAIL"); + }); + + test("fails when p95 latency rises past the ratio budget", () => { + const observed = { + ...extractObservedMetrics(fakeReport()), + latencyMs: { p50: 1000, p95: populatedBaseline.latencyMs.p95 * (1 + LATENCY_P95_MAX_INCREASE_RATIO) + 1 }, + }; + const comparison = compareMetrics(populatedBaseline, observed); + expect(comparison.rows.find((row) => row.metric === "p95 latency (ms)")?.verdict).toBe("FAIL"); + }); +}); diff --git a/bench/src/compare-baseline.ts b/bench/src/compare-baseline.ts new file mode 100644 index 0000000..2c3b2d6 --- /dev/null +++ b/bench/src/compare-baseline.ts @@ -0,0 +1,427 @@ +#!/usr/bin/env bun +// Release-gate regression check for the diff-file live benchmark (see live.ts). +// +// Consumes a LiveReport JSON artifact (the file `bun run bench:live --json-out +// ` writes, or the report the runner already writes under +// `.runs/live//report.json`) and compares its metrics against the +// committed `bench/baseline.json`. Exits non-zero on a material regression, so +// the release pipeline can refuse to ship a CLI that reviews worse than the +// last recorded baseline. +// +// Compare mode (default): +// +// bun run bench:compare -- --result +// bun run bench:compare -- --run-id # resolves the runner's own path +// +// Record mode writes the observed metrics into baseline.json as the new +// baseline for the report's model. This is the deliberate re-baseline path: +// nothing updates baseline.json except an explicit --record invocation. +// +// bun run bench:compare -- --result --record + +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { z } from "zod"; + +// --------------------------------------------------------------------------- +// Regression tolerances. Each constant is the largest change from baseline +// that still passes; anything past it is a material regression and fails the +// gate. Values are deliberately loose enough to absorb ordinary run-to-run +// inference variance (live mode is a single nondeterministic model run per +// case) while still catching a real behavioral or cost regression. + +/** Detection rate may drop at most this many percentage points below baseline + * before the gate fails. */ +export const DETECTION_RATE_MAX_DROP_PP = 2; + +/** The false/unrelated finding count may rise by at most this many findings + * above baseline before the gate fails. An absolute count, not a rate: the + * corpus size is fixed, so a rate would just rescale the same tolerance. */ +export const FALSE_FINDINGS_MAX_INCREASE = 2; + +/** Gate-verdict correctness (does the CLI's exit code agree with the + * authored classification: block must-block, pass everything else) may drop + * at most this many percentage points below baseline. */ +export const GATE_VERDICT_MAX_DROP_PP = 2; + +/** Mean provider cost per case may rise at most this fraction above baseline + * before the gate fails. Provider pricing and routing drift more than model + * behavior does, so this tolerance is wider than the detection tolerances. */ +export const MEAN_COST_MAX_INCREASE_RATIO = 0.25; + +/** p95 review latency may rise at most this fraction above baseline. Wide + * because provider latency is the least controllable metric here. */ +export const LATENCY_P95_MAX_INCREASE_RATIO = 0.5; + +// --------------------------------------------------------------------------- +// Live report shape (the subset of live.ts's LiveReport this gate needs). +// Loosely validated: this script reads an artifact the same run produced +// moments earlier, not untrusted input, so it checks shape, not full schema. + +const liveCaseResultSchema = z.object({ + id: z.string(), + type: z.enum(["defect", "clean"]), + scored: z.boolean(), + truthSeverity: z.string().nullable(), + durationMs: z.number().nullable(), + exitCode: z.number().optional(), +}); + +const liveReportSchema = z.object({ + summary: z.object({ + model: z.string(), + reviewMode: z.enum(["exhaustive", "bounded"]), + fixtureCorpusSha256: z.string(), + evaluatorSha256: z.string(), + totalCases: z.number().int().nonnegative(), + scoredCases: z.number().int().nonnegative(), + defectCases: z.number().int().nonnegative(), + detected: z.number().int().nonnegative(), + falsePositives: z.number().int().nonnegative(), + observedProviderCostUsdDecimal: z.string(), + ranAt: z.string(), + }), + results: z.array(liveCaseResultSchema), +}); + +export type LiveReportForComparison = z.infer; + +// --------------------------------------------------------------------------- +// Baseline file shape. + +const baselineProfileSchema = z.discriminatedUnion("populated", [ + z.object({ + populated: z.literal(false), + instructions: z.string(), + }), + z.object({ + populated: z.literal(true), + generatedAt: z.string(), + reviewMode: z.enum(["exhaustive", "bounded"]), + sourceRunAt: z.string(), + totalCases: z.number().int().positive(), + scoredCases: z.number().int().positive(), + detectionRate: z.number().min(0).max(1), + falsePositives: z.number().int().nonnegative(), + gateVerdictCorrectness: z.number().min(0).max(1), + meanCostUsdPerCase: z.number().nonnegative(), + latencyMs: z.object({ + p50: z.number().nonnegative(), + p95: z.number().nonnegative(), + }), + }), +]); + +const baselineFileSchema = z.object({ + schemaVersion: z.literal(1), + corpus: z.object({ + fixtureCorpusSha256: z.string(), + evaluatorSha256: z.string(), + }), + profiles: z.record(z.string(), baselineProfileSchema), +}); + +export type BaselineFile = z.infer; +export type BaselineProfile = z.infer; + +// --------------------------------------------------------------------------- +// Metric extraction from a live report. + +export interface ObservedMetrics { + model: string; + reviewMode: "exhaustive" | "bounded"; + fixtureCorpusSha256: string; + evaluatorSha256: string; + ranAt: string; + totalCases: number; + scoredCases: number; + detectionRate: number; + falsePositives: number; + gateVerdictCorrectness: number; + meanCostUsdPerCase: number; + latencyMs: { p50: number; p95: number }; +} + +/** True when a scored case's postil exit code should be 1: the authored + * ground truth carries an error-severity (must-block) finding. Mirrors + * harness.ts's expectedGateFailing for the mock suite. */ +function expectedGateFailing(c: Pick, "type" | "truthSeverity">): boolean { + return c.type === "defect" && c.truthSeverity === "error"; +} + +/** Nearest-rank percentile over a pre-sorted ascending array. */ +export function percentile(sorted: readonly number[], p: number): number { + if (sorted.length === 0) throw new Error("percentile of an empty sample is undefined"); + const rank = Math.ceil((p / 100) * sorted.length); + const index = Math.min(Math.max(rank, 1), sorted.length) - 1; + return sorted[index]!; +} + +export function extractObservedMetrics(report: LiveReportForComparison): ObservedMetrics { + const s = report.summary; + const scoredResults = report.results.filter((r) => r.scored); + if (scoredResults.length === 0) { + throw new Error("live report has no scored cases; nothing to compare"); + } + + const gateCorrect = scoredResults.filter( + (r) => (r.exitCode === 1) === expectedGateFailing(r), + ).length; + + const durations = scoredResults + .map((r) => r.durationMs) + .filter((v): v is number => v !== null) + .sort((a, b) => a - b); + if (durations.length === 0) { + throw new Error("live report has no case with a recorded duration; nothing to compare"); + } + + return { + model: s.model, + reviewMode: s.reviewMode, + fixtureCorpusSha256: s.fixtureCorpusSha256, + evaluatorSha256: s.evaluatorSha256, + ranAt: s.ranAt, + totalCases: s.totalCases, + scoredCases: s.scoredCases, + detectionRate: s.detected / s.defectCases, + falsePositives: s.falsePositives, + gateVerdictCorrectness: gateCorrect / scoredResults.length, + // Mean cost across every case the run attempted, not only scored ones: + // a case that burned a provider call and then failed to parse still cost + // money, and a regression that starts producing unparseable output + // should not get a cost pass for it. + meanCostUsdPerCase: Number(s.observedProviderCostUsdDecimal) / s.totalCases, + latencyMs: { + p50: percentile(durations, 50), + p95: percentile(durations, 95), + }, + }; +} + +// --------------------------------------------------------------------------- +// Comparison + +interface MetricVerdict { + metric: string; + baseline: string; + observed: string; + verdict: "PASS" | "FAIL"; + detail?: string; +} + +export interface ComparisonResult { + ok: boolean; + rows: MetricVerdict[]; +} + +function pct(v: number): string { + return `${(v * 100).toFixed(1)}%`; +} + +function usd(v: number): string { + return `$${v.toFixed(6)}`; +} + +export function compareMetrics(baseline: Extract, observed: ObservedMetrics): ComparisonResult { + const rows: MetricVerdict[] = []; + + const detectionFloor = baseline.detectionRate - DETECTION_RATE_MAX_DROP_PP / 100; + rows.push({ + metric: "detection rate", + baseline: pct(baseline.detectionRate), + observed: pct(observed.detectionRate), + verdict: observed.detectionRate >= detectionFloor ? "PASS" : "FAIL", + detail: `floor ${pct(detectionFloor)} (baseline - ${DETECTION_RATE_MAX_DROP_PP}pp)`, + }); + + const falsePositiveCeiling = baseline.falsePositives + FALSE_FINDINGS_MAX_INCREASE; + rows.push({ + metric: "false/unrelated findings", + baseline: String(baseline.falsePositives), + observed: String(observed.falsePositives), + verdict: observed.falsePositives <= falsePositiveCeiling ? "PASS" : "FAIL", + detail: `ceiling ${falsePositiveCeiling} (baseline + ${FALSE_FINDINGS_MAX_INCREASE})`, + }); + + const gateFloor = baseline.gateVerdictCorrectness - GATE_VERDICT_MAX_DROP_PP / 100; + rows.push({ + metric: "gate verdict correctness", + baseline: pct(baseline.gateVerdictCorrectness), + observed: pct(observed.gateVerdictCorrectness), + verdict: observed.gateVerdictCorrectness >= gateFloor ? "PASS" : "FAIL", + detail: `floor ${pct(gateFloor)} (baseline - ${GATE_VERDICT_MAX_DROP_PP}pp)`, + }); + + const costCeiling = baseline.meanCostUsdPerCase * (1 + MEAN_COST_MAX_INCREASE_RATIO); + rows.push({ + metric: "mean cost per case", + baseline: usd(baseline.meanCostUsdPerCase), + observed: usd(observed.meanCostUsdPerCase), + verdict: observed.meanCostUsdPerCase <= costCeiling ? "PASS" : "FAIL", + detail: `ceiling ${usd(costCeiling)} (baseline x ${(1 + MEAN_COST_MAX_INCREASE_RATIO).toFixed(2)})`, + }); + + const latencyCeiling = baseline.latencyMs.p95 * (1 + LATENCY_P95_MAX_INCREASE_RATIO); + rows.push({ + metric: "p95 latency (ms)", + baseline: baseline.latencyMs.p95.toFixed(0), + observed: observed.latencyMs.p95.toFixed(0), + verdict: observed.latencyMs.p95 <= latencyCeiling ? "PASS" : "FAIL", + detail: `ceiling ${latencyCeiling.toFixed(0)} (baseline x ${(1 + LATENCY_P95_MAX_INCREASE_RATIO).toFixed(2)})`, + }); + + // p50 latency and total case counts are reported for context; they are not + // independently gated (p95 already bounds the tail, and the corpus-hash + // check below already guarantees the case count did not silently change). + rows.push({ + metric: "p50 latency (ms), informational", + baseline: "n/a", + observed: observed.latencyMs.p50.toFixed(0), + verdict: "PASS", + }); + + return { ok: rows.every((r) => r.verdict === "PASS"), rows }; +} + +export function formatComparisonTable(rows: readonly MetricVerdict[]): string { + const widths = { + metric: Math.max(...rows.map((r) => r.metric.length), "metric".length), + baseline: Math.max(...rows.map((r) => r.baseline.length), "baseline".length), + observed: Math.max(...rows.map((r) => r.observed.length), "observed".length), + verdict: "verdict".length, + }; + const pad = (v: string, width: number) => v + " ".repeat(Math.max(0, width - v.length)); + const lines = [ + [pad("metric", widths.metric), pad("baseline", widths.baseline), pad("observed", widths.observed), pad("verdict", widths.verdict)].join(" "), + ]; + for (const row of rows) { + lines.push( + [pad(row.metric, widths.metric), pad(row.baseline, widths.baseline), pad(row.observed, widths.observed), pad(row.verdict, widths.verdict)].join(" ") + + (row.detail ? ` (${row.detail})` : ""), + ); + } + return lines.join("\n"); +} + +// --------------------------------------------------------------------------- +// CLI + +function flagValue(args: string[], flag: string): string | undefined { + const index = args.indexOf(flag); + const value = index === -1 ? undefined : args[index + 1]; + return value?.startsWith("--") === true ? undefined : value; +} + +async function readJson(path: string): Promise { + return JSON.parse(await readFile(path, "utf8")); +} + +function defaultBaselinePath(): string { + return resolve(import.meta.dir, "..", "baseline.json"); +} + +function liveRunnerReportPath(runId: string): string { + return resolve(import.meta.dir, "..", ".runs", "live", runId, "report.json"); +} + +async function main() { + const args = process.argv.slice(2); + const record = args.includes("--record"); + const baselinePath = flagValue(args, "--baseline") ?? defaultBaselinePath(); + const resultFlag = flagValue(args, "--result"); + const runIdFlag = flagValue(args, "--run-id"); + if (resultFlag === undefined && runIdFlag === undefined) { + throw new Error("bench:compare needs either --result or --run-id "); + } + const resultPath = resultFlag ?? liveRunnerReportPath(runIdFlag!); + + const report = liveReportSchema.parse(await readJson(resultPath)); + const observed = extractObservedMetrics(report); + + const baselineRaw = await readJson(baselinePath).catch((error) => { + throw new Error(`could not read baseline at ${baselinePath}: ${error instanceof Error ? error.message : String(error)}`); + }); + const baselineFile = baselineFileSchema.parse(baselineRaw); + + if (record) { + baselineFile.profiles[observed.model] = { + populated: true, + generatedAt: new Date().toISOString(), + reviewMode: observed.reviewMode, + sourceRunAt: observed.ranAt, + totalCases: observed.totalCases, + scoredCases: observed.scoredCases, + detectionRate: observed.detectionRate, + falsePositives: observed.falsePositives, + gateVerdictCorrectness: observed.gateVerdictCorrectness, + meanCostUsdPerCase: observed.meanCostUsdPerCase, + latencyMs: observed.latencyMs, + }; + baselineFile.corpus = { + fixtureCorpusSha256: observed.fixtureCorpusSha256, + evaluatorSha256: observed.evaluatorSha256, + }; + await writeFile(baselinePath, `${JSON.stringify(baselineFile, null, 2)}\n`); + console.log(`Recorded baseline for ${observed.model} at ${baselinePath}`); + return; + } + + if (baselineFile.corpus.fixtureCorpusSha256 !== observed.fixtureCorpusSha256 || + baselineFile.corpus.evaluatorSha256 !== observed.evaluatorSha256) { + console.error( + "FIXTURE CORPUS MISMATCH: this report was scored against a different fixture set or\n" + + "evaluator source than baseline.json was recorded against, so its metrics are not\n" + + "comparable.\n" + + ` baseline fixtureCorpusSha256 ${baselineFile.corpus.fixtureCorpusSha256}\n` + + ` observed fixtureCorpusSha256 ${observed.fixtureCorpusSha256}\n` + + ` baseline evaluatorSha256 ${baselineFile.corpus.evaluatorSha256}\n` + + ` observed evaluatorSha256 ${observed.evaluatorSha256}\n\n` + + "This is not a regression the gate can evaluate; it means the fixtures or scoring\n" + + "code changed since the baseline was recorded. Re-baseline deliberately once you\n" + + "have confirmed the new corpus is intentional:\n" + + ` bun run bench:compare -- --result ${resultPath} --record`, + ); + process.exitCode = 1; + return; + } + + const profile = baselineFile.profiles[observed.model]; + if (profile === undefined) { + console.error( + `No baseline is recorded for model ${observed.model}. Populate one with:\n` + + ` bun run bench:compare -- --result ${resultPath} --record`, + ); + process.exitCode = 1; + return; + } + if (!profile.populated) { + console.error( + `The baseline for ${observed.model} is a placeholder, not yet populated: ${profile.instructions}`, + ); + process.exitCode = 1; + return; + } + + const comparison = compareMetrics(profile, observed); + console.log(`postil bench-live release gate: ${observed.model} (${observed.reviewMode})`); + console.log(formatComparisonTable(comparison.rows)); + if (!comparison.ok) { + console.error( + "\nRELEASE BLOCKED: the live benchmark regressed past tolerance against bench/baseline.json.\n" + + "Fix the regression, or if the new numbers are an accepted tradeoff, re-baseline\n" + + "deliberately with:\n" + + ` bun run bench:compare -- --result ${resultPath} --record`, + ); + process.exitCode = 1; + return; + } + console.log("\nPASS: no metric regressed past tolerance."); +} + +if (import.meta.main) { + main().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; + }); +} diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 91c0fb5..8490d04 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1293,8 +1293,13 @@ describe("managed admission workflow", () => { const release = await Bun.file( resolve(import.meta.dir, "..", "..", ".github", "workflows", "release.yml"), ).text(); - expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?\n build:\n/u); - expect(release).toMatch(/build:\n\s+needs: validate-tag/u); + expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?\n bench-live:\n/u); + expect(release).toMatch(/bench-live:\n\s+needs: validate-tag\n/u); + expect(release).toContain("REVIEW_MODEL: z-ai/glm-5.2"); + expect(release).toContain("OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}"); + expect(release).toContain("bun run bench:live --"); + expect(release).toContain("bun run bench:compare --"); + expect(release).toMatch(/build:\n\s+needs: \[validate-tag, bench-live\]/u); let checkedReferences = 0; const workflowGlob = new Bun.Glob("*.yml"); for await (const workflowName of workflowGlob.scan(resolve(import.meta.dir, "..", "..", ".github", "workflows"))) {