Gate releases on a live benchmark regression check

.github/workflows/release.yml

@@ -33,8 +33,53 @@ jobs:
             exit 1
           }
 
-  build:
+  bench-live:
     needs: validate-tag
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
+      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.14
+
+      - name: Require OpenRouter credential
+        env:
+          COMPLETION_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+        run: test -n "$COMPLETION_API_KEY"
+
+      # Plain release profile: the same build the release job ships, so the
+      # gate measures the binary users will actually get.
+      - run: cargo build --quiet --release
+      - name: Install benchmark dependencies
+        working-directory: bench
+        run: bun install --frozen-lockfile
+
+      - name: Run diff-file live benchmark
+        working-directory: bench
+        env:
+          REVIEW_MODEL: z-ai/glm-5.2
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+        run: bun run bench:live -- --run-id "release-${{ github.ref_name }}" --json-out "${{ runner.temp }}/bench-live-report.json"
+
+      - name: Compare against the recorded baseline
+        id: compare-baseline
+        working-directory: bench
+        run: bun run bench:compare -- --result "${{ runner.temp }}/bench-live-report.json"
+
+      - name: Report benchmark regression
+        if: steps.compare-baseline.outcome == 'failure'
+        run: |
+          echo "::error::Release blocked by a benchmark regression. See the 'Compare against the recorded baseline' step above for the failing metric(s)."
+          echo "## Release blocked" >> "$GITHUB_STEP_SUMMARY"
+          echo "The live benchmark regressed past tolerance against \`bench/baseline.json\`. See the 'Compare against the recorded baseline' step log for the failing metric(s)." >> "$GITHUB_STEP_SUMMARY"
+
+  build:
+    needs: [validate-tag, bench-live]
     strategy:
       matrix:
         include:

bench/README.md

@@ -358,9 +358,11 @@ scorer call or scorer identity. A finding that is later suppressed does
 exercise the scorer and must retain its exact identity and usage record.
 
 It refuses to run without `POSTIL_API_KEY`, `OPENROUTER_API_KEY`, `MODEL_API_KEY`,
-or `LLM_API_KEY` and never logs or prints the key value. Live mode is
-**not run in CI**: it spends real tokens and depends on an
-external provider. Every live run writes its JSON report under
+or `LLM_API_KEY` and never logs or prints the key value. Live mode spends real
+tokens and depends on an external provider, so it is **not run on ordinary
+pushes or pull requests**; the release pipeline runs a full-corpus live pass
+against `bench/baseline.json` before every tagged release (see "Release gate"
+below). Every live run writes its JSON report under
 `.runs/live/<run-id>/` (gitignored), beside raw per-attempt stdout and stderr.
 Set `--run-id <id>` or `POSTIL_BENCH_SCREEN_RUN_ID` to name the immutable
 namespace; an omitted ID gets a unique generated value. Reusing an ID fails
@@ -434,3 +436,40 @@ per case, diff-only with no repository context or policy docs. **Neither
 severity metric is a peer-comparison claim**: no competitor has been run on the
 same fixtures. Results vary across runs because model inference is
 nondeterministic. Treat them as internal evidence, not a published benchmark.
+
+## Release gate
+
+The `Release` workflow runs a full-corpus diff-file live pass
+(`REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live`) against the plain release
+binary before it builds any target, then checks the result with
+`bun run bench:compare`. A material regression blocks the release: `build`
+depends on the `bench-live` job.
+
+`compare-baseline.ts` computes five metrics from the live report and compares
+each against the matching model entry in `bench/baseline.json`: authored-target
+detection rate, false/unrelated finding count, gate-verdict correctness (does
+the CLI's exit code agree with the authored must-block/advisory/clean
+classification), mean provider cost per case, and p95 review latency. Each
+metric has its own tolerance (see the exported `*_MAX_*` constants at the top
+of `compare-baseline.ts`), wide enough to absorb ordinary run-to-run inference
+variance but not a real behavioral, cost, or latency regression. The baseline
+also records the fixture-corpus and evaluator-source SHA-256 digests the live
+report already computes; a mismatch fails loudly instead of comparing metrics
+across an unrelated fixture set.
+
+```sh
+# Compare an existing report against the committed baseline.
+bun run bench:compare -- --result <path-to-report.json>
+bun run bench:compare -- --run-id <run-id>   # resolves .runs/live/<run-id>/report.json
+
+# Re-baseline deliberately, after confirming the new numbers are an accepted
+# tradeoff (a model change, a fixture change, an intentional pipeline change):
+REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live -- --json-out live-report.json
+bun run bench:compare -- --result live-report.json --record
+```
+
+`--record` is the only thing that writes `bench/baseline.json`; nothing else
+updates it implicitly. When the gate fails in CI, the printed table shows
+baseline vs. observed vs. verdict for every metric, so the failing metric and
+by how much is visible directly in the job log without downloading the report
+artifact.

bench/baseline.json

@@ -0,0 +1,25 @@
+{
+  "schemaVersion": 1,
+  "corpus": {
+    "fixtureCorpusSha256": "3c4f952f9411bf2c4e5f0a61b5975e47d203f7681a8235d53d717c743ea9cb54",
+    "evaluatorSha256": "64dd1975bd5743e15b073b8c6dbad91e37cef3388f793073b216d002de80d78a"
+  },
+  "profiles": {
+    "z-ai/glm-5.2": {
+      "populated": true,
+      "generatedAt": "2026-07-30T01:06:07.029Z",
+      "reviewMode": "exhaustive",
+      "sourceRunAt": "2026-07-30T01:05:48.700Z",
+      "totalCases": 70,
+      "scoredCases": 69,
+      "detectionRate": 0.9649122807017544,
+      "falsePositives": 4,
+      "gateVerdictCorrectness": 0.7681159420289855,
+      "meanCostUsdPerCase": 0.004135275999999999,
+      "latencyMs": {
+        "p50": 4528,
+        "p95": 48585
+      }
+    }
+  }
+}

bench/package.json

@@ -8,6 +8,7 @@
     "bench": "bun run src/run.ts",
     "bench:live": "bun run src/run.ts --live",
     "bench:live-models": "POSTIL_BENCH_MODE=live bun run src/run.ts",
+    "bench:compare": "bun run src/compare-baseline.ts",
     "verify-admission": "bun run src/verify-admission.ts",
     "scorer-eval": "bun run src/scorer-eval.ts"
   },

bench/src/compare-baseline.test.ts

@@ -0,0 +1,193 @@
+import { describe, expect, test } from "bun:test";
+import {
+  DETECTION_RATE_MAX_DROP_PP,
+  FALSE_FINDINGS_MAX_INCREASE,
+  GATE_VERDICT_MAX_DROP_PP,
+  LATENCY_P95_MAX_INCREASE_RATIO,
+  MEAN_COST_MAX_INCREASE_RATIO,
+  compareMetrics,
+  extractObservedMetrics,
+  percentile,
+  type BaselineProfile,
+  type LiveReportForComparison,
+} from "./compare-baseline";
+
+function fakeReport(overrides: {
+  falsePositives?: number;
+  observedProviderCostUsdDecimal?: string;
+  results?: LiveReportForComparison["results"];
+  detected?: number;
+} = {}): LiveReportForComparison {
+  const results: LiveReportForComparison["results"] = overrides.results ?? [
+    { id: "must-block-1", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 1 },
+    { id: "must-block-2", type: "defect", scored: true, truthSeverity: "error", durationMs: 2000, exitCode: 1 },
+    { id: "advisory-1", type: "defect", scored: true, truthSeverity: "warn", durationMs: 1500, exitCode: 0 },
+    { id: "clean-1", type: "clean", scored: true, truthSeverity: null, durationMs: 500, exitCode: 0 },
+  ];
+  const defectCases = results.filter((r) => r.type === "defect").length;
+  return {
+    summary: {
+      model: "z-ai/glm-5.2",
+      reviewMode: "exhaustive",
+      fixtureCorpusSha256: "corpus-sha",
+      evaluatorSha256: "evaluator-sha",
+      totalCases: results.length,
+      scoredCases: results.filter((r) => r.scored).length,
+      defectCases,
+      detected: overrides.detected ?? defectCases,
+      falsePositives: overrides.falsePositives ?? 0,
+      observedProviderCostUsdDecimal: overrides.observedProviderCostUsdDecimal ?? "0.004",
+      ranAt: "2026-01-01T00:00:00.000Z",
+    },
+    results,
+  };
+}
+
+const populatedBaseline: Extract<BaselineProfile, { populated: true }> = {
+  populated: true,
+  generatedAt: "2026-01-01T00:00:00.000Z",
+  reviewMode: "exhaustive",
+  sourceRunAt: "2026-01-01T00:00:00.000Z",
+  totalCases: 70,
+  scoredCases: 70,
+  detectionRate: 0.95,
+  falsePositives: 2,
+  gateVerdictCorrectness: 0.97,
+  meanCostUsdPerCase: 0.003,
+  latencyMs: { p50: 4000, p95: 9000 },
+};
+
+describe("percentile", () => {
+  test("nearest-rank over a sorted sample", () => {
+    const sorted = [10, 20, 30, 40, 50];
+    expect(percentile(sorted, 50)).toBe(30);
+    expect(percentile(sorted, 95)).toBe(50);
+    expect(percentile(sorted, 1)).toBe(10);
+  });
+
+  test("single-element sample returns that element at any percentile", () => {
+    expect(percentile([42], 50)).toBe(42);
+    expect(percentile([42], 95)).toBe(42);
+  });
+
+  test("rejects an empty sample", () => {
+    expect(() => percentile([], 50)).toThrow("empty sample");
+  });
+});
+
+describe("extractObservedMetrics", () => {
+  test("derives detection rate, gate-verdict correctness, cost, and latency from a live report", () => {
+    const metrics = extractObservedMetrics(fakeReport());
+    // 2 defects, 2 detected.
+    expect(metrics.detectionRate).toBeCloseTo(1, 5);
+    // All four scored cases: two must-block exit 1 (correct), one advisory
+    // exit 0 (correct), one clean exit 0 (correct) => 4/4 correct.
+    expect(metrics.gateVerdictCorrectness).toBeCloseTo(1, 5);
+    expect(metrics.meanCostUsdPerCase).toBeCloseTo(0.004 / 4, 6);
+    expect(metrics.latencyMs.p50).toBeGreaterThan(0);
+    expect(metrics.latencyMs.p95).toBe(2000);
+  });
+
+  test("scores a wrongly-blocked advisory case as an incorrect gate verdict", () => {
+    const report = fakeReport({
+      results: [
+        { id: "must-block-1", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 1 },
+        // An advisory (warn) case that the model over-blocked: exit 1 when it
+        // should have exited 0.
+        { id: "advisory-1", type: "defect", scored: true, truthSeverity: "warn", durationMs: 1500, exitCode: 1 },
+      ],
+    });
+    const metrics = extractObservedMetrics(report);
+    expect(metrics.gateVerdictCorrectness).toBeCloseTo(0.5, 5);
+  });
+
+  test("excludes unscored (errored) cases from gate-verdict and latency scoring", () => {
+    const report = fakeReport({
+      results: [
+        { id: "must-block-1", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 1 },
+        { id: "errored-1", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined },
+      ],
+    });
+    const metrics = extractObservedMetrics(report);
+    expect(metrics.gateVerdictCorrectness).toBeCloseTo(1, 5);
+    expect(metrics.latencyMs.p50).toBe(1000);
+  });
+
+  test("throws when every case is unscored", () => {
+    const report = fakeReport({
+      results: [
+        { id: "errored-1", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined },
+      ],
+    });
+    expect(() => extractObservedMetrics(report)).toThrow("no scored cases");
+  });
+});
+
+describe("compareMetrics", () => {
+  test("passes when observed metrics match baseline exactly", () => {
+    const baseline = { ...populatedBaseline, detectionRate: 1, gateVerdictCorrectness: 1 };
+    const observed = extractObservedMetrics(fakeReport({
+      falsePositives: baseline.falsePositives,
+      observedProviderCostUsdDecimal: String(baseline.meanCostUsdPerCase * 4),
+    }));
+    const comparison = compareMetrics(baseline, observed);
+    expect(comparison.ok).toBe(true);
+    expect(comparison.rows.every((row) => row.verdict === "PASS")).toBe(true);
+  });
+
+  test("fails when detection rate drops more than the tolerance", () => {
+    const baseline = { ...populatedBaseline, detectionRate: 0.98 };
+    const observed = extractObservedMetrics(fakeReport({
+      detected: 1,
+      results: [
+        { id: "must-block-1", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 1 },
+        { id: "must-block-2", type: "defect", scored: true, truthSeverity: "error", durationMs: 1000, exitCode: 0 },
+      ],
+    }));
+    expect(observed.detectionRate).toBeCloseTo(0.5, 5);
+    expect(0.98 - observed.detectionRate).toBeGreaterThan(DETECTION_RATE_MAX_DROP_PP / 100);
+    const comparison = compareMetrics(baseline, observed);
+    expect(comparison.ok).toBe(false);
+    expect(comparison.rows.find((row) => row.metric === "detection rate")?.verdict).toBe("FAIL");
+  });
+
+  test("tolerates a detection-rate drop within the percentage-point budget", () => {
+    // Baseline 69/70, observed 68/70: a ~1.4pp drop, inside the 2pp budget.
+    const baseline = { ...populatedBaseline, detectionRate: 69 / 70 };
+    const observed = extractObservedMetrics(fakeReport()); // detectionRate 1.0 here; force via override below
+    const nearlyEqual = { ...observed, detectionRate: 68 / 70 };
+    const comparison = compareMetrics(baseline, nearlyEqual);
+    expect(comparison.rows.find((row) => row.metric === "detection rate")?.verdict).toBe("PASS");
+  });
+
+  test("fails when false/unrelated findings rise past the absolute budget", () => {
+    const observed = extractObservedMetrics(fakeReport({
+      falsePositives: populatedBaseline.falsePositives + FALSE_FINDINGS_MAX_INCREASE + 1,
+    }));
+    const comparison = compareMetrics(populatedBaseline, observed);
+    expect(comparison.rows.find((row) => row.metric === "false/unrelated findings")?.verdict).toBe("FAIL");
+    expect(comparison.ok).toBe(false);
+  });
+
+  test("fails when gate-verdict correctness drops more than the tolerance", () => {
+    const baseline = { ...populatedBaseline, gateVerdictCorrectness: 1 };
+    const observed = { ...extractObservedMetrics(fakeReport()), gateVerdictCorrectness: 1 - (GATE_VERDICT_MAX_DROP_PP / 100) - 0.01 };
+    const comparison = compareMetrics(baseline, observed);
+    expect(comparison.rows.find((row) => row.metric === "gate verdict correctness")?.verdict).toBe("FAIL");
+  });
+
+  test("fails when mean cost rises past the ratio budget", () => {
+    const observed = { ...extractObservedMetrics(fakeReport()), meanCostUsdPerCase: populatedBaseline.meanCostUsdPerCase * (1 + MEAN_COST_MAX_INCREASE_RATIO) + 0.001 };
+    const comparison = compareMetrics(populatedBaseline, observed);
+    expect(comparison.rows.find((row) => row.metric === "mean cost per case")?.verdict).toBe("FAIL");
+  });
+
+  test("fails when p95 latency rises past the ratio budget", () => {
+    const observed = {
+      ...extractObservedMetrics(fakeReport()),
+      latencyMs: { p50: 1000, p95: populatedBaseline.latencyMs.p95 * (1 + LATENCY_P95_MAX_INCREASE_RATIO) + 1 },
+    };
+    const comparison = compareMetrics(populatedBaseline, observed);
+    expect(comparison.rows.find((row) => row.metric === "p95 latency (ms)")?.verdict).toBe("FAIL");
+  });
+});