From d152eff052381d98f0cb33124c63d1dbf6c9b70b Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 30 Jul 2026 22:15:48 +0000 Subject: [PATCH] Tell a benchmark outage apart from a quality regression The v0.8.5 release gate blocked on "live report has no scored cases; nothing to compare". Every one of the 70 cases had failed with a provider error, at zero tokens and zero cost, about three minutes after the run started. Nothing in that message names a credential, and the run step itself exited 0, so the only signal that the benchmark never reached the model was an empty report arriving at the comparison step. The live run now exits non-zero when it scores no case, and says that a run which measured nothing is an operational failure rather than a quality result. The comparison reports the same distinction, since whoever reads a blocked release needs to know whether the model got worse or was never called. The credential check now probes the key instead of testing that the variable is non-empty, and rejects one whose remaining credit is below the cost of a single run. A present-but-rejected key previously spent three minutes failing every case before surfacing anything. --- .github/workflows/release.yml | 28 ++++++++++++++++++++++++++-- bench/src/compare-baseline.test.ts | 13 +++++++++++-- bench/src/compare-baseline.ts | 11 ++++++++++- bench/src/run.ts | 13 +++++++++++++ 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c4aeb4..9f310ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,10 +47,34 @@ jobs: with: bun-version: 1.3.14 - - name: Require OpenRouter credential + # A present-but-rejected key fails all 70 cases about three minutes later + # with nothing that names the credential. Probing it costs one request and + # turns that into an immediate, accurate failure. + - name: Require a working OpenRouter credential env: COMPLETION_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - run: test -n "$COMPLETION_API_KEY" + run: | + set -euo pipefail + if [[ -z "${COMPLETION_API_KEY}" ]]; then + echo "::error::OPENROUTER_API_KEY is not set." + exit 1 + fi + status="$(curl --fail-with-body --silent --show-error \ + --output /tmp/openrouter-key.json --write-out '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + -H "Authorization: Bearer ${COMPLETION_API_KEY}" \ + https://openrouter.ai/api/v1/key)" || { + echo "::error::OPENROUTER_API_KEY was rejected by OpenRouter (HTTP ${status:-none}). Rotate the repository secret." + exit 1 + } + # A key with a spent limit authenticates and then fails every call. + remaining="$(jq -r '.data.limit_remaining // "unlimited"' /tmp/openrouter-key.json)" + if [[ "${remaining}" != "unlimited" ]] && \ + awk "BEGIN { exit !(${remaining} < 1) }"; then + echo "::error::OPENROUTER_API_KEY has ${remaining} credit remaining, below the cost of one benchmark run." + exit 1 + fi + echo "OpenRouter credential accepted (remaining: ${remaining})." # Plain release profile: the same build the release job ships, so the # gate measures the binary users will actually get. diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index 3e5e1b4..4cce14d 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -113,13 +113,22 @@ describe("extractObservedMetrics", () => { expect(metrics.latencyMs.p50).toBe(1000); }); - test("throws when every case is unscored", () => { + test("names an all-unscored run an operational failure, not a regression", () => { + // A release blocked because the model got worse and one blocked because the + // run never reached the model need different responses from whoever reads + // the log, so the message has to say which happened. const report = fakeReport({ results: [ { id: "errored-1", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined }, + { id: "errored-2", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined }, ], }); - expect(() => extractObservedMetrics(report)).toThrow("no scored cases"); + expect(() => extractObservedMetrics(report)).toThrow( + "scored none of its 2 cases", + ); + expect(() => extractObservedMetrics(report)).toThrow( + "operational failure rather than a quality regression", + ); }); }); diff --git a/bench/src/compare-baseline.ts b/bench/src/compare-baseline.ts index 2c3b2d6..5c76614 100644 --- a/bench/src/compare-baseline.ts +++ b/bench/src/compare-baseline.ts @@ -161,7 +161,16 @@ export function extractObservedMetrics(report: LiveReportForComparison): Observe const s = report.summary; const scoredResults = report.results.filter((r) => r.scored); if (scoredResults.length === 0) { - throw new Error("live report has no scored cases; nothing to compare"); + // Distinguishable on purpose. A release blocked because the model got worse + // and a release blocked because the run never reached the model call for + // different responses, and the second must not be mistaken for the first. + throw new Error( + `live report scored none of its ${report.results.length} cases, so no metric ` + + "could be computed. This is an operational failure rather than a quality " + + "regression: every case failed before producing a valid envelope. Check " + + "the provider credential, the account's remaining credit, and the model's " + + "availability, then rerun the benchmark.", + ); } const gateCorrect = scoredResults.filter( diff --git a/bench/src/run.ts b/bench/src/run.ts index 268d192..ff72444 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -345,6 +345,19 @@ async function main() { }); await writeReport(jsonOut, JSON.stringify(report, null, 2), runId); console.log(json ? JSON.stringify(report, null, 2) : formatLiveReport(report)); + // A run that scored nothing measured nothing. Exiting 0 here leaves the + // release gate to infer an outage from an empty report, which reads as a + // tooling problem rather than as "the provider was never reached". + if (report.results.length > 0 && !report.results.some((result) => result.scored)) { + console.error( + `live benchmark produced no scored case out of ${report.results.length}: ` + + "every case failed before a valid envelope was produced. This is an " + + "operational failure, not a quality measurement. Check the provider " + + "credential, the account's remaining credit, and the model's " + + "availability before rerunning.", + ); + process.exitCode = 1; + } return; }