diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c4aeb4..9f310ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,10 +47,34 @@ jobs: with: bun-version: 1.3.14 - - name: Require OpenRouter credential + # A present-but-rejected key fails all 70 cases about three minutes later + # with nothing that names the credential. Probing it costs one request and + # turns that into an immediate, accurate failure. + - name: Require a working OpenRouter credential env: COMPLETION_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - run: test -n "$COMPLETION_API_KEY" + run: | + set -euo pipefail + if [[ -z "${COMPLETION_API_KEY}" ]]; then + echo "::error::OPENROUTER_API_KEY is not set." + exit 1 + fi + status="$(curl --fail-with-body --silent --show-error \ + --output /tmp/openrouter-key.json --write-out '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + -H "Authorization: Bearer ${COMPLETION_API_KEY}" \ + https://openrouter.ai/api/v1/key)" || { + echo "::error::OPENROUTER_API_KEY was rejected by OpenRouter (HTTP ${status:-none}). Rotate the repository secret." + exit 1 + } + # A key with a spent limit authenticates and then fails every call. + remaining="$(jq -r '.data.limit_remaining // "unlimited"' /tmp/openrouter-key.json)" + if [[ "${remaining}" != "unlimited" ]] && \ + awk "BEGIN { exit !(${remaining} < 1) }"; then + echo "::error::OPENROUTER_API_KEY has ${remaining} credit remaining, below the cost of one benchmark run." + exit 1 + fi + echo "OpenRouter credential accepted (remaining: ${remaining})." # Plain release profile: the same build the release job ships, so the # gate measures the binary users will actually get. diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index 3e5e1b4..4cce14d 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -113,13 +113,22 @@ describe("extractObservedMetrics", () => { expect(metrics.latencyMs.p50).toBe(1000); }); - test("throws when every case is unscored", () => { + test("names an all-unscored run an operational failure, not a regression", () => { + // A release blocked because the model got worse and one blocked because the + // run never reached the model need different responses from whoever reads + // the log, so the message has to say which happened. const report = fakeReport({ results: [ { id: "errored-1", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined }, + { id: "errored-2", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined }, ], }); - expect(() => extractObservedMetrics(report)).toThrow("no scored cases"); + expect(() => extractObservedMetrics(report)).toThrow( + "scored none of its 2 cases", + ); + expect(() => extractObservedMetrics(report)).toThrow( + "operational failure rather than a quality regression", + ); }); }); diff --git a/bench/src/compare-baseline.ts b/bench/src/compare-baseline.ts index 2c3b2d6..5c76614 100644 --- a/bench/src/compare-baseline.ts +++ b/bench/src/compare-baseline.ts @@ -161,7 +161,16 @@ export function extractObservedMetrics(report: LiveReportForComparison): Observe const s = report.summary; const scoredResults = report.results.filter((r) => r.scored); if (scoredResults.length === 0) { - throw new Error("live report has no scored cases; nothing to compare"); + // Distinguishable on purpose. A release blocked because the model got worse + // and a release blocked because the run never reached the model call for + // different responses, and the second must not be mistaken for the first. + throw new Error( + `live report scored none of its ${report.results.length} cases, so no metric ` + + "could be computed. This is an operational failure rather than a quality " + + "regression: every case failed before producing a valid envelope. Check " + + "the provider credential, the account's remaining credit, and the model's " + + "availability, then rerun the benchmark.", + ); } const gateCorrect = scoredResults.filter( diff --git a/bench/src/run.ts b/bench/src/run.ts index 268d192..ff72444 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -345,6 +345,19 @@ async function main() { }); await writeReport(jsonOut, JSON.stringify(report, null, 2), runId); console.log(json ? JSON.stringify(report, null, 2) : formatLiveReport(report)); + // A run that scored nothing measured nothing. Exiting 0 here leaves the + // release gate to infer an outage from an empty report, which reads as a + // tooling problem rather than as "the provider was never reached". + if (report.results.length > 0 && !report.results.some((result) => result.scored)) { + console.error( + `live benchmark produced no scored case out of ${report.results.length}: ` + + "every case failed before a valid envelope was produced. This is an " + + "operational failure, not a quality measurement. Check the provider " + + "credential, the account's remaining credit, and the model's " + + "availability before rerunning.", + ); + process.exitCode = 1; + } return; }