Tell a benchmark outage apart from a quality regression

.github/workflows/release.yml

@@ -47,10 +47,34 @@ jobs:
         with:
           bun-version: 1.3.14
 
-      - name: Require OpenRouter credential
+      # A present-but-rejected key fails all 70 cases about three minutes later
+      # with nothing that names the credential. Probing it costs one request and
+      # turns that into an immediate, accurate failure.
+      - name: Require a working OpenRouter credential
         env:
           COMPLETION_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
-        run: test -n "$COMPLETION_API_KEY"
+        run: |
+          set -euo pipefail
+          if [[ -z "${COMPLETION_API_KEY}" ]]; then
+            echo "::error::OPENROUTER_API_KEY is not set."
+            exit 1
+          fi
+          status="$(curl --fail-with-body --silent --show-error \
+            --output /tmp/openrouter-key.json --write-out '%{http_code}' \
+            --connect-timeout 10 --max-time 30 \
+            -H "Authorization: Bearer ${COMPLETION_API_KEY}" \
+            https://openrouter.ai/api/v1/key)" || {
+            echo "::error::OPENROUTER_API_KEY was rejected by OpenRouter (HTTP ${status:-none}). Rotate the repository secret."
+            exit 1
+          }
+          # A key with a spent limit authenticates and then fails every call.
+          remaining="$(jq -r '.data.limit_remaining // "unlimited"' /tmp/openrouter-key.json)"
+          if [[ "${remaining}" != "unlimited" ]] && \
+             awk "BEGIN { exit !(${remaining} < 1) }"; then
+            echo "::error::OPENROUTER_API_KEY has ${remaining} credit remaining, below the cost of one benchmark run."
+            exit 1
+          fi
+          echo "OpenRouter credential accepted (remaining: ${remaining})."
 
       # Plain release profile: the same build the release job ships, so the
       # gate measures the binary users will actually get.

bench/src/compare-baseline.test.ts

@@ -113,13 +113,22 @@ describe("extractObservedMetrics", () => {
     expect(metrics.latencyMs.p50).toBe(1000);
   });
 
-  test("throws when every case is unscored", () => {
+  test("names an all-unscored run an operational failure, not a regression", () => {
+    // A release blocked because the model got worse and one blocked because the
+    // run never reached the model need different responses from whoever reads
+    // the log, so the message has to say which happened.
     const report = fakeReport({
       results: [
         { id: "errored-1", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined },
+        { id: "errored-2", type: "defect", scored: false, truthSeverity: "error", durationMs: null, exitCode: undefined },
       ],
     });
-    expect(() => extractObservedMetrics(report)).toThrow("no scored cases");
+    expect(() => extractObservedMetrics(report)).toThrow(
+      "scored none of its 2 cases",
+    );
+    expect(() => extractObservedMetrics(report)).toThrow(
+      "operational failure rather than a quality regression",
+    );
   });
 });
 

bench/src/compare-baseline.ts

@@ -161,7 +161,16 @@ export function extractObservedMetrics(report: LiveReportForComparison): Observe
   const s = report.summary;
   const scoredResults = report.results.filter((r) => r.scored);
   if (scoredResults.length === 0) {
-    throw new Error("live report has no scored cases; nothing to compare");
+    // Distinguishable on purpose. A release blocked because the model got worse
+    // and a release blocked because the run never reached the model call for
+    // different responses, and the second must not be mistaken for the first.
+    throw new Error(
+      `live report scored none of its ${report.results.length} cases, so no metric ` +
+        "could be computed. This is an operational failure rather than a quality " +
+        "regression: every case failed before producing a valid envelope. Check " +
+        "the provider credential, the account's remaining credit, and the model's " +
+        "availability, then rerun the benchmark.",
+    );
   }
 
   const gateCorrect = scoredResults.filter(

bench/src/run.ts

@@ -345,6 +345,19 @@ async function main() {
     });
     await writeReport(jsonOut, JSON.stringify(report, null, 2), runId);
     console.log(json ? JSON.stringify(report, null, 2) : formatLiveReport(report));
+    // A run that scored nothing measured nothing. Exiting 0 here leaves the
+    // release gate to infer an outage from an empty report, which reads as a
+    // tooling problem rather than as "the provider was never reached".
+    if (report.results.length > 0 && !report.results.some((result) => result.scored)) {
+      console.error(
+        `live benchmark produced no scored case out of ${report.results.length}: ` +
+          "every case failed before a valid envelope was produced. This is an " +
+          "operational failure, not a quality measurement. Check the provider " +
+          "credential, the account's remaining credit, and the model's " +
+          "availability before rerunning.",
+      );
+      process.exitCode = 1;
+    }
     return;
   }