From f6137c69931721bcf86692e3fd7110af57232666 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 5 Oct 2026 18:56:58 -0600 Subject: [PATCH] fix(runtime): surface provider HTTP rejections instead of RUNTIME_FINAL_ERROR The protected live-tangle check failed with only RUNTIME_FINAL_ERROR. The runtime's real message was a Router 401 invalid_key, but the run-event mapper reduced every free-text final error to the bare code. Classify Router HTTP failures into a public diagnostic that keeps the status class, HTTP status, and the provider's own error type and code tokens. Tokens that look like credentials are dropped, and provider free text still never reaches durable state. The same classifier now covers backend_error events and the reconnect and replay terminal paths. --- src/app/application-support.ts | 4 +- src/app/provider-values.ts | 6 ++ src/app/run-event-mapper.ts | 14 +++- src/app/run-replay.ts | 4 +- src/domain/provider-values.ts | 2 +- src/domain/runtime-diagnostics.ts | 55 +++++++++++++++ test/application.test.ts | 110 ++++++++++++++++++++++++++++++ 7 files changed, 187 insertions(+), 8 deletions(-) diff --git a/src/app/application-support.ts b/src/app/application-support.ts index 1eb5a8bc..833e3f04 100644 --- a/src/app/application-support.ts +++ b/src/app/application-support.ts @@ -25,7 +25,7 @@ import { DEFAULT_CANCEL_REASON, shutdownRequestDigest, } from './operation-ledger.js' -import { safeProviderDiagnostic } from './provider-values.js' +import { runtimeFailureDiagnostic, safeProviderDiagnostic } from './provider-values.js' import { RUN_EFFECT_KIND, runEffectRequest } from './run-admission.js' import { executeRun } from './run-execution.js' import type { RunExecutionSnapshot } from './run-execution-snapshot.js' @@ -363,7 +363,7 @@ async function reconcileSnapshot( usage: snapshot.usage ?? usageSnapshotForRun(current), ...(snapshot.error === undefined ? {} - : { error: safeProviderDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }), + : { error: runtimeFailureDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }), ...(snapshot.detail === undefined ? {} : { reason: safeProviderDiagnostic(snapshot.detail, 'RUNTIME_RECONCILIATION_STATUS') }), diff --git a/src/app/provider-values.ts b/src/app/provider-values.ts index cd4fe29d..1fa5b0a6 100644 --- a/src/app/provider-values.ts +++ b/src/app/provider-values.ts @@ -1,4 +1,5 @@ import { safeProviderDiagnostic } from '../domain/provider-values.js' +import { providerHttpFailureDiagnostic } from '../domain/runtime-diagnostics.js' export { finiteNonNegativeNumber, @@ -8,6 +9,11 @@ export { safePublicIdentifier, } from '../domain/provider-values.js' +/** A runtime failure keeps its provider HTTP status and error code; other text falls back. */ +export function runtimeFailureDiagnostic(value: unknown, fallback: string): string { + return providerHttpFailureDiagnostic(value) ?? safeProviderDiagnostic(value, fallback) +} + function safeProperty(value: object, key: string): unknown { try { return Reflect.get(value, key) diff --git a/src/app/run-event-mapper.ts b/src/app/run-event-mapper.ts index cd98c9f1..47c9a4c4 100644 --- a/src/app/run-event-mapper.ts +++ b/src/app/run-event-mapper.ts @@ -15,7 +15,10 @@ import type { import type { ExecutionEnvironmentObservation } from '../domain/execution-observation.js' import { localInteractionId } from '../domain/interaction-identity.js' import { redactSensitiveText, redactStructuredValue } from '../domain/redaction.js' -import { publicRuntimeDiagnostic } from '../domain/runtime-diagnostics.js' +import { + providerHttpFailureDiagnostic, + publicRuntimeDiagnostic, +} from '../domain/runtime-diagnostics.js' import type { BraidRuntimeEvent } from '../domain/runtime-events.js' import type { BraidMessagePart, RunStatus } from '../domain/state.js' import { isCanonicalIsoDateTime } from '../domain/text.js' @@ -28,6 +31,7 @@ import { import { finiteNonNegativeNumber, optionalFiniteNonNegativeNumber, + runtimeFailureDiagnostic, safeDiagnostic, safeProviderDiagnostic, safePublicIdentifier, @@ -414,7 +418,7 @@ export function providerEventFor( return { kind: 'run.error', runId, - message: safeProviderDiagnostic(event.message, 'RUNTIME_BACKEND_ERROR'), + message: runtimeFailureDiagnostic(event.message, 'RUNTIME_BACKEND_ERROR'), recoverable: event.recoverable, provider, } @@ -430,7 +434,11 @@ export function providerEventFor( usage: usageFromMetadata(event.metadata), ...(event.error === undefined ? {} - : { error: safeDiagnostic(event.error.message, 'RUNTIME_FINAL_ERROR') }), + : { + error: + providerHttpFailureDiagnostic(event.error.message) ?? + safeDiagnostic(event.error.message, 'RUNTIME_FINAL_ERROR'), + }), ...(event.reason === undefined ? {} : { diff --git a/src/app/run-replay.ts b/src/app/run-replay.ts index 51aa23bd..ab87a8cf 100644 --- a/src/app/run-replay.ts +++ b/src/app/run-replay.ts @@ -4,7 +4,7 @@ import type { ProviderRunSnapshot } from '../ports/execution.js' import type { ReconcileInput, ReconnectInput, ReplayPort } from './application-ports.js' import { AppError } from './errors.js' import { safeSnapshotDetail, safeSnapshotText, safeSnapshotUsage } from './provider-snapshot.js' -import { safeRuntimeDiagnostic } from './provider-values.js' +import { runtimeFailureDiagnostic, safeRuntimeDiagnostic } from './provider-values.js' import { retainedExecutionRecoveryContext } from './run-recovery-context.js' interface RecoveryReconnectInput extends ReconnectInput { @@ -195,7 +195,7 @@ export async function reconcileRun( }), ...(snapshot.error === undefined ? {} - : { error: safeSnapshotDetail(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }), + : { error: runtimeFailureDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }), ...(snapshot.detail === undefined ? {} : { reason: safeSnapshotDetail(snapshot.detail, 'RUNTIME_RECONCILIATION_STATUS') }), diff --git a/src/domain/provider-values.ts b/src/domain/provider-values.ts index 8026d2f4..f89fcbd2 100644 --- a/src/domain/provider-values.ts +++ b/src/domain/provider-values.ts @@ -1,7 +1,7 @@ import { redactSensitiveText } from './secret-sanitizer.js' import { containsUnsafeControlCharacter } from './text.js' -const SENSITIVE_DIAGNOSTIC = +export const SENSITIVE_DIAGNOSTIC = /(?:secret|password|passphrase|token|bearer|authorization|credential|private(?:[_-]?key)?|api[-_]?key|session(?:[_-]?key)?|access[_-]?key|client[_-]?secret|signature|signed[_-]?url|nonce)/iu const SAFE_DIAGNOSTIC = /^[A-Z][A-Z0-9._:-]{0,63}$/u const TYPED_PROVIDER_DIAGNOSTIC = /^[A-Z][A-Z0-9]*(?:[._][A-Z0-9]+)*$/u diff --git a/src/domain/runtime-diagnostics.ts b/src/domain/runtime-diagnostics.ts index 745c0baf..449c0aa5 100644 --- a/src/domain/runtime-diagnostics.ts +++ b/src/domain/runtime-diagnostics.ts @@ -1,3 +1,6 @@ +import { SENSITIVE_DIAGNOSTIC } from './provider-values.js' +import { redactSensitiveText } from './secret-sanitizer.js' + export const BRAID_SANDBOX_INTERACTION_UNSUPPORTED = 'BRAID_SANDBOX_INTERACTION_UNSUPPORTED' as const export const BRAID_SANDBOX_CLEANUP_UNCONFIRMED = 'BRAID_SANDBOX_CLEANUP_UNCONFIRMED' as const @@ -15,3 +18,55 @@ export function publicRuntimeDiagnostic(value: unknown): string | undefined { } return PUBLIC_RUNTIME_DIAGNOSTICS[value as keyof typeof PUBLIC_RUNTIME_DIAGNOSTICS] } + +// Runtime's Router executor reports a non-success upstream response as +// `router : `, either as the whole message or after a +// `: transport failed: ` prefix. +const PROVIDER_HTTP_FAILURE = /(?:^|:\s)router ([45]\d{2}): ([^\n]*)/u +const PROVIDER_ERROR_OBJECT = /"error"\s*:\s*\{([^{}]*)/u +const providerErrorField = (name: string) => + new RegExp(String.raw`"${name}"\s*:\s*"([A-Za-z][A-Za-z0-9_.-]{0,63})"`, 'u') +const PROVIDER_ERROR_TYPE = providerErrorField('type') +const PROVIDER_ERROR_CODE = providerErrorField('code') + +function providerFailureClass(status: number): string { + if (status === 401 || status === 403) return 'RUNTIME_PROVIDER_UNAUTHORIZED' + if (status === 402) return 'RUNTIME_PROVIDER_PAYMENT_REQUIRED' + if (status === 404) return 'RUNTIME_PROVIDER_NOT_FOUND' + if (status === 429) return 'RUNTIME_PROVIDER_RATE_LIMITED' + if (status >= 500) return 'RUNTIME_PROVIDER_UNAVAILABLE' + return 'RUNTIME_PROVIDER_REJECTED' +} + +// A provider-declared token is public only when it cannot carry credential material. +function providerToken(scope: string, field: RegExp): string | undefined { + const value = field.exec(scope)?.[1] + if (value === undefined || value === 'error') return undefined + if (SENSITIVE_DIAGNOSTIC.test(value) || redactSensitiveText(value) !== value) return undefined + return value +} + +/** + * Classify a runtime failure message into a public diagnostic. + * + * Provider text is untrusted, so only the HTTP status and the provider's own + * error type and code tokens survive; the free-text body never reaches state. + * Returns undefined when the message does not carry a provider HTTP failure. + */ +export function providerHttpFailureDiagnostic(value: unknown): string | undefined { + if (typeof value !== 'string') return undefined + const match = PROVIDER_HTTP_FAILURE.exec(value) + if (match === null) return undefined + const status = Number(match[1]) + const body = match[2] ?? '' + const scopes = [PROVIDER_ERROR_OBJECT.exec(body)?.[1], body].filter( + (scope): scope is string => scope !== undefined, + ) + const token = (field: RegExp) => + scopes.map((scope) => providerToken(scope, field)).find((found) => found !== undefined) + const tokens = [token(PROVIDER_ERROR_TYPE), token(PROVIDER_ERROR_CODE)] + .filter((found): found is string => found !== undefined) + .filter((found, index, all) => all.indexOf(found) === index) + const detail = tokens.length === 0 ? '' : ` (${tokens.join('/')})` + return `${providerFailureClass(status)}: provider returned HTTP ${status}${detail}` +} diff --git a/test/application.test.ts b/test/application.test.ts index 98089608..f7036840 100644 --- a/test/application.test.ts +++ b/test/application.test.ts @@ -11,6 +11,7 @@ import { createBraidApplication, DETERMINISTIC_PROFILE } from '../src/app/compos import { cancelRequestDigest } from '../src/app/operation-ledger.js' import { effectRequestDigest } from '../src/app/effect-coordinator.js' import { MemoryJournal } from '../src/app/journal.js' +import { providerEventFor } from '../src/app/run-event-mapper.js' import { createProfileRecord } from '../src/app/profiles.js' import { createInteractionRequest, @@ -22,6 +23,7 @@ import type { ConnectionRecord } from '../src/domain/entities.js' import type { BraidEventEnvelope } from '../src/domain/events.js' import { createConnectionId } from '../src/domain/ids.js' import { assertBraidState } from '../src/domain/invariants.js' +import { providerHttpFailureDiagnostic } from '../src/domain/runtime-diagnostics.js' import type { RuntimeEventEnvelope } from '../src/domain/runtime-events.js' import { FixedClock } from '../src/ports/clock.js' import type { JournalPort } from '../src/ports/effect-storage.js' @@ -1280,6 +1282,114 @@ test('provider diagnostics and model metadata cannot persist credential material assert.equal(state.runs[0]?.outputTokens, 0) }) +test('a provider HTTP rejection surfaces its status and error code instead of RUNTIME_FINAL_ERROR', async () => { + const canary = 'sk-tan-never-persist-this-provider-key-value' + const journal = new MemoryJournal(new FixedClock()) + const app = new BraidApplication({ + profile: DETERMINISTIC_PROFILE, + execution: { + async *streamTurn(): AsyncIterable { + yield { + type: 'final', + status: 'failed', + reason: 'failed', + text: '', + error: { + kind: 'backend', + message: `routerInlineExecutor: transport failed: router 401: {"error":{"message":"Invalid API key ${canary}","type":"authentication_error","code":"invalid_key"}}`, + }, + task: { id: 'task-router-401', intent: 'router-401' }, + timestamp: '2026-08-01T00:00:00.000Z', + } + }, + }, + clock: new FixedClock(), + ids: new SequenceIds(), + journal, + effectStorage: journal, + }) + + app.initialize('/workspace') + const state = await app.send({ operationId: 'op-router-401', text: 'hello' }).completion + assert.equal( + state.runs[0]?.error, + 'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 401 (authentication_error/invalid_key)', + ) + assert.equal(JSON.stringify({ state, events: app.events() }).includes(canary), false) +}) + +test('a backend error event keeps the provider HTTP rejection class', () => { + const event = providerEventFor( + 'run-backend-401', + { + type: 'backend_error', + message: 'routerInlineExecutor: transport failed: router 403: {"error":{"code":"forbidden"}}', + recoverable: false, + } as RuntimeStreamEvent, + { eventId: 'backend-401', providerSequence: 1, receivedAt: '2026-08-01T00:00:00.000Z' }, + ) + assert.equal(event.kind, 'run.error') + assert.equal( + event.message, + 'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 403 (forbidden)', + ) +}) + +test('provider HTTP failure diagnostics keep only status and safe provider tokens', () => { + assert.equal( + providerHttpFailureDiagnostic( + 'router 402: {"error":{"message":"Add funds","type":"insufficient_funds","code":"payment_required"}}', + ), + 'RUNTIME_PROVIDER_PAYMENT_REQUIRED: provider returned HTTP 402 (insufficient_funds/payment_required)', + ) + assert.equal( + providerHttpFailureDiagnostic('x: router 503: upstream unavailable'), + 'RUNTIME_PROVIDER_UNAVAILABLE: provider returned HTTP 503', + ) + assert.equal( + providerHttpFailureDiagnostic( + 'router 429: {"error":{"type":"rate_limit_error","code":"rate_limit_error"}}', + ), + 'RUNTIME_PROVIDER_RATE_LIMITED: provider returned HTTP 429 (rate_limit_error)', + ) + assert.equal( + providerHttpFailureDiagnostic( + 'router 400: {"error":{"type":"secret_abc","code":"model_not_found"}}', + ), + 'RUNTIME_PROVIDER_REJECTED: provider returned HTTP 400 (model_not_found)', + ) + assert.equal( + providerHttpFailureDiagnostic( + 'router 401: {"type":"error","error":{"type":"authentication_error","message":"bad"}}', + ), + 'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 401 (authentication_error)', + ) + for (const secret of [ + 'sk-tan-abcdef0123456789abcdef0123', + 'ghp_abcdefghijklmnopqrstuvwxyz0123', + `AKIA${'ABCDEFGHIJKLMNOP'}`, + 'client_secret_value', + ]) { + assert.equal( + providerHttpFailureDiagnostic( + `router 400: {"error":{"type":"${secret}","code":"${secret}"}}`, + ), + 'RUNTIME_PROVIDER_REJECTED: provider returned HTTP 400', + ) + } + assert.equal( + providerHttpFailureDiagnostic('tool said the router 404: {"error":{"type":"fake"}}'), + undefined, + ) + assert.equal( + providerHttpFailureDiagnostic('router 500: upstream\n{"error":{"type":"injected"}}'), + 'RUNTIME_PROVIDER_UNAVAILABLE: provider returned HTTP 500', + ) + assert.equal(providerHttpFailureDiagnostic('router 200: ok'), undefined) + assert.equal(providerHttpFailureDiagnostic('transport failed: fetch failed'), undefined) + assert.equal(providerHttpFailureDiagnostic(undefined), undefined) +}) + test('provider diagnostic getters cannot break execution failure handling', () => { const hostile = new Proxy( {},