diff --git a/src/app/application-support.ts b/src/app/application-support.ts index 1eb5a8bc..833e3f04 100644 --- a/src/app/application-support.ts +++ b/src/app/application-support.ts @@ -25,7 +25,7 @@ import { DEFAULT_CANCEL_REASON, shutdownRequestDigest, } from './operation-ledger.js' -import { safeProviderDiagnostic } from './provider-values.js' +import { runtimeFailureDiagnostic, safeProviderDiagnostic } from './provider-values.js' import { RUN_EFFECT_KIND, runEffectRequest } from './run-admission.js' import { executeRun } from './run-execution.js' import type { RunExecutionSnapshot } from './run-execution-snapshot.js' @@ -363,7 +363,7 @@ async function reconcileSnapshot( usage: snapshot.usage ?? usageSnapshotForRun(current), ...(snapshot.error === undefined ? {} - : { error: safeProviderDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }), + : { error: runtimeFailureDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }), ...(snapshot.detail === undefined ? {} : { reason: safeProviderDiagnostic(snapshot.detail, 'RUNTIME_RECONCILIATION_STATUS') }), diff --git a/src/app/provider-values.ts b/src/app/provider-values.ts index cd4fe29d..1fa5b0a6 100644 --- a/src/app/provider-values.ts +++ b/src/app/provider-values.ts @@ -1,4 +1,5 @@ import { safeProviderDiagnostic } from '../domain/provider-values.js' +import { providerHttpFailureDiagnostic } from '../domain/runtime-diagnostics.js' export { finiteNonNegativeNumber, @@ -8,6 +9,11 @@ export { safePublicIdentifier, } from '../domain/provider-values.js' +/** A runtime failure keeps its provider HTTP status and error code; other text falls back. */ +export function runtimeFailureDiagnostic(value: unknown, fallback: string): string { + return providerHttpFailureDiagnostic(value) ?? safeProviderDiagnostic(value, fallback) +} + function safeProperty(value: object, key: string): unknown { try { return Reflect.get(value, key) diff --git a/src/app/run-event-mapper.ts b/src/app/run-event-mapper.ts index cd98c9f1..47c9a4c4 100644 --- a/src/app/run-event-mapper.ts +++ b/src/app/run-event-mapper.ts @@ -15,7 +15,10 @@ import type { import type { ExecutionEnvironmentObservation } from '../domain/execution-observation.js' import { localInteractionId } from '../domain/interaction-identity.js' import { redactSensitiveText, redactStructuredValue } from '../domain/redaction.js' -import { publicRuntimeDiagnostic } from '../domain/runtime-diagnostics.js' +import { + providerHttpFailureDiagnostic, + publicRuntimeDiagnostic, +} from '../domain/runtime-diagnostics.js' import type { BraidRuntimeEvent } from '../domain/runtime-events.js' import type { BraidMessagePart, RunStatus } from '../domain/state.js' import { isCanonicalIsoDateTime } from '../domain/text.js' @@ -28,6 +31,7 @@ import { import { finiteNonNegativeNumber, optionalFiniteNonNegativeNumber, + runtimeFailureDiagnostic, safeDiagnostic, safeProviderDiagnostic, safePublicIdentifier, @@ -414,7 +418,7 @@ export function providerEventFor( return { kind: 'run.error', runId, - message: safeProviderDiagnostic(event.message, 'RUNTIME_BACKEND_ERROR'), + message: runtimeFailureDiagnostic(event.message, 'RUNTIME_BACKEND_ERROR'), recoverable: event.recoverable, provider, } @@ -430,7 +434,11 @@ export function providerEventFor( usage: usageFromMetadata(event.metadata), ...(event.error === undefined ? {} - : { error: safeDiagnostic(event.error.message, 'RUNTIME_FINAL_ERROR') }), + : { + error: + providerHttpFailureDiagnostic(event.error.message) ?? + safeDiagnostic(event.error.message, 'RUNTIME_FINAL_ERROR'), + }), ...(event.reason === undefined ? {} : { diff --git a/src/app/run-replay.ts b/src/app/run-replay.ts index 51aa23bd..ab87a8cf 100644 --- a/src/app/run-replay.ts +++ b/src/app/run-replay.ts @@ -4,7 +4,7 @@ import type { ProviderRunSnapshot } from '../ports/execution.js' import type { ReconcileInput, ReconnectInput, ReplayPort } from './application-ports.js' import { AppError } from './errors.js' import { safeSnapshotDetail, safeSnapshotText, safeSnapshotUsage } from './provider-snapshot.js' -import { safeRuntimeDiagnostic } from './provider-values.js' +import { runtimeFailureDiagnostic, safeRuntimeDiagnostic } from './provider-values.js' import { retainedExecutionRecoveryContext } from './run-recovery-context.js' interface RecoveryReconnectInput extends ReconnectInput { @@ -195,7 +195,7 @@ export async function reconcileRun( }), ...(snapshot.error === undefined ? {} - : { error: safeSnapshotDetail(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }), + : { error: runtimeFailureDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }), ...(snapshot.detail === undefined ? {} : { reason: safeSnapshotDetail(snapshot.detail, 'RUNTIME_RECONCILIATION_STATUS') }), diff --git a/src/domain/provider-values.ts b/src/domain/provider-values.ts index 8026d2f4..f89fcbd2 100644 --- a/src/domain/provider-values.ts +++ b/src/domain/provider-values.ts @@ -1,7 +1,7 @@ import { redactSensitiveText } from './secret-sanitizer.js' import { containsUnsafeControlCharacter } from './text.js' -const SENSITIVE_DIAGNOSTIC = +export const SENSITIVE_DIAGNOSTIC = /(?:secret|password|passphrase|token|bearer|authorization|credential|private(?:[_-]?key)?|api[-_]?key|session(?:[_-]?key)?|access[_-]?key|client[_-]?secret|signature|signed[_-]?url|nonce)/iu const SAFE_DIAGNOSTIC = /^[A-Z][A-Z0-9._:-]{0,63}$/u const TYPED_PROVIDER_DIAGNOSTIC = /^[A-Z][A-Z0-9]*(?:[._][A-Z0-9]+)*$/u diff --git a/src/domain/runtime-diagnostics.ts b/src/domain/runtime-diagnostics.ts index 745c0baf..449c0aa5 100644 --- a/src/domain/runtime-diagnostics.ts +++ b/src/domain/runtime-diagnostics.ts @@ -1,3 +1,6 @@ +import { SENSITIVE_DIAGNOSTIC } from './provider-values.js' +import { redactSensitiveText } from './secret-sanitizer.js' + export const BRAID_SANDBOX_INTERACTION_UNSUPPORTED = 'BRAID_SANDBOX_INTERACTION_UNSUPPORTED' as const export const BRAID_SANDBOX_CLEANUP_UNCONFIRMED = 'BRAID_SANDBOX_CLEANUP_UNCONFIRMED' as const @@ -15,3 +18,55 @@ export function publicRuntimeDiagnostic(value: unknown): string | undefined { } return PUBLIC_RUNTIME_DIAGNOSTICS[value as keyof typeof PUBLIC_RUNTIME_DIAGNOSTICS] } + +// Runtime's Router executor reports a non-success upstream response as +// `router : `, either as the whole message or after a +// `: transport failed: ` prefix. +const PROVIDER_HTTP_FAILURE = /(?:^|:\s)router ([45]\d{2}): ([^\n]*)/u +const PROVIDER_ERROR_OBJECT = /"error"\s*:\s*\{([^{}]*)/u +const providerErrorField = (name: string) => + new RegExp(String.raw`"${name}"\s*:\s*"([A-Za-z][A-Za-z0-9_.-]{0,63})"`, 'u') +const PROVIDER_ERROR_TYPE = providerErrorField('type') +const PROVIDER_ERROR_CODE = providerErrorField('code') + +function providerFailureClass(status: number): string { + if (status === 401 || status === 403) return 'RUNTIME_PROVIDER_UNAUTHORIZED' + if (status === 402) return 'RUNTIME_PROVIDER_PAYMENT_REQUIRED' + if (status === 404) return 'RUNTIME_PROVIDER_NOT_FOUND' + if (status === 429) return 'RUNTIME_PROVIDER_RATE_LIMITED' + if (status >= 500) return 'RUNTIME_PROVIDER_UNAVAILABLE' + return 'RUNTIME_PROVIDER_REJECTED' +} + +// A provider-declared token is public only when it cannot carry credential material. +function providerToken(scope: string, field: RegExp): string | undefined { + const value = field.exec(scope)?.[1] + if (value === undefined || value === 'error') return undefined + if (SENSITIVE_DIAGNOSTIC.test(value) || redactSensitiveText(value) !== value) return undefined + return value +} + +/** + * Classify a runtime failure message into a public diagnostic. + * + * Provider text is untrusted, so only the HTTP status and the provider's own + * error type and code tokens survive; the free-text body never reaches state. + * Returns undefined when the message does not carry a provider HTTP failure. + */ +export function providerHttpFailureDiagnostic(value: unknown): string | undefined { + if (typeof value !== 'string') return undefined + const match = PROVIDER_HTTP_FAILURE.exec(value) + if (match === null) return undefined + const status = Number(match[1]) + const body = match[2] ?? '' + const scopes = [PROVIDER_ERROR_OBJECT.exec(body)?.[1], body].filter( + (scope): scope is string => scope !== undefined, + ) + const token = (field: RegExp) => + scopes.map((scope) => providerToken(scope, field)).find((found) => found !== undefined) + const tokens = [token(PROVIDER_ERROR_TYPE), token(PROVIDER_ERROR_CODE)] + .filter((found): found is string => found !== undefined) + .filter((found, index, all) => all.indexOf(found) === index) + const detail = tokens.length === 0 ? '' : ` (${tokens.join('/')})` + return `${providerFailureClass(status)}: provider returned HTTP ${status}${detail}` +} diff --git a/test/application.test.ts b/test/application.test.ts index 98089608..f7036840 100644 --- a/test/application.test.ts +++ b/test/application.test.ts @@ -11,6 +11,7 @@ import { createBraidApplication, DETERMINISTIC_PROFILE } from '../src/app/compos import { cancelRequestDigest } from '../src/app/operation-ledger.js' import { effectRequestDigest } from '../src/app/effect-coordinator.js' import { MemoryJournal } from '../src/app/journal.js' +import { providerEventFor } from '../src/app/run-event-mapper.js' import { createProfileRecord } from '../src/app/profiles.js' import { createInteractionRequest, @@ -22,6 +23,7 @@ import type { ConnectionRecord } from '../src/domain/entities.js' import type { BraidEventEnvelope } from '../src/domain/events.js' import { createConnectionId } from '../src/domain/ids.js' import { assertBraidState } from '../src/domain/invariants.js' +import { providerHttpFailureDiagnostic } from '../src/domain/runtime-diagnostics.js' import type { RuntimeEventEnvelope } from '../src/domain/runtime-events.js' import { FixedClock } from '../src/ports/clock.js' import type { JournalPort } from '../src/ports/effect-storage.js' @@ -1280,6 +1282,114 @@ test('provider diagnostics and model metadata cannot persist credential material assert.equal(state.runs[0]?.outputTokens, 0) }) +test('a provider HTTP rejection surfaces its status and error code instead of RUNTIME_FINAL_ERROR', async () => { + const canary = 'sk-tan-never-persist-this-provider-key-value' + const journal = new MemoryJournal(new FixedClock()) + const app = new BraidApplication({ + profile: DETERMINISTIC_PROFILE, + execution: { + async *streamTurn(): AsyncIterable { + yield { + type: 'final', + status: 'failed', + reason: 'failed', + text: '', + error: { + kind: 'backend', + message: `routerInlineExecutor: transport failed: router 401: {"error":{"message":"Invalid API key ${canary}","type":"authentication_error","code":"invalid_key"}}`, + }, + task: { id: 'task-router-401', intent: 'router-401' }, + timestamp: '2026-08-01T00:00:00.000Z', + } + }, + }, + clock: new FixedClock(), + ids: new SequenceIds(), + journal, + effectStorage: journal, + }) + + app.initialize('/workspace') + const state = await app.send({ operationId: 'op-router-401', text: 'hello' }).completion + assert.equal( + state.runs[0]?.error, + 'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 401 (authentication_error/invalid_key)', + ) + assert.equal(JSON.stringify({ state, events: app.events() }).includes(canary), false) +}) + +test('a backend error event keeps the provider HTTP rejection class', () => { + const event = providerEventFor( + 'run-backend-401', + { + type: 'backend_error', + message: 'routerInlineExecutor: transport failed: router 403: {"error":{"code":"forbidden"}}', + recoverable: false, + } as RuntimeStreamEvent, + { eventId: 'backend-401', providerSequence: 1, receivedAt: '2026-08-01T00:00:00.000Z' }, + ) + assert.equal(event.kind, 'run.error') + assert.equal( + event.message, + 'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 403 (forbidden)', + ) +}) + +test('provider HTTP failure diagnostics keep only status and safe provider tokens', () => { + assert.equal( + providerHttpFailureDiagnostic( + 'router 402: {"error":{"message":"Add funds","type":"insufficient_funds","code":"payment_required"}}', + ), + 'RUNTIME_PROVIDER_PAYMENT_REQUIRED: provider returned HTTP 402 (insufficient_funds/payment_required)', + ) + assert.equal( + providerHttpFailureDiagnostic('x: router 503: upstream unavailable'), + 'RUNTIME_PROVIDER_UNAVAILABLE: provider returned HTTP 503', + ) + assert.equal( + providerHttpFailureDiagnostic( + 'router 429: {"error":{"type":"rate_limit_error","code":"rate_limit_error"}}', + ), + 'RUNTIME_PROVIDER_RATE_LIMITED: provider returned HTTP 429 (rate_limit_error)', + ) + assert.equal( + providerHttpFailureDiagnostic( + 'router 400: {"error":{"type":"secret_abc","code":"model_not_found"}}', + ), + 'RUNTIME_PROVIDER_REJECTED: provider returned HTTP 400 (model_not_found)', + ) + assert.equal( + providerHttpFailureDiagnostic( + 'router 401: {"type":"error","error":{"type":"authentication_error","message":"bad"}}', + ), + 'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 401 (authentication_error)', + ) + for (const secret of [ + 'sk-tan-abcdef0123456789abcdef0123', + 'ghp_abcdefghijklmnopqrstuvwxyz0123', + `AKIA${'ABCDEFGHIJKLMNOP'}`, + 'client_secret_value', + ]) { + assert.equal( + providerHttpFailureDiagnostic( + `router 400: {"error":{"type":"${secret}","code":"${secret}"}}`, + ), + 'RUNTIME_PROVIDER_REJECTED: provider returned HTTP 400', + ) + } + assert.equal( + providerHttpFailureDiagnostic('tool said the router 404: {"error":{"type":"fake"}}'), + undefined, + ) + assert.equal( + providerHttpFailureDiagnostic('router 500: upstream\n{"error":{"type":"injected"}}'), + 'RUNTIME_PROVIDER_UNAVAILABLE: provider returned HTTP 500', + ) + assert.equal(providerHttpFailureDiagnostic('router 200: ok'), undefined) + assert.equal(providerHttpFailureDiagnostic('transport failed: fetch failed'), undefined) + assert.equal(providerHttpFailureDiagnostic(undefined), undefined) +}) + test('provider diagnostic getters cannot break execution failure handling', () => { const hostile = new Proxy( {},