Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/app/application-support.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import {
DEFAULT_CANCEL_REASON,
shutdownRequestDigest,
} from './operation-ledger.js'
import { safeProviderDiagnostic } from './provider-values.js'
import { runtimeFailureDiagnostic, safeProviderDiagnostic } from './provider-values.js'
import { RUN_EFFECT_KIND, runEffectRequest } from './run-admission.js'
import { executeRun } from './run-execution.js'
import type { RunExecutionSnapshot } from './run-execution-snapshot.js'
Expand Down Expand Up @@ -363,7 +363,7 @@ async function reconcileSnapshot(
usage: snapshot.usage ?? usageSnapshotForRun(current),
...(snapshot.error === undefined
? {}
: { error: safeProviderDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }),
: { error: runtimeFailureDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }),
...(snapshot.detail === undefined
? {}
: { reason: safeProviderDiagnostic(snapshot.detail, 'RUNTIME_RECONCILIATION_STATUS') }),
Expand Down
6 changes: 6 additions & 0 deletions src/app/provider-values.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { safeProviderDiagnostic } from '../domain/provider-values.js'
import { providerHttpFailureDiagnostic } from '../domain/runtime-diagnostics.js'

export {
finiteNonNegativeNumber,
Expand All @@ -8,6 +9,11 @@ export {
safePublicIdentifier,
} from '../domain/provider-values.js'

/** A runtime failure keeps its provider HTTP status and error code; other text falls back. */
export function runtimeFailureDiagnostic(value: unknown, fallback: string): string {
return providerHttpFailureDiagnostic(value) ?? safeProviderDiagnostic(value, fallback)
}

function safeProperty(value: object, key: string): unknown {
try {
return Reflect.get(value, key)
Expand Down
14 changes: 11 additions & 3 deletions src/app/run-event-mapper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@ import type {
import type { ExecutionEnvironmentObservation } from '../domain/execution-observation.js'
import { localInteractionId } from '../domain/interaction-identity.js'
import { redactSensitiveText, redactStructuredValue } from '../domain/redaction.js'
import { publicRuntimeDiagnostic } from '../domain/runtime-diagnostics.js'
import {
providerHttpFailureDiagnostic,
publicRuntimeDiagnostic,
} from '../domain/runtime-diagnostics.js'
import type { BraidRuntimeEvent } from '../domain/runtime-events.js'
import type { BraidMessagePart, RunStatus } from '../domain/state.js'
import { isCanonicalIsoDateTime } from '../domain/text.js'
Expand All @@ -28,6 +31,7 @@ import {
import {
finiteNonNegativeNumber,
optionalFiniteNonNegativeNumber,
runtimeFailureDiagnostic,
safeDiagnostic,
safeProviderDiagnostic,
safePublicIdentifier,
Expand Down Expand Up @@ -414,7 +418,7 @@ export function providerEventFor(
return {
kind: 'run.error',
runId,
message: safeProviderDiagnostic(event.message, 'RUNTIME_BACKEND_ERROR'),
message: runtimeFailureDiagnostic(event.message, 'RUNTIME_BACKEND_ERROR'),
recoverable: event.recoverable,
provider,
}
Expand All @@ -430,7 +434,11 @@ export function providerEventFor(
usage: usageFromMetadata(event.metadata),
...(event.error === undefined
? {}
: { error: safeDiagnostic(event.error.message, 'RUNTIME_FINAL_ERROR') }),
: {
error:
providerHttpFailureDiagnostic(event.error.message) ??
safeDiagnostic(event.error.message, 'RUNTIME_FINAL_ERROR'),
}),
...(event.reason === undefined
? {}
: {
Expand Down
4 changes: 2 additions & 2 deletions src/app/run-replay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type { ProviderRunSnapshot } from '../ports/execution.js'
import type { ReconcileInput, ReconnectInput, ReplayPort } from './application-ports.js'
import { AppError } from './errors.js'
import { safeSnapshotDetail, safeSnapshotText, safeSnapshotUsage } from './provider-snapshot.js'
import { safeRuntimeDiagnostic } from './provider-values.js'
import { runtimeFailureDiagnostic, safeRuntimeDiagnostic } from './provider-values.js'
import { retainedExecutionRecoveryContext } from './run-recovery-context.js'

interface RecoveryReconnectInput extends ReconnectInput {
Expand Down Expand Up @@ -195,7 +195,7 @@ export async function reconcileRun(
}),
...(snapshot.error === undefined
? {}
: { error: safeSnapshotDetail(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }),
: { error: runtimeFailureDiagnostic(snapshot.error, 'RUNTIME_RECONCILIATION_ERROR') }),
...(snapshot.detail === undefined
? {}
: { reason: safeSnapshotDetail(snapshot.detail, 'RUNTIME_RECONCILIATION_STATUS') }),
Expand Down
2 changes: 1 addition & 1 deletion src/domain/provider-values.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { redactSensitiveText } from './secret-sanitizer.js'
import { containsUnsafeControlCharacter } from './text.js'

const SENSITIVE_DIAGNOSTIC =
export const SENSITIVE_DIAGNOSTIC =
/(?:secret|password|passphrase|token|bearer|authorization|credential|private(?:[_-]?key)?|api[-_]?key|session(?:[_-]?key)?|access[_-]?key|client[_-]?secret|signature|signed[_-]?url|nonce)/iu
const SAFE_DIAGNOSTIC = /^[A-Z][A-Z0-9._:-]{0,63}$/u
const TYPED_PROVIDER_DIAGNOSTIC = /^[A-Z][A-Z0-9]*(?:[._][A-Z0-9]+)*$/u
Expand Down
55 changes: 55 additions & 0 deletions src/domain/runtime-diagnostics.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import { SENSITIVE_DIAGNOSTIC } from './provider-values.js'
import { redactSensitiveText } from './secret-sanitizer.js'

export const BRAID_SANDBOX_INTERACTION_UNSUPPORTED =
'BRAID_SANDBOX_INTERACTION_UNSUPPORTED' as const
export const BRAID_SANDBOX_CLEANUP_UNCONFIRMED = 'BRAID_SANDBOX_CLEANUP_UNCONFIRMED' as const
Expand All @@ -15,3 +18,55 @@ export function publicRuntimeDiagnostic(value: unknown): string | undefined {
}
return PUBLIC_RUNTIME_DIAGNOSTICS[value as keyof typeof PUBLIC_RUNTIME_DIAGNOSTICS]
}

// Runtime's Router executor reports a non-success upstream response as
// `router <status>: <body prefix>`, either as the whole message or after a
// `<context>: transport failed: ` prefix.
const PROVIDER_HTTP_FAILURE = /(?:^|:\s)router ([45]\d{2}): ([^\n]*)/u
const PROVIDER_ERROR_OBJECT = /"error"\s*:\s*\{([^{}]*)/u
const providerErrorField = (name: string) =>
new RegExp(String.raw`"${name}"\s*:\s*"([A-Za-z][A-Za-z0-9_.-]{0,63})"`, 'u')
const PROVIDER_ERROR_TYPE = providerErrorField('type')
const PROVIDER_ERROR_CODE = providerErrorField('code')

function providerFailureClass(status: number): string {
if (status === 401 || status === 403) return 'RUNTIME_PROVIDER_UNAUTHORIZED'
if (status === 402) return 'RUNTIME_PROVIDER_PAYMENT_REQUIRED'
if (status === 404) return 'RUNTIME_PROVIDER_NOT_FOUND'
if (status === 429) return 'RUNTIME_PROVIDER_RATE_LIMITED'
if (status >= 500) return 'RUNTIME_PROVIDER_UNAVAILABLE'
return 'RUNTIME_PROVIDER_REJECTED'
}

// A provider-declared token is public only when it cannot carry credential material.
function providerToken(scope: string, field: RegExp): string | undefined {
const value = field.exec(scope)?.[1]
if (value === undefined || value === 'error') return undefined
if (SENSITIVE_DIAGNOSTIC.test(value) || redactSensitiveText(value) !== value) return undefined
return value
}

/**
* Classify a runtime failure message into a public diagnostic.
*
* Provider text is untrusted, so only the HTTP status and the provider's own
* error type and code tokens survive; the free-text body never reaches state.
* Returns undefined when the message does not carry a provider HTTP failure.
*/
export function providerHttpFailureDiagnostic(value: unknown): string | undefined {
if (typeof value !== 'string') return undefined
const match = PROVIDER_HTTP_FAILURE.exec(value)
if (match === null) return undefined
const status = Number(match[1])
const body = match[2] ?? ''
const scopes = [PROVIDER_ERROR_OBJECT.exec(body)?.[1], body].filter(
(scope): scope is string => scope !== undefined,
)
const token = (field: RegExp) =>
scopes.map((scope) => providerToken(scope, field)).find((found) => found !== undefined)
const tokens = [token(PROVIDER_ERROR_TYPE), token(PROVIDER_ERROR_CODE)]
.filter((found): found is string => found !== undefined)
.filter((found, index, all) => all.indexOf(found) === index)
const detail = tokens.length === 0 ? '' : ` (${tokens.join('/')})`
return `${providerFailureClass(status)}: provider returned HTTP ${status}${detail}`
}
110 changes: 110 additions & 0 deletions test/application.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { createBraidApplication, DETERMINISTIC_PROFILE } from '../src/app/compos
import { cancelRequestDigest } from '../src/app/operation-ledger.js'
import { effectRequestDigest } from '../src/app/effect-coordinator.js'
import { MemoryJournal } from '../src/app/journal.js'
import { providerEventFor } from '../src/app/run-event-mapper.js'
import { createProfileRecord } from '../src/app/profiles.js'
import {
createInteractionRequest,
Expand All @@ -22,6 +23,7 @@ import type { ConnectionRecord } from '../src/domain/entities.js'
import type { BraidEventEnvelope } from '../src/domain/events.js'
import { createConnectionId } from '../src/domain/ids.js'
import { assertBraidState } from '../src/domain/invariants.js'
import { providerHttpFailureDiagnostic } from '../src/domain/runtime-diagnostics.js'
import type { RuntimeEventEnvelope } from '../src/domain/runtime-events.js'
import { FixedClock } from '../src/ports/clock.js'
import type { JournalPort } from '../src/ports/effect-storage.js'
Expand Down Expand Up @@ -1280,6 +1282,114 @@ test('provider diagnostics and model metadata cannot persist credential material
assert.equal(state.runs[0]?.outputTokens, 0)
})

test('a provider HTTP rejection surfaces its status and error code instead of RUNTIME_FINAL_ERROR', async () => {
const canary = 'sk-tan-never-persist-this-provider-key-value'
const journal = new MemoryJournal(new FixedClock())
const app = new BraidApplication({
profile: DETERMINISTIC_PROFILE,
execution: {
async *streamTurn(): AsyncIterable<RuntimeStreamEvent> {
yield {
type: 'final',
status: 'failed',
reason: 'failed',
text: '',
error: {
kind: 'backend',
message: `routerInlineExecutor: transport failed: router 401: {"error":{"message":"Invalid API key ${canary}","type":"authentication_error","code":"invalid_key"}}`,
},
task: { id: 'task-router-401', intent: 'router-401' },
timestamp: '2026-08-01T00:00:00.000Z',
}
},
},
clock: new FixedClock(),
ids: new SequenceIds(),
journal,
effectStorage: journal,
})

app.initialize('/workspace')
const state = await app.send({ operationId: 'op-router-401', text: 'hello' }).completion
assert.equal(
state.runs[0]?.error,
'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 401 (authentication_error/invalid_key)',
)
assert.equal(JSON.stringify({ state, events: app.events() }).includes(canary), false)
})

test('a backend error event keeps the provider HTTP rejection class', () => {
const event = providerEventFor(
'run-backend-401',
{
type: 'backend_error',
message: 'routerInlineExecutor: transport failed: router 403: {"error":{"code":"forbidden"}}',
recoverable: false,
} as RuntimeStreamEvent,
{ eventId: 'backend-401', providerSequence: 1, receivedAt: '2026-08-01T00:00:00.000Z' },
)
assert.equal(event.kind, 'run.error')
assert.equal(
event.message,
'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 403 (forbidden)',
)
})

test('provider HTTP failure diagnostics keep only status and safe provider tokens', () => {
assert.equal(
providerHttpFailureDiagnostic(
'router 402: {"error":{"message":"Add funds","type":"insufficient_funds","code":"payment_required"}}',
),
'RUNTIME_PROVIDER_PAYMENT_REQUIRED: provider returned HTTP 402 (insufficient_funds/payment_required)',
)
assert.equal(
providerHttpFailureDiagnostic('x: router 503: upstream unavailable'),
'RUNTIME_PROVIDER_UNAVAILABLE: provider returned HTTP 503',
)
assert.equal(
providerHttpFailureDiagnostic(
'router 429: {"error":{"type":"rate_limit_error","code":"rate_limit_error"}}',
),
'RUNTIME_PROVIDER_RATE_LIMITED: provider returned HTTP 429 (rate_limit_error)',
)
assert.equal(
providerHttpFailureDiagnostic(
'router 400: {"error":{"type":"secret_abc","code":"model_not_found"}}',
),
'RUNTIME_PROVIDER_REJECTED: provider returned HTTP 400 (model_not_found)',
)
assert.equal(
providerHttpFailureDiagnostic(
'router 401: {"type":"error","error":{"type":"authentication_error","message":"bad"}}',
),
'RUNTIME_PROVIDER_UNAUTHORIZED: provider returned HTTP 401 (authentication_error)',
)
for (const secret of [
'sk-tan-abcdef0123456789abcdef0123',
'ghp_abcdefghijklmnopqrstuvwxyz0123',
`AKIA${'ABCDEFGHIJKLMNOP'}`,
'client_secret_value',
]) {
assert.equal(
providerHttpFailureDiagnostic(
`router 400: {"error":{"type":"${secret}","code":"${secret}"}}`,
),
'RUNTIME_PROVIDER_REJECTED: provider returned HTTP 400',
)
}
assert.equal(
providerHttpFailureDiagnostic('tool said the router 404: {"error":{"type":"fake"}}'),
undefined,
)
assert.equal(
providerHttpFailureDiagnostic('router 500: upstream\n{"error":{"type":"injected"}}'),
'RUNTIME_PROVIDER_UNAVAILABLE: provider returned HTTP 500',
)
assert.equal(providerHttpFailureDiagnostic('router 200: ok'), undefined)
assert.equal(providerHttpFailureDiagnostic('transport failed: fetch failed'), undefined)
assert.equal(providerHttpFailureDiagnostic(undefined), undefined)
})

test('provider diagnostic getters cannot break execution failure handling', () => {
const hostile = new Proxy(
{},
Expand Down