From 66a9ed31bd891137939867d39b65c2b00a9f093e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 20:53:00 -0800 Subject: [PATCH 1/4] fix(cli): show the server's request id on API errors --- packages/sim-cli/src/commands/credentials.ts | 3 +- .../protocol/workspace-operation-wait.ts | 3 +- packages/sim-cli/src/embed.ts | 1 + packages/sim-cli/src/http/client.ts | 48 ++++++---- packages/sim-cli/src/runtime/execute.ts | 3 +- packages/sim-cli/src/runtime/naming.ts | 4 +- packages/sim-cli/src/terminal.test.ts | 87 +++++++++++++++++++ packages/sim-cli/src/terminal.ts | 2 + 8 files changed, 130 insertions(+), 21 deletions(-) create mode 100644 packages/sim-cli/src/terminal.test.ts diff --git a/packages/sim-cli/src/commands/credentials.ts b/packages/sim-cli/src/commands/credentials.ts index c40062e796f..fce2df4f295 100644 --- a/packages/sim-cli/src/commands/credentials.ts +++ b/packages/sim-cli/src/commands/credentials.ts @@ -186,7 +186,8 @@ function withKeylessRetryHint(error: unknown, providerId: string): unknown { error.status, error.code, error.details, - error.exitCode + error.exitCode, + error.requestId ) } diff --git a/packages/sim-cli/src/commands/protocol/workspace-operation-wait.ts b/packages/sim-cli/src/commands/protocol/workspace-operation-wait.ts index 9aa2672d9ed..f6213a530d1 100644 --- a/packages/sim-cli/src/commands/protocol/workspace-operation-wait.ts +++ b/packages/sim-cli/src/commands/protocol/workspace-operation-wait.ts @@ -152,7 +152,8 @@ export async function waitWorkspaceOperation( requestId: report?.requestId, applied: report?.applied, }, - failure.exitCode + failure.exitCode, + failure.requestId ) } if (report.status !== 'processing') return { report, timedOut: false } diff --git a/packages/sim-cli/src/embed.ts b/packages/sim-cli/src/embed.ts index a13f36747eb..984c44f4b1e 100644 --- a/packages/sim-cli/src/embed.ts +++ b/packages/sim-cli/src/embed.ts @@ -196,6 +196,7 @@ function renderEmbeddedError(ctx: EmbedContext, error: unknown): number { if (error.details !== undefined) { for (const line of formatApiErrorDetails(error.details)) ctx.stderr.diagnostic(sanitize(line)) } + if (error.requestId) ctx.stderr.diagnostic(` request id: ${sanitize(error.requestId)}`) return error.exitCode } ctx.stderr.diagnostic(`Error: ${sanitize(getErrorMessage(error))}`) diff --git a/packages/sim-cli/src/http/client.ts b/packages/sim-cli/src/http/client.ts index e6a82701dca..c1bca7336a1 100644 --- a/packages/sim-cli/src/http/client.ts +++ b/packages/sim-cli/src/http/client.ts @@ -17,7 +17,9 @@ export class SimApiError extends Error { readonly status: number, readonly code: string | null = null, readonly details?: unknown, - readonly exitCode = 1 + readonly exitCode = 1, + /** The server's id for the failed request (`x-request-id`), to quote to support. */ + readonly requestId: string | null = null ) { super(message) this.name = 'SimApiError' @@ -125,7 +127,8 @@ function toNonJsonError( url: string, status: number, contentType: string | null, - raw: string + raw: string, + requestId: string | null = null ): SimApiError { const type = contentType?.split(';')[0]?.trim().toLowerCase() const isMarkup = @@ -142,7 +145,11 @@ function toNonJsonError( `${url} returned ${kind}, not JSON (HTTP ${status}) — check your endpoint.${ keepSnippet ? ` Response: ${truncate(text, 200, '…')}` : '' }`, - status + status, + null, + undefined, + 1, + requestId ) } @@ -154,36 +161,43 @@ function toNonJsonError( * (`{ error }`), or by a proxy that returns HTML. Each of those still has to * produce a sentence rather than `[object Object]`. */ -function toApiError( - url: string, - status: number, - contentType: string | null, - raw: string -): SimApiError { +function toApiError(url: string, response: Response, raw: string): SimApiError { + const { status } = response + const headerRequestId = response.headers.get('x-request-id')?.trim() || null + const failure = (message: string) => + new SimApiError(message, status, null, undefined, 1, headerRequestId) + let parsed: unknown try { parsed = JSON.parse(raw) } catch { - if (!raw.trim()) return new SimApiError(`Request failed with status ${status}`, status) - return toNonJsonError(url, status, contentType, raw) + if (!raw.trim()) return failure(`Request failed with status ${status}`) + return toNonJsonError(url, status, response.headers.get('content-type'), raw, headerRequestId) } const body = parsed as { error?: unknown; message?: unknown } if (body.error && typeof body.error === 'object') { - const error = body.error as { code?: unknown; message?: unknown; details?: unknown } + const error = body.error as { + code?: unknown + message?: unknown + details?: unknown + requestId?: unknown + } return new SimApiError( typeof error.message === 'string' ? error.message : `Request failed with status ${status}`, status, typeof error.code === 'string' ? error.code : null, - error.details + error.details, + 1, + typeof error.requestId === 'string' && error.requestId ? error.requestId : headerRequestId ) } - if (typeof body.error === 'string') return new SimApiError(body.error, status) - if (typeof body.message === 'string') return new SimApiError(body.message, status) + if (typeof body.error === 'string') return failure(body.error) + if (typeof body.message === 'string') return failure(body.message) - return new SimApiError(`Request failed with status ${status}`, status) + return failure(`Request failed with status ${status}`) } /** @@ -673,7 +687,7 @@ export class SimClient { if (!response.ok) { const raw = await readResponseText(response) - const error = toApiError(url, response.status, response.headers.get('content-type'), raw) + const error = toApiError(url, response, raw) if (response.status === 401) { error.message = `${error.message} — run: sim login --profile ${this.profile.authProfile}` } diff --git a/packages/sim-cli/src/runtime/execute.ts b/packages/sim-cli/src/runtime/execute.ts index d6eba513290..d49d7b6a3b8 100644 --- a/packages/sim-cli/src/runtime/execute.ts +++ b/packages/sim-cli/src/runtime/execute.ts @@ -584,7 +584,8 @@ export async function executeOperation( reconciliation: 'Find the operation using this requestId, or retry identical inputs with the same requestId.', }, - failure.exitCode + failure.exitCode, + failure.requestId ) } throw error diff --git a/packages/sim-cli/src/runtime/naming.ts b/packages/sim-cli/src/runtime/naming.ts index 57f6aacb457..a15dd6248f1 100644 --- a/packages/sim-cli/src/runtime/naming.ts +++ b/packages/sim-cli/src/runtime/naming.ts @@ -179,6 +179,8 @@ export function retypeApiError( retypeLocatedMessage(error.message, spellings), error.status, error.code, - error.details === undefined ? undefined : retypeDetails(error.details, spellings) + error.details === undefined ? undefined : retypeDetails(error.details, spellings), + error.exitCode, + error.requestId ) } diff --git a/packages/sim-cli/src/terminal.test.ts b/packages/sim-cli/src/terminal.test.ts new file mode 100644 index 00000000000..bbf412d8a1f --- /dev/null +++ b/packages/sim-cli/src/terminal.test.ts @@ -0,0 +1,87 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { buildProgram } from './program' +import { runTerminalCli } from './terminal' + +/** + * Runs `sim ` the way the installed binary does against a server that + * answers with `response`, and returns what it wrote to stderr and its exit code. + */ +async function runAgainst( + argv: string[], + response: Response +): Promise<{ stderr: string; exitCode: number }> { + const configDir = mkdtempSync(join(tmpdir(), 'sim-cli-terminal-')) + vi.stubEnv('SIM_CONFIG_DIR', configDir) + vi.stubEnv('SIM_ENDPOINT', 'https://sim.example') + vi.stubEnv('SIM_API_KEY', 'sk-test') + vi.stubEnv('SIM_WORKSPACE', 'ws_1') + vi.stubEnv('SIM_NO_UPDATE_CHECK', '1') + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response)) + + let stderr = '' + vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => { + stderr += String(chunk) + return true + }) + vi.spyOn(console, 'error').mockImplementation((...lines: unknown[]) => { + stderr += `${lines.join(' ')}\n` + }) + vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new ExitCalled(code ?? 0) + }) as never) + + const originalArgv = process.argv + process.argv = ['node', 'sim', ...argv] + try { + await runTerminalCli(buildProgram()) + throw new Error('the CLI did not exit') + } catch (error) { + if (!(error instanceof ExitCalled)) throw error + return { stderr, exitCode: error.code } + } finally { + process.argv = originalArgv + rmSync(configDir, { recursive: true, force: true }) + } +} + +class ExitCalled extends Error { + constructor(readonly code: number) { + super(`exit ${code}`) + } +} + +describe('API errors at the terminal', () => { + it('names the server request id in JSON output', async () => { + const { stderr, exitCode } = await runAgainst( + ['--output', 'json', 'workflows', 'get', 'wf-missing'], + new Response( + JSON.stringify({ + error: { code: 'NOT_FOUND', message: 'Workflow not found', requestId: 'req-body' }, + }), + { status: 404, headers: { 'content-type': 'application/json' } } + ) + ) + + expect(exitCode).toBe(1) + expect(JSON.parse(stderr)).toEqual({ + error: { code: 'NOT_FOUND', message: 'Workflow not found', requestId: 'req-body' }, + }) + }) + + it('falls back to the x-request-id header when the body has none', async () => { + const { stderr, exitCode } = await runAgainst( + ['--output', 'text', 'workflows', 'get', 'wf-missing'], + new Response(JSON.stringify({ error: 'Unauthorized' }), { + status: 403, + headers: { 'content-type': 'application/json', 'x-request-id': 'req-header' }, + }) + ) + + expect(exitCode).toBe(1) + expect(stderr).toContain('Unauthorized') + expect(stderr).toMatch(/request id: req-header/) + }) +}) diff --git a/packages/sim-cli/src/terminal.ts b/packages/sim-cli/src/terminal.ts index f19dd41e7b2..d5d8f217d43 100644 --- a/packages/sim-cli/src/terminal.ts +++ b/packages/sim-cli/src/terminal.ts @@ -46,6 +46,7 @@ function explainFailure(error: unknown, program: Command): number | null { code: error.code ?? 'CLI_ERROR', message: error.message, ...(error.details === undefined ? {} : { details: error.details }), + ...(error.requestId ? { requestId: error.requestId } : {}), }, } process.stderr.write(output === 'json' ? `${JSON.stringify(payload)}\n` : dump(payload)) @@ -58,6 +59,7 @@ function explainFailure(error: unknown, program: Command): number | null { console.error(styles().dim(sanitize(line))) } } + if (error.requestId) console.error(styles().dim(` request id: ${sanitize(error.requestId)}`)) return error.exitCode } return null From d8adbf9c297e52d65fbbce782e635ce3d5e3c6c2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 20:53:34 -0800 Subject: [PATCH 2/4] fix(api): report a finished run's status once none of its pauses can be resumed --- apps/docs/openapi-v2-workflows.json | 4 +- apps/sim/lib/api/contracts/v2/workflows.ts | 2 +- .../read-workflow-run-pause.integration.ts | 56 ++++++++- .../workflows/executor/execution-queries.ts | 15 ++- .../executor/execution-status.test.ts | 115 ++++++++++++++++++ .../workflows/executor/execution-status.ts | 14 ++- .../lib/workflows/executor/pause-point-sql.ts | 21 ++++ .../sim/lib/workflows/executor/paused-runs.ts | 22 +--- 8 files changed, 224 insertions(+), 25 deletions(-) create mode 100644 apps/sim/lib/workflows/executor/pause-point-sql.ts diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 27407f0f794..1a5724727c8 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -15969,7 +15969,7 @@ "failed", "cancelled" ], - "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." + "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status and its error. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." }, "trigger": { "type": "string", @@ -16468,7 +16468,7 @@ "cancelled", "queued" ], - "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." + "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status and its error. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." }, "trigger": { "anyOf": [ diff --git a/apps/sim/lib/api/contracts/v2/workflows.ts b/apps/sim/lib/api/contracts/v2/workflows.ts index 0aa62b0001d..688a3151f96 100644 --- a/apps/sim/lib/api/contracts/v2/workflows.ts +++ b/apps/sim/lib/api/contracts/v2/workflows.ts @@ -1983,7 +1983,7 @@ export const v2ResumeWorkflowContract = defineRouteContract({ }) const RUN_STATUS_DESCRIPTION = - "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." + "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status and its error. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." /** * The list projection passes `workflow_execution_logs.status` through except where it diff --git a/apps/sim/lib/workflows/application/read-workflow-run-pause.integration.ts b/apps/sim/lib/workflows/application/read-workflow-run-pause.integration.ts index 9d1a2dd89fe..cd250609e64 100644 --- a/apps/sim/lib/workflows/application/read-workflow-run-pause.integration.ts +++ b/apps/sim/lib/workflows/application/read-workflow-run-pause.integration.ts @@ -3,7 +3,8 @@ * PostgreSQL: the pause's reviewer content is stored and read back with the run's resolved * secret redacted, and the run detail withholds that content on the same terms as the run's * output, while the pause's input fields stay readable. The workspace's paused-run list shows - * each point on exactly the same terms. + * each point on exactly the same terms. A run whose resumed execution failed reads failed, not + * paused, once no point is left to resume. */ import { db } from '@sim/db' import { @@ -32,6 +33,7 @@ import { encryptSecret } from '@/lib/core/security/encryption' import { executeWorkflowOperation } from '@/lib/workflows/application/execute-workflow' import { listPausedRuns } from '@/lib/workflows/application/list-paused-runs' import { readWorkflowRun } from '@/lib/workflows/application/read-workflow-run' +import { listWorkflowExecutions } from '@/lib/workflows/executor/execution-queries' import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager' import type { PausePoint } from '@/executor/types' @@ -475,3 +477,55 @@ describe("the workspace's paused-run list", () => { expect(second.runs[0].pausedAt.getTime()).toBeLessThanOrEqual(first.runs[0].pausedAt.getTime()) }) }) + +describe('a run whose resumed execution failed', () => { + /** What a failed resume leaves behind: the log failed, its point failed, the record still open. */ + async function settleAsFailed(runId: string) { + const [row] = await db + .select({ id: pausedExecutions.id, pausePoints: pausedExecutions.pausePoints }) + .from(pausedExecutions) + .where(eq(pausedExecutions.executionId, runId)) + const points = row.pausePoints as Record + await db + .update(pausedExecutions) + .set({ + pausePoints: Object.fromEntries( + Object.entries(points).map(([id, point]) => [id, { ...point, resumeStatus: 'failed' }]) + ), + }) + .where(eq(pausedExecutions.id, row.id)) + await db + .update(workflowExecutionLogs) + .set({ status: 'failed', level: 'error', endedAt: new Date() }) + .where(eq(workflowExecutionLogs.executionId, runId)) + } + + async function listedStatus(runId: string) { + const rows = await listWorkflowExecutions({ workflowId, limit: 100, order: 'desc' }) + return rows.data.find((row) => row.executionId === runId)?.status + } + + it('reads failed in the run detail and the run list once no point can be resumed', async () => { + const runId = await pausedRun() + await settleAsFailed(runId) + + const run = await readRun(owner, runId, { includeOutput: false }) + expect(run).toMatchObject({ status: 'failed', paused: null, pausePoints: null }) + expect(await listedStatus(runId)).toBe('failed') + }) + + it('still reads paused while a point is waiting, even though the log has finished', async () => { + const runId = await pausedRun() + await db + .update(workflowExecutionLogs) + .set({ status: 'failed', level: 'error', endedAt: new Date() }) + .where(eq(workflowExecutionLogs.executionId, runId)) + + const run = await readRun(owner, runId, { includeOutput: false }) + expect(run.status).toBe('paused') + expect(run.pausePoints).toEqual([ + expect.objectContaining({ contextId: 'approval', status: 'paused' }), + ]) + expect(await listedStatus(runId)).toBe('paused') + }) +}) diff --git a/apps/sim/lib/workflows/executor/execution-queries.ts b/apps/sim/lib/workflows/executor/execution-queries.ts index 0835acd023f..03bef36954a 100644 --- a/apps/sim/lib/workflows/executor/execution-queries.ts +++ b/apps/sim/lib/workflows/executor/execution-queries.ts @@ -1,6 +1,7 @@ import { db } from '@sim/db' import { pausedExecutions, workflowExecutionLogs } from '@sim/db/schema' import { and, asc, desc, eq, gt, gte, lt, lte, or, sql } from 'drizzle-orm' +import { PAUSE_POINT_ROWS } from '@/lib/workflows/executor/pause-point-sql' export type WorkflowExecutionStatus = | 'pending' @@ -26,8 +27,20 @@ export interface ListWorkflowExecutionsInput { order: 'asc' | 'desc' } +/** + * An open pause record reads `paused` unless the log has finished and no point is left to resume, + * the same rule `readWorkflowExecutionStatus` applies to a single run. + */ const executionStatus = sql`CASE - WHEN ${pausedExecutions.status} IN ('paused', 'partially_resumed') THEN 'paused' + WHEN ${pausedExecutions.status} IN ('paused', 'partially_resumed') + AND NOT ( + ${workflowExecutionLogs.status} IN ('completed', 'failed', 'cancelled') + AND NOT EXISTS ( + select 1 from ${PAUSE_POINT_ROWS} + where coalesce(point.value ->> 'resumeStatus', 'paused') NOT IN ('resumed', 'failed') + ) + ) + THEN 'paused' ELSE ${workflowExecutionLogs.status} END` diff --git a/apps/sim/lib/workflows/executor/execution-status.test.ts b/apps/sim/lib/workflows/executor/execution-status.test.ts index fa3e92c1770..6fe2d592dc0 100644 --- a/apps/sim/lib/workflows/executor/execution-status.test.ts +++ b/apps/sim/lib/workflows/executor/execution-status.test.ts @@ -16,6 +16,7 @@ vi.mock('@/lib/workflows/executor/paused-execution-metadata', () => ({ import { FunctionalOutputsUnavailableError } from '@/lib/logs/execution/functional-outputs' import { isWaitingHumanPausePoint } from '@/lib/workflows/executor/execution-state' import { + getProjectedWorkflowExecutionStatus, getWorkflowExecutionStatus, readPausedRunPoints, } from '@/lib/workflows/executor/execution-status' @@ -665,6 +666,120 @@ describe('getWorkflowExecutionStatus polls that read no execution data', () => { }) }) +describe('getWorkflowExecutionStatus once a pause has settled', () => { + const FAILED_TIME_POINT = { + contextId: 'wait-five', + blockId: 'wait-five', + registeredAt: '2026-08-05T12:00:01.000Z', + resumeStatus: 'failed', + pauseKind: 'time', + resumeAt: '2026-08-05T12:05:01.000Z', + } + + /** A run whose resumed execution failed: the log is terminal, the pause record still open. */ + function queueFailedRunWithOpenPause(pausePoints: Record) { + queueTableRows(schemaMock.workflowExecutionLogs, [ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + status: 'failed', + level: 'error', + trigger: 'api', + startedAt: new Date('2026-08-05T12:00:00.000Z'), + endedAt: new Date('2026-08-05T12:05:03.000Z'), + totalDurationMs: 303000, + executionData: {}, + costTotal: null, + }, + ]) + queueTableRows(schemaMock.resumeQueue, []) + queueTableRows(schemaMock.pausedExecutions, [ + { + id: 'paused-execution-1', + status: 'paused', + pausePoints, + metadata: {}, + resumedCount: 0, + pausedAt: new Date('2026-08-05T12:00:01.000Z'), + nextResumeAt: null, + }, + ]) + } + + beforeEach(() => { + resetDbChainMock() + mockGetJob.mockResolvedValue(null) + mockMaterializeForDisplayWithBlockOutputs.mockResolvedValue({ + executionData: { + error: 'Lookup: upstream returned 500', + traceSpans: [ + { + id: 'span-1', + name: 'Lookup', + type: 'api', + status: 'error', + blockId: 'lookup', + executionOrder: 2, + }, + ], + }, + blockOutputs: new Map(), + }) + }) + + it('reports a run whose resumed execution failed as failed, with its error and failing block', async () => { + queueFailedRunWithOpenPause({ 'wait-five': FAILED_TIME_POINT }) + + const projected = await getProjectedWorkflowExecutionStatus(input) + + expect(projected?.status).toMatchObject({ + status: 'failed', + paused: null, + error: 'Lookup: upstream returned 500', + }) + expect(projected?.pausePoints).toBeNull() + expect(projected?.failedBlock).toMatchObject({ blockId: 'lookup', blockName: 'Lookup' }) + }) + + it.each([ + [ + 'a human pause', + { + contextId: 'approval', + blockId: 'approval', + resumeStatus: 'paused', + pauseKind: 'human', + resumeAt: null, + }, + ], + [ + 'a time pause', + { + contextId: 'wait-ten', + blockId: 'wait-ten', + resumeStatus: 'paused', + pauseKind: 'time', + resumeAt: '2026-08-05T12:10:01.000Z', + }, + ], + ])('still reads paused while %s is waiting beside the failed one', async (_, waiting) => { + queueFailedRunWithOpenPause({ 'wait-five': FAILED_TIME_POINT, [waiting.contextId]: waiting }) + + const status = await getWorkflowExecutionStatus(input) + + expect(status).toMatchObject({ + status: 'paused', + error: null, + paused: { + contextId: waiting.contextId, + pauseKind: waiting.pauseKind, + resumeAt: waiting.resumeAt, + }, + }) + }) +}) + describe('readPausedRunPoints cap', () => { beforeEach(() => { resetDbChainMock() diff --git a/apps/sim/lib/workflows/executor/execution-status.ts b/apps/sim/lib/workflows/executor/execution-status.ts index 16309c385ba..c6387f60d1e 100644 --- a/apps/sim/lib/workflows/executor/execution-status.ts +++ b/apps/sim/lib/workflows/executor/execution-status.ts @@ -83,6 +83,11 @@ function pickEarliestPausePoint(points: PausePoint[]): PausePoint | null { }, null) } +/** A point no resume can act on any more: it resumed, or its resumed execution failed. */ +function isSettledPausePoint(point: PausePoint): boolean { + return point.resumeStatus === 'resumed' || point.resumeStatus === 'failed' +} + function extractError(executionData: unknown): string | null { if (!executionData || typeof executionData !== 'object') return null const data = executionData as ExecutionDataShape @@ -519,8 +524,15 @@ async function readWorkflowExecutionStatus( ) .limit(1) + /** + * An open pause record makes the run read `paused` unless the log has finished and no point + * is left to resume: a resume that ran and failed marks its point `failed` and fails the log + * but leaves the record open, and that point can never be resumed again. + */ const isCurrentlyPaused = - !!pausedRow && (pausedRow.status === 'paused' || pausedRow.status === 'partially_resumed') + !!pausedRow && + (pausedRow.status === 'paused' || pausedRow.status === 'partially_resumed') && + !(hasTerminalLog && normalizePausePoints(pausedRow.pausePoints).every(isSettledPausePoint)) let status: WorkflowExecutionStatusResponse['status'] if (isCurrentlyPaused) { diff --git a/apps/sim/lib/workflows/executor/pause-point-sql.ts b/apps/sim/lib/workflows/executor/pause-point-sql.ts new file mode 100644 index 00000000000..372b84b9b01 --- /dev/null +++ b/apps/sim/lib/workflows/executor/pause-point-sql.ts @@ -0,0 +1,21 @@ +import { pausedExecutions } from '@sim/db/schema' +import { sql } from 'drizzle-orm' + +/** + * Every pause point of a `paused_executions` row, as the derived table `point(value)`. + * + * `pause_points` is an object keyed by context id, or an array on legacy rows; both shapes are + * read, as `normalizePausePoints` reads them, and each set function only ever receives the shape + * it accepts. + */ +export const PAUSE_POINT_ROWS = sql`( + select value from jsonb_each( + case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'object' + then ${pausedExecutions.pausePoints} else '{}'::jsonb end + ) + union all + select value from jsonb_array_elements( + case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'array' + then ${pausedExecutions.pausePoints} else '[]'::jsonb end + ) +) as point(value)` diff --git a/apps/sim/lib/workflows/executor/paused-runs.ts b/apps/sim/lib/workflows/executor/paused-runs.ts index 91a90dbf37b..07219f5c584 100644 --- a/apps/sim/lib/workflows/executor/paused-runs.ts +++ b/apps/sim/lib/workflows/executor/paused-runs.ts @@ -14,6 +14,7 @@ import { resumeKeyset, textKey, } from '@/lib/api/list-query' +import { PAUSE_POINT_ROWS } from '@/lib/workflows/executor/pause-point-sql' /** A paused run of an active workflow, with the raw `pause_points` its detail is built from. */ interface WorkspacePausedRunRow { @@ -35,25 +36,8 @@ const PAUSED_RUN_KEYS: readonly KeysetKey[] = [ const POINT_WAITS_ON_HUMAN = sql`coalesce(point.value ->> 'pauseKind', 'human') <> 'time' and coalesce(point.value ->> 'resumeStatus', 'paused') = 'paused'` -/** - * A paused run with a pause point still waiting on a human. `pause_points` is an object keyed - * by context id, or an array on legacy rows; both shapes are read, as `normalizePausePoints` - * reads them, and each set function only ever receives the shape it accepts. - */ -const WAITS_ON_HUMAN = sql`exists ( - select 1 from ( - select value from jsonb_each( - case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'object' - then ${pausedExecutions.pausePoints} else '{}'::jsonb end - ) - union all - select value from jsonb_array_elements( - case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'array' - then ${pausedExecutions.pausePoints} else '[]'::jsonb end - ) - ) as point(value) - where ${POINT_WAITS_ON_HUMAN} -)` +/** A paused run with a pause point still waiting on a human. */ +const WAITS_ON_HUMAN = sql`exists (select 1 from ${PAUSE_POINT_ROWS} where ${POINT_WAITS_ON_HUMAN})` /** The cursor that resumes the list after `row`. */ export function pausedRunCursorKeys(row: WorkspacePausedRunRow): CursorKey[] { From b91f4227f09cb65a7d62d75eb5f79d119bde0210 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 21:04:40 -0800 Subject: [PATCH 3/4] fix(cli): keep the request id on every error built from a response --- apps/docs/openapi-v2-workflows.json | 4 +-- apps/sim/lib/api/contracts/v2/workflows.ts | 2 +- packages/sim-cli/src/http/client.test.ts | 32 ++++++++++++++++++ packages/sim-cli/src/http/client.ts | 39 ++++++++++++++++++---- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index 1a5724727c8..7918cdc1183 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -15969,7 +15969,7 @@ "failed", "cancelled" ], - "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status and its error. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." + "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." }, "trigger": { "type": "string", @@ -16468,7 +16468,7 @@ "cancelled", "queued" ], - "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status and its error. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." + "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." }, "trigger": { "anyOf": [ diff --git a/apps/sim/lib/api/contracts/v2/workflows.ts b/apps/sim/lib/api/contracts/v2/workflows.ts index 688a3151f96..816a0fd78b7 100644 --- a/apps/sim/lib/api/contracts/v2/workflows.ts +++ b/apps/sim/lib/api/contracts/v2/workflows.ts @@ -1983,7 +1983,7 @@ export const v2ResumeWorkflowContract = defineRouteContract({ }) const RUN_STATUS_DESCRIPTION = - "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status and its error. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." + "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason." /** * The list projection passes `workflow_execution_logs.status` through except where it diff --git a/packages/sim-cli/src/http/client.test.ts b/packages/sim-cli/src/http/client.test.ts index c44b287324b..bc0e39c7d3b 100644 --- a/packages/sim-cli/src/http/client.test.ts +++ b/packages/sim-cli/src/http/client.test.ts @@ -122,6 +122,38 @@ describe('non-JSON responses', () => { }) }) +describe('request ids on errors', () => { + it('keeps the x-request-id of a 200 that is not JSON', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response('', { + status: 200, + headers: { 'content-type': 'text/html', 'x-request-id': 'req-200' }, + }) + ) + ) + await expect(client().request('/api/v2/workflows')).rejects.toMatchObject({ + requestId: 'req-200', + }) + }) + + it('keeps the x-request-id of a redirect', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(null, { + status: 308, + headers: { location: 'https://www.sim.example/api/v2/tables', 'x-request-id': 'req-308' }, + }) + ) + ) + await expect(client().request('/api/v2/tables')).rejects.toMatchObject({ + requestId: 'req-308', + }) + }) +}) + describe('a request that never answers', () => { it('bounds a request by default, above every timeout the server itself applies', async () => { // A synchronous workflow run is allowed 3000s on a paid plan, so a tighter diff --git a/packages/sim-cli/src/http/client.ts b/packages/sim-cli/src/http/client.ts index c1bca7336a1..a527c818da7 100644 --- a/packages/sim-cli/src/http/client.ts +++ b/packages/sim-cli/src/http/client.ts @@ -163,7 +163,7 @@ function toNonJsonError( */ function toApiError(url: string, response: Response, raw: string): SimApiError { const { status } = response - const headerRequestId = response.headers.get('x-request-id')?.trim() || null + const headerRequestId = responseRequestId(response) const failure = (message: string) => new SimApiError(message, status, null, undefined, 1, headerRequestId) @@ -225,6 +225,11 @@ function transportErrorMessage(error: unknown): string { return messages.join(': ') || 'Unknown network error' } +/** The server's id for a response (`x-request-id`), or null when it sent none. */ +function responseRequestId(response: Response): string | null { + return response.headers.get('x-request-id')?.trim() || null +} + async function readResponseText(response: Response): Promise { try { return await response.text() @@ -232,7 +237,10 @@ async function readResponseText(response: Response): Promise { throw new SimApiError( `Unable to read the response: ${transportErrorMessage(error)}`, response.status, - 'RESPONSE_READ_FAILED' + 'RESPONSE_READ_FAILED', + undefined, + 1, + responseRequestId(response) ) } } @@ -594,7 +602,13 @@ export class SimClient { try { return JSON.parse(raw) as T } catch { - throw toNonJsonError(url, response.status, response.headers.get('content-type'), raw) + throw toNonJsonError( + url, + response.status, + response.headers.get('content-type'), + raw, + responseRequestId(response) + ) } } @@ -709,6 +723,7 @@ export class SimClient { * sentence — the redirect is the finding either way. */ private toRedirectError(url: string, path: string, response: Response): SimApiError { + const requestId = responseRequestId(response) const location = response.headers.get('location')?.trim() let target: URL | null = null if (location) { @@ -722,19 +737,31 @@ export class SimClient { if (!target) { return new SimApiError( `${url} answered HTTP ${response.status} with no usable redirect target. Check the endpoint for profile "${this.profile.name}".`, - response.status + response.status, + null, + undefined, + 1, + requestId ) } const suggested = redirectEndpoint(this.profile.endpoint, path, target) if (!suggested) { return new SimApiError( `${url} redirected to ${target.href}. The CLI does not follow redirects, because a redirect can drop the request body and turn a write into a silent no-op.`, - response.status + response.status, + null, + undefined, + 1, + requestId ) } return new SimApiError( `Endpoint redirected to ${suggested}. Run: sim configure --profile ${this.profile.name} --set-endpoint ${suggested}`, - response.status + response.status, + null, + undefined, + 1, + requestId ) } } From 4464cc5c54f1edf955cb0d8d33b0b89f1e61f6df Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 21:15:31 -0800 Subject: [PATCH 4/4] fix(cli): read an internal route's request id beside the error --- packages/sim-cli/src/http/client.test.ts | 16 ++++++++ packages/sim-cli/src/http/client.ts | 52 +++++++++++++++--------- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/packages/sim-cli/src/http/client.test.ts b/packages/sim-cli/src/http/client.test.ts index bc0e39c7d3b..314789270c5 100644 --- a/packages/sim-cli/src/http/client.test.ts +++ b/packages/sim-cli/src/http/client.test.ts @@ -123,6 +123,22 @@ describe('non-JSON responses', () => { }) describe('request ids on errors', () => { + it("reads an internal route's request id stamped beside the error", async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ error: 'Workflow not found', requestId: 'req-body' }), { + status: 404, + headers: { 'content-type': 'application/json' }, + }) + ) + ) + await expect(client().request('/api/workflows/missing')).rejects.toMatchObject({ + message: 'Workflow not found', + requestId: 'req-body', + }) + }) + it('keeps the x-request-id of a 200 that is not JSON', async () => { vi.stubGlobal( 'fetch', diff --git a/packages/sim-cli/src/http/client.ts b/packages/sim-cli/src/http/client.ts index a527c818da7..3241e038c4b 100644 --- a/packages/sim-cli/src/http/client.ts +++ b/packages/sim-cli/src/http/client.ts @@ -164,42 +164,56 @@ function toNonJsonError( function toApiError(url: string, response: Response, raw: string): SimApiError { const { status } = response const headerRequestId = responseRequestId(response) - const failure = (message: string) => - new SimApiError(message, status, null, undefined, 1, headerRequestId) let parsed: unknown try { parsed = JSON.parse(raw) } catch { - if (!raw.trim()) return failure(`Request failed with status ${status}`) + if (!raw.trim()) { + return new SimApiError( + `Request failed with status ${status}`, + status, + null, + undefined, + 1, + headerRequestId + ) + } return toNonJsonError(url, status, response.headers.get('content-type'), raw, headerRequestId) } - const body = parsed as { error?: unknown; message?: unknown } - - if (body.error && typeof body.error === 'object') { - const error = body.error as { - code?: unknown - message?: unknown - details?: unknown - requestId?: unknown - } - return new SimApiError( + const body = parsed as { error?: unknown; message?: unknown; requestId?: unknown } + const error = + body.error && typeof body.error === 'object' + ? (body.error as { + code?: unknown + message?: unknown + details?: unknown + requestId?: unknown + }) + : null + /** v2 nests the id in `error`; internal routes stamp it beside `error`; the header backs both. */ + const requestId = + nonEmptyString(error?.requestId) ?? nonEmptyString(body.requestId) ?? headerRequestId + const failure = (message: string, code: string | null = null, details?: unknown) => + new SimApiError(message, status, code, details, 1, requestId) + + if (error) { + return failure( typeof error.message === 'string' ? error.message : `Request failed with status ${status}`, - status, typeof error.code === 'string' ? error.code : null, - error.details, - 1, - typeof error.requestId === 'string' && error.requestId ? error.requestId : headerRequestId + error.details ) } - if (typeof body.error === 'string') return failure(body.error) if (typeof body.message === 'string') return failure(body.message) - return failure(`Request failed with status ${status}`) } +function nonEmptyString(value: unknown): string | null { + return typeof value === 'string' && value ? value : null +} + /** * Keeps the useful nested reason from Node/Undici transport failures without * serializing request options, headers, socket objects, or credentials.