fix(api): report a finished run's real status once its pauses have settled; show the request id in the CLI (#9924)

* fix(cli): show the server's request id on API errors

* fix(api): report a finished run's status once none of its pauses can be resumed

* fix(cli): keep the request id on every error built from a response

* fix(cli): read an internal route's request id beside the error

Author: waleedlatif1

apps/docs/openapi-v2-workflows.json

@@ -15969,7 +15969,7 @@
               "failed",
               "cancelled"
             ],
-            "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
+            "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
           },
           "trigger": {
             "type": "string",
@@ -16468,7 +16468,7 @@
               "cancelled",
               "queued"
             ],
-            "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
+            "description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
           },
           "trigger": {
             "anyOf": [

apps/sim/lib/api/contracts/v2/workflows.ts

@@ -1983,7 +1983,7 @@ export const v2ResumeWorkflowContract = defineRouteContract({
 })
 
 const RUN_STATUS_DESCRIPTION =
-  "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
+  "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
 
 /**
  * The list projection passes `workflow_execution_logs.status` through except where it

apps/sim/lib/workflows/application/read-workflow-run-pause.integration.ts

@@ -3,7 +3,8 @@
  * PostgreSQL: the pause's reviewer content is stored and read back with the run's resolved
  * secret redacted, and the run detail withholds that content on the same terms as the run's
  * output, while the pause's input fields stay readable. The workspace's paused-run list shows
- * each point on exactly the same terms.
+ * each point on exactly the same terms. A run whose resumed execution failed reads failed, not
+ * paused, once no point is left to resume.
  */
 import { db } from '@sim/db'
 import {
@@ -32,6 +33,7 @@ import { encryptSecret } from '@/lib/core/security/encryption'
 import { executeWorkflowOperation } from '@/lib/workflows/application/execute-workflow'
 import { listPausedRuns } from '@/lib/workflows/application/list-paused-runs'
 import { readWorkflowRun } from '@/lib/workflows/application/read-workflow-run'
+import { listWorkflowExecutions } from '@/lib/workflows/executor/execution-queries'
 import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager'
 import type { PausePoint } from '@/executor/types'
 
@@ -475,3 +477,55 @@ describe("the workspace's paused-run list", () => {
     expect(second.runs[0].pausedAt.getTime()).toBeLessThanOrEqual(first.runs[0].pausedAt.getTime())
   })
 })
+
+describe('a run whose resumed execution failed', () => {
+  /** What a failed resume leaves behind: the log failed, its point failed, the record still open. */
+  async function settleAsFailed(runId: string) {
+    const [row] = await db
+      .select({ id: pausedExecutions.id, pausePoints: pausedExecutions.pausePoints })
+      .from(pausedExecutions)
+      .where(eq(pausedExecutions.executionId, runId))
+    const points = row.pausePoints as Record<string, PausePoint>
+    await db
+      .update(pausedExecutions)
+      .set({
+        pausePoints: Object.fromEntries(
+          Object.entries(points).map(([id, point]) => [id, { ...point, resumeStatus: 'failed' }])
+        ),
+      })
+      .where(eq(pausedExecutions.id, row.id))
+    await db
+      .update(workflowExecutionLogs)
+      .set({ status: 'failed', level: 'error', endedAt: new Date() })
+      .where(eq(workflowExecutionLogs.executionId, runId))
+  }
+
+  async function listedStatus(runId: string) {
+    const rows = await listWorkflowExecutions({ workflowId, limit: 100, order: 'desc' })
+    return rows.data.find((row) => row.executionId === runId)?.status
+  }
+
+  it('reads failed in the run detail and the run list once no point can be resumed', async () => {
+    const runId = await pausedRun()
+    await settleAsFailed(runId)
+
+    const run = await readRun(owner, runId, { includeOutput: false })
+    expect(run).toMatchObject({ status: 'failed', paused: null, pausePoints: null })
+    expect(await listedStatus(runId)).toBe('failed')
+  })
+
+  it('still reads paused while a point is waiting, even though the log has finished', async () => {
+    const runId = await pausedRun()
+    await db
+      .update(workflowExecutionLogs)
+      .set({ status: 'failed', level: 'error', endedAt: new Date() })
+      .where(eq(workflowExecutionLogs.executionId, runId))
+
+    const run = await readRun(owner, runId, { includeOutput: false })
+    expect(run.status).toBe('paused')
+    expect(run.pausePoints).toEqual([
+      expect.objectContaining({ contextId: 'approval', status: 'paused' }),
+    ])
+    expect(await listedStatus(runId)).toBe('paused')
+  })
+})

apps/sim/lib/workflows/executor/execution-queries.ts

@@ -1,6 +1,7 @@
 import { db } from '@sim/db'
 import { pausedExecutions, workflowExecutionLogs } from '@sim/db/schema'
 import { and, asc, desc, eq, gt, gte, lt, lte, or, sql } from 'drizzle-orm'
+import { PAUSE_POINT_ROWS } from '@/lib/workflows/executor/pause-point-sql'
 
 export type WorkflowExecutionStatus =
   | 'pending'
@@ -26,8 +27,20 @@ export interface ListWorkflowExecutionsInput {
   order: 'asc' | 'desc'
 }
 
+/**
+ * An open pause record reads `paused` unless the log has finished and no point is left to resume,
+ * the same rule `readWorkflowExecutionStatus` applies to a single run.
+ */
 const executionStatus = sql<string>`CASE
-  WHEN ${pausedExecutions.status} IN ('paused', 'partially_resumed') THEN 'paused'
+  WHEN ${pausedExecutions.status} IN ('paused', 'partially_resumed')
+    AND NOT (
+      ${workflowExecutionLogs.status} IN ('completed', 'failed', 'cancelled')
+      AND NOT EXISTS (
+        select 1 from ${PAUSE_POINT_ROWS}
+        where coalesce(point.value ->> 'resumeStatus', 'paused') NOT IN ('resumed', 'failed')
+      )
+    )
+  THEN 'paused'
   ELSE ${workflowExecutionLogs.status}
 END`
 

apps/sim/lib/workflows/executor/execution-status.test.ts

@@ -16,6 +16,7 @@ vi.mock('@/lib/workflows/executor/paused-execution-metadata', () => ({
 import { FunctionalOutputsUnavailableError } from '@/lib/logs/execution/functional-outputs'
 import { isWaitingHumanPausePoint } from '@/lib/workflows/executor/execution-state'
 import {
+  getProjectedWorkflowExecutionStatus,
   getWorkflowExecutionStatus,
   readPausedRunPoints,
 } from '@/lib/workflows/executor/execution-status'
@@ -665,6 +666,120 @@ describe('getWorkflowExecutionStatus polls that read no execution data', () => {
   })
 })
 
+describe('getWorkflowExecutionStatus once a pause has settled', () => {
+  const FAILED_TIME_POINT = {
+    contextId: 'wait-five',
+    blockId: 'wait-five',
+    registeredAt: '2026-08-05T12:00:01.000Z',
+    resumeStatus: 'failed',
+    pauseKind: 'time',
+    resumeAt: '2026-08-05T12:05:01.000Z',
+  }
+
+  /** A run whose resumed execution failed: the log is terminal, the pause record still open. */
+  function queueFailedRunWithOpenPause(pausePoints: Record<string, unknown>) {
+    queueTableRows(schemaMock.workflowExecutionLogs, [
+      {
+        executionId: 'execution-1',
+        workflowId: 'workflow-1',
+        workspaceId: 'workspace-1',
+        status: 'failed',
+        level: 'error',
+        trigger: 'api',
+        startedAt: new Date('2026-08-05T12:00:00.000Z'),
+        endedAt: new Date('2026-08-05T12:05:03.000Z'),
+        totalDurationMs: 303000,
+        executionData: {},
+        costTotal: null,
+      },
+    ])
+    queueTableRows(schemaMock.resumeQueue, [])
+    queueTableRows(schemaMock.pausedExecutions, [
+      {
+        id: 'paused-execution-1',
+        status: 'paused',
+        pausePoints,
+        metadata: {},
+        resumedCount: 0,
+        pausedAt: new Date('2026-08-05T12:00:01.000Z'),
+        nextResumeAt: null,
+      },
+    ])
+  }
+
+  beforeEach(() => {
+    resetDbChainMock()
+    mockGetJob.mockResolvedValue(null)
+    mockMaterializeForDisplayWithBlockOutputs.mockResolvedValue({
+      executionData: {
+        error: 'Lookup: upstream returned 500',
+        traceSpans: [
+          {
+            id: 'span-1',
+            name: 'Lookup',
+            type: 'api',
+            status: 'error',
+            blockId: 'lookup',
+            executionOrder: 2,
+          },
+        ],
+      },
+      blockOutputs: new Map(),
+    })
+  })
+
+  it('reports a run whose resumed execution failed as failed, with its error and failing block', async () => {
+    queueFailedRunWithOpenPause({ 'wait-five': FAILED_TIME_POINT })
+
+    const projected = await getProjectedWorkflowExecutionStatus(input)
+
+    expect(projected?.status).toMatchObject({
+      status: 'failed',
+      paused: null,
+      error: 'Lookup: upstream returned 500',
+    })
+    expect(projected?.pausePoints).toBeNull()
+    expect(projected?.failedBlock).toMatchObject({ blockId: 'lookup', blockName: 'Lookup' })
+  })
+
+  it.each([
+    [
+      'a human pause',
+      {
+        contextId: 'approval',
+        blockId: 'approval',
+        resumeStatus: 'paused',
+        pauseKind: 'human',
+        resumeAt: null,
+      },
+    ],
+    [
+      'a time pause',
+      {
+        contextId: 'wait-ten',
+        blockId: 'wait-ten',
+        resumeStatus: 'paused',
+        pauseKind: 'time',
+        resumeAt: '2026-08-05T12:10:01.000Z',
+      },
+    ],
+  ])('still reads paused while %s is waiting beside the failed one', async (_, waiting) => {
+    queueFailedRunWithOpenPause({ 'wait-five': FAILED_TIME_POINT, [waiting.contextId]: waiting })
+
+    const status = await getWorkflowExecutionStatus(input)
+
+    expect(status).toMatchObject({
+      status: 'paused',
+      error: null,
+      paused: {
+        contextId: waiting.contextId,
+        pauseKind: waiting.pauseKind,
+        resumeAt: waiting.resumeAt,
+      },
+    })
+  })
+})
+
 describe('readPausedRunPoints cap', () => {
   beforeEach(() => {
     resetDbChainMock()

apps/sim/lib/workflows/executor/execution-status.ts

@@ -83,6 +83,11 @@ function pickEarliestPausePoint(points: PausePoint[]): PausePoint | null {
   }, null)
 }
 
+/** A point no resume can act on any more: it resumed, or its resumed execution failed. */
+function isSettledPausePoint(point: PausePoint): boolean {
+  return point.resumeStatus === 'resumed' || point.resumeStatus === 'failed'
+}
+
 function extractError(executionData: unknown): string | null {
   if (!executionData || typeof executionData !== 'object') return null
   const data = executionData as ExecutionDataShape
@@ -519,8 +524,15 @@ async function readWorkflowExecutionStatus(
     )
     .limit(1)
 
+  /**
+   * An open pause record makes the run read `paused` unless the log has finished and no point
+   * is left to resume: a resume that ran and failed marks its point `failed` and fails the log
+   * but leaves the record open, and that point can never be resumed again.
+   */
   const isCurrentlyPaused =
-    !!pausedRow && (pausedRow.status === 'paused' || pausedRow.status === 'partially_resumed')
+    !!pausedRow &&
+    (pausedRow.status === 'paused' || pausedRow.status === 'partially_resumed') &&
+    !(hasTerminalLog && normalizePausePoints(pausedRow.pausePoints).every(isSettledPausePoint))
 
   let status: WorkflowExecutionStatusResponse['status']
   if (isCurrentlyPaused) {

apps/sim/lib/workflows/executor/pause-point-sql.ts

@@ -0,0 +1,21 @@
+import { pausedExecutions } from '@sim/db/schema'
+import { sql } from 'drizzle-orm'
+
+/**
+ * Every pause point of a `paused_executions` row, as the derived table `point(value)`.
+ *
+ * `pause_points` is an object keyed by context id, or an array on legacy rows; both shapes are
+ * read, as `normalizePausePoints` reads them, and each set function only ever receives the shape
+ * it accepts.
+ */
+export const PAUSE_POINT_ROWS = sql`(
+  select value from jsonb_each(
+    case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'object'
+      then ${pausedExecutions.pausePoints} else '{}'::jsonb end
+  )
+  union all
+  select value from jsonb_array_elements(
+    case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'array'
+      then ${pausedExecutions.pausePoints} else '[]'::jsonb end
+  )
+) as point(value)`

apps/sim/lib/workflows/executor/paused-runs.ts

@@ -14,6 +14,7 @@ import {
   resumeKeyset,
   textKey,
 } from '@/lib/api/list-query'
+import { PAUSE_POINT_ROWS } from '@/lib/workflows/executor/pause-point-sql'
 
 /** A paused run of an active workflow, with the raw `pause_points` its detail is built from. */
 interface WorkspacePausedRunRow {
@@ -35,25 +36,8 @@ const PAUSED_RUN_KEYS: readonly KeysetKey<WorkspacePausedRunRow>[] = [
 const POINT_WAITS_ON_HUMAN = sql`coalesce(point.value ->> 'pauseKind', 'human') <> 'time'
     and coalesce(point.value ->> 'resumeStatus', 'paused') = 'paused'`
 
-/**
- * A paused run with a pause point still waiting on a human. `pause_points` is an object keyed
- * by context id, or an array on legacy rows; both shapes are read, as `normalizePausePoints`
- * reads them, and each set function only ever receives the shape it accepts.
- */
-const WAITS_ON_HUMAN = sql`exists (
-  select 1 from (
-    select value from jsonb_each(
-      case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'object'
-        then ${pausedExecutions.pausePoints} else '{}'::jsonb end
-    )
-    union all
-    select value from jsonb_array_elements(
-      case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'array'
-        then ${pausedExecutions.pausePoints} else '[]'::jsonb end
-    )
-  ) as point(value)
-  where ${POINT_WAITS_ON_HUMAN}
-)`
+/** A paused run with a pause point still waiting on a human. */
+const WAITS_ON_HUMAN = sql`exists (select 1 from ${PAUSE_POINT_ROWS} where ${POINT_WAITS_ON_HUMAN})`
 
 /** The cursor that resumes the list after `row`. */
 export function pausedRunCursorKeys(row: WorkspacePausedRunRow): CursorKey[] {

packages/sim-cli/src/commands/credentials.ts

@@ -186,7 +186,8 @@ function withKeylessRetryHint(error: unknown, providerId: string): unknown {
     error.status,
     error.code,
     error.details,
-    error.exitCode
+    error.exitCode,
+    error.requestId
   )
 }
 

packages/sim-cli/src/commands/protocol/workspace-operation-wait.ts

@@ -152,7 +152,8 @@ export async function waitWorkspaceOperation(
           requestId: report?.requestId,
           applied: report?.applied,
         },
-        failure.exitCode
+        failure.exitCode,
+        failure.requestId
       )
     }
     if (report.status !== 'processing') return { report, timedOut: false }