fix(resume): retry automatic resumes after read deadlines and an unreadable usage ledger (#9046)

* fix(resume): retry automatic resumes after read deadlines and an unreadable usage ledger

* fix(resume): keep the stack on the usage-status failure log

* fix(resume): log usage-status failures with the logger's redacted stack

Author: waleedlatif1

apps/sim/lib/billing/calculations/usage-monitor.ts

@@ -1,7 +1,7 @@
 import { db } from '@sim/db'
 import { userStats } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
-import { toError } from '@sim/utils/errors'
+import { describeError, toError } from '@sim/utils/errors'
 import { eq } from 'drizzle-orm'
 import { USAGE_UNAVAILABLE_MESSAGE } from '@/lib/billing/constants'
 import { isOrganizationBillingBlocked } from '@/lib/billing/core/access'
@@ -171,14 +171,10 @@ export async function checkUsageStatus(
 
     return buildUsageData({ currentUsage, limit, scope, organizationId })
   } catch (error) {
-    logger.error('Error checking usage status', {
-      error: error instanceof Error ? { message: error.message, stack: error.stack } : error,
-      userId,
-    })
-
     logger.error('Cannot determine usage status - blocking execution', {
       userId,
-      error: toError(error).message,
+      error,
+      cause: describeError(error),
     })
 
     return {

apps/sim/lib/concurrency/singleflight.ts

@@ -13,6 +13,8 @@ const DEFAULT_SETTLE_TIMEOUT_MS = 30_000
  * fresh producer instead of joining the wedged one.
  */
 export class CoalesceSettleTimeoutError extends Error {
+  readonly code = 'COALESCE_SETTLE_TIMEOUT' as const
+
   constructor(key: string, timeoutMs: number) {
     super(`Coalesced producer for "${key}" did not settle within ${timeoutMs}ms`)
     this.name = 'CoalesceSettleTimeoutError'

apps/sim/lib/core/errors/retryable-infrastructure.ts

@@ -55,6 +55,17 @@ const RETRYABLE_PG_CLIENT_ERROR_CODES = new Set([
   'CONNECTION_DESTROYED',
 ])
 
+/**
+ * Deadlines the app enforces on its own reads: a read whose budget ran out before its query
+ * began (`DatabaseReadDeadlineError`) and a coalesced read that never settled
+ * (`CoalesceSettleTimeoutError`). Neither read produced an answer or an effect, the same
+ * class as `ETIMEDOUT` above.
+ */
+const RETRYABLE_READ_DEADLINE_ERROR_CODES = new Set([
+  'DATABASE_READ_DEADLINE_EXCEEDED',
+  'COALESCE_SETTLE_TIMEOUT',
+])
+
 function getErrorChain(error: unknown): Array<Error & Record<string, unknown>> {
   const chain: Array<Error & Record<string, unknown>> = []
   let current: unknown = error
@@ -79,6 +90,7 @@ export function describeRetryableInfrastructureError(
       (code && RETRYABLE_NETWORK_ERROR_CODES.has(code)) ||
       (code && RETRYABLE_APP_ERROR_CODES.has(code)) ||
       (code && RETRYABLE_PG_CLIENT_ERROR_CODES.has(code)) ||
+      (code && RETRYABLE_READ_DEADLINE_ERROR_CODES.has(code)) ||
       (errno && RETRYABLE_NETWORK_ERROR_CODES.has(errno))
     ) {
       return {

apps/sim/lib/db/read-retry.ts

@@ -29,6 +29,8 @@ const TRANSIENT_READ_CODES = new Set([
 
 /** A read exhausted its budget before its query could begin. */
 export class DatabaseReadDeadlineError extends Error {
+  readonly code = 'DATABASE_READ_DEADLINE_EXCEEDED' as const
+
   constructor() {
     super('Database read deadline exceeded')
     this.name = 'DatabaseReadDeadlineError'

apps/sim/lib/execution/preprocessing.test.ts

@@ -20,8 +20,10 @@ import { createMockRedis } from '@sim/testing/mocks/redis.mock'
 import { redisConfigMockFns, resetRedisConfigMock } from '@sim/testing/mocks/redis-config.mock'
 import { utilsHelpersMock } from '@sim/testing/mocks/utils-helpers.mock'
 import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { coalesceLocally } from '@/lib/concurrency/singleflight'
 import { ADMISSION_REJECTION_CODE } from '@/lib/core/admission/rejection'
 import { ADMISSION_ERROR_CODE } from '@/lib/core/admission/transient-failure'
+import { DatabaseReadDeadlineError } from '@/lib/db/read-retry'
 import type { LoggingSession } from '@/lib/logs/execution/logging-session'
 
 const { mockCheckRateLimit } = vi.hoisted(() => ({
@@ -1099,3 +1101,82 @@ describe('preprocessExecution admission rejection codes and blocked-run log thro
     expect(loggingSession.safeCompleteWithError).toHaveBeenCalledOnce()
   })
 })
+
+describe('preprocessExecution usage gate retry classification for resumes', () => {
+  const resumeAdmission = () =>
+    preprocessExecution({
+      workflowId: 'workflow-1',
+      userId: 'actor-1',
+      triggerType: 'manual',
+      executionId: 'execution-1',
+      requestId: 'request-1',
+      checkRateLimit: false,
+      checkDeployment: false,
+      logPreprocessingErrors: false,
+      workspaceId: 'workspace-1',
+      billingAttribution: { ...ORGANIZATION_ATTRIBUTION, actorUserId: 'actor-1' },
+      executionType: 'async',
+    })
+
+  beforeEach(() => {
+    mockGetActivelyBannedUserIds.mockResolvedValue([])
+  })
+
+  it('marks a usage read that outlived its coalesced settle deadline retryable', async () => {
+    mockCheckAttributedUsageLimits.mockImplementationOnce(() =>
+      coalesceLocally('usage-gate:org-1', () => new Promise<never>(() => {}), 1)
+    )
+
+    const result = await resumeAdmission()
+
+    expect(result).toMatchObject({
+      success: false,
+      error: {
+        message: 'Unable to determine usage limits. Execution blocked for security.',
+        statusCode: 500,
+        retryable: true,
+      },
+    })
+  })
+
+  it('marks a usage read that exhausted its database read budget retryable', async () => {
+    mockCheckAttributedUsageLimits.mockRejectedValueOnce(new DatabaseReadDeadlineError())
+
+    const result = await resumeAdmission()
+
+    expect(result).toMatchObject({
+      success: false,
+      error: { statusCode: 500, retryable: true },
+    })
+  })
+
+  it('marks an unreadable usage ledger retryable without tagging it as over the limit', async () => {
+    mockCheckAttributedUsageLimits.mockResolvedValueOnce({
+      isExceeded: true,
+      reason: 'usage_unavailable',
+      message: 'Usage is temporarily unavailable',
+      payerUsage: { currentUsage: 0, limit: 0 },
+    })
+
+    const result = await resumeAdmission()
+
+    expect(result).toMatchObject({ success: false, error: { statusCode: 402, retryable: true } })
+    expect(result.success === false && result.error.code).toBeUndefined()
+  })
+
+  it('keeps a payer genuinely over its limit non-retryable', async () => {
+    mockCheckAttributedUsageLimits.mockResolvedValueOnce({
+      isExceeded: true,
+      message: 'Usage limit exceeded',
+      payerUsage: { currentUsage: 12, limit: 10 },
+    })
+
+    const result = await resumeAdmission()
+
+    expect(result).toMatchObject({
+      success: false,
+      error: { statusCode: 402, code: ADMISSION_REJECTION_CODE.USAGE_LIMIT_EXCEEDED },
+    })
+    expect(result.success === false && result.error.retryable).not.toBe(true)
+  })
+})

apps/sim/lib/execution/preprocessing.ts

@@ -769,7 +769,7 @@ export async function preprocessExecution(
                 statusCode: 402,
                 // An unreadable ledger fails closed; that is no verdict on the payer, so senders retry.
                 ...(usageCheck.reason === 'usage_unavailable'
-                  ? {}
+                  ? { retryable: true }
                   : { code: ADMISSION_REJECTION_CODE.USAGE_LIMIT_EXCEEDED }),
               },
             },