fix(credentials): honor every minter's Retry-After and close token fallback gaps

- Carry the token endpoint's Retry-After on 429/503 failures from every
  bespoke client-credential minter (Box, Zoom, DocuSign, NetSuite, Help
  Scout, Ramp, Jama, Zendesk, Salesforce, Zoho Desk) and through the
  CrowdStrike, SailPoint and ZoomInfo client errors, so their throttled
  answers wait as long as the provider asks instead of the fixed backoff.
- Recheck the 30 s expiry margin when the per-instance cache falls back
  to a still-valid token after a slow failed refresh.
- Map an unreadable Vanta token body to provider_unavailable so it is
  remembered for every worker and paced at 15 s like any failed exchange.
- Remember a refusal whose code the shared failure row cannot store with
  an additive `refusal` flag, so a replay never serves the stale token.
- Rename the credentials helper to throttledRetryAfterMs so it no longer
  shares a name with providers/retry.ts.

Author: waleedlatif1

apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts

@@ -99,12 +99,18 @@ describe('mintBoxServiceAccountToken', () => {
     })
   })
 
-  it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => {
-    mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'rate_limit_exceeded' }, 429))
+  it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => {
+    mockFetch.mockResolvedValueOnce(
+      jsonResponse(
+        { error: 'rate_limit_exceeded' },
+        { status: 429, headers: { 'Retry-After': '7' } }
+      )
+    )
 
     await expect(mintBoxServiceAccountToken(FIELDS)).rejects.toMatchObject({
       code: 'provider_unavailable',
       status: 429,
+      retryAfterMs: 7_000,
     })
   })
 })

apps/sim/lib/credentials/client-credential-accounts/minters/box.ts

@@ -14,6 +14,7 @@ import {
   readProviderErrorSnippet,
   requireClientSecret,
   TokenServiceAccountValidationError,
+  throttledRetryAfterMs,
 } from '@/lib/credentials/token-service-accounts/errors'
 
 const logger = createLogger('BoxServiceAccountMinter')
@@ -176,10 +177,12 @@ export async function mintBoxServiceAccountToken(
         ...(hint ? { hint } : {}),
       })
     }
-    throw new TokenServiceAccountValidationError('provider_unavailable', res.status, {
-      step: 'box_token_mint',
-      body,
-    })
+    throw new TokenServiceAccountValidationError(
+      'provider_unavailable',
+      res.status,
+      { step: 'box_token_mint', body },
+      { retryAfterMs: throttledRetryAfterMs(res) }
+    )
   }
 
   const payload = await parseProviderJson<BoxTokenResponse>(res, 'box_token_mint')

apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts

@@ -84,6 +84,20 @@ describe('CrowdStrike API client token exchange', () => {
     await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({ code })
   })
 
+  it("carries the token endpoint's Retry-After on a throttled answer", async () => {
+    mockFetch.mockResolvedValue(
+      jsonResponse(
+        { errors: [{ code: 429, message: 'rate limited' }] },
+        { status: 429, headers: { 'Retry-After': '7' } }
+      )
+    )
+
+    await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({
+      code: 'provider_unavailable',
+      retryAfterMs: 7_000,
+    })
+  })
+
   it("reports a redirect to the API client's own cloud as a wrong cloud and never follows it", async () => {
     mockFetch.mockResolvedValue(
       new Response(null, {

apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts

@@ -82,7 +82,8 @@ export async function mintCrowdStrikeServiceAccountToken(
       throw new TokenServiceAccountValidationError(
         providerFailureCode(error.status),
         error.status,
-        { step: STEP, cloud }
+        { step: STEP, cloud },
+        { retryAfterMs: error.retryAfterMs }
       )
     }
     throw new TokenServiceAccountValidationError('provider_unavailable', 502, {

apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts

@@ -254,6 +254,17 @@ describe('mintDocuSignServiceAccountToken', () => {
     })
   })
 
+  it("carries the token endpoint's Retry-After on a throttled answer", async () => {
+    mockFetch.mockResolvedValueOnce(
+      jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } })
+    )
+
+    await expect(mintDocuSignServiceAccountToken(FIELDS)).rejects.toMatchObject({
+      code: 'provider_unavailable',
+      retryAfterMs: 7_000,
+    })
+  })
+
   it('keeps the key, assertion, and access token out of every failure', async () => {
     const failures: unknown[] = []
     mockFetch.mockResolvedValueOnce(

apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts

@@ -19,6 +19,7 @@ import {
   parseProviderJson,
   readProviderErrorSnippet,
   TokenServiceAccountValidationError,
+  throttledRetryAfterMs,
 } from '@/lib/credentials/token-service-accounts/errors'
 
 const TOKEN_STEP = 'docusign_token_mint'
@@ -266,11 +267,12 @@ export async function mintDocuSignServiceAccountToken(
         { step: TOKEN_STEP, environment, body, ...(hint ? { hint } : {}) }
       )
     }
-    throw new TokenServiceAccountValidationError('provider_unavailable', tokenRes.status, {
-      step: TOKEN_STEP,
-      environment,
-      body,
-    })
+    throw new TokenServiceAccountValidationError(
+      'provider_unavailable',
+      tokenRes.status,
+      { step: TOKEN_STEP, environment, body },
+      { retryAfterMs: throttledRetryAfterMs(tokenRes) }
+    )
   }
 
   const payload = await parseProviderJson<DocuSignTokenResponse>(tokenRes, TOKEN_STEP)
@@ -297,11 +299,12 @@ export async function mintDocuSignServiceAccountToken(
   )
   if (!userinfoRes.ok) {
     const body = await readProviderErrorSnippet(userinfoRes)
-    throw new TokenServiceAccountValidationError('provider_unavailable', userinfoRes.status, {
-      step: USERINFO_STEP,
-      environment,
-      body,
-    })
+    throw new TokenServiceAccountValidationError(
+      'provider_unavailable',
+      userinfoRes.status,
+      { step: USERINFO_STEP, environment, body },
+      { retryAfterMs: throttledRetryAfterMs(userinfoRes) }
+    )
   }
   const userinfo = await parseProviderJson<DocuSignUserinfoResponse>(userinfoRes, USERINFO_STEP)
   const accounts = (Array.isArray(userinfo.accounts) ? userinfo.accounts : []).filter(

apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.test.ts

@@ -0,0 +1,24 @@
+import { jsonResponse } from '@sim/testing/helpers/http'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { mintHelpScoutServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/helpscout'
+
+const FIELDS = { clientId: 'helpscout-client', clientSecret: 'helpscout-secret', orgId: '' }
+
+const mockFetch = vi.fn<typeof fetch>()
+
+describe('Help Scout client credential exchange', () => {
+  beforeEach(() => {
+    mockFetch.mockReset()
+    vi.stubGlobal('fetch', mockFetch)
+  })
+
+  it("carries the token endpoint's Retry-After on a throttled answer", async () => {
+    mockFetch.mockResolvedValueOnce(
+      jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } })
+    )
+
+    await expect(
+      mintHelpScoutServiceAccountToken(FIELDS, { skipIdentity: true })
+    ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, retryAfterMs: 7_000 })
+  })
+})

apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts

@@ -11,6 +11,7 @@ import {
   isTransientProviderStatus,
   requireClientSecret,
   TokenServiceAccountValidationError,
+  throttledRetryAfterMs,
 } from '@/lib/credentials/token-service-accounts/errors'
 
 const MAX_HELPSCOUT_AUTH_RESPONSE_BYTES = 64 * 1024
@@ -28,7 +29,8 @@ async function readHelpScoutAuthResponse(
         ? 'invalid_credentials'
         : 'provider_unavailable',
       response.status,
-      { step }
+      { step },
+      { retryAfterMs: throttledRetryAfterMs(response) }
     )
   }
 

apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts

@@ -30,4 +30,18 @@ describe('Jama credential validation', () => {
       ).rejects.toMatchObject({ code: expectedCode, status })
     }
   )
+
+  it("carries the token endpoint's Retry-After on a throttled answer", async () => {
+    inputValidationMockFns.mockSecureFetchWithValidation.mockResolvedValueOnce(
+      Response.json({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } })
+    )
+
+    await expect(
+      mintJamaServiceAccountToken({
+        orgId: 'https://example.jamacloud.com',
+        clientId: 'client-id',
+        clientSecret: 'client-secret',
+      })
+    ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, retryAfterMs: 7_000 })
+  })
 })

apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts

@@ -10,6 +10,7 @@ import {
   isTransientProviderStatus,
   requireClientSecret,
   TokenServiceAccountValidationError,
+  throttledRetryAfterMs,
 } from '@/lib/credentials/token-service-accounts/errors'
 import { normalizeJamaInstanceUrl } from '@/lib/jama/client'
 
@@ -49,7 +50,9 @@ export async function mintJamaServiceAccountToken(
             !isTransientProviderStatus(response.status)
             ? 'invalid_credentials'
             : 'provider_unavailable',
-          response.status
+          response.status,
+          undefined,
+          { retryAfterMs: throttledRetryAfterMs(response) }
         )
       }
       return toRecord(await response.json())