fix(credentials): end a refused token, read legacy refusals as refusals, and keep throttle delays through SailPoint and CrowdStrike

- A refusal of the credentials drops the per-instance cached token and expires the shared
  token minted with that secret, so a later outage cannot serve it again.
- A previous build's provider_unavailable memo with a non-transient 4xx status replays as a
  refusal instead of serving the stored token.
- The SailPoint saved-credential minter returns the first 429 with its Retry-After instead of
  retrying until the exchange timeout swallows it.
- A CrowdStrike error whose body cannot be read keeps its status and Retry-After.
- Vanta logs a malformed or oversized token response apart from an unreadable stream.

Author: waleedlatif1

apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts

@@ -98,6 +98,25 @@ describe('CrowdStrike API client token exchange', () => {
     })
   })
 
+  it("keeps a throttled answer's status and Retry-After when its error body cannot be read", async () => {
+    mockFetch.mockResolvedValue(
+      new Response(
+        new ReadableStream({
+          start(controller) {
+            controller.error(new TypeError('terminated'))
+          },
+        }),
+        { status: 429, headers: { 'Retry-After': '7' } }
+      )
+    )
+
+    await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({
+      code: 'provider_unavailable',
+      status: 429,
+      retryAfterMs: 7_000,
+    })
+  })
+
   it("reports a redirect to the API client's own cloud as a wrong cloud and never follows it", async () => {
     mockFetch.mockResolvedValue(
       new Response(null, {

apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts

@@ -113,6 +113,19 @@ describe('SailPoint API client token exchange', () => {
     })
   })
 
+  it('returns the first throttled answer with its Retry-After instead of retrying it', async () => {
+    mockFetch.mockImplementation(async () =>
+      jsonResponse({ error: 'rate limited' }, { status: 429, headers: { 'Retry-After': '120' } })
+    )
+
+    await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({
+      code: 'provider_unavailable',
+      status: 429,
+      retryAfterMs: 120_000,
+    })
+    expect(mockFetch).toHaveBeenCalledOnce()
+  })
+
   it('treats a network failure or refused redirect as a provider failure', async () => {
     mockFetch.mockRejectedValue(new TypeError('fetch failed: unexpected redirect'))
 

apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts

@@ -18,7 +18,7 @@ import {
 } from '@/lib/internal/sailpoint/client'
 
 const STEP = 'sailpoint_token_mint'
-/** Bounds the whole exchange, including its 429 backoff retries. */
+/** Bounds the whole exchange. */
 const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000
 
 /**
@@ -32,6 +32,8 @@ const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000
  * from `expires_in`; the credential cache re-mints before it lapses.
  *
  * The token response names no identity, so the connect-time check is the mint.
+ * A 429 is returned at once with its `Retry-After` rather than retried here, so
+ * the credential resolver's throttling backoff waits as long as SailPoint asks.
  *
  * @see https://developer.sailpoint.com/docs/api/authentication
  */
@@ -57,7 +59,8 @@ export async function mintSailPointServiceAccountToken(
       tokenUrl,
       fields.clientId,
       clientSecret,
-      options?.signal ? AbortSignal.any([options.signal, timeout]) : timeout
+      options?.signal ? AbortSignal.any([options.signal, timeout]) : timeout,
+      { maxThrottleRetries: 0 }
     )
   } catch (error) {
     options?.signal?.throwIfAborted()

apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts

@@ -1,5 +1,5 @@
 import { isRecordLike } from '@sim/utils/object'
-import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits'
+import { PayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits'
 import { VANTA_PERMISSION_OPTIONS } from '@/lib/credentials/client-credential-accounts/descriptors'
 import {
   invalidateCoordinatedClientToken,
@@ -108,9 +108,13 @@ async function exchangeToken(
       label: 'Vanta authentication response',
       signal,
     })
-  } catch {
+  } catch (error) {
     signal?.throwIfAborted()
-    throw unavailable('provider returned an unreadable token response')
+    throw unavailable(
+      error instanceof SyntaxError || error instanceof PayloadSizeLimitError
+        ? 'provider returned an invalid token response'
+        : 'provider token response could not be read'
+    )
   }
   if (
     !isRecordLike(payload) ||

apps/sim/lib/credentials/coordinated-client-token.integration.ts

@@ -333,6 +333,32 @@ describe('coordinated token resolution while the provider fails', () => {
     expect(exchange).toHaveBeenCalledTimes(2)
   })
 
+  it('never serves a refused token again once the refusal memo ends and an outage follows', async () => {
+    const exchange = issuingOnce('refused-then-outage', 4 * 60_000, rejecting())
+    await resolveCoordinatedClientToken(input('refused-then-outage', exchange))
+    await expect(
+      resolveCoordinatedClientToken(input('refused-then-outage', exchange))
+    ).rejects.toMatchObject({ code: 'invalid_credentials' })
+    await expireFailureRowsOf('refused-then-outage')
+    exchange.mockImplementation(failing(503))
+
+    await expect(
+      resolveCoordinatedClientToken(input('refused-then-outage', exchange))
+    ).rejects.toMatchObject({ code: 'provider_unavailable' })
+  })
+
+  it('keeps the token another secret minted when a different secret is refused', async () => {
+    const exchange = issuingOnce('refused-other-secret', 4 * 60_000, failing(503))
+    await resolveCoordinatedClientToken(input('refused-other-secret', exchange))
+    await expect(
+      resolveCoordinatedClientToken(input('refused-other-secret', rejecting(), 'revoked'))
+    ).rejects.toMatchObject({ code: 'invalid_credentials' })
+
+    await expect(
+      resolveCoordinatedClientToken(input('refused-other-secret', exchange))
+    ).resolves.toMatchObject({ accessToken: 'refused-other-secret:token-1' })
+  })
+
   it('keeps refusing for every worker a refusal whose code the shared row cannot store', async () => {
     const exchange = issuingOnce('consent-refresh', 4 * 60_000, async () => {
       throw new TokenServiceAccountValidationError('consent_required', 400, { step: 'mint' })
@@ -562,16 +588,14 @@ describe('coordinated token resolution while the provider fails', () => {
 })
 
 describe('default token rows shared with the previous build during a rolling deploy', () => {
-  it('serves a still-valid token when the previous build remembered an outage', async () => {
-    const application = 'legacy-outage'
+  /** Seeds a still-valid token and the previous build's failure memo of `status` for one secret. */
+  async function seedPreviousBuildFailure(application: string, status: number) {
     const appKey = applicationKey(NAMESPACE, [application])
     const fingerprint = sha256Json([application, 'secret-1'])
     const token = await encryptSecret(
       JSON.stringify({ accessToken: 'legacy-token', secretFingerprint: fingerprint })
     )
-    const outage = await encryptSecret(
-      JSON.stringify({ code: 'provider_unavailable', status: 503 })
-    )
+    const failure = await encryptSecret(JSON.stringify({ code: 'provider_unavailable', status }))
     await db.insert(clientCredentialToken).values([
       {
         id: appKey,
@@ -581,10 +605,15 @@ describe('default token rows shared with the previous build during a rolling dep
       },
       {
         id: `${appKey}:${sha256Json(fingerprint)}`,
-        encryptedValue: outage.encrypted,
+        encryptedValue: failure.encrypted,
         expiresAt: new Date(Date.now() + 20_000),
       },
     ])
+  }
+
+  it('serves a still-valid token when the previous build remembered an outage', async () => {
+    const application = 'legacy-outage'
+    await seedPreviousBuildFailure(application, 503)
     const { exchange } = provider(application)
 
     await expect(
@@ -593,6 +622,17 @@ describe('default token rows shared with the previous build during a rolling dep
     expect(exchange).not.toHaveBeenCalled()
   })
 
+  it('replays a refusal the previous build stored as provider_unavailable without serving the token', async () => {
+    const application = 'legacy-unmapped-refusal'
+    await seedPreviousBuildFailure(application, 400)
+    const { exchange } = provider(application)
+
+    await expect(resolveCoordinatedClientToken(input(application, exchange))).rejects.toMatchObject(
+      { code: 'provider_unavailable', status: 400 }
+    )
+    expect(exchange).not.toHaveBeenCalled()
+  })
+
   it('stores every outage and throttled answer as a failure the previous build reads', async () => {
     for (const [application, exchange] of [
       ['legacy-readable-outage', failing(502)],

apps/sim/lib/credentials/coordinated-client-token.ts

@@ -13,7 +13,10 @@ import {
   TOKEN_FAILURE_STREAK_RETENTION_MS,
   type TokenRequestFailure,
 } from '@/lib/credentials/token-request-resilience'
-import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors'
+import {
+  providerFailureCode,
+  TokenServiceAccountValidationError,
+} from '@/lib/credentials/token-service-accounts/errors'
 import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks'
 
 const logger = createLogger('CoordinatedClientToken')
@@ -214,9 +217,18 @@ function decodeFailure(value: string, step: string): StoredFailure {
   }
 }
 
-/** Whether a remembered failure refused the credentials, so no stored token is served for it. */
+/**
+ * Whether a remembered failure refused the credentials, so no stored token is served for it. A
+ * build before failure kinds stored a refusal whose code it could not store as
+ * `provider_unavailable` with the provider's status and no streak count, so such a row whose
+ * status names a refusal (a non-transient 4xx) is one too.
+ */
 function isRememberedRefusal(failure: StoredFailure): boolean {
-  return failure.refusal === true || isPermanentTokenFailureCode(failure.code)
+  if (failure.refusal === true || isPermanentTokenFailureCode(failure.code)) return true
+  return (
+    failure.consecutiveFailures === undefined &&
+    isPermanentTokenFailureCode(providerFailureCode(failure.status))
+  )
 }
 
 /** The error a remembered failure is replayed as, with how long it is still replayed. */
@@ -279,7 +291,8 @@ function isServable(expiresAt: Date, minRemainingMs: number): boolean {
  * for a minute growing to five without one, and an outage for 5 seconds growing to 30 while it
  * continues, so a hot workflow neither hammers the token endpoint with a revoked secret nor floods
  * a struggling one. While an outage or throttling is remembered, the stored token for the same
- * secret is still served until shortly before it actually expires; a refusal is reported at once.
+ * secret is still served until shortly before it actually expires; a refusal is reported at once
+ * and expires that secret's stored token, so no later outage serves it again.
  * A remembered failure, and that fallback, are served from committed rows without taking the
  * application lock. Tokens are encrypted at rest.
  */
@@ -335,6 +348,13 @@ export async function resolveCoordinatedClientToken(
   const fallbackFrom = (row: StoredRow | undefined) =>
     serveStored(row, fallbackRemainingMs).catch(() => undefined)
 
+  /**
+   * Whether `row` holds a token minted with this secret, expired or not, so a refusal of the
+   * secret ends that token too and a token another secret minted stays in service.
+   */
+  const holdsTokenForSecret = async (row: StoredRow | undefined) =>
+    (await serveStored(row, Number.NEGATIVE_INFINITY).catch(() => undefined)) !== undefined
+
   /**
    * Replays the application's unexpired failure memo for this secret: the still-valid token for a
    * remembered outage or throttling, the failure otherwise. `undefined` when nothing is remembered.
@@ -453,16 +473,23 @@ export async function resolveCoordinatedClientToken(
             target: clientCredentialToken.id,
             set: { encryptedValue: encrypted, expiresAt, updatedAt: new Date() },
           })
-        if (assessed.kind !== 'permanent') {
-          const fallback = await fallbackFrom(applicationRow)
-          if (fallback) {
-            logger.warn('Serving a still-valid client-credentials token after its refresh failed', {
-              namespace,
-              failure: assessed.kind,
-              retryInMs: replayMs,
-            })
-            return fallback
+        if (assessed.kind === 'permanent') {
+          if (await holdsTokenForSecret(applicationRow)) {
+            await tx
+              .update(clientCredentialToken)
+              .set({ expiresAt: new Date(0), updatedAt: new Date() })
+              .where(eq(clientCredentialToken.id, applicationKey))
           }
+          return { failure: replayedFailure(failure, step, replayMs) }
+        }
+        const fallback = await fallbackFrom(applicationRow)
+        if (fallback) {
+          logger.warn('Serving a still-valid client-credentials token after its refresh failed', {
+            namespace,
+            failure: assessed.kind,
+            retryInMs: replayMs,
+          })
+          return fallback
         }
         return { failure: replayedFailure(failure, step, replayMs) }
       }

apps/sim/lib/internal/crowdstrike/client.ts

@@ -226,7 +226,8 @@ export class CrowdStrikeTokenRedirectError extends Error {
  * With `followRedirects: false` a redirect is never followed and throws
  * {@link CrowdStrikeTokenRedirectError}, so the secret reaches no other host;
  * the legacy block's inline path keeps following one as it always has. A
- * refusal throws {@link CrowdStrikeAuthError} with Falcon's status.
+ * refusal throws {@link CrowdStrikeAuthError} with Falcon's status and
+ * `Retry-After`, even when its error body cannot be read.
  *
  * @see https://developer.crowdstrike.com/api-reference/collections/oauth2.md
  */
@@ -264,7 +265,9 @@ export async function requestCrowdStrikeToken(
     )
   }
 
-  const data = await readFalconJson(response, signal)
+  const data = response.ok
+    ? await readFalconJson(response, signal)
+    : await readFalconJson(response, signal).catch(() => null)
   signal?.throwIfAborted()
   if (!response.ok) {
     throw new CrowdStrikeAuthError(

apps/sim/lib/internal/sailpoint/client.ts

@@ -183,16 +183,18 @@ async function readBoundedBody(
 /**
  * Exchanges API client credentials for an access token at a tenant's token
  * endpoint (`client_credentials` grant, credentials in the form body), with
- * redirects refused and up to four backoff retries on 429. A refusal throws
- * {@link SailPointAuthError} with the provider's status and message.
+ * redirects refused and up to four backoff retries on 429 (`maxThrottleRetries`).
+ * A refusal throws {@link SailPointAuthError} with the provider's status,
+ * message, and `Retry-After`.
  *
  * @see https://developer.sailpoint.com/docs/api/authentication
  */
 export async function requestSailPointToken(
   tokenUrl: string,
   clientId: string,
   clientSecret: string,
-  signal: AbortSignal
+  signal: AbortSignal,
+  { maxThrottleRetries = MAX_FETCH_RETRIES }: { maxThrottleRetries?: number } = {}
 ): Promise<{ token: string; expiresInSeconds: number }> {
   let attempt = 0
 
@@ -214,7 +216,7 @@ export async function requestSailPointToken(
       signal,
     })
 
-    if (response.status === 429 && attempt < MAX_FETCH_RETRIES) {
+    if (response.status === 429 && attempt < maxThrottleRetries) {
       const retryAfterMs = parseRetryAfter(response.headers.get('retry-after'))
       await consumeOrCancelBody(response, DEFAULT_MAX_ERROR_BODY_BYTES)
       attempt += 1

apps/sim/lib/oauth/credential-service-cancellation.test.ts

@@ -257,6 +257,20 @@ describe('client credential token refresh after a failed mint', () => {
     expect(mint).toHaveBeenCalledTimes(2)
   })
 
+  it('never serves a refused token again when a later refresh fails with an outage', async () => {
+    const mint = minter()
+    mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 })
+    await resolveFor('ramp-refused-then-outage')
+    mint.mockRejectedValueOnce(refused()).mockRejectedValue(unavailable())
+    vi.advanceTimersByTime(6 * 60_000)
+    await resolveFor('ramp-refused-then-outage')
+    vi.advanceTimersByTime(31_000)
+
+    await expect(resolveFor('ramp-refused-then-outage')).resolves.toEqual({
+      error: expect.objectContaining({ code: 'provider_unavailable' }),
+    })
+  })
+
   it('retries a quick transient failure once inside the mint', async () => {
     const mint = minter()
     mint.mockRejectedValueOnce(unavailable(503)).mockResolvedValueOnce(token)

apps/sim/lib/oauth/credential-service.ts

@@ -746,7 +746,8 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string {
  * per-instance cache: re-reads the credential row, serves a cached token minted
  * from the same ciphertext while enough of it remains, replays a recent mint
  * failure, and otherwise decrypts and mints, falling back to the cached token
- * while it is still valid when the refresh fails with an outage. Wrapped in
+ * while it is still valid when the refresh fails with an outage and dropping it
+ * when the refresh is refused, so no later outage serves it again. Wrapped in
  * `coalesceLocally` so concurrent block executions on one instance share a
  * single mint. `sharedToken` providers coordinate one token per application
  * themselves (including the in-request retry and the fallback), so their tokens
@@ -829,7 +830,11 @@ async function resolveMintedServiceAccountToken(
         retryAtMs,
         expiresAtMs: retryAtMs + TOKEN_FAILURE_STREAK_RETENTION_MS,
       })
-      const fallback = failure.kind === 'permanent' ? undefined : stillValid()
+      if (failure.kind === 'permanent') {
+        clientCredentialTokenCache.delete(cacheIdentity)
+        throw error
+      }
+      const fallback = stillValid()
       if (fallback) {
         logger.warn('Serving a still-valid client-credential token after its refresh failed', {
           ...(options?.privacyMode === 'selector' ? {} : { credentialId }),