fix(credentials): keep SailPoint throttle delays through unreadable error bodies and replay only coordinated failures locally

- A SailPoint token error whose body cannot be read keeps its status and Retry-After,
  as CrowdStrike's does; caller cancellation still propagates.
- A shared-token (Vanta) mint failure is replayed per instance only when the coordinated
  layer remembers it (replayForMs); a failure it does not remember asks again at once.

Author: waleedlatif1

apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts

@@ -126,6 +126,25 @@ describe('SailPoint API client token exchange', () => {
     expect(mockFetch).toHaveBeenCalledOnce()
   })
 
+  it("keeps a throttled answer's status and Retry-After when its error body cannot be read", async () => {
+    mockFetch.mockResolvedValue(
+      new Response(
+        new ReadableStream({
+          start(controller) {
+            controller.error(new TypeError('terminated'))
+          },
+        }),
+        { status: 429, headers: { 'Retry-After': '120' } }
+      )
+    )
+
+    await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({
+      code: 'provider_unavailable',
+      status: 429,
+      retryAfterMs: 120_000,
+    })
+  })
+
   it('treats a network failure or refused redirect as a provider failure', async () => {
     mockFetch.mockRejectedValue(new TypeError('fetch failed: unexpected redirect'))
 

apps/sim/lib/internal/sailpoint/client.ts

@@ -185,7 +185,7 @@ async function readBoundedBody(
  * endpoint (`client_credentials` grant, credentials in the form body), with
  * redirects refused and up to four backoff retries on 429 (`maxThrottleRetries`).
  * A refusal throws {@link SailPointAuthError} with the provider's status,
- * message, and `Retry-After`.
+ * message, and `Retry-After`, even when its error body cannot be read.
  *
  * @see https://developer.sailpoint.com/docs/api/authentication
  */
@@ -225,11 +225,10 @@ export async function requestSailPointToken(
       continue
     }
 
-    const data = await readBoundedBody(
-      response,
-      response.ok ? MAX_TOKEN_RESPONSE_BYTES : DEFAULT_MAX_ERROR_BODY_BYTES,
-      signal
-    )
+    const data = response.ok
+      ? await readBoundedBody(response, MAX_TOKEN_RESPONSE_BYTES, signal)
+      : await readBoundedBody(response, DEFAULT_MAX_ERROR_BODY_BYTES, signal).catch(() => null)
+    signal?.throwIfAborted()
     if (!response.ok) {
       throw new SailPointAuthError(
         getSailPointErrorMessage(data, 'Failed to authenticate with SailPoint'),

apps/sim/lib/oauth/credential-service-cancellation.test.ts

@@ -446,4 +446,27 @@ describe('client credential token refresh after a failed mint', () => {
       resolveFor('vanta-coordinated-replay', VANTA_SERVICE_ACCOUNT_PROVIDER_ID)
     ).resolves.toEqual({ value: { accessToken: token.accessToken } })
   })
+
+  it('asks the coordinated layer again at once after a failure it does not remember', async () => {
+    encryptionMockFns.mockDecryptSecret.mockResolvedValue({
+      decrypted: JSON.stringify({
+        type: CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE,
+        providerId: VANTA_SERVICE_ACCOUNT_PROVIDER_ID,
+        clientId: 'vanta-client',
+        clientSecret: 'vanta-client-secret',
+      }),
+    })
+    const mint = minter()
+    mint
+      .mockRejectedValueOnce(
+        new TokenServiceAccountValidationError('provider_unavailable', 503, { step: 'mint' })
+      )
+      .mockResolvedValueOnce(token)
+
+    await resolveFor('vanta-coordination-failed', VANTA_SERVICE_ACCOUNT_PROVIDER_ID)
+
+    await expect(
+      resolveFor('vanta-coordination-failed', VANTA_SERVICE_ACCOUNT_PROVIDER_ID)
+    ).resolves.toEqual({ value: { accessToken: token.accessToken } })
+  })
 })

apps/sim/lib/oauth/credential-service.ts

@@ -35,6 +35,7 @@ import {
   isTokenServiceAccountProviderId,
   SNOWFLAKE_SERVICE_ACCOUNT_PROVIDER_ID,
 } from '@/lib/credentials/token-service-accounts/descriptors'
+import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors'
 import {
   parseStoredSecretMode,
   parseTokenServiceAccountSecret,
@@ -741,6 +742,16 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string {
   return encryptedServiceAccountKey.slice(0, 32)
 }
 
+/**
+ * Whether a `sharedToken` mint failed with a failure the coordinated layer remembers for every
+ * worker. Only those are replayed here, for exactly as long as that layer still replays them; a
+ * failure it does not remember (a lock timeout, a database error) may already be over for
+ * another worker, so the next resolution asks again.
+ */
+function isCoordinatedReplay(error: unknown): boolean {
+  return error instanceof TokenServiceAccountValidationError && error.replayForMs !== undefined
+}
+
 /**
  * Resolves a stored secret that mints short-lived tokens, read-through the
  * per-instance cache: re-reads the credential row, serves a cached token minted
@@ -751,8 +762,8 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string {
  * `coalesceLocally` so concurrent block executions on one instance share a
  * single mint. `sharedToken` providers coordinate one token per application
  * themselves (including the in-request retry and the fallback), so their tokens
- * are never cached here and a failure is replayed only as long as their own
- * replay reports; a caller that already read the secret to choose its mode
+ * are never cached here and only a failure their own layer remembers is
+ * replayed, for as long as it reports; a caller that already read the secret to choose its mode
  * passes it as `stored` rather than reading it twice.
  */
 async function resolveMintedServiceAccountToken(
@@ -820,6 +831,10 @@ async function resolveMintedServiceAccountToken(
       return { ...minted.token }
     } catch (error) {
       if (error instanceof Error && error.name === 'AbortError') throw error
+      if (sharedToken && !isCoordinatedReplay(error)) {
+        clientCredentialMintFailureCache.delete(cacheIdentity)
+        throw error
+      }
       const failure = assessTokenRequestFailure(error, failed)
       const retryAtMs = Date.now() + failure.replayMs
       clientCredentialMintFailureCache.set(cacheIdentity, {