fix(credentials): age-gate 403 token eviction and stop memoizing transient mint failures

An internal operation's 403 now drops this instance's cached client-credential
token only once it is older than a minute, matching the external-request path,
so a 403 from a file-access refusal or per-record permission no longer re-mints
on every call. The coordinated token's failure memo keeps only refusals of the
credentials themselves; provider_unavailable propagates with no memo row.

Author: waleedlatif1

apps/sim/lib/credentials/coordinated-client-token.test.ts

@@ -0,0 +1,78 @@
+/**
+ * @vitest-environment node
+ */
+import { clientCredentialToken } from '@sim/db/schema'
+import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing/mocks/database.mock'
+import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+  type CoordinatedClientTokenInput,
+  resolveCoordinatedClientToken,
+} from '@/lib/credentials/coordinated-client-token'
+import {
+  type TokenServiceAccountValidationCode,
+  TokenServiceAccountValidationError,
+} from '@/lib/credentials/token-service-accounts/errors'
+
+vi.mock('@/lib/core/security/encryption', () => encryptionMock)
+
+beforeEach(() => {
+  resetDbChainMock()
+  encryptionMockFns.mockEncryptSecret.mockImplementation(async (secret: string) => ({
+    encrypted: secret,
+  }))
+  encryptionMockFns.mockDecryptSecret.mockImplementation(async (value: string) => ({
+    decrypted: value,
+  }))
+})
+
+function refusing(code: TokenServiceAccountValidationCode, status: number) {
+  return vi.fn<CoordinatedClientTokenInput['exchange']>(async () => {
+    throw new TokenServiceAccountValidationError(code, status, { step: 'mint' })
+  })
+}
+
+/**
+ * Resolves once, then again against the rows the first resolution wrote, so the second call sees
+ * any failure memo exactly as another worker reading the shared table would.
+ */
+async function resolveTwice(exchange: CoordinatedClientTokenInput['exchange']) {
+  const input: CoordinatedClientTokenInput = {
+    namespace: 'fixture',
+    application: ['application'],
+    clientSecret: 'secret',
+    step: 'mint',
+    exchange,
+  }
+  const first = await resolveCoordinatedClientToken(input).catch((error: unknown) => error)
+  const written = dbChainMockFns.values.mock.calls.map(([row]) => row)
+  queueTableRows(clientCredentialToken, [])
+  queueTableRows(clientCredentialToken, written)
+  const second = await resolveCoordinatedClientToken(input).catch((error: unknown) => error)
+  return { first, second }
+}
+
+describe('resolveCoordinatedClientToken failure memo', () => {
+  it('replays a refused exchange instead of calling the provider again', async () => {
+    const exchange = refusing('invalid_credentials', 401)
+
+    const { first, second } = await resolveTwice(exchange)
+
+    expect(first).toMatchObject({ code: 'invalid_credentials', status: 401 })
+    expect(second).toMatchObject({ code: 'invalid_credentials', status: 401 })
+    expect(exchange).toHaveBeenCalledOnce()
+  })
+
+  it.each([
+    ['a timeout or 5xx', 503],
+    ['a 429', 429],
+  ])('asks the provider again after %s instead of replaying it', async (_label, status) => {
+    const exchange = refusing('provider_unavailable', status)
+
+    const { first, second } = await resolveTwice(exchange)
+
+    expect(first).toMatchObject({ code: 'provider_unavailable', status })
+    expect(second).toMatchObject({ code: 'provider_unavailable', status })
+    expect(exchange).toHaveBeenCalledTimes(2)
+  })
+})

apps/sim/lib/credentials/coordinated-client-token.ts

@@ -44,18 +44,27 @@ interface StoredApplicationToken {
   secretFingerprint: string
 }
 
+type StoredFailureCode = 'invalid_credentials' | 'grant_not_permitted' | 'site_not_found'
+
 interface StoredFailure {
-  code: 'invalid_credentials' | 'grant_not_permitted' | 'site_not_found' | 'provider_unavailable'
+  code: StoredFailureCode
   status: number
 }
 
-const STORED_FAILURE_CODES: ReadonlySet<string> = new Set([
+/**
+ * Refusals that say the credentials themselves are wrong, so replaying them is safe. A transient
+ * failure (`provider_unavailable`: a timeout, 5xx, 429, or malformed reply) is never remembered.
+ */
+const STORED_FAILURE_CODES: ReadonlySet<string> = new Set<StoredFailureCode>([
   'invalid_credentials',
   'grant_not_permitted',
   'site_not_found',
-  'provider_unavailable',
 ])
 
+function isStoredFailureCode(code: string): code is StoredFailureCode {
+  return STORED_FAILURE_CODES.has(code)
+}
+
 function digest(value: unknown): string {
   return createHash('sha256').update(JSON.stringify(value)).digest('hex')
 }
@@ -132,11 +141,11 @@ function decodeFailure(value: string, step: string): StoredFailure {
   if (
     !isRecordLike(parsed) ||
     typeof parsed.code !== 'string' ||
-    !STORED_FAILURE_CODES.has(parsed.code) ||
+    !isStoredFailureCode(parsed.code) ||
     typeof parsed.status !== 'number'
   )
     throw unavailable(step, 'stored token failure is malformed')
-  return { code: parsed.code as StoredFailure['code'], status: parsed.status }
+  return { code: parsed.code, status: parsed.status }
 }
 
 function remainingSeconds(expiresAt: Date): number {
@@ -154,9 +163,10 @@ function isServable(expiresAt: Date): boolean {
  * to reuse one token rather than accumulate them. Minting happens only under a
  * transaction-scoped advisory lock keyed by the application, so two workers never race two
  * tokens into existence; a token minted with a different secret is never served, and a token the
- * provider rejects is expired through {@link invalidateCoordinatedClientToken}. A rejected
- * exchange is remembered for 30 seconds so a hot workflow on a revoked secret does not hammer the
- * token endpoint. Tokens are encrypted at rest.
+ * provider rejects is expired through {@link invalidateCoordinatedClientToken}. An exchange the
+ * provider refuses for the credentials themselves is remembered for 30 seconds so a hot workflow on
+ * a revoked secret does not hammer the token endpoint; a transient failure is not remembered.
+ * Tokens are encrypted at rest.
  */
 export async function resolveCoordinatedClientToken(
   input: CoordinatedClientTokenInput
@@ -225,13 +235,12 @@ export async function resolveCoordinatedClientToken(
             expiresInSeconds: remainingSeconds(exchanged.expiresAt),
           }
         } catch (error) {
-          if (!(error instanceof TokenServiceAccountValidationError)) throw error
-          const failure: StoredFailure = {
-            code: STORED_FAILURE_CODES.has(error.code)
-              ? (error.code as StoredFailure['code'])
-              : 'provider_unavailable',
-            status: error.status,
-          }
+          if (
+            !(error instanceof TokenServiceAccountValidationError) ||
+            !isStoredFailureCode(error.code)
+          )
+            throw error
+          const failure: StoredFailure = { code: error.code, status: error.status }
           const { encrypted } = await encryptSecret(JSON.stringify(failure))
           const expiresAt = new Date(Date.now() + FAILURE_TTL_MS)
           await tx

apps/sim/lib/oauth/credential-service-cancellation.test.ts

@@ -125,4 +125,30 @@ describe('client credential token cache', () => {
     await expect(resolve()).resolves.toEqual({ accessToken: 'replacement-token' })
     expect(mint).toHaveBeenCalledTimes(2)
   })
+  it('keeps a token minted within the minimum age, and drops it once it is older', async () => {
+    vi.useFakeTimers({ toFake: ['Date'] })
+    try {
+      const mint = vi.fn<clientAccounts.ClientCredentialAccountMinter>()
+      mint
+        .mockResolvedValueOnce({ ...token, accessToken: 'refused-token' })
+        .mockResolvedValueOnce({ ...token, accessToken: 'replacement-token' })
+      vi.spyOn(clientAccounts, 'getClientCredentialAccountMinter').mockReturnValue(mint)
+      const resolve = () => {
+        queueTableRows(credential, [storedCredential])
+        return resolveServiceAccountToken('ramp-refused-aged', RAMP_SERVICE_ACCOUNT_PROVIDER_ID)
+      }
+
+      await expect(resolve()).resolves.toEqual({ accessToken: 'refused-token' })
+      vi.advanceTimersByTime(30_000)
+      forgetRejectedClientCredentialToken('refused-token', { minimumAgeMs: 60_000 })
+      await expect(resolve()).resolves.toEqual({ accessToken: 'refused-token' })
+
+      vi.advanceTimersByTime(31_000)
+      forgetRejectedClientCredentialToken('refused-token', { minimumAgeMs: 60_000 })
+      await expect(resolve()).resolves.toEqual({ accessToken: 'replacement-token' })
+      expect(mint).toHaveBeenCalledTimes(2)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
 })

apps/sim/lib/oauth/credential-service.ts

@@ -624,6 +624,7 @@ export interface ServiceAccountTokenResult {
 interface CachedClientCredentialToken {
   /** The minted token and the destination fields it is bound to, exactly as tools receive them. */
   token: ServiceAccountTokenResult
+  mintedAtMs: number
   expiresAtMs: number
   /**
    * Fingerprint of the encrypted secret the token was minted from (see
@@ -694,11 +695,19 @@ function pruneExpiredClientCredentialCaches(nowMs: number): void {
  * refused it (a revoked API client, or a token minted before the client was
  * granted a scope), so the next resolution mints a replacement instead of
  * serving the refused token until it nears expiry. Matching on the token value
- * leaves alone a replacement another call has already minted.
+ * leaves alone a replacement another call has already minted. With
+ * `minimumAgeMs`, a token minted more recently is kept: a refusal that may come
+ * from the endpoint rather than the token then re-mints at most once per that
+ * interval instead of once per call.
  */
-export function forgetRejectedClientCredentialToken(accessToken: string): void {
+export function forgetRejectedClientCredentialToken(
+  accessToken: string,
+  options: { minimumAgeMs?: number } = {}
+): void {
+  const mintedBeforeMs = Date.now() - (options.minimumAgeMs ?? 0)
   for (const [identity, entry] of clientCredentialTokenCache) {
-    if (entry.token.accessToken === accessToken) clientCredentialTokenCache.delete(identity)
+    if (entry.token.accessToken === accessToken && entry.mintedAtMs <= mintedBeforeMs)
+      clientCredentialTokenCache.delete(identity)
   }
 }
 
@@ -778,6 +787,7 @@ async function resolveMintedServiceAccountToken(
       if (!sharedToken)
         clientCredentialTokenCache.set(cacheIdentity, {
           token: minted.token,
+          mintedAtMs: Date.now(),
           expiresAtMs: Date.now() + minted.expiresInSeconds * 1000,
           secretFingerprint,
         })

apps/sim/tools/index.test.ts

@@ -3823,15 +3823,21 @@ describe('Internal Route Trust', () => {
       )
     }
 
-    it.each([401, 403])(
-      'drops the cached token after a %i so the next call mints one',
-      async (status) => {
-        const result = await runWith('service_account', status)
+    it('drops the cached token after a 401 so the next call mints one', async () => {
+      const result = await runWith('service_account', 401)
 
-        expect(result.success).toBe(false)
-        expect(mockForgetRejectedClientCredentialToken).toHaveBeenCalledWith('minted-token')
-      }
-    )
+      expect(result.success).toBe(false)
+      expect(mockForgetRejectedClientCredentialToken).toHaveBeenCalledWith('minted-token')
+    })
+
+    it('drops the cached token after a 403 only once it is old enough to blame', async () => {
+      const result = await runWith('service_account', 403)
+
+      expect(result.success).toBe(false)
+      expect(mockForgetRejectedClientCredentialToken).toHaveBeenCalledWith('minted-token', {
+        minimumAgeMs: 60_000,
+      })
+    })
 
     it('keeps the token after a call the provider accepts', async () => {
       const result = await runWith('service_account', 200)

apps/sim/tools/index.ts

@@ -2957,7 +2957,11 @@ async function executeDeclaredInternalOperation({
         authorizedServiceAccountCredential &&
         typeof params.accessToken === 'string'
       ) {
-        await forgetRefusedServiceAccountToken(params.accessToken, requestId)
+        await forgetRefusedServiceAccountToken(
+          params.accessToken,
+          requestId,
+          result.status === 403 ? REJECTED_TOKEN_MINIMUM_AGE_MS : undefined
+        )
       }
       if (result instanceof Response && result.status === 401 && refreshCredential) {
         await result.body?.cancel()
@@ -3042,8 +3046,8 @@ async function executeDeclaredInternalOperation({
  */
 const SCOPE_REFUSAL_PATTERN = /scope does not match|insufficient_scope/i
 /**
- * A tool call's 401 can come from the endpoint rather than the token, so a token minted within the
- * last minute is kept: a call that keeps failing re-mints its application's token at most once a
+ * A tool call's 401 (and an internal operation's 403) can come from the endpoint rather than the
+ * token, so a token minted within the last minute is kept: a call that keeps failing re-mints its application's token at most once a
  * minute, instead of once per call.
  */
 const REJECTED_TOKEN_MINIMUM_AGE_MS = 60_000
@@ -3094,15 +3098,19 @@ async function expireRejectedServiceAccountToken(
  * Drops a refused service-account token from this instance's client-credential cache: an internal
  * operation answers 401 or 403, and an external request 401, when the provider refused it. The
  * next call mints a replacement: providers such as CrowdStrike Falcon and SailPoint fix a token's
- * scopes when it is minted, so a scope granted since then only reaches a new token.
+ * scopes when it is minted, so a scope granted since then only reaches a new token. A 403 can also
+ * come from Sim's own file-access refusal or a per-record permission, so the internal-operation
+ * path passes `minimumAgeMs` for it and a token minted within that window is kept.
  */
 async function forgetRefusedServiceAccountToken(
   accessToken: string,
-  requestId: string
+  requestId: string,
+  minimumAgeMs?: number
 ): Promise<void> {
   try {
     const { forgetRejectedClientCredentialToken } = await import('@/lib/oauth/credential-service')
-    forgetRejectedClientCredentialToken(accessToken)
+    if (minimumAgeMs === undefined) forgetRejectedClientCredentialToken(accessToken)
+    else forgetRejectedClientCredentialToken(accessToken, { minimumAgeMs })
   } catch (error) {
     logger.warn('Could not drop a service-account token the provider refused', {
       requestId,