fix(credentials): keep the coordinated token's 30-second backoff for every failed exchange

Remembering only credential refusals in the shared row left the per-instance
mint cache and the Vanta chain still backing off on outages, so the policy
differed by layer. Keep the long-standing 30-second backoff everywhere and
document it.

Author: waleedlatif1

apps/sim/lib/credentials/coordinated-client-token.test.ts

@@ -44,27 +44,18 @@ interface StoredApplicationToken {
   secretFingerprint: string
 }
 
-type StoredFailureCode = 'invalid_credentials' | 'grant_not_permitted' | 'site_not_found'
-
 interface StoredFailure {
-  code: StoredFailureCode
+  code: 'invalid_credentials' | 'grant_not_permitted' | 'site_not_found' | 'provider_unavailable'
   status: number
 }
 
-/**
- * Refusals that say the credentials themselves are wrong, so replaying them is safe. A transient
- * failure (`provider_unavailable`: a timeout, 5xx, 429, or malformed reply) is never remembered.
- */
-const STORED_FAILURE_CODES: ReadonlySet<string> = new Set<StoredFailureCode>([
+const STORED_FAILURE_CODES: ReadonlySet<string> = new Set([
   'invalid_credentials',
   'grant_not_permitted',
   'site_not_found',
+  'provider_unavailable',
 ])
 
-function isStoredFailureCode(code: string): code is StoredFailureCode {
-  return STORED_FAILURE_CODES.has(code)
-}
-
 function digest(value: unknown): string {
   return createHash('sha256').update(JSON.stringify(value)).digest('hex')
 }
@@ -141,11 +132,11 @@ function decodeFailure(value: string, step: string): StoredFailure {
   if (
     !isRecordLike(parsed) ||
     typeof parsed.code !== 'string' ||
-    !isStoredFailureCode(parsed.code) ||
+    !STORED_FAILURE_CODES.has(parsed.code) ||
     typeof parsed.status !== 'number'
   )
     throw unavailable(step, 'stored token failure is malformed')
-  return { code: parsed.code, status: parsed.status }
+  return { code: parsed.code as StoredFailure['code'], status: parsed.status }
 }
 
 function remainingSeconds(expiresAt: Date): number {
@@ -163,10 +154,11 @@ function isServable(expiresAt: Date): boolean {
  * to reuse one token rather than accumulate them. Minting happens only under a
  * transaction-scoped advisory lock keyed by the application, so two workers never race two
  * tokens into existence; a token minted with a different secret is never served, and a token the
- * provider rejects is expired through {@link invalidateCoordinatedClientToken}. An exchange the
- * provider refuses for the credentials themselves is remembered for 30 seconds so a hot workflow on
- * a revoked secret does not hammer the token endpoint; a transient failure is not remembered.
- * Tokens are encrypted at rest.
+ * provider rejects is expired through {@link invalidateCoordinatedClientToken}. An exchange that
+ * fails validation, whether refused for the credentials or answered with an outage, is remembered
+ * for 30 seconds, the same backoff the per-instance mint cache applies, so a hot workflow neither
+ * hammers the token endpoint with a revoked secret nor floods a struggling one. Tokens are
+ * encrypted at rest.
  */
 export async function resolveCoordinatedClientToken(
   input: CoordinatedClientTokenInput
@@ -235,12 +227,13 @@ export async function resolveCoordinatedClientToken(
             expiresInSeconds: remainingSeconds(exchanged.expiresAt),
           }
         } catch (error) {
-          if (
-            !(error instanceof TokenServiceAccountValidationError) ||
-            !isStoredFailureCode(error.code)
-          )
-            throw error
-          const failure: StoredFailure = { code: error.code, status: error.status }
+          if (!(error instanceof TokenServiceAccountValidationError)) throw error
+          const failure: StoredFailure = {
+            code: STORED_FAILURE_CODES.has(error.code)
+              ? (error.code as StoredFailure['code'])
+              : 'provider_unavailable',
+            status: error.status,
+          }
           const { encrypted } = await encryptSecret(JSON.stringify(failure))
           const expiresAt = new Date(Date.now() + FAILURE_TTL_MS)
           await tx