fix(credentials): replay remembered refusals as credential errors, derive the delegable Google services list, and correct a fallback TSDoc (#9993)

Author: waleedlatif1

apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/connect-service-account-modal.tsx

@@ -31,6 +31,7 @@ import {
   getDatabaseConnectionDescriptor,
 } from '@/lib/credentials/database-connections/descriptors'
 import {
+  GOOGLE_DELEGATION_SERVICE_NAMES,
   googleDelegationServiceFor,
   googleKeylessOffer,
   isGoogleKeylessService,
@@ -61,7 +62,6 @@ import { DatabricksAccountModal } from '@/app/workspace/[workspaceId]/integratio
 import {
   buildGoogleKeylessConfig,
   emptyGoogleKeylessInput,
-  GOOGLE_DELEGATION_SERVICE_NAMES,
   GoogleKeylessFields,
   type GoogleKeylessInput,
   isGoogleKeylessInputComplete,

apps/sim/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal/google-keyless-fields.tsx

@@ -56,12 +56,6 @@ const DELEGATION_OPTIONS = GOOGLE_DELEGATION_SERVICES.map((service) => ({
   label: getServiceConfigByServiceId(service)?.name ?? service,
 }))
 
-/** "Gmail, Google Drive, …, and Google Calendar": the services keyless credentials can delegate. */
-export const GOOGLE_DELEGATION_SERVICE_NAMES = new Intl.ListFormat('en', {
-  style: 'long',
-  type: 'conjunction',
-}).format(DELEGATION_OPTIONS.map((option) => option.label))
-
 /** Whether the provider and service account fields have Google's documented format. */
 function isTrustComplete(value: GoogleKeylessInput): boolean {
   return (

apps/sim/executor/utils/model-fallback-request.ts

@@ -29,7 +29,10 @@ interface ModelFallbackRequestInput {
   fallbackSystemPrompt: string
   retry?: BlockRetryAttempt
   resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined
-  /** The block's saved Vertex or Bedrock credential selection, resolved per attempt. */
+  /**
+   * The block's saved Vertex or Bedrock credential selection, resolved once per call inside the
+   * first attempt on the primary's provider and shared by the fallbacks on that provider.
+   */
   credentials?: { inputs: BlockModelCredentialInputs; callerLabel: string }
 }
 

apps/sim/lib/credentials/coordinated-client-token.integration.ts

@@ -391,6 +391,24 @@ describe('coordinated token resolution while the provider fails', () => {
     expect(exchange).toHaveBeenCalledTimes(2)
   })
 
+  it('replays a refusal the shared row cannot store as a credential to fix, never as an outage', async () => {
+    const exchange = vi.fn(async (): Promise<{ accessToken: string; expiresAt: Date }> => {
+      throw new TokenServiceAccountValidationError('consent_required', 400, { step: 'mint' })
+    })
+    await expect(
+      resolveCoordinatedClientToken(input('consent-replayed', exchange))
+    ).rejects.toMatchObject({ code: 'consent_required' })
+
+    await expect(
+      resolveCoordinatedClientToken(input('consent-replayed', exchange))
+    ).rejects.toMatchObject({
+      code: 'invalid_credentials',
+      status: 400,
+      replayForMs: expect.any(Number),
+    })
+    expect(exchange).toHaveBeenCalledOnce()
+  })
+
   it('never serves a still-valid token to a different secret whose refresh fails', async () => {
     const exchange = issuingOnce('rotated-refresh', 4 * 60_000, failing(429))
     await resolveCoordinatedClientToken(input('rotated-refresh', exchange))
@@ -433,7 +451,11 @@ describe('coordinated token resolution while the provider fails', () => {
       ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429 })
       await expect(
         resolveCoordinatedClientToken(input(application, exchange))
-      ).rejects.toMatchObject({ status: 429, replayForMs: expect.any(Number) })
+      ).rejects.toMatchObject({
+        code: 'provider_unavailable',
+        status: 429,
+        replayForMs: expect.any(Number),
+      })
 
       expect(exchange).toHaveBeenCalledOnce()
       const stored = await failureRowOf(application)
@@ -644,7 +666,7 @@ describe('default token rows shared with the previous build during a rolling dep
     const { exchange } = provider(application)
 
     await expect(resolveCoordinatedClientToken(input(application, exchange))).rejects.toMatchObject(
-      { code: 'provider_unavailable', status: 400 }
+      { code: 'invalid_credentials', status: 400 }
     )
     expect(exchange).not.toHaveBeenCalled()
   })

apps/sim/lib/credentials/coordinated-client-token.ts

@@ -231,14 +231,22 @@ function isRememberedRefusal(failure: StoredFailure): boolean {
   )
 }
 
-/** The error a remembered failure is replayed as, with how long it is still replayed. */
+/**
+ * The error a remembered failure is replayed as, with how long it is still replayed. A refusal
+ * stored as `provider_unavailable` because the memo cannot hold its own code is replayed as
+ * `invalid_credentials`, so it still asks for the credential to be fixed rather than waited out.
+ */
 function replayedFailure(
   failure: StoredFailure,
   step: string,
   replayMs: number
 ): TokenServiceAccountValidationError {
+  const code =
+    failure.code === 'provider_unavailable' && isRememberedRefusal(failure)
+      ? 'invalid_credentials'
+      : failure.code
   return new TokenServiceAccountValidationError(
-    failure.code,
+    code,
     failure.status,
     { step },
     { replayForMs: Math.max(replayMs, 0) }

apps/sim/lib/credentials/google-workload-identity.server.ts

@@ -5,6 +5,7 @@ import { z } from 'zod'
 import { coalesceLocally } from '@/lib/concurrency/singleflight'
 import { encryptSecret } from '@/lib/core/security/encryption'
 import {
+  GOOGLE_DELEGATION_SERVICE_NAMES,
   GOOGLE_DELEGATION_SERVICES,
   GOOGLE_KEYLESS_SERVICES,
   GOOGLE_WORKLOAD_IDENTITY_SECRET_TYPE,
@@ -964,12 +965,12 @@ export async function resolveGoogleWorkloadIdentityToken(
         ? unsupportedUse(
             'google_keyless_service',
             'this keyless Google credential works only with Google Cloud APIs and the Workspace services it was delegated',
-            'This keyless Google credential works only with Google Cloud tools such as BigQuery and, through domain-wide delegation, with Gmail, Google Drive, Google Docs, Google Sheets, and Google Calendar. Reconnect it to delegate this service, or use a JSON key credential instead'
+            `This keyless Google credential works only with Google Cloud tools such as BigQuery and, through domain-wide delegation, with ${GOOGLE_DELEGATION_SERVICE_NAMES}. Reconnect it to delegate this service, or use a JSON key credential instead`
           )
         : unsupportedUse(
             'google_keyless_service_unsupported',
             'keyless Google credentials cannot serve this service',
-            'Keyless Google credentials work only with Google Cloud tools such as BigQuery and, through domain-wide delegation, with Gmail, Google Drive, Google Docs, Google Sheets, and Google Calendar. Use a JSON key credential for this service'
+            `Keyless Google credentials work only with Google Cloud tools such as BigQuery and, through domain-wide delegation, with ${GOOGLE_DELEGATION_SERVICE_NAMES}. Use a JSON key credential for this service`
           )
     }
   }

apps/sim/lib/credentials/google-workload-identity.ts

@@ -18,7 +18,11 @@
 
 import { impersonateEmailSchema } from '@/lib/api/contracts/oauth-connections'
 import { storedAudienceField } from '@/lib/credentials/workload-identity/audiences'
-import { getProviderIdFromServiceId, getScopesForService } from '@/lib/oauth/utils'
+import {
+  getProviderIdFromServiceId,
+  getScopesForService,
+  getServiceConfigByServiceId,
+} from '@/lib/oauth/utils'
 
 /**
  * The Google services a keyless credential may mint for as the service account
@@ -44,6 +48,17 @@ export const GOOGLE_DELEGATION_SERVICES = [
 
 export type GoogleDelegationService = (typeof GOOGLE_DELEGATION_SERVICES)[number]
 
+/**
+ * "Gmail, Google Drive, …, and Google Calendar": the services a keyless
+ * credential can delegate, by the names the delegation picker shows.
+ */
+export const GOOGLE_DELEGATION_SERVICE_NAMES = new Intl.ListFormat('en', {
+  style: 'long',
+  type: 'conjunction',
+}).format(
+  GOOGLE_DELEGATION_SERVICES.map((service) => getServiceConfigByServiceId(service)?.name ?? service)
+)
+
 const DELEGATION_SERVICE_SET: ReadonlySet<string> = new Set(GOOGLE_DELEGATION_SERVICES)
 
 function isGoogleDelegationService(value: string): value is GoogleDelegationService {