From b18d662f2eaf13776573d1b6f704420d878767cf Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 19:17:13 -0800 Subject: [PATCH 1/9] feat(credentials): carry a provider's Retry-After on token request failures A 429 or 503 token answer now records its Retry-After (delay-seconds or HTTP-date, RFC 9110 section 10.2.3) as an optional retryAfterMs on TokenServiceAccountValidationError, through throwForProviderResponse and the shared client-credentials grant, whose provider classifier receives it. --- .../client-credentials-grant.test.ts | 30 ++++++++++++- .../credentials/client-credentials-grant.ts | 17 +++++-- .../token-service-accounts/errors.test.ts | 44 +++++++++++++++++++ .../token-service-accounts/errors.ts | 40 ++++++++++++++--- .../token-service-accounts/grants/shopify.ts | 13 ++++-- 5 files changed, 129 insertions(+), 15 deletions(-) diff --git a/apps/sim/lib/credentials/client-credentials-grant.test.ts b/apps/sim/lib/credentials/client-credentials-grant.test.ts index 9cbaf6432c1..53c099ea3a0 100644 --- a/apps/sim/lib/credentials/client-credentials-grant.test.ts +++ b/apps/sim/lib/credentials/client-credentials-grant.test.ts @@ -140,7 +140,11 @@ describe('exchangeClientCredentialsGrant', () => { await expect( exchangeClientCredentialsGrant({ ...REQUEST, classifyRefusal }) ).rejects.toMatchObject({ code: 'grant_not_permitted', status: 400 }) - expect(classifyRefusal).toHaveBeenCalledWith(400, expect.stringContaining('org_mismatch')) + expect(classifyRefusal).toHaveBeenCalledWith( + 400, + expect.stringContaining('org_mismatch'), + undefined + ) }) it.each([ @@ -158,6 +162,30 @@ describe('exchangeClientCredentialsGrant', () => { }) }) + it('carries the Retry-After of a throttled exchange, also to a provider classification', async () => { + mockFetch.mockImplementation(async () => + jsonResponse({ error: 'slow_down' }, { status: 429, headers: { 'Retry-After': '45' } }) + ) + + await expect(exchangeClientCredentialsGrant(REQUEST)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + retryAfterMs: 45_000, + }) + const classifyRefusal = vi.fn( + (status: number, _body: string, retryAfterMs?: number) => + new TokenServiceAccountValidationError( + 'provider_unavailable', + status, + { step: REQUEST.step }, + { retryAfterMs } + ) + ) + await expect( + exchangeClientCredentialsGrant({ ...REQUEST, classifyRefusal }) + ).rejects.toMatchObject({ retryAfterMs: 45_000 }) + }) + it('refuses a token response that is not JSON', async () => { mockFetch.mockResolvedValue(new Response('gateway', { status: 200 })) diff --git a/apps/sim/lib/credentials/client-credentials-grant.ts b/apps/sim/lib/credentials/client-credentials-grant.ts index 93522121d78..e97e932e9ff 100644 --- a/apps/sim/lib/credentials/client-credentials-grant.ts +++ b/apps/sim/lib/credentials/client-credentials-grant.ts @@ -5,6 +5,7 @@ import { fetchProvider, providerFailureCode, providerFailureReason, + providerRetryAfterMs, readProviderErrorSnippet, TokenServiceAccountValidationError, } from '@/lib/credentials/token-service-accounts/errors' @@ -34,8 +35,14 @@ export interface ClientCredentialsGrantRequest { /** * Maps a refused exchange to a provider-specific error. Without it, a refusal is * `invalid_credentials` for a non-transient 4xx and `provider_unavailable` otherwise. + * `retryAfterMs` is the refusal's `Retry-After` (see {@link providerRetryAfterMs}), which an + * error the provider would answer differently later should carry. */ - classifyRefusal?: (status: number, body: string) => TokenServiceAccountValidationError + classifyRefusal?: ( + status: number, + body: string, + retryAfterMs: number | undefined + ) => TokenServiceAccountValidationError signal?: AbortSignal } @@ -44,7 +51,7 @@ export interface ClientCredentialsGrantRequest { * with the client ID and secret in the body, redirects refused so the secret reaches no other host. * A response without a non-blank `access_token` and a positive integer `expires_in` is * `provider_unavailable`, and the token expires `min(expires_in, maxLifetimeSeconds)` after the - * request started. + * request started. A throttled or unavailable refusal carries its `Retry-After` as `retryAfterMs`. * * @see https://datatracker.ietf.org/doc/html/rfc6749#section-4.4 */ @@ -79,12 +86,14 @@ export async function exchangeClientCredentialsGrant( if (!response.ok) { const body = await readProviderErrorSnippet(response) signal?.throwIfAborted() + const retryAfterMs = providerRetryAfterMs(response) throw request.classifyRefusal - ? request.classifyRefusal(response.status, body) + ? request.classifyRefusal(response.status, body, retryAfterMs) : new TokenServiceAccountValidationError( providerFailureCode(response.status), response.status, - { step, body } + { step, body }, + { retryAfterMs } ) } let payload: unknown diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts index f12b7bae6e6..5c5f9dc205d 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts @@ -133,6 +133,50 @@ describe('token service-account error helpers', () => { expect(error.code).toBe('provider_unavailable') }) + + it.each([ + [429, '120', 120_000], + [503, '7', 7_000], + ])( + 'carries the Retry-After a %i asked for in seconds', + async (status, retryAfter, retryAfterMs) => { + const res = new Response('slow down', { status, headers: { 'Retry-After': retryAfter } }) + + const error = await expectValidationError(throwForProviderResponse(res, 'self')) + + expect(error.code).toBe('provider_unavailable') + expect(error.retryAfterMs).toBe(retryAfterMs) + } + ) + + it('carries the Retry-After a 429 asked for as an HTTP date', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + try { + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')) + const res = new Response('slow down', { + status: 429, + headers: { 'Retry-After': 'Thu, 01 Jan 2026 00:01:30 GMT' }, + }) + + const error = await expectValidationError(throwForProviderResponse(res, 'self')) + + expect(error.retryAfterMs).toBe(90_000) + } finally { + vi.useRealTimers() + } + }) + + it.each([ + [500, '30'], + [401, '30'], + [429, 'soon'], + ])('carries no Retry-After for a %i with Retry-After %s', async (status, retryAfter) => { + const res = new Response('nope', { status, headers: { 'Retry-After': retryAfter } }) + + const error = await expectValidationError(throwForProviderResponse(res, 'self')) + + expect(error.retryAfterMs).toBeUndefined() + }) }) describe('readProviderErrorSnippet', () => { diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.ts b/apps/sim/lib/credentials/token-service-accounts/errors.ts index 0dff3b071b1..1a0a03e31d8 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '@sim/utils/errors' +import { parseRetryAfter } from '@sim/utils/retry' import { truncate } from '@sim/utils/string' import { DEFAULT_MAX_ERROR_BODY_BYTES, @@ -33,14 +34,28 @@ export type TokenServiceAccountValidationCode = */ | 'grant_not_permitted' +export interface TokenServiceAccountValidationErrorOptions { + /** See {@link TokenServiceAccountValidationError.retryAfterMs}. */ + retryAfterMs?: number +} + export class TokenServiceAccountValidationError extends Error { + /** + * The earliest a repeat of the failed request can be expected to succeed, in milliseconds: the + * provider's `Retry-After` on a throttled or unavailable answer (uncapped, as the provider sent + * it), or how long a remembered failure is still replayed. Absent when nothing is known. + */ + public readonly retryAfterMs?: number + constructor( public readonly code: TokenServiceAccountValidationCode, public readonly status: number, - public readonly logDetail?: Record + public readonly logDetail?: Record, + options: TokenServiceAccountValidationErrorOptions = {} ) { super(code) this.name = 'TokenServiceAccountValidationError' + if (options.retryAfterMs !== undefined) this.retryAfterMs = options.retryAfterMs } } @@ -127,6 +142,16 @@ export function providerFailureCode( : 'provider_unavailable' } +/** + * The `Retry-After` of a 429 (RFC 6585) or 503 (RFC 9110 section 15.6.4) answer in milliseconds, + * as delay-seconds or an HTTP-date (RFC 9110 section 10.2.3); `undefined` for any other status or + * a missing or unparsable header. Uncapped: the caller that waits decides how long is too long. + */ +export function providerRetryAfterMs(response: ProviderResponse): number | undefined { + if (response.status !== 429 && response.status !== 503) return undefined + return parseRetryAfter(response.headers.get('retry-after'), Number.POSITIVE_INFINITY) ?? undefined +} + export interface FetchProviderOptions { /** * Validation code thrown when the host does not resolve (`ENOTFOUND` only — @@ -254,7 +279,7 @@ export async function readProviderErrorSnippet(res: ProviderResponse): Promise { if (res.ok) return const body = await readProviderErrorSnippet(res) - throw new TokenServiceAccountValidationError(providerFailureCode(res.status), res.status, { - step, - body, - ...context, - }) + throw new TokenServiceAccountValidationError( + providerFailureCode(res.status), + res.status, + { step, body, ...context }, + { retryAfterMs: providerRetryAfterMs(res) } + ) } diff --git a/apps/sim/lib/credentials/token-service-accounts/grants/shopify.ts b/apps/sim/lib/credentials/token-service-accounts/grants/shopify.ts index 440cdb0e4da..3672a6878b8 100644 --- a/apps/sim/lib/credentials/token-service-accounts/grants/shopify.ts +++ b/apps/sim/lib/credentials/token-service-accounts/grants/shopify.ts @@ -21,7 +21,12 @@ const MAX_TOKEN_LIFETIME_SECONDS = 86_400 * Shopify answers `shop_not_permitted` when the app and store are in different organizations, * which no secret fixes, and 404 for a store domain that does not exist. */ -function classifyShopifyRefusal(shop: string, status: number, body: string) { +function classifyShopifyRefusal( + shop: string, + status: number, + body: string, + retryAfterMs: number | undefined +) { if (/shop_not_permitted/i.test(body)) { return new TokenServiceAccountValidationError('grant_not_permitted', status, { step: STEP, @@ -32,7 +37,8 @@ function classifyShopifyRefusal(shop: string, status: number, body: string) { return new TokenServiceAccountValidationError( status === 404 ? 'site_not_found' : providerFailureCode(status), status, - { step: STEP, domain: shop, reason: 'provider rejected token exchange' } + { step: STEP, domain: shop, reason: 'provider rejected token exchange' }, + { retryAfterMs } ) } @@ -82,7 +88,8 @@ export async function mintShopifyClientCredentialsToken( dnsFailureCode: 'site_not_found', dnsFailureReason: 'store domain does not resolve', }, - classifyRefusal: (status, body) => classifyShopifyRefusal(shop, status, body), + classifyRefusal: (status, body, retryAfterMs) => + classifyShopifyRefusal(shop, status, body, retryAfterMs), signal, }), signal: options?.signal, From 1ecf770eea5d32ba9271a70d5c9dffc1c704002c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 19:20:43 -0800 Subject: [PATCH 2/9] feat(credentials): keep valid client-credential tokens through refresh outages The per-instance client-credential mint cache now: - serves a cached token that is inside its five-minute early-refresh window but not expired when the refresh fails with an outage or throttling, as MSAL, Azure.Core and botocore do; a refusal of the credentials still fails - retries a quick transient mint failure once after about a second - replays failures by kind instead of a flat 30 s: a refusal 30 s, a throttled answer for its Retry-After (1 s to 5 min), an outage 5 s growing to 30 s while it continues The policy lives in lib/credentials/token-request-resilience.ts so the coordinated resolver can share it. --- .../credentials/token-request-resilience.ts | 124 +++++++++++++ .../credential-service-cancellation.test.ts | 173 +++++++++++++++++- apps/sim/lib/oauth/credential-service.ts | 95 +++++++--- 3 files changed, 362 insertions(+), 30 deletions(-) create mode 100644 apps/sim/lib/credentials/token-request-resilience.ts diff --git a/apps/sim/lib/credentials/token-request-resilience.ts b/apps/sim/lib/credentials/token-request-resilience.ts new file mode 100644 index 00000000000..ad8b58a8353 --- /dev/null +++ b/apps/sim/lib/credentials/token-request-resilience.ts @@ -0,0 +1,124 @@ +import { interruptibleSleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' +import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' + +/** + * How a failed token request is retried: + * - `permanent`: the provider refused the credentials or the account (or the failure is not a + * provider answer at all); asking again cannot help until the secret changes. + * - `throttled`: a 429, or an outage with a `Retry-After`; the provider said when to come back. + * - `transient`: any other outage (5xx, timeout, network failure, malformed token response). + */ +export type TokenRequestFailureKind = 'permanent' | 'throttled' | 'transient' + +export interface TokenRequestFailure { + kind: TokenRequestFailureKind + /** Non-permanent failures in a row, this one included; 1 for a permanent failure. */ + consecutiveFailures: number + /** How long the failure is replayed before the provider is asked again. */ + replayMs: number +} + +/** + * A token with less than this left is treated as expired and never served, so clock skew and a + * call already in flight cannot carry it past its expiry. + */ +export const TOKEN_EXPIRY_MARGIN_MS = 30_000 + +/** + * How long a remembered failure outlives its replay, so the next failure continues the streak + * and backs off longer. A failure after a quieter gap starts a new streak. + */ +export const TOKEN_FAILURE_STREAK_RETENTION_MS = 5 * 60_000 + +/** A refusal of the credentials is replayed this long, so a revoked secret does not hammer the provider. */ +const PERMANENT_FAILURE_REPLAY_MS = 30_000 +/** + * A provider's `Retry-After` is honored for at most five minutes. Microsoft Entra ID, the most + * explicit documented case, throttles client-credentials token requests with `Retry-After: 60`; + * five minutes leaves room for per-minute provider windows while a proxy or a misbehaving provider + * answering with hours cannot lock every workflow on an application out for that long. + */ +const RETRY_AFTER_BOUNDS = { baseMs: 1_000, maxMs: 5 * 60_000 } as const +/** + * An outage is retried after 5 s at first (Entra's resilience guidance: the first back-off retry at + * least 5 s after the response), doubling while it continues, up to 30 s: the flat replay every + * failure used to get, so no outage is replayed longer than before. + */ +const OUTAGE_BACKOFF = { baseMs: 5_000, maxMs: 30_000 } as const +/** The pause before the one in-request retry: MSAL's documented one-second retry-once. */ +const IN_REQUEST_RETRY_DELAY = { baseMs: 1_000, maxMs: 1_000 } as const +/** + * A failed attempt is retried only when it failed this quickly. A timed-out attempt (10 s) is + * never retried in-request, and a retried exchange holding a coordinated application lock stays + * within the 15 s its waiters allow: 2 s, the 1.2 s pause, and one more 10 s attempt. + */ +const IN_REQUEST_RETRY_BUDGET_MS = 2_000 + +/** Classifies a failed token request; see {@link TokenRequestFailureKind}. */ +export function classifyTokenRequestFailure(error: unknown): TokenRequestFailureKind { + if (!(error instanceof TokenServiceAccountValidationError)) return 'permanent' + if (error.code !== 'provider_unavailable') return 'permanent' + return error.status === 429 || error.retryAfterMs !== undefined ? 'throttled' : 'transient' +} + +/** + * Decides how long a failed token request is replayed. A failure carrying `retryAfterMs` (the + * provider's `Retry-After`, or what remains of a replay another layer already decided) is replayed + * that long within 1 s to 5 min. Otherwise a permanent failure is replayed 30 s and an outage backs + * off from 5 s to 30 s with jitter as it continues. `previousConsecutiveFailures` counts the + * non-permanent failures immediately before this one (0 after a success or a permanent failure). + */ +export function assessTokenRequestFailure( + error: unknown, + previousConsecutiveFailures = 0 +): TokenRequestFailure { + const kind = classifyTokenRequestFailure(error) + const consecutiveFailures = kind === 'permanent' ? 1 : previousConsecutiveFailures + 1 + const retryAfterMs = + error instanceof TokenServiceAccountValidationError ? error.retryAfterMs : undefined + if (retryAfterMs !== undefined) { + return { + kind, + consecutiveFailures, + replayMs: backoffWithJitter(1, retryAfterMs, RETRY_AFTER_BOUNDS), + } + } + if (kind === 'permanent') { + return { kind, consecutiveFailures, replayMs: PERMANENT_FAILURE_REPLAY_MS } + } + return { + kind, + consecutiveFailures, + replayMs: Math.min( + backoffWithJitter(consecutiveFailures, null, OUTAGE_BACKOFF), + OUTAGE_BACKOFF.maxMs + ), + } +} + +/** + * Runs a token request, retrying a transient failure once after about a second when the failed + * attempt was quick (see {@link IN_REQUEST_RETRY_BUDGET_MS}). Throttled and permanent failures, + * and an aborted `signal`, are never retried. + */ +export async function requestTokenWithRetry( + request: () => Promise, + signal?: AbortSignal +): Promise { + const startedAt = Date.now() + try { + return await request() + } catch (error) { + signal?.throwIfAborted() + if ( + classifyTokenRequestFailure(error) !== 'transient' || + Date.now() - startedAt > IN_REQUEST_RETRY_BUDGET_MS + ) { + throw error + } + await interruptibleSleep(backoffWithJitter(1, null, IN_REQUEST_RETRY_DELAY), signal) + signal?.throwIfAborted() + return request() + } +} diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index a3ee4f787d7..abb4cd33bf0 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -2,12 +2,13 @@ import { credential } from '@sim/db/schema' import { createDeferred } from '@sim/testing/helpers/deferred' import { queueTableRows, resetDbChainMock } from '@sim/testing/mocks/database.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, RAMP_SERVICE_ACCOUNT_PROVIDER_ID, } from '@/lib/credentials/client-credential-accounts/descriptors' import * as clientAccounts from '@/lib/credentials/client-credential-accounts/server' +import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { forgetRejectedClientCredentialToken, resolveServiceAccountToken, @@ -152,3 +153,173 @@ describe('client credential token cache', () => { } }) }) + +describe('client credential token refresh after a failed mint', () => { + const refused = () => + new TokenServiceAccountValidationError('invalid_credentials', 401, { step: 'mint' }) + const unavailable = (status = 502, retryAfterMs?: number) => + new TokenServiceAccountValidationError( + 'provider_unavailable', + status, + { step: 'mint' }, + { + retryAfterMs, + } + ) + + /** Resolves with fake timers running, so an in-mint retry pause elapses without real waiting. */ + async function resolveFor(credentialId: string) { + queueTableRows(credential, [storedCredential]) + const outcome = resolveServiceAccountToken(credentialId, RAMP_SERVICE_ACCOUNT_PROVIDER_ID).then( + (value) => ({ value }), + (error: unknown) => ({ error }) + ) + await vi.runAllTimersAsync() + return outcome + } + + function minter() { + const mint = vi.fn() + vi.spyOn(clientAccounts, 'getClientCredentialAccountMinter').mockReturnValue(mint) + return mint + } + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('keeps serving a still-valid token while its early refresh fails transiently', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await expect(resolveFor('ramp-refresh-outage')).resolves.toEqual({ + value: { accessToken: 'current-token' }, + }) + mint.mockRejectedValue(unavailable()) + vi.advanceTimersByTime(6 * 60_000) + + await expect(resolveFor('ramp-refresh-outage')).resolves.toEqual({ + value: { accessToken: 'current-token' }, + }) + const askedDuringRefresh = mint.mock.calls.length + await expect(resolveFor('ramp-refresh-outage')).resolves.toEqual({ + value: { accessToken: 'current-token' }, + }) + expect(askedDuringRefresh).toBeGreaterThan(1) + expect(mint).toHaveBeenCalledTimes(askedDuringRefresh) + }) + + it('reports the failed refresh once the cached token has run out', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refresh-expired') + mint.mockRejectedValue(unavailable()) + vi.advanceTimersByTime(600_000 - 20_000) + + await expect(resolveFor('ramp-refresh-expired')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + }) + + it('stops serving the cached token once the provider refuses the credentials', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refresh-refused') + mint.mockRejectedValue(refused()) + vi.advanceTimersByTime(6 * 60_000) + + await expect(resolveFor('ramp-refresh-refused')).resolves.toEqual({ + error: expect.objectContaining({ code: 'invalid_credentials' }), + }) + expect(mint).toHaveBeenCalledTimes(2) + }) + + it('retries a quick transient failure once inside the mint', async () => { + const mint = minter() + mint.mockRejectedValueOnce(unavailable(503)).mockResolvedValueOnce(token) + + await expect(resolveFor('ramp-retry-once')).resolves.toEqual({ + value: { accessToken: token.accessToken }, + }) + expect(mint).toHaveBeenCalledTimes(2) + }) + + it('does not retry an attempt that took as long as a provider timeout', async () => { + const mint = minter() + mint.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 10_000) + throw unavailable() + }) + + await expect(resolveFor('ramp-timed-out')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + expect(mint).toHaveBeenCalledOnce() + }) + + it('replays a refusal of the credentials for 30 seconds without retrying it', async () => { + const mint = minter() + mint.mockRejectedValue(refused()) + + await resolveFor('ramp-refused') + vi.advanceTimersByTime(29_000) + await expect(resolveFor('ramp-refused')).resolves.toEqual({ + error: expect.objectContaining({ code: 'invalid_credentials' }), + }) + expect(mint).toHaveBeenCalledOnce() + + vi.advanceTimersByTime(2_000) + await resolveFor('ramp-refused') + expect(mint).toHaveBeenCalledTimes(2) + }) + + it.each([ + ['as long as a throttled provider asks', 120_000, 120_000], + ['no longer than five minutes, whatever the provider asks', 3_600_000, 300_000], + ])('waits %s before asking again', async (_case, retryAfterMs, waitMs) => { + const mint = minter() + const credentialId = `ramp-throttled-${retryAfterMs}` + mint.mockRejectedValueOnce(unavailable(429, retryAfterMs)).mockResolvedValueOnce(token) + + await expect(resolveFor(credentialId)).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable', status: 429 }), + }) + vi.advanceTimersByTime(waitMs - 1_000) + await expect(resolveFor(credentialId)).resolves.toEqual({ + error: expect.objectContaining({ status: 429 }), + }) + expect(mint).toHaveBeenCalledOnce() + + vi.advanceTimersByTime(2_000) + await expect(resolveFor(credentialId)).resolves.toEqual({ + value: { accessToken: token.accessToken }, + }) + }) + + it('backs off an outage briefly at first, longer while it lasts, and never past 30 seconds', async () => { + const mint = minter() + mint.mockRejectedValue(unavailable()) + + await resolveFor('ramp-outage') + const firstAttempt = mint.mock.calls.length + vi.advanceTimersByTime(6_001) + await resolveFor('ramp-outage') + const secondAttempt = mint.mock.calls.length + expect(secondAttempt).toBeGreaterThan(firstAttempt) + + vi.advanceTimersByTime(6_001) + await resolveFor('ramp-outage') + expect(mint).toHaveBeenCalledTimes(secondAttempt) + + for (let attempt = 0; attempt < 6; attempt++) { + const asked = mint.mock.calls.length + vi.advanceTimersByTime(30_001) + await resolveFor('ramp-outage') + expect(mint.mock.calls.length).toBeGreaterThan(asked) + } + }) +}) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 93d428ac8b4..557181baf2a 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -24,6 +24,13 @@ import { GOOGLE_SERVICE_ACCOUNT_EXCLUDED_SCOPES, isGoogleWorkloadIdentitySecret, } from '@/lib/credentials/google-workload-identity' +import { + assessTokenRequestFailure, + requestTokenWithRetry, + TOKEN_EXPIRY_MARGIN_MS, + TOKEN_FAILURE_STREAK_RETENTION_MS, + type TokenRequestFailureKind, +} from '@/lib/credentials/token-request-resilience' import { getTokenClientCredentialMinter } from '@/lib/credentials/token-service-accounts/client-credentials' import { type CLIENT_CREDENTIALS_AUTH_METHOD, @@ -631,7 +638,13 @@ interface MintedServiceAccountToken { interface FailedClientCredentialMint { /** The error the failed mint threw, re-thrown to callers while memoized. */ error: unknown + kind: TokenRequestFailureKind + /** Non-permanent failures in a row; the next one backs off longer. */ + consecutiveFailures: number secretFingerprint: string + /** Until when the failure is replayed instead of minting again. */ + retryAtMs: number + /** When the entry is dropped: after its replay, once a later failure can no longer extend the streak. */ expiresAtMs: number } @@ -652,10 +665,18 @@ interface FailedClientCredentialMint { * multiple concurrently valid tokens, so each instance minting its own token * is correct. * - * Failed mints are never cached as tokens; instead they are memoized for - * {@link CLIENT_CREDENTIAL_MINT_FAILURE_TTL_MS} so a hot workflow on a - * revoked/invalid secret doesn't hammer the provider's token endpoint once - * per block execution. + * A token inside its last {@link CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS} is refreshed + * early, and a refresh that fails with an outage or throttling (not a refusal of + * the credentials) keeps the still-valid token in service until + * {@link TOKEN_EXPIRY_MARGIN_MS} before it expires, as MSAL, Azure.Core and + * botocore do with a failed proactive refresh. + * + * Failed mints are never cached as tokens; instead they are replayed (see + * {@link assessTokenRequestFailure}) so a hot workflow neither hammers the + * provider's token endpoint with a revoked secret nor floods a struggling one: + * a refusal for 30 s, a throttled answer as long as its `Retry-After` asks (at + * most five minutes), and an outage for 5 s growing to 30 s while it lasts. A + * quick outage is first retried once inside the mint. * * Both maps are pruned of expired entries on each resolution * ({@link pruneExpiredClientCredentialCaches}), so their size is bounded by the @@ -665,7 +686,6 @@ interface FailedClientCredentialMint { const clientCredentialTokenCache = new Map() const clientCredentialMintFailureCache = new Map() const CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS = 5 * 60 * 1000 -const CLIENT_CREDENTIAL_MINT_FAILURE_TTL_MS = 30 * 1000 /** Drops fully-expired token and failure entries so the maps stay bounded. */ function pruneExpiredClientCredentialCaches(nowMs: number): void { @@ -711,11 +731,14 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string { * Resolves a stored secret that mints short-lived tokens, read-through the * per-instance cache: re-reads the credential row, serves a cached token minted * from the same ciphertext while enough of it remains, replays a recent mint - * failure, and otherwise decrypts and mints. Wrapped in `coalesceLocally` so - * concurrent block executions on one instance share a single mint. - * `sharedToken` providers coordinate one token per application themselves, so - * their tokens are never cached here, and a caller that already read the - * secret to choose its mode passes it as `stored` rather than reading it twice. + * failure, and otherwise decrypts and mints, falling back to the cached token + * while it is still valid when the refresh fails with an outage. Wrapped in + * `coalesceLocally` so concurrent block executions on one instance share a + * single mint. `sharedToken` providers coordinate one token per application + * themselves (including the in-request retry and the fallback), so their tokens + * are never cached here and a failure is replayed only as long as their own + * replay reports; a caller that already read the secret to choose its mode + * passes it as `stored` rather than reading it twice. */ async function resolveMintedServiceAccountToken( credentialId: string, @@ -748,29 +771,28 @@ async function resolveMintedServiceAccountToken( } const secretFingerprint = secretFingerprintOf(encryptedServiceAccountKey) - const cached = sharedToken ? undefined : clientCredentialTokenCache.get(cacheIdentity) - if ( - cached && - cached.secretFingerprint === secretFingerprint && - cached.expiresAtMs - Date.now() > CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS - ) { + const cachedEntry = sharedToken ? undefined : clientCredentialTokenCache.get(cacheIdentity) + const cached = cachedEntry?.secretFingerprint === secretFingerprint ? cachedEntry : undefined + if (cached && cached.expiresAtMs - Date.now() > CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS) { return { ...cached.token } } + const stillValid = + cached && cached.expiresAtMs - Date.now() > TOKEN_EXPIRY_MARGIN_MS ? cached : undefined - const failed = clientCredentialMintFailureCache.get(cacheIdentity) - if ( - failed && - failed.secretFingerprint === secretFingerprint && - Date.now() < failed.expiresAtMs - ) { + const failedEntry = clientCredentialMintFailureCache.get(cacheIdentity) + const failed = failedEntry?.secretFingerprint === secretFingerprint ? failedEntry : undefined + if (failed && Date.now() < failed.retryAtMs) { + if (stillValid && failed.kind !== 'permanent') return { ...stillValid.token } throw failed.error } - clientCredentialMintFailureCache.delete(cacheIdentity) try { const decrypted = stored?.decrypted ?? (await decryptSecret(encryptedServiceAccountKey)).decrypted - const minted = await mint(decrypted) + const minted = sharedToken + ? await mint(decrypted) + : await requestTokenWithRetry(() => mint(decrypted)) + clientCredentialMintFailureCache.delete(cacheIdentity) if (!sharedToken) clientCredentialTokenCache.set(cacheIdentity, { token: minted.token, @@ -780,12 +802,27 @@ async function resolveMintedServiceAccountToken( }) return { ...minted.token } } catch (error) { - if (!(error instanceof Error && error.name === 'AbortError')) { - clientCredentialMintFailureCache.set(cacheIdentity, { - error: options?.privacyMode === 'selector' ? new Error('Credential mint failed') : error, - secretFingerprint, - expiresAtMs: Date.now() + CLIENT_CREDENTIAL_MINT_FAILURE_TTL_MS, + if (error instanceof Error && error.name === 'AbortError') throw error + const failure = assessTokenRequestFailure( + error, + failed && failed.kind !== 'permanent' ? failed.consecutiveFailures : 0 + ) + const retryAtMs = Date.now() + failure.replayMs + clientCredentialMintFailureCache.set(cacheIdentity, { + error: options?.privacyMode === 'selector' ? new Error('Credential mint failed') : error, + kind: failure.kind, + consecutiveFailures: failure.consecutiveFailures, + secretFingerprint, + retryAtMs, + expiresAtMs: retryAtMs + TOKEN_FAILURE_STREAK_RETENTION_MS, + }) + if (stillValid && failure.kind !== 'permanent') { + logger.warn('Serving a still-valid client-credential token after its refresh failed', { + credentialId, + failure: failure.kind, + retryInMs: failure.replayMs, }) + return { ...stillValid.token } } throw error } From 84e4f91733c0cbbf8e2db95e7debb189c96df383 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 19:31:54 -0800 Subject: [PATCH 3/9] feat(credentials): share token refresh resilience across workers in the coordinated resolver The coordinated client-credentials resolver (Shopify, Linear, Atlassian, Vanta) now applies the same policy as the per-instance mint cache, through the shared row every worker reads: - a quick outage is retried once under the application lock, within the 15 s its waiters allow - a refusal is remembered 30 s, a throttled answer for its Retry-After (1 s to 5 min, Vanta's now included), an outage 5 s growing to 30 s while it lasts - while an outage or throttling is remembered, the stored token for the same secret is served until 30 s before it expires; a refusal fails at once Rolling deploys stay safe: a refusal is stored exactly as before, and outages add only a consecutiveFailures key that previous decoders ignore (both prior decoders are pinned in the integration tests). The replay length rides on the row's existing expiresAt. Expired memos are kept five more minutes so the next failure can extend the backoff; an older build's sweep only resets it. --- .../vanta-token.integration.ts | 23 ++ .../client-credential-accounts/vanta-token.ts | 4 +- .../coordinated-client-token.integration.ts | 271 +++++++++++++++++- .../credentials/coordinated-client-token.ts | 263 +++++++++++------ 4 files changed, 464 insertions(+), 97 deletions(-) diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts index 2ba5cd6f308..b45bc12c373 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts @@ -137,6 +137,11 @@ beforeAll(async () => { ) return } + if (parsed.client_id.endsWith('-throttled')) { + response.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '90' }) + response.end(JSON.stringify({ error: 'rate_limited' })) + return + } await sleep(15) const identity = `${request.headers['x-provider-origin']}:${parsed.client_id}` const issued = (applications.get(identity)?.issued ?? 0) + 1 @@ -321,6 +326,24 @@ describe('Vanta application token lifecycle against shared storage', () => { expect(await isActive(next.accessToken)).toBe(true) }) + it('waits as long as a throttled Vanta asks before any worker asks again', async () => { + const input = fields('throttled') + const keys = previousBuildKeys(input, READ) + const count = tokenRequests + const before = Date.now() + + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + }) + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ status: 429 }) + + expect(tokenRequests).toBe(count + 1) + const stored = await readStoredRow(keys.failureKey) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(90_000) + expect(stored.expiresAt.getTime() - Date.now()).toBeLessThanOrEqual(90_000) + }) + it('fails closed when coordination storage fails', async () => { const count = tokenRequests vi.spyOn(db, 'transaction').mockRejectedValueOnce(new Error('database unavailable')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts index 2d1bc48ec70..ec0c46221e8 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts @@ -8,6 +8,7 @@ import { import { fetchProvider, isTransientProviderStatus, + providerRetryAfterMs, requireClientSecret, TokenServiceAccountValidationError, } from '@/lib/credentials/token-service-accounts/errors' @@ -91,7 +92,8 @@ async function exchangeToken( ? 'invalid_credentials' : 'provider_unavailable', response.status, - { step: STEP, reason: 'provider rejected token exchange' } + { step: STEP, reason: 'provider rejected token exchange' }, + { retryAfterMs: providerRetryAfterMs(response) } ) } const payload = await readResponseJsonWithLimit(response, { diff --git a/apps/sim/lib/credentials/coordinated-client-token.integration.ts b/apps/sim/lib/credentials/coordinated-client-token.integration.ts index 714466025d9..3a35e062aea 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.integration.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.integration.ts @@ -3,9 +3,10 @@ import { db } from '@sim/db' import { clientCredentialToken } from '@sim/db/schema' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' import { eq, like, or } from 'drizzle-orm' import { afterAll, afterEach, describe, expect, it, vi } from 'vitest' -import { decryptSecret } from '@/lib/core/security/encryption' +import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { type CoordinatedClientTokenInput, invalidateCoordinatedClientToken, @@ -63,6 +64,68 @@ function rejecting() { }) } +function failing(status: number, retryAfterMs?: number) { + return vi.fn(async (): Promise<{ accessToken: string; expiresAt: Date }> => { + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + status, + { step: 'mint' }, + { + retryAfterMs, + } + ) + }) +} + +/** An exchange that issues one token valid for `lifetimeMs`, then fails with `then`. */ +function issuingOnce( + application: string, + lifetimeMs: number, + then: CoordinatedClientTokenInput['exchange'] +) { + return vi + .fn() + .mockResolvedValueOnce({ + accessToken: `${application}:token-1`, + expiresAt: new Date(Date.now() + lifetimeMs), + }) + .mockImplementation(then) +} + +async function failureRowOf(application: string) { + const rows = await db + .select() + .from(clientCredentialToken) + .where(like(clientCredentialToken.id, `${applicationKey(NAMESPACE, [application])}:%`)) + if (rows.length !== 1) throw new Error(`Expected one failure row, found ${rows.length}`) + return readStoredRow(rows[0].id) +} + +async function expireFailureRowsOf(application: string) { + await db + .update(clientCredentialToken) + .set({ expiresAt: new Date(Date.now() - 1000) }) + .where(like(clientCredentialToken.id, `${applicationKey(NAMESPACE, [application])}:%`)) +} + +/** The previous build's failure decoders, which must keep accepting every failure this build stores. */ +const previousBuildDecodesFailure = { + coordinated: (value: unknown) => + isRecordLike(value) && + typeof value.code === 'string' && + [ + 'invalid_credentials', + 'grant_not_permitted', + 'site_not_found', + 'provider_unavailable', + ].includes(value.code) && + typeof value.status === 'number', + vantaBeforeCoordination: (value: unknown) => + isRecordLike(value) && + (value.code === 'invalid_credentials' || value.code === 'provider_unavailable') && + typeof value.status === 'number', +} + afterEach(() => { vi.unstubAllGlobals() }) @@ -204,27 +267,207 @@ describe('coordinated client-credentials token against shared storage', () => { ).resolves.toMatchObject({ accessToken: 'fresh-rejection:token-2' }) }) - it('sweeps expired failure memos', async () => { + it('sweeps failure memos once they can no longer extend a backoff', async () => { + for (const application of ['swept', 'retained']) { + await expect( + resolveCoordinatedClientToken(input(application, rejecting(), 'revoked')) + ).rejects.toMatchObject({ code: 'invalid_credentials' }) + } + const failurePrefix = (application: string) => `${applicationKey(NAMESPACE, [application])}:` + const expire = (application: string, agoMs: number) => + db + .update(clientCredentialToken) + .set({ expiresAt: new Date(Date.now() - agoMs) }) + .where(like(clientCredentialToken.id, `${failurePrefix(application)}%`)) + await expire('swept', 6 * 60_000) + await expire('retained', 60_000) + + await resolveCoordinatedClientToken(input('sweeper', provider('sweeper').exchange)) + + const remaining = async (application: string) => + db + .select({ id: clientCredentialToken.id }) + .from(clientCredentialToken) + .where(like(clientCredentialToken.id, `${failurePrefix(application)}%`)) + expect(await remaining('swept')).toEqual([]) + expect(await remaining('retained')).toHaveLength(1) + }) +}) + +describe('coordinated token resolution while the provider fails', () => { + it('serves the stored token while it is still valid when its early refresh is throttled', async () => { + const exchange = issuingOnce('throttled-refresh', 4 * 60_000, failing(429)) + await resolveCoordinatedClientToken(input('throttled-refresh', exchange)) + + for (let call = 0; call < 3; call++) { + await expect( + resolveCoordinatedClientToken(input('throttled-refresh', exchange)) + ).resolves.toMatchObject({ accessToken: 'throttled-refresh:token-1' }) + } + expect(exchange).toHaveBeenCalledTimes(2) + }) + + it('reports the failed refresh once the stored token has run out', async () => { + const exchange = issuingOnce('expired-refresh', 20_000, failing(429)) + await resolveCoordinatedClientToken(input('expired-refresh', exchange)) + await expect( - resolveCoordinatedClientToken(input('swept', rejecting(), 'revoked')) + resolveCoordinatedClientToken(input('expired-refresh', exchange)) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429 }) + }) + + it('stops serving the stored token once the provider refuses the credentials', async () => { + const exchange = issuingOnce('refused-refresh', 4 * 60_000, rejecting()) + await resolveCoordinatedClientToken(input('refused-refresh', exchange)) + + await expect( + resolveCoordinatedClientToken(input('refused-refresh', exchange)) ).rejects.toMatchObject({ code: 'invalid_credentials' }) - const failurePrefix = `${applicationKey(NAMESPACE, ['swept'])}:` - await db - .update(clientCredentialToken) - .set({ expiresAt: new Date(Date.now() - 1000) }) - .where(like(clientCredentialToken.id, `${failurePrefix}%`)) + await expect( + resolveCoordinatedClientToken(input('refused-refresh', exchange)) + ).rejects.toMatchObject({ code: 'invalid_credentials' }) + expect(exchange).toHaveBeenCalledTimes(2) + }) - await resolveCoordinatedClientToken(input('sweeper', provider('sweeper').exchange)) + it('never serves a still-valid token to a different secret whose refresh fails', async () => { + const exchange = issuingOnce('rotated-refresh', 4 * 60_000, failing(429)) + await resolveCoordinatedClientToken(input('rotated-refresh', exchange)) - const rows = await db - .select({ id: clientCredentialToken.id }) - .from(clientCredentialToken) - .where(like(clientCredentialToken.id, `${failurePrefix}%`)) - expect(rows).toEqual([]) + await expect( + resolveCoordinatedClientToken(input('rotated-refresh', exchange, 'secret-2')) + ).rejects.toMatchObject({ code: 'provider_unavailable' }) + }) + + it('retries a quick outage once under the application lock, for every waiting resolution', async () => { + const { exchange: issue } = provider('retried') + const exchange = vi + .fn() + .mockRejectedValueOnce( + new TokenServiceAccountValidationError('provider_unavailable', 503, { step: 'mint' }) + ) + .mockImplementation(issue) + + const results = await Promise.all( + Array.from({ length: 5 }, () => resolveCoordinatedClientToken(input('retried', exchange))) + ) + + expect(new Set(results.map((result) => result.accessToken))).toEqual( + new Set(['retried:token-1']) + ) + expect(exchange).toHaveBeenCalledTimes(2) + }) + + it.each([ + ['as long as the provider asks', 'throttled-90s', 90_000, 90_000], + ['for at most five minutes', 'throttled-1h', 3_600_000, 300_000], + ])( + "shares a throttled exchange's Retry-After with every worker %s", + async (_case, application, retryAfterMs, replayMs) => { + const exchange = failing(429, retryAfterMs) + const before = Date.now() + + await expect( + resolveCoordinatedClientToken(input(application, exchange)) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429 }) + await expect( + resolveCoordinatedClientToken(input(application, exchange)) + ).rejects.toMatchObject({ status: 429, retryAfterMs: expect.any(Number) }) + + expect(exchange).toHaveBeenCalledOnce() + const stored = await failureRowOf(application) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(replayMs) + expect(stored.expiresAt.getTime() - Date.now()).toBeLessThanOrEqual(replayMs) + } + ) + + it('backs off an outage from a few seconds, longer while it continues, never past 30 seconds', async () => { + const exchange = failing(502) + + const replays: number[] = [] + for (let attempt = 0; attempt < 5; attempt++) { + const before = Date.now() + await expect(resolveCoordinatedClientToken(input('outage', exchange))).rejects.toMatchObject({ + code: 'provider_unavailable', + }) + replays.push((await failureRowOf('outage')).expiresAt.getTime() - before) + await expireFailureRowsOf('outage') + } + + expect(replays[0]).toBeLessThanOrEqual(6_000 + 2_500) + expect(replays[1]).toBeGreaterThanOrEqual(8_000) + expect(Math.max(...replays)).toBeLessThanOrEqual(30_000 + 2_500) + }) + + it('starts the backoff over after the provider recovers', async () => { + const { exchange: issue } = provider('recovered') + const exchange = vi + .fn() + .mockImplementation(failing(502)) + for (let attempt = 0; attempt < 3; attempt++) { + await expect(resolveCoordinatedClientToken(input('recovered', exchange))).rejects.toThrow() + await expireFailureRowsOf('recovered') + } + exchange.mockImplementationOnce(issue) + const { accessToken } = await resolveCoordinatedClientToken(input('recovered', exchange)) + await invalidateCoordinatedClientToken(accessToken) + exchange.mockImplementation(failing(502)) + + const before = Date.now() + await expect(resolveCoordinatedClientToken(input('recovered', exchange))).rejects.toThrow() + + expect((await failureRowOf('recovered')).expiresAt.getTime() - before).toBeLessThanOrEqual( + 6_000 + 2_500 + ) }) }) describe('default token rows shared with the previous build during a rolling deploy', () => { + it('serves a still-valid token when the previous build remembered an outage', async () => { + const application = 'legacy-outage' + const appKey = applicationKey(NAMESPACE, [application]) + const fingerprint = sha256Json([application, 'secret-1']) + const token = await encryptSecret( + JSON.stringify({ accessToken: 'legacy-token', secretFingerprint: fingerprint }) + ) + const outage = await encryptSecret( + JSON.stringify({ code: 'provider_unavailable', status: 503 }) + ) + await db.insert(clientCredentialToken).values([ + { + id: appKey, + encryptedValue: token.encrypted, + accessTokenDigest: sha256Json(['coordinated_client_token', 'legacy-token']), + expiresAt: new Date(Date.now() + 4 * 60_000), + }, + { + id: `${appKey}:${sha256Json(fingerprint)}`, + encryptedValue: outage.encrypted, + expiresAt: new Date(Date.now() + 20_000), + }, + ]) + const { exchange } = provider(application) + + await expect( + resolveCoordinatedClientToken(input(application, exchange)) + ).resolves.toMatchObject({ accessToken: 'legacy-token' }) + expect(exchange).not.toHaveBeenCalled() + }) + + it('stores every outage and throttled answer as a failure the previous build reads', async () => { + for (const [application, exchange] of [ + ['legacy-readable-outage', failing(502)], + ['legacy-readable-throttle', failing(429, 60_000)], + ] as const) { + await expect(resolveCoordinatedClientToken(input(application, exchange))).rejects.toThrow() + + const stored = await failureRowOf(application) + const payload: unknown = JSON.parse(stored.decrypted) + expect(previousBuildDecodesFailure.coordinated(payload)).toBe(true) + expect(previousBuildDecodesFailure.vantaBeforeCoordination(payload)).toBe(true) + expect(stored.accessTokenDigest).toBeNull() + } + }) + it('stores tokens and refusals under the ids, digest, and payloads the previous build used', async () => { const application = ['legacy.myshopify.com', 'legacy-client'] const appKey = applicationKey(NAMESPACE, application) diff --git a/apps/sim/lib/credentials/coordinated-client-token.ts b/apps/sim/lib/credentials/coordinated-client-token.ts index 0fea2e04b00..2e565134055 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.ts @@ -1,15 +1,24 @@ import { createHash } from 'node:crypto' import { db } from '@sim/db' import { clientCredentialToken } from '@sim/db/schema' +import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { and, eq, inArray, lt, sql } from 'drizzle-orm' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' +import { + assessTokenRequestFailure, + classifyTokenRequestFailure, + requestTokenWithRetry, + TOKEN_EXPIRY_MARGIN_MS, + TOKEN_FAILURE_STREAK_RETENTION_MS, +} from '@/lib/credentials/token-request-resilience' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' -/** A stored token is only handed out while at least this much of its lifetime remains. */ +const logger = createLogger('CoordinatedClientToken') + +/** A stored token is refreshed once less than this much of its lifetime remains. */ const MIN_REMAINING_MS = 5 * 60 * 1000 -const FAILURE_TTL_MS = 30_000 const DEFAULT_ACCESS_TOKEN_DIGEST_TAG = 'coordinated_client_token' /** The provider token a coordinated exchange produced. */ @@ -33,7 +42,11 @@ export interface CoordinatedClientTokenInput { /** Performs the provider exchange. Throws `TokenServiceAccountValidationError` on refusal. */ exchange: (signal?: AbortSignal) => Promise signal?: AbortSignal - /** Shortest remaining lifetime a stored token is still served with. Defaults to five minutes. */ + /** + * Remaining lifetime below which a stored token is refreshed. Defaults to five minutes. If the + * refresh fails with an outage or throttling, the token is still served until 30 seconds (or + * this value, when smaller) before it expires. + */ minRemainingMs?: number /** * A request setting beyond the secret that a remembered exchange failure is specific to, so a @@ -67,9 +80,15 @@ interface StoredApplicationToken { record: Readonly> } +/** + * A remembered exchange failure, replayed until its row's `expiresAt`. Builds before failure + * kinds stored `{ code, status }` only and ignore other keys, so `consecutiveFailures` is + * written only for outages and throttling, to grow their backoff while they continue. + */ interface StoredFailure { code: 'invalid_credentials' | 'grant_not_permitted' | 'site_not_found' | 'provider_unavailable' status: number + consecutiveFailures?: number } const STORED_FAILURE_CODES: ReadonlySet = new Set([ @@ -89,16 +108,18 @@ function accessTokenDigestOf(accessToken: string, tag = DEFAULT_ACCESS_TOKEN_DIG } /** - * Deletes a bounded batch of expired failure memos (rows without a token digest). Run it outside - * the transaction that takes an application lock, so its row locks are released before any - * provider call. + * Deletes a bounded batch of failure memos (rows without a token digest) that expired more than + * {@link TOKEN_FAILURE_STREAK_RETENTION_MS} ago; a more recently expired one is kept so the next + * failure continues its backoff. Run it outside the transaction that takes an application lock, + * so its row locks are released before any provider call. */ async function deleteExpiredClientTokenFailures(): Promise { + const retentionSeconds = TOKEN_FAILURE_STREAK_RETENTION_MS / 1000 await db.execute(sql` DELETE FROM ${clientCredentialToken} WHERE ${clientCredentialToken.id} IN ( SELECT ${clientCredentialToken.id} FROM ${clientCredentialToken} WHERE ${clientCredentialToken.accessTokenDigest} IS NULL - AND ${clientCredentialToken.expiresAt} < now() + AND ${clientCredentialToken.expiresAt} < now() - make_interval(secs => ${retentionSeconds}) ORDER BY ${clientCredentialToken.expiresAt} LIMIT 100 FOR UPDATE SKIP LOCKED )`) } @@ -168,7 +189,45 @@ function decodeFailure(value: string, step: string): StoredFailure { typeof parsed.status !== 'number' ) throw unavailable(step, 'stored token failure is malformed') - return { code: parsed.code as StoredFailure['code'], status: parsed.status } + return { + code: parsed.code as StoredFailure['code'], + status: parsed.status, + ...(Number.isSafeInteger(parsed.consecutiveFailures) && Number(parsed.consecutiveFailures) > 0 + ? { consecutiveFailures: Number(parsed.consecutiveFailures) } + : {}), + } +} + +/** The error a remembered failure is replayed as, with how long it is still replayed. */ +function replayedFailure( + failure: StoredFailure, + step: string, + replayMs: number +): TokenServiceAccountValidationError { + return new TokenServiceAccountValidationError( + failure.code, + failure.status, + { step }, + { retryAfterMs: Math.max(replayMs, 0) } + ) +} + +/** + * Outages and throttled answers in a row before this exchange, read from the application's last + * failure memo for this secret once its replay has ended. Anything unreadable starts a new streak. + */ +async function consecutiveFailuresBefore( + row: typeof clientCredentialToken.$inferSelect | undefined, + step: string +): Promise { + if (!row || Date.now() - row.expiresAt.getTime() > TOKEN_FAILURE_STREAK_RETENTION_MS) return 0 + try { + const failure = decodeFailure((await decryptSecret(row.encryptedValue)).decrypted, step) + if (classifyTokenRequestFailure(replayedFailure(failure, step, 0)) === 'permanent') return 0 + return failure.consecutiveFailures ?? 1 + } catch { + return 0 + } } function remainingSeconds(expiresAt: Date): number { @@ -186,11 +245,16 @@ function isServable(expiresAt: Date, minRemainingMs: number): boolean { * to reuse one token rather than accumulate them. Minting happens only under a * transaction-scoped advisory lock keyed by the application, so two workers never race two * tokens into existence; a token minted with a different secret is never served, and a token the - * provider rejects is expired through {@link invalidateCoordinatedClientToken}. An exchange that - * fails validation, whether refused for the credentials or answered with an outage, is remembered - * for 30 seconds, the same backoff the per-instance mint cache applies, so a hot workflow neither - * hammers the token endpoint with a revoked secret nor floods a struggling one. Tokens are - * encrypted at rest. + * provider rejects is expired through {@link invalidateCoordinatedClientToken}. + * + * A quick outage is retried once inside the exchange. An exchange that still fails validation is + * remembered for every worker, with the policy the per-instance mint cache applies (see + * `assessTokenRequestFailure`): a refusal of the credentials for 30 seconds, a throttled answer as + * long as its `Retry-After` asks (at most five minutes), and an outage for 5 seconds growing to 30 + * while it continues, so a hot workflow neither hammers the token endpoint with a revoked secret + * nor floods a struggling one. While an outage or throttling is remembered, the stored token for + * the same secret is still served until shortly before it actually expires; a refusal is + * reported at once. Tokens are encrypted at rest. */ export async function resolveCoordinatedClientToken( input: CoordinatedClientTokenInput @@ -219,6 +283,27 @@ export async function resolveCoordinatedClientToken( return { accessToken: stored.accessToken, expiresInSeconds: remainingSeconds(row.expiresAt) } } + /** + * The stored token for this secret while it has not yet run out, to serve when its refresh + * fails with an outage. Anything that would not be served fresh (a record that does not decode + * or verify) is never a fallback. + */ + const stillValid = async (row: typeof clientCredentialToken.$inferSelect | undefined) => { + if (!row || !isServable(row.expiresAt, Math.min(TOKEN_EXPIRY_MARGIN_MS, minRemainingMs))) + return undefined + try { + const stored = decodeApplicationToken( + (await decryptSecret(row.encryptedValue)).decrypted, + step + ) + pinnedState?.verify(stored.record) + if (stored.secretFingerprint !== secretFingerprint) return undefined + return { accessToken: stored.accessToken, expiresInSeconds: remainingSeconds(row.expiresAt) } + } catch { + return undefined + } + } + try { const [current] = await db .select() @@ -229,83 +314,97 @@ export async function resolveCoordinatedClientToken( if (fresh) return fresh await deleteExpiredClientTokenFailures() - const outcome = await db.transaction( - async (tx) => { - await tx.execute(sql`SET LOCAL lock_timeout = '15s'`) - await acquireAdvisoryXactLock(tx, `${namespace}_client_token`, applicationKey) - signal?.throwIfAborted() - const entries = await tx - .select() - .from(clientCredentialToken) - .where(inArray(clientCredentialToken.id, [applicationKey, failureKey])) - const minted = await serveIfFresh( - entries.find((entry) => entry.id === applicationKey), - true + const outcome = await db.transaction< + CoordinatedClientToken | { failure: TokenServiceAccountValidationError } + >(async (tx) => { + await tx.execute(sql`SET LOCAL lock_timeout = '15s'`) + await acquireAdvisoryXactLock(tx, `${namespace}_client_token`, applicationKey) + signal?.throwIfAborted() + const entries = await tx + .select() + .from(clientCredentialToken) + .where(inArray(clientCredentialToken.id, [applicationKey, failureKey])) + const applicationRow = entries.find((entry) => entry.id === applicationKey) + const minted = await serveIfFresh(applicationRow, true) + if (minted) return minted + const failureRow = entries.find((entry) => entry.id === failureKey) + if (failureRow && failureRow.expiresAt.getTime() > Date.now()) { + const failure = replayedFailure( + decodeFailure((await decryptSecret(failureRow.encryptedValue)).decrypted, step), + step, + failureRow.expiresAt.getTime() - Date.now() ) - if (minted) return minted - const failure = entries.find( - (entry) => entry.id === failureKey && entry.expiresAt.getTime() > Date.now() + if (classifyTokenRequestFailure(failure) !== 'permanent') { + const fallback = await stillValid(applicationRow) + if (fallback) return fallback + } + return { failure } + } + signal?.throwIfAborted() + try { + const exchanged = await requestTokenWithRetry(() => input.exchange(signal), signal) + const { encrypted } = await encryptSecret( + JSON.stringify({ + accessToken: exchanged.accessToken, + secretFingerprint, + ...pinnedState?.values, + }) ) - if (failure) { - return { - failure: decodeFailure((await decryptSecret(failure.encryptedValue)).decrypted, step), - } + const values = { + encryptedValue: encrypted, + accessTokenDigest: accessTokenDigestOf(exchanged.accessToken, input.accessTokenDigestTag), + expiresAt: exchanged.expiresAt, + updatedAt: new Date(), + } + await tx + .insert(clientCredentialToken) + .values({ id: applicationKey, ...values }) + .onConflictDoUpdate({ target: clientCredentialToken.id, set: values }) + await tx.delete(clientCredentialToken).where(eq(clientCredentialToken.id, failureKey)) + return { + accessToken: exchanged.accessToken, + expiresInSeconds: remainingSeconds(exchanged.expiresAt), } - signal?.throwIfAborted() - try { - const exchanged = await input.exchange(signal) - const { encrypted } = await encryptSecret( - JSON.stringify({ - accessToken: exchanged.accessToken, - secretFingerprint, - ...pinnedState?.values, + } catch (error) { + if (!(error instanceof TokenServiceAccountValidationError)) throw error + const assessed = assessTokenRequestFailure( + error, + await consecutiveFailuresBefore(failureRow, step) + ) + const failure: StoredFailure = { + code: STORED_FAILURE_CODES.has(error.code) + ? (error.code as StoredFailure['code']) + : 'provider_unavailable', + status: error.status, + ...(assessed.kind === 'permanent' + ? {} + : { consecutiveFailures: assessed.consecutiveFailures }), + } + const { encrypted } = await encryptSecret(JSON.stringify(failure)) + const expiresAt = new Date(Date.now() + assessed.replayMs) + await tx + .insert(clientCredentialToken) + .values({ id: failureKey, encryptedValue: encrypted, expiresAt }) + .onConflictDoUpdate({ + target: clientCredentialToken.id, + set: { encryptedValue: encrypted, expiresAt, updatedAt: new Date() }, + }) + if (assessed.kind !== 'permanent') { + const fallback = await stillValid(applicationRow) + if (fallback) { + logger.warn('Serving a still-valid client-credentials token after its refresh failed', { + namespace, + failure: assessed.kind, + retryInMs: assessed.replayMs, }) - ) - const values = { - encryptedValue: encrypted, - accessTokenDigest: accessTokenDigestOf( - exchanged.accessToken, - input.accessTokenDigestTag - ), - expiresAt: exchanged.expiresAt, - updatedAt: new Date(), - } - await tx - .insert(clientCredentialToken) - .values({ id: applicationKey, ...values }) - .onConflictDoUpdate({ target: clientCredentialToken.id, set: values }) - await tx.delete(clientCredentialToken).where(eq(clientCredentialToken.id, failureKey)) - return { - accessToken: exchanged.accessToken, - expiresInSeconds: remainingSeconds(exchanged.expiresAt), + return fallback } - } catch (error) { - if (!(error instanceof TokenServiceAccountValidationError)) throw error - const failure: StoredFailure = { - code: STORED_FAILURE_CODES.has(error.code) - ? (error.code as StoredFailure['code']) - : 'provider_unavailable', - status: error.status, - } - const { encrypted } = await encryptSecret(JSON.stringify(failure)) - const expiresAt = new Date(Date.now() + FAILURE_TTL_MS) - await tx - .insert(clientCredentialToken) - .values({ id: failureKey, encryptedValue: encrypted, expiresAt }) - .onConflictDoUpdate({ - target: clientCredentialToken.id, - set: { encryptedValue: encrypted, expiresAt, updatedAt: new Date() }, - }) - return { failure } } + return { failure: replayedFailure(failure, step, assessed.replayMs) } } - ) + }) signal?.throwIfAborted() - if ('failure' in outcome) { - throw new TokenServiceAccountValidationError(outcome.failure.code, outcome.failure.status, { - step, - }) - } + if ('failure' in outcome) throw outcome.failure return outcome } catch (error) { signal?.throwIfAborted() From 80f337a9f49a9c4f035e17c7bdd2b529bd184a25 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 19:55:02 -0800 Subject: [PATCH 4/9] fix(credentials): pace throttled and Vanta token retries and replay remembered failures without the lock - A 429 that names no Retry-After is replayed for at least a minute, doubling up to the five-minute cap, instead of the 5 s outage curve. - The outage backoff never undercuts Entra's 5 s first-retry floor through jitter. - Vanta asks for a token at most once per 15 s while exchanges fail (no in-request retry, replay floored), within its documented 5 tokens a minute, through a new optional minExchangeIntervalMs on the coordinated resolver. - A remembered failure and the still-valid fallback are served from committed rows before the application lock is taken, so a refresh outage no longer serializes every resolution behind the lock. - retryAfterMs now only carries a provider's Retry-After; a replayed memo reports its remainder as the additive replayForMs, which the per-instance layer honors exactly. - One decode/verify path for stored tokens; the stored-failure permanence check uses isPermanentTokenFailureCode instead of building an error. --- .../vanta-token.integration.ts | 36 ++++ .../client-credential-accounts/vanta-token.ts | 6 + .../coordinated-client-token.integration.ts | 94 ++++++++++- .../credentials/coordinated-client-token.ts | 158 +++++++++++------- .../credentials/token-request-resilience.ts | 73 ++++---- .../token-service-accounts/errors.ts | 16 +- .../credential-service-cancellation.test.ts | 72 +++++++- apps/sim/lib/oauth/credential-service.ts | 5 +- 8 files changed, 361 insertions(+), 99 deletions(-) diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts index b45bc12c373..855b98b82e0 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts @@ -142,6 +142,15 @@ beforeAll(async () => { response.end(JSON.stringify({ error: 'rate_limited' })) return } + if (parsed.client_id.endsWith('-rate-limited')) { + response.writeHead(429, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ error: 'rate_limited' })) + return + } + if (parsed.client_id.endsWith('-outage')) { + response.writeHead(502).end() + return + } await sleep(15) const identity = `${request.headers['x-provider-origin']}:${parsed.client_id}` const issued = (applications.get(identity)?.issued ?? 0) + 1 @@ -344,6 +353,33 @@ describe('Vanta application token lifecycle against shared storage', () => { expect(stored.expiresAt.getTime() - Date.now()).toBeLessThanOrEqual(90_000) }) + it('waits at least a minute after a Vanta 429 that names no Retry-After', async () => { + const input = fields('rate-limited') + const keys = previousBuildKeys(input, READ) + const before = Date.now() + + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ status: 429 }) + + const stored = await readStoredRow(keys.failureKey) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(60_000) + }) + + it('asks a failing Vanta for a token at most once per 15 seconds, within its 5 tokens a minute', async () => { + const input = fields('outage') + const keys = previousBuildKeys(input, READ) + const count = tokenRequests + const before = Date.now() + + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 502, + }) + + expect(tokenRequests).toBe(count + 1) + const stored = await readStoredRow(keys.failureKey) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(15_000) + }) + it('fails closed when coordination storage fails', async () => { const count = tokenRequests vi.spyOn(db, 'transaction').mockRejectedValueOnce(new Error('database unavailable')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts index ec0c46221e8..7f9a50862f2 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts @@ -15,6 +15,11 @@ import { const SCOPES = VANTA_PERMISSION_OPTIONS.map((option) => option.value) const EXPIRY_BUFFER_MS = 30_000 +/** + * Vanta answers 429 when an application mints more than 5 tokens a minute, and every token it + * issues revokes the previous one, so a failing exchange is retried at most every 15 s (4 a minute). + */ +const MIN_EXCHANGE_INTERVAL_MS = 15_000 const STEP = 'vanta_token_mint' export interface VantaClientTokenFields { @@ -152,6 +157,7 @@ export async function resolveVantaClientToken( step: STEP, signal: options.signal, minRemainingMs: EXPIRY_BUFFER_MS, + minExchangeIntervalMs: MIN_EXCHANGE_INTERVAL_MS, failureKeyQualifier: scope, accessTokenDigestTag: apiDomain, pinnedState: { diff --git a/apps/sim/lib/credentials/coordinated-client-token.integration.ts b/apps/sim/lib/credentials/coordinated-client-token.integration.ts index 3a35e062aea..572fe56ec91 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.integration.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.integration.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto' import { db } from '@sim/db' import { clientCredentialToken } from '@sim/db/schema' +import { createDeferred } from '@sim/testing/helpers/deferred' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' @@ -14,6 +15,7 @@ import { } from '@/lib/credentials/coordinated-client-token' import { verifyAndBuildServiceAccountSecret } from '@/lib/credentials/service-account-secret' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' +import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' const NAMESPACE = `fixture${generateId().replaceAll('-', '')}` const ownedKeyPrefixes = new Set([`${NAMESPACE}:`]) @@ -181,9 +183,11 @@ describe('coordinated client-credentials token against shared storage', () => { await expect( resolveCoordinatedClientToken(input('memo', exchange, 'revoked')) ).rejects.toMatchObject({ code: 'invalid_credentials' }) - await expect( - resolveCoordinatedClientToken(input('memo', exchange, 'revoked')) - ).rejects.toMatchObject({ code: 'invalid_credentials' }) + const replayed = await resolveCoordinatedClientToken(input('memo', exchange, 'revoked')).catch( + (error: unknown) => error + ) + expect(replayed).toMatchObject({ code: 'invalid_credentials', replayForMs: expect.any(Number) }) + expect(replayed).not.toHaveProperty('retryAfterMs', expect.anything()) expect(exchange).toHaveBeenCalledOnce() await db @@ -371,7 +375,7 @@ describe('coordinated token resolution while the provider fails', () => { ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429 }) await expect( resolveCoordinatedClientToken(input(application, exchange)) - ).rejects.toMatchObject({ status: 429, retryAfterMs: expect.any(Number) }) + ).rejects.toMatchObject({ status: 429, replayForMs: expect.any(Number) }) expect(exchange).toHaveBeenCalledOnce() const stored = await failureRowOf(application) @@ -380,6 +384,87 @@ describe('coordinated token resolution while the provider fails', () => { } ) + it('replays a throttled exchange that names no Retry-After for at least a minute', async () => { + const exchange = failing(429) + const before = Date.now() + + await expect( + resolveCoordinatedClientToken(input('throttled-bare', exchange)) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429 }) + + expect(exchange).toHaveBeenCalledOnce() + expect( + (await failureRowOf('throttled-bare')).expiresAt.getTime() - before + ).toBeGreaterThanOrEqual(60_000) + }) + + it.each([ + [ + 'the still-valid token while its refresh is remembered failing', + 'unlocked-fallback', + 'outage', + ], + ['a remembered refusal', 'unlocked-refusal', 'refusal'], + ] as const)( + 'serves %s without waiting for the application lock', + async (_case, application, remembered) => { + const appKey = applicationKey(NAMESPACE, [application]) + const fingerprint = sha256Json([application, 'secret-1']) + const token = await encryptSecret( + JSON.stringify({ accessToken: `${application}:token-1`, secretFingerprint: fingerprint }) + ) + const failure = await encryptSecret( + JSON.stringify( + remembered === 'outage' + ? { code: 'provider_unavailable', status: 503, consecutiveFailures: 1 } + : { code: 'invalid_credentials', status: 401 } + ) + ) + await db.insert(clientCredentialToken).values([ + { + id: appKey, + encryptedValue: token.encrypted, + accessTokenDigest: sha256Json(['coordinated_client_token', `${application}:token-1`]), + expiresAt: new Date(Date.now() + 4 * 60_000), + }, + { + id: `${appKey}:${sha256Json(fingerprint)}`, + encryptedValue: failure.encrypted, + expiresAt: new Date(Date.now() + 60_000), + }, + ]) + const { exchange } = provider(application) + const lockHeld = createDeferred() + const release = createDeferred() + const holder = db.transaction(async (tx) => { + await acquireAdvisoryXactLock(tx, `${NAMESPACE}_client_token`, appKey) + lockHeld.resolve() + await release.promise + }) + + try { + await lockHeld.promise + const outcome = await Promise.race([ + resolveCoordinatedClientToken(input(application, exchange)).then( + (value) => ({ value }), + (error: unknown) => ({ error }) + ), + sleep(3_000).then(() => 'waited on the lock' as const), + ]) + + expect(outcome).toEqual( + remembered === 'outage' + ? { value: expect.objectContaining({ accessToken: `${application}:token-1` }) } + : { error: expect.objectContaining({ code: 'invalid_credentials' }) } + ) + expect(exchange).not.toHaveBeenCalled() + } finally { + release.resolve() + await holder + } + } + ) + it('backs off an outage from a few seconds, longer while it continues, never past 30 seconds', async () => { const exchange = failing(502) @@ -393,6 +478,7 @@ describe('coordinated token resolution while the provider fails', () => { await expireFailureRowsOf('outage') } + expect(replays[0]).toBeGreaterThanOrEqual(5_000) expect(replays[0]).toBeLessThanOrEqual(6_000 + 2_500) expect(replays[1]).toBeGreaterThanOrEqual(8_000) expect(Math.max(...replays)).toBeLessThanOrEqual(30_000 + 2_500) diff --git a/apps/sim/lib/credentials/coordinated-client-token.ts b/apps/sim/lib/credentials/coordinated-client-token.ts index 2e565134055..406eb0b8dbf 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.ts @@ -7,7 +7,7 @@ import { and, eq, inArray, lt, sql } from 'drizzle-orm' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { assessTokenRequestFailure, - classifyTokenRequestFailure, + isPermanentTokenFailureCode, requestTokenWithRetry, TOKEN_EXPIRY_MARGIN_MS, TOKEN_FAILURE_STREAK_RETENTION_MS, @@ -48,6 +48,13 @@ export interface CoordinatedClientTokenInput { * this value, when smaller) before it expires. */ minRemainingMs?: number + /** + * The provider is asked for a token at most once per this interval while its exchanges fail: + * a quick outage is not retried inside the exchange, and every failure is replayed at least this + * long. For providers that rate-limit token minting itself (Vanta: 5 tokens a minute), where + * every exchange the provider processes also revokes the application's previous token. + */ + minExchangeIntervalMs?: number /** * A request setting beyond the secret that a remembered exchange failure is specific to, so a * refusal for one setting does not block another. @@ -74,6 +81,8 @@ export interface CoordinatedClientToken { expiresInSeconds: number } +type StoredRow = typeof clientCredentialToken.$inferSelect + interface StoredApplicationToken { accessToken: string secretFingerprint: string @@ -208,7 +217,7 @@ function replayedFailure( failure.code, failure.status, { step }, - { retryAfterMs: Math.max(replayMs, 0) } + { replayForMs: Math.max(replayMs, 0) } ) } @@ -217,13 +226,13 @@ function replayedFailure( * failure memo for this secret once its replay has ended. Anything unreadable starts a new streak. */ async function consecutiveFailuresBefore( - row: typeof clientCredentialToken.$inferSelect | undefined, + row: StoredRow | undefined, step: string ): Promise { if (!row || Date.now() - row.expiresAt.getTime() > TOKEN_FAILURE_STREAK_RETENTION_MS) return 0 try { const failure = decodeFailure((await decryptSecret(row.encryptedValue)).decrypted, step) - if (classifyTokenRequestFailure(replayedFailure(failure, step, 0)) === 'permanent') return 0 + if (isPermanentTokenFailureCode(failure.code)) return 0 return failure.consecutiveFailures ?? 1 } catch { return 0 @@ -247,14 +256,16 @@ function isServable(expiresAt: Date, minRemainingMs: number): boolean { * tokens into existence; a token minted with a different secret is never served, and a token the * provider rejects is expired through {@link invalidateCoordinatedClientToken}. * - * A quick outage is retried once inside the exchange. An exchange that still fails validation is - * remembered for every worker, with the policy the per-instance mint cache applies (see - * `assessTokenRequestFailure`): a refusal of the credentials for 30 seconds, a throttled answer as - * long as its `Retry-After` asks (at most five minutes), and an outage for 5 seconds growing to 30 - * while it continues, so a hot workflow neither hammers the token endpoint with a revoked secret - * nor floods a struggling one. While an outage or throttling is remembered, the stored token for - * the same secret is still served until shortly before it actually expires; a refusal is - * reported at once. Tokens are encrypted at rest. + * A quick outage is retried once inside the exchange (unless `minExchangeIntervalMs` is set). An + * exchange that still fails validation is remembered for every worker, with the policy the + * per-instance mint cache applies (see `assessTokenRequestFailure`): a refusal of the credentials + * for 30 seconds, a throttled answer as long as its `Retry-After` asks (at most five minutes) or + * for a minute growing to five without one, and an outage for 5 seconds growing to 30 while it + * continues, so a hot workflow neither hammers the token endpoint with a revoked secret nor floods + * a struggling one. While an outage or throttling is remembered, the stored token for the same + * secret is still served until shortly before it actually expires; a refusal is reported at once. + * A remembered failure, and that fallback, are served from committed rows without taking the + * application lock. Tokens are encrypted at rest. */ export async function resolveCoordinatedClientToken( input: CoordinatedClientTokenInput @@ -270,48 +281,85 @@ export async function resolveCoordinatedClientToken( : [secretFingerprint, input.failureKeyQualifier] )}` - const serveIfFresh = async ( - row: typeof clientCredentialToken.$inferSelect | undefined, + const fallbackRemainingMs = Math.min(TOKEN_EXPIRY_MARGIN_MS, minRemainingMs) + const decodedRows = new WeakMap>() + + /** + * The token `row` holds for this secret while more than `remainingMs` of it is left. The record + * is decoded and its pinned state verified (throwing on a malformed record or a mismatch) once + * per row, whenever it could be served, and for an expired record too when `verifyExpired` is + * set and settings are pinned. + */ + const serveStored = async ( + row: StoredRow | undefined, + remainingMs: number, verifyExpired = false - ) => { + ): Promise => { if (!row) return undefined - const servable = isServable(row.expiresAt, minRemainingMs) + const servable = isServable(row.expiresAt, remainingMs) if (!servable && !(verifyExpired && pinnedState)) return undefined - const stored = decodeApplicationToken((await decryptSecret(row.encryptedValue)).decrypted, step) - pinnedState?.verify(stored.record) + let decoded = decodedRows.get(row) + if (!decoded) { + decoded = decryptSecret(row.encryptedValue).then(({ decrypted }) => { + const stored = decodeApplicationToken(decrypted, step) + pinnedState?.verify(stored.record) + return stored + }) + decodedRows.set(row, decoded) + } + const stored = await decoded if (!servable || stored.secretFingerprint !== secretFingerprint) return undefined return { accessToken: stored.accessToken, expiresInSeconds: remainingSeconds(row.expiresAt) } } /** - * The stored token for this secret while it has not yet run out, to serve when its refresh - * fails with an outage. Anything that would not be served fresh (a record that does not decode - * or verify) is never a fallback. + * The stored token for this secret while it has not yet run out, served while its refresh fails + * with an outage or throttling. A record that does not decode or verify is never a fallback. */ - const stillValid = async (row: typeof clientCredentialToken.$inferSelect | undefined) => { - if (!row || !isServable(row.expiresAt, Math.min(TOKEN_EXPIRY_MARGIN_MS, minRemainingMs))) - return undefined - try { - const stored = decodeApplicationToken( - (await decryptSecret(row.encryptedValue)).decrypted, - step - ) - pinnedState?.verify(stored.record) - if (stored.secretFingerprint !== secretFingerprint) return undefined - return { accessToken: stored.accessToken, expiresInSeconds: remainingSeconds(row.expiresAt) } - } catch { - return undefined + const fallbackFrom = (row: StoredRow | undefined) => + serveStored(row, fallbackRemainingMs).catch(() => undefined) + + /** + * Replays the application's unexpired failure memo for this secret: the still-valid token for a + * remembered outage or throttling, the failure otherwise. `undefined` when nothing is remembered. + * Reading committed rows needs no lock, so this also runs before the application lock is taken. + */ + const replayRemembered = async ( + applicationRow: StoredRow | undefined, + failureRow: StoredRow | undefined + ): Promise< + CoordinatedClientToken | { failure: TokenServiceAccountValidationError } | undefined + > => { + if (!failureRow || failureRow.expiresAt.getTime() <= Date.now()) return undefined + const failure = decodeFailure((await decryptSecret(failureRow.encryptedValue)).decrypted, step) + if (!isPermanentTokenFailureCode(failure.code)) { + const fallback = await fallbackFrom(applicationRow) + if (fallback) return fallback } + return { failure: replayedFailure(failure, step, failureRow.expiresAt.getTime() - Date.now()) } + } + + const settle = ( + outcome: CoordinatedClientToken | { failure: TokenServiceAccountValidationError } + ): CoordinatedClientToken => { + signal?.throwIfAborted() + if ('failure' in outcome) throw outcome.failure + return outcome } try { - const [current] = await db + const current = await db .select() .from(clientCredentialToken) - .where(eq(clientCredentialToken.id, applicationKey)) - .limit(1) - const fresh = await serveIfFresh(current) + .where(inArray(clientCredentialToken.id, [applicationKey, failureKey])) + const currentApplication = current.find((entry) => entry.id === applicationKey) + const fresh = await serveStored(currentApplication, minRemainingMs, true) if (fresh) return fresh + const remembered = await replayRemembered( + currentApplication, + current.find((entry) => entry.id === failureKey) + ) + if (remembered) return settle(remembered) await deleteExpiredClientTokenFailures() const outcome = await db.transaction< @@ -325,24 +373,17 @@ export async function resolveCoordinatedClientToken( .from(clientCredentialToken) .where(inArray(clientCredentialToken.id, [applicationKey, failureKey])) const applicationRow = entries.find((entry) => entry.id === applicationKey) - const minted = await serveIfFresh(applicationRow, true) + const minted = await serveStored(applicationRow, minRemainingMs, true) if (minted) return minted const failureRow = entries.find((entry) => entry.id === failureKey) - if (failureRow && failureRow.expiresAt.getTime() > Date.now()) { - const failure = replayedFailure( - decodeFailure((await decryptSecret(failureRow.encryptedValue)).decrypted, step), - step, - failureRow.expiresAt.getTime() - Date.now() - ) - if (classifyTokenRequestFailure(failure) !== 'permanent') { - const fallback = await stillValid(applicationRow) - if (fallback) return fallback - } - return { failure } - } + const rememberedUnderLock = await replayRemembered(applicationRow, failureRow) + if (rememberedUnderLock) return rememberedUnderLock signal?.throwIfAborted() try { - const exchanged = await requestTokenWithRetry(() => input.exchange(signal), signal) + const exchanged = + input.minExchangeIntervalMs === undefined + ? await requestTokenWithRetry(() => input.exchange(signal), signal) + : await input.exchange(signal) const { encrypted } = await encryptSecret( JSON.stringify({ accessToken: exchanged.accessToken, @@ -371,6 +412,7 @@ export async function resolveCoordinatedClientToken( error, await consecutiveFailuresBefore(failureRow, step) ) + const replayMs = Math.max(assessed.replayMs, input.minExchangeIntervalMs ?? 0) const failure: StoredFailure = { code: STORED_FAILURE_CODES.has(error.code) ? (error.code as StoredFailure['code']) @@ -381,7 +423,7 @@ export async function resolveCoordinatedClientToken( : { consecutiveFailures: assessed.consecutiveFailures }), } const { encrypted } = await encryptSecret(JSON.stringify(failure)) - const expiresAt = new Date(Date.now() + assessed.replayMs) + const expiresAt = new Date(Date.now() + replayMs) await tx .insert(clientCredentialToken) .values({ id: failureKey, encryptedValue: encrypted, expiresAt }) @@ -390,22 +432,20 @@ export async function resolveCoordinatedClientToken( set: { encryptedValue: encrypted, expiresAt, updatedAt: new Date() }, }) if (assessed.kind !== 'permanent') { - const fallback = await stillValid(applicationRow) + const fallback = await fallbackFrom(applicationRow) if (fallback) { logger.warn('Serving a still-valid client-credentials token after its refresh failed', { namespace, failure: assessed.kind, - retryInMs: assessed.replayMs, + retryInMs: replayMs, }) return fallback } } - return { failure: replayedFailure(failure, step, assessed.replayMs) } + return { failure: replayedFailure(failure, step, replayMs) } } }) - signal?.throwIfAborted() - if ('failure' in outcome) throw outcome.failure - return outcome + return settle(outcome) } catch (error) { signal?.throwIfAborted() if (error instanceof TokenServiceAccountValidationError) throw error diff --git a/apps/sim/lib/credentials/token-request-resilience.ts b/apps/sim/lib/credentials/token-request-resilience.ts index ad8b58a8353..0bae5667732 100644 --- a/apps/sim/lib/credentials/token-request-resilience.ts +++ b/apps/sim/lib/credentials/token-request-resilience.ts @@ -6,7 +6,7 @@ import { TokenServiceAccountValidationError } from '@/lib/credentials/token-serv * How a failed token request is retried: * - `permanent`: the provider refused the credentials or the account (or the failure is not a * provider answer at all); asking again cannot help until the secret changes. - * - `throttled`: a 429, or an outage with a `Retry-After`; the provider said when to come back. + * - `throttled`: a 429, or a 503 with a `Retry-After`; the provider is rate limiting the caller. * - `transient`: any other outage (5xx, timeout, network failure, malformed token response). */ export type TokenRequestFailureKind = 'permanent' | 'throttled' | 'transient' @@ -41,9 +41,17 @@ const PERMANENT_FAILURE_REPLAY_MS = 30_000 */ const RETRY_AFTER_BOUNDS = { baseMs: 1_000, maxMs: 5 * 60_000 } as const /** - * An outage is retried after 5 s at first (Entra's resilience guidance: the first back-off retry at - * least 5 s after the response), doubling while it continues, up to 30 s: the flat replay every - * failure used to get, so no outage is replayed longer than before. + * A throttled answer that names no `Retry-After` is replayed for at least a minute, doubling while + * throttling continues, up to the five-minute `Retry-After` cap. Token endpoints document their + * limits per minute (Vanta: 5 tokens and 25 token requests a minute), so asking again sooner lands + * in the window that was just exhausted. + */ +const THROTTLED_BACKOFF = { baseMs: 60_000, maxMs: RETRY_AFTER_BOUNDS.maxMs } as const +/** + * An outage is retried no sooner than 5 s after it (Entra's resilience guidance: without a + * `Retry-After`, the first back-off retry at least 5 s after the response), doubling with jitter + * while it continues, up to 30 s: the flat replay every failure used to get, so no outage is + * replayed longer than before. */ const OUTAGE_BACKOFF = { baseMs: 5_000, maxMs: 30_000 } as const /** The pause before the one in-request retry: MSAL's documented one-second retry-once. */ @@ -55,19 +63,32 @@ const IN_REQUEST_RETRY_DELAY = { baseMs: 1_000, maxMs: 1_000 } as const */ const IN_REQUEST_RETRY_BUDGET_MS = 2_000 +/** Whether a token request failure with this code is a refusal that asking again cannot fix. */ +export function isPermanentTokenFailureCode(code: string): boolean { + return code !== 'provider_unavailable' +} + /** Classifies a failed token request; see {@link TokenRequestFailureKind}. */ -export function classifyTokenRequestFailure(error: unknown): TokenRequestFailureKind { +function classifyTokenRequestFailure(error: unknown): TokenRequestFailureKind { if (!(error instanceof TokenServiceAccountValidationError)) return 'permanent' - if (error.code !== 'provider_unavailable') return 'permanent' - return error.status === 429 || error.retryAfterMs !== undefined ? 'throttled' : 'transient' + if (isPermanentTokenFailureCode(error.code)) return 'permanent' + return error.status === 429 || (error.status === 503 && error.retryAfterMs !== undefined) + ? 'throttled' + : 'transient' +} + +/** Exponential backoff with ±20% jitter, kept within `bounds` so jitter never undercuts the floor. */ +function boundedBackoff(attempt: number, bounds: { baseMs: number; maxMs: number }): number { + return Math.min(Math.max(backoffWithJitter(attempt, null, bounds), bounds.baseMs), bounds.maxMs) } /** - * Decides how long a failed token request is replayed. A failure carrying `retryAfterMs` (the - * provider's `Retry-After`, or what remains of a replay another layer already decided) is replayed - * that long within 1 s to 5 min. Otherwise a permanent failure is replayed 30 s and an outage backs - * off from 5 s to 30 s with jitter as it continues. `previousConsecutiveFailures` counts the - * non-permanent failures immediately before this one (0 after a success or a permanent failure). + * Decides how long a failed token request is replayed. A remembered failure another layer is + * replaying (`replayForMs`) is replayed exactly as long as that layer still does. Otherwise a + * provider's `Retry-After` is honored within 1 s to 5 min; without one, a throttled answer backs + * off from 1 min to 5 min, an outage from 5 s to 30 s, and a refusal is replayed 30 s. + * `previousConsecutiveFailures` counts the non-permanent failures immediately before this one (0 + * after a success or a permanent failure). */ export function assessTokenRequestFailure( error: unknown, @@ -75,26 +96,20 @@ export function assessTokenRequestFailure( ): TokenRequestFailure { const kind = classifyTokenRequestFailure(error) const consecutiveFailures = kind === 'permanent' ? 1 : previousConsecutiveFailures + 1 - const retryAfterMs = - error instanceof TokenServiceAccountValidationError ? error.retryAfterMs : undefined - if (retryAfterMs !== undefined) { - return { - kind, - consecutiveFailures, - replayMs: backoffWithJitter(1, retryAfterMs, RETRY_AFTER_BOUNDS), - } - } - if (kind === 'permanent') { - return { kind, consecutiveFailures, replayMs: PERMANENT_FAILURE_REPLAY_MS } - } - return { + const replay = (replayMs: number): TokenRequestFailure => ({ kind, consecutiveFailures, - replayMs: Math.min( - backoffWithJitter(consecutiveFailures, null, OUTAGE_BACKOFF), - OUTAGE_BACKOFF.maxMs - ), + replayMs, + }) + if (error instanceof TokenServiceAccountValidationError) { + if (error.replayForMs !== undefined) return replay(Math.max(error.replayForMs, 0)) + if (error.retryAfterMs !== undefined && kind !== 'permanent') { + return replay(backoffWithJitter(1, error.retryAfterMs, RETRY_AFTER_BOUNDS)) + } } + if (kind === 'permanent') return replay(PERMANENT_FAILURE_REPLAY_MS) + if (kind === 'throttled') return replay(boundedBackoff(consecutiveFailures, THROTTLED_BACKOFF)) + return replay(boundedBackoff(consecutiveFailures, OUTAGE_BACKOFF)) } /** diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.ts b/apps/sim/lib/credentials/token-service-accounts/errors.ts index 1a0a03e31d8..1cf8a805568 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.ts @@ -37,16 +37,25 @@ export type TokenServiceAccountValidationCode = export interface TokenServiceAccountValidationErrorOptions { /** See {@link TokenServiceAccountValidationError.retryAfterMs}. */ retryAfterMs?: number + /** See {@link TokenServiceAccountValidationError.replayForMs}. */ + replayForMs?: number } export class TokenServiceAccountValidationError extends Error { /** - * The earliest a repeat of the failed request can be expected to succeed, in milliseconds: the - * provider's `Retry-After` on a throttled or unavailable answer (uncapped, as the provider sent - * it), or how long a remembered failure is still replayed. Absent when nothing is known. + * The provider's `Retry-After` on a 429 or 503 answer, in milliseconds, uncapped as the provider + * sent it. Absent when the provider named none. Only ever set from a provider's own answer. */ public readonly retryAfterMs?: number + /** + * Set when this is a remembered failure being replayed rather than a fresh provider answer: how + * much longer, in milliseconds, it is replayed before the provider is asked again. It says when + * the provider will next be asked, not that asking will then succeed (a replayed refusal of the + * credentials carries it too). + */ + public readonly replayForMs?: number + constructor( public readonly code: TokenServiceAccountValidationCode, public readonly status: number, @@ -56,6 +65,7 @@ export class TokenServiceAccountValidationError extends Error { super(code) this.name = 'TokenServiceAccountValidationError' if (options.retryAfterMs !== undefined) this.retryAfterMs = options.retryAfterMs + if (options.replayForMs !== undefined) this.replayForMs = options.replayForMs } } diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index abb4cd33bf0..d9b92d612d0 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, RAMP_SERVICE_ACCOUNT_PROVIDER_ID, + VANTA_SERVICE_ACCOUNT_PROVIDER_ID, } from '@/lib/credentials/client-credential-accounts/descriptors' import * as clientAccounts from '@/lib/credentials/client-credential-accounts/server' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' @@ -168,9 +169,9 @@ describe('client credential token refresh after a failed mint', () => { ) /** Resolves with fake timers running, so an in-mint retry pause elapses without real waiting. */ - async function resolveFor(credentialId: string) { + async function resolveFor(credentialId: string, providerId = RAMP_SERVICE_ACCOUNT_PROVIDER_ID) { queueTableRows(credential, [storedCredential]) - const outcome = resolveServiceAccountToken(credentialId, RAMP_SERVICE_ACCOUNT_PROVIDER_ID).then( + const outcome = resolveServiceAccountToken(credentialId, providerId).then( (value) => ({ value }), (error: unknown) => ({ error }) ) @@ -322,4 +323,71 @@ describe('client credential token refresh after a failed mint', () => { expect(mint.mock.calls.length).toBeGreaterThan(asked) } }) + + it('waits at least a minute after a throttled answer that gave no Retry-After', async () => { + const mint = minter() + mint.mockRejectedValueOnce(unavailable(429)).mockResolvedValueOnce(token) + + await resolveFor('ramp-throttled-bare') + vi.advanceTimersByTime(59_000) + await expect(resolveFor('ramp-throttled-bare')).resolves.toEqual({ + error: expect.objectContaining({ status: 429 }), + }) + expect(mint).toHaveBeenCalledOnce() + + vi.advanceTimersByTime(14_000) + await expect(resolveFor('ramp-throttled-bare')).resolves.toEqual({ + value: { accessToken: token.accessToken }, + }) + }) + + it('waits at least 5 seconds before asking again after an outage, whatever the jitter', async () => { + const lowestJitter = vi + .spyOn(globalThis.crypto, 'getRandomValues') + .mockImplementation((array) => array) + try { + const mint = minter() + mint.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 10_000) + throw unavailable() + }) + + await resolveFor('ramp-outage-floor') + vi.advanceTimersByTime(4_900) + await resolveFor('ramp-outage-floor') + + expect(mint).toHaveBeenCalledOnce() + } finally { + lowestJitter.mockRestore() + } + }) + + it('replays a coordinated failure exactly as long as the coordinated layer still replays it', async () => { + encryptionMockFns.mockDecryptSecret.mockResolvedValue({ + decrypted: JSON.stringify({ + type: CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, + providerId: VANTA_SERVICE_ACCOUNT_PROVIDER_ID, + clientId: 'vanta-client', + clientSecret: 'vanta-client-secret', + }), + }) + const mint = minter() + mint + .mockRejectedValueOnce( + new TokenServiceAccountValidationError( + 'provider_unavailable', + 502, + { step: 'mint' }, + { replayForMs: 300 } + ) + ) + .mockResolvedValueOnce(token) + + await resolveFor('vanta-coordinated-replay', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) + vi.advanceTimersByTime(301) + + await expect( + resolveFor('vanta-coordinated-replay', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) + ).resolves.toEqual({ value: { accessToken: token.accessToken } }) + }) }) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 557181baf2a..7e5949002e6 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -675,8 +675,9 @@ interface FailedClientCredentialMint { * {@link assessTokenRequestFailure}) so a hot workflow neither hammers the * provider's token endpoint with a revoked secret nor floods a struggling one: * a refusal for 30 s, a throttled answer as long as its `Retry-After` asks (at - * most five minutes), and an outage for 5 s growing to 30 s while it lasts. A - * quick outage is first retried once inside the mint. + * most five minutes) or for 1 min growing to 5 min without one, and an outage + * for 5 s growing to 30 s while it lasts. A quick outage is first retried once + * inside the mint. * * Both maps are pruned of expired entries on each resolution * ({@link pruneExpiredClientCredentialCaches}), so their size is bounded by the From 3e85f962d637c1295e05290645c6e28d73a9e5ae Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 21:19:00 -0800 Subject: [PATCH 5/9] fix(credentials): honor every minter's Retry-After and close token fallback gaps - Carry the token endpoint's Retry-After on 429/503 failures from every bespoke client-credential minter (Box, Zoom, DocuSign, NetSuite, Help Scout, Ramp, Jama, Zendesk, Salesforce, Zoho Desk) and through the CrowdStrike, SailPoint and ZoomInfo client errors, so their throttled answers wait as long as the provider asks instead of the fixed backoff. - Recheck the 30 s expiry margin when the per-instance cache falls back to a still-valid token after a slow failed refresh. - Map an unreadable Vanta token body to provider_unavailable so it is remembered for every worker and paced at 15 s like any failed exchange. - Remember a refusal whose code the shared failure row cannot store with an additive `refusal` flag, so a replay never serves the stale token. - Rename the credentials helper to throttledRetryAfterMs so it no longer shares a name with providers/retry.ts. --- .../minters/box.test.ts | 10 +++++-- .../client-credential-accounts/minters/box.ts | 11 ++++--- .../minters/crowdstrike.test.ts | 14 +++++++++ .../minters/crowdstrike.ts | 3 +- .../minters/docusign.test.ts | 11 +++++++ .../minters/docusign.ts | 23 ++++++++------- .../minters/helpscout.test.ts | 24 +++++++++++++++ .../minters/helpscout.ts | 4 ++- .../minters/jama.test.ts | 14 +++++++++ .../minters/jama.ts | 5 +++- .../minters/netsuite.test.ts | 16 ++++++++++ .../minters/netsuite.ts | 11 ++++--- .../minters/ramp.test.ts | 10 +++++++ .../minters/ramp.ts | 4 ++- .../minters/sailpoint.test.ts | 11 +++++++ .../minters/sailpoint.ts | 3 +- .../minters/salesforce.test.ts | 10 +++++-- .../minters/salesforce.ts | 12 ++++---- .../minters/zendesk.test.ts | 11 +++++++ .../minters/zendesk.ts | 4 ++- .../minters/zoho-desk.test.ts | 7 +++-- .../minters/zoho-desk.ts | 18 ++++++++---- .../minters/zoom.test.ts | 10 +++++-- .../minters/zoom.ts | 11 ++++--- .../minters/zoominfo.test.ts | 11 +++++++ .../minters/zoominfo.ts | 3 +- .../vanta-token.integration.ts | 23 +++++++++++++++ .../client-credential-accounts/vanta-token.ts | 20 ++++++++----- .../credentials/client-credentials-grant.ts | 6 ++-- .../coordinated-client-token.integration.ts | 16 ++++++++++ .../credentials/coordinated-client-token.ts | 29 +++++++++++++------ .../token-service-accounts/errors.ts | 4 +-- apps/sim/lib/internal/crowdstrike/client.ts | 11 +++++-- apps/sim/lib/internal/sailpoint/client.ts | 8 +++-- apps/sim/lib/internal/zoominfo/client.ts | 9 ++++-- .../credential-service-cancellation.test.ts | 15 ++++++++++ apps/sim/lib/oauth/credential-service.ts | 11 ++++--- 37 files changed, 344 insertions(+), 79 deletions(-) create mode 100644 apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.test.ts diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts index 4a46a16d2bb..e7ff2e4c558 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts @@ -99,12 +99,18 @@ describe('mintBoxServiceAccountToken', () => { }) }) - it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'rate_limit_exceeded' }, 429)) + it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse( + { error: 'rate_limit_exceeded' }, + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) await expect(mintBoxServiceAccountToken(FIELDS)).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, + retryAfterMs: 7_000, }) }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/box.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/box.ts index 58a17d48fc3..0cf59991317 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/box.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/box.ts @@ -14,6 +14,7 @@ import { readProviderErrorSnippet, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const logger = createLogger('BoxServiceAccountMinter') @@ -176,10 +177,12 @@ export async function mintBoxServiceAccountToken( ...(hint ? { hint } : {}), }) } - throw new TokenServiceAccountValidationError('provider_unavailable', res.status, { - step: 'box_token_mint', - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + res.status, + { step: 'box_token_mint', body }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } const payload = await parseProviderJson(res, 'box_token_mint') diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts index 8a6b2bbf57a..8b636f864c1 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts @@ -84,6 +84,20 @@ describe('CrowdStrike API client token exchange', () => { await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({ code }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValue( + jsonResponse( + { errors: [{ code: 429, message: 'rate limited' }] }, + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) + + await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it("reports a redirect to the API client's own cloud as a wrong cloud and never follows it", async () => { mockFetch.mockResolvedValue( new Response(null, { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts index 6bf77c4c8fb..85474bf2121 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts @@ -82,7 +82,8 @@ export async function mintCrowdStrikeServiceAccountToken( throw new TokenServiceAccountValidationError( providerFailureCode(error.status), error.status, - { step: STEP, cloud } + { step: STEP, cloud }, + { retryAfterMs: error.retryAfterMs } ) } throw new TokenServiceAccountValidationError('provider_unavailable', 502, { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts index 51912c73c5a..ed68e865831 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts @@ -254,6 +254,17 @@ describe('mintDocuSignServiceAccountToken', () => { }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintDocuSignServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('keeps the key, assertion, and access token out of every failure', async () => { const failures: unknown[] = [] mockFetch.mockResolvedValueOnce( diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts index 2a0aee7ee39..f7e7e300541 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts @@ -19,6 +19,7 @@ import { parseProviderJson, readProviderErrorSnippet, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const TOKEN_STEP = 'docusign_token_mint' @@ -266,11 +267,12 @@ export async function mintDocuSignServiceAccountToken( { step: TOKEN_STEP, environment, body, ...(hint ? { hint } : {}) } ) } - throw new TokenServiceAccountValidationError('provider_unavailable', tokenRes.status, { - step: TOKEN_STEP, - environment, - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + tokenRes.status, + { step: TOKEN_STEP, environment, body }, + { retryAfterMs: throttledRetryAfterMs(tokenRes) } + ) } const payload = await parseProviderJson(tokenRes, TOKEN_STEP) @@ -297,11 +299,12 @@ export async function mintDocuSignServiceAccountToken( ) if (!userinfoRes.ok) { const body = await readProviderErrorSnippet(userinfoRes) - throw new TokenServiceAccountValidationError('provider_unavailable', userinfoRes.status, { - step: USERINFO_STEP, - environment, - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + userinfoRes.status, + { step: USERINFO_STEP, environment, body }, + { retryAfterMs: throttledRetryAfterMs(userinfoRes) } + ) } const userinfo = await parseProviderJson(userinfoRes, USERINFO_STEP) const accounts = (Array.isArray(userinfo.accounts) ? userinfo.accounts : []).filter( diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.test.ts new file mode 100644 index 00000000000..9266a12175c --- /dev/null +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.test.ts @@ -0,0 +1,24 @@ +import { jsonResponse } from '@sim/testing/helpers/http' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mintHelpScoutServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/helpscout' + +const FIELDS = { clientId: 'helpscout-client', clientSecret: 'helpscout-secret', orgId: '' } + +const mockFetch = vi.fn() + +describe('Help Scout client credential exchange', () => { + beforeEach(() => { + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) + }) + + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect( + mintHelpScoutServiceAccountToken(FIELDS, { skipIdentity: true }) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, retryAfterMs: 7_000 }) + }) +}) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts index 3ae63521c17..8b256d8dad7 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts @@ -11,6 +11,7 @@ import { isTransientProviderStatus, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const MAX_HELPSCOUT_AUTH_RESPONSE_BYTES = 64 * 1024 @@ -28,7 +29,8 @@ async function readHelpScoutAuthResponse( ? 'invalid_credentials' : 'provider_unavailable', response.status, - { step } + { step }, + { retryAfterMs: throttledRetryAfterMs(response) } ) } diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts index 66592bf3efa..e7163b7dda9 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts @@ -30,4 +30,18 @@ describe('Jama credential validation', () => { ).rejects.toMatchObject({ code: expectedCode, status }) } ) + + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + inputValidationMockFns.mockSecureFetchWithValidation.mockResolvedValueOnce( + Response.json({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect( + mintJamaServiceAccountToken({ + orgId: 'https://example.jamacloud.com', + clientId: 'client-id', + clientSecret: 'client-secret', + }) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, retryAfterMs: 7_000 }) + }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts index 22038795a26..463b9371dea 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts @@ -10,6 +10,7 @@ import { isTransientProviderStatus, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' import { normalizeJamaInstanceUrl } from '@/lib/jama/client' @@ -49,7 +50,9 @@ export async function mintJamaServiceAccountToken( !isTransientProviderStatus(response.status) ? 'invalid_credentials' : 'provider_unavailable', - response.status + response.status, + undefined, + { retryAfterMs: throttledRetryAfterMs(response) } ) } return toRecord(await response.json()) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts index d211bde2cf0..57ffd1443f7 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts @@ -148,6 +148,22 @@ describe('mintNetSuiteServiceAccountToken', () => { await expect(mintNetSuiteServiceAccountToken(FIELDS)).rejects.toMatchObject({ code, status }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValue( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + ) + + await expect(mintNetSuiteServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('redacts credentials and a reflected assertion from provider error metadata', async () => { let assertion = '' vi.stubGlobal( diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts index d0737e1d68a..9e3738e5970 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts @@ -17,6 +17,7 @@ import { tenantPrincipal } from '@/lib/credentials/principal' import { isTransientProviderStatus, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const TOKEN_PATH = '/services/rest/auth/oauth2/v1/token' @@ -141,10 +142,12 @@ async function exchangeNetSuiteToken( response.status >= 400 && response.status < 500 && !isTransientProviderStatus(response.status) ? 'invalid_credentials' : 'provider_unavailable' - throw new TokenServiceAccountValidationError(code, response.status, { - step: TOKEN_MINT_STEP, - body: sanitizeNetSuiteTokenError(body, fields, assertion), - }) + throw new TokenServiceAccountValidationError( + code, + response.status, + { step: TOKEN_MINT_STEP, body: sanitizeNetSuiteTokenError(body, fields, assertion) }, + { retryAfterMs: throttledRetryAfterMs(response) } + ) } let payload: NetSuiteTokenResponse diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.test.ts index f620f90fd59..4a32ae86b6a 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.test.ts @@ -207,6 +207,16 @@ describe('Ramp client credential exchange', () => { } ) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintRampServiceAccountToken(FIELDS, { skipIdentity: true })).rejects.toMatchObject( + { code: 'provider_unavailable', retryAfterMs: 7_000 } + ) + }) + it.each([400, 401, 403, 408, 429, 500])( 'classifies provider failure %s without retaining its secret-bearing body', async (status) => { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.ts index 65921857245..994c2b6bfdd 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.ts @@ -11,6 +11,7 @@ import { isTransientProviderStatus, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' import { getScopesForService } from '@/lib/oauth/utils' @@ -29,7 +30,8 @@ async function readRampAuthResponse( ? 'invalid_credentials' : 'provider_unavailable', response.status, - { step } + { step }, + { retryAfterMs: throttledRetryAfterMs(response) } ) } diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts index 186612668a7..cf7443dc65e 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts @@ -102,6 +102,17 @@ describe('SailPoint API client token exchange', () => { await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({ code }) }) + it("carries the token endpoint's Retry-After on an unavailable answer", async () => { + mockFetch.mockResolvedValue( + jsonResponse({ error: 'unavailable' }, { status: 503, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('treats a network failure or refused redirect as a provider failure', async () => { mockFetch.mockRejectedValue(new TypeError('fetch failed: unexpected redirect')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts index 9a8683ae38a..e4d9e187ba9 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts @@ -68,7 +68,8 @@ export async function mintSailPointServiceAccountToken( { step: STEP, tenant: host, - } + }, + { retryAfterMs: error.retryAfterMs } ) } if (isHostNotFound(error)) { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts index d918dac7a6c..7516fe0cf35 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts @@ -130,12 +130,18 @@ describe('mintSalesforceServiceAccountToken', () => { }) }) - it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'rate_limit_exceeded' }, 429)) + it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse( + { error: 'rate_limit_exceeded' }, + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) await expect(mintSalesforceServiceAccountToken(FIELDS)).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, + retryAfterMs: 7_000, }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.ts index 495dbb48310..87cc11ad645 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.ts @@ -21,6 +21,7 @@ import { providerFailureReason, readProviderErrorSnippet, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' /** @@ -463,11 +464,12 @@ export async function mintSalesforceServiceAccountToken( ...(hint ? { hint } : {}), }) } - throw new TokenServiceAccountValidationError('provider_unavailable', res.status, { - step: TOKEN_MINT_STEP, - host, - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + res.status, + { step: TOKEN_MINT_STEP, host, body }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } const payload = await parseProviderJson(res, TOKEN_MINT_STEP) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.test.ts index 00ccdccd258..a22b744cdd5 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.test.ts @@ -153,6 +153,17 @@ describe('Zendesk credential minting', () => { await expect(mintZendeskServiceAccountToken(CLIENT)).rejects.toMatchObject({ code }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintZendeskServiceAccountToken(CLIENT)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('refuses a token response without an access token', async () => { mockFetch.mockResolvedValueOnce(jsonResponse({ token_type: 'bearer' })) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts index ad7bae47c3a..062ad14b05d 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts @@ -18,6 +18,7 @@ import { providerFailureCode, readProviderErrorSnippet, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' import { zendeskApiTokenCredential } from '@/tools/zendesk/types' @@ -64,7 +65,8 @@ async function throwForZendeskResponse(response: Response, step: string): Promis throw new TokenServiceAccountValidationError( response.status === 404 ? 'site_not_found' : providerFailureCode(response.status), response.status, - { step, body } + { step, body }, + { retryAfterMs: throttledRetryAfterMs(response) } ) } diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts index a0d87d76ee7..a5342b83433 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts @@ -187,12 +187,15 @@ describe('mintZohoDeskServiceAccountToken', () => { }) }) - it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'too_many_requests' }, 429)) + it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'too_many_requests' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) await expect(mintZohoDeskServiceAccountToken(FIELDS)).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, + retryAfterMs: 7_000, }) }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts index 4113ce36120..6960d0583f4 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts @@ -20,6 +20,7 @@ import { readProviderErrorSnippet, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' import { tryDeriveZohoDeskBaseFromApiDomain } from '@/tools/zoho_desk/host-allowlist' @@ -199,12 +200,17 @@ export async function mintZohoDeskServiceAccountToken( if (res.status >= 400 && res.status < 500 && !isTransientProviderStatus(res.status)) { throw invalidCredentials(res.status, soid, dataCenter.id, body, zohoBodyError(body)) } - throw new TokenServiceAccountValidationError('provider_unavailable', res.status, { - step: STEP, - soid, - dataCenter: dataCenter.id, - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + res.status, + { + step: STEP, + soid, + dataCenter: dataCenter.id, + body, + }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } const payload = await parseProviderJson(res, STEP) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts index 63cf63dd9a8..6421913c8d1 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts @@ -69,12 +69,18 @@ describe('mintZoomServiceAccountToken', () => { }) }) - it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'rate_limit_exceeded' }, 429)) + it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse( + { error: 'rate_limit_exceeded' }, + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) await expect(mintZoomServiceAccountToken(FIELDS)).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, + retryAfterMs: 7_000, }) }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.ts index 4d86f95da32..89daf45c2a7 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.ts @@ -11,6 +11,7 @@ import { readProviderErrorSnippet, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const ZOOM_TOKEN_URL = 'https://zoom.us/oauth/token' @@ -88,10 +89,12 @@ export async function mintZoomServiceAccountToken( ...(hint ? { hint } : {}), }) } - throw new TokenServiceAccountValidationError('provider_unavailable', res.status, { - step: 'zoom_token_mint', - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + res.status, + { step: 'zoom_token_mint', body }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } const payload = await parseProviderJson(res, 'zoom_token_mint') diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.test.ts index 446f1ec8a7e..ddd9baf0210 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.test.ts @@ -64,6 +64,17 @@ describe('ZoomInfo client credentials exchange', () => { await expect(mintZoomInfoServiceAccountToken(FIELDS)).rejects.toMatchObject({ code }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockSecureFetchWithValidation.mockResolvedValue( + Response.json({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintZoomInfoServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('treats a refused redirect or network failure as a provider failure', async () => { mockSecureFetchWithValidation.mockRejectedValue(new Error('Too many redirects (max: 0)')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.ts index c7bda0f1c68..9b72f30d82c 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.ts @@ -56,7 +56,8 @@ export async function mintZoomInfoServiceAccountToken( throw new TokenServiceAccountValidationError( providerFailureCode(error.tokenStatus), error.tokenStatus, - { step: STEP } + { step: STEP }, + { retryAfterMs: error.retryAfterMs } ) } throw new TokenServiceAccountValidationError('provider_unavailable', 502, { diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts index 855b98b82e0..f015a7bf441 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts @@ -151,6 +151,11 @@ beforeAll(async () => { response.writeHead(502).end() return } + if (parsed.client_id.endsWith('-garbled')) { + response.writeHead(200, { 'Content-Type': 'text/html' }) + response.end('upstream proxy error') + return + } await sleep(15) const identity = `${request.headers['x-provider-origin']}:${parsed.client_id}` const issued = (applications.get(identity)?.issued ?? 0) + 1 @@ -380,6 +385,24 @@ describe('Vanta application token lifecycle against shared storage', () => { expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(15_000) }) + it('paces a Vanta that answers 200 with an unreadable body like any other failing exchange', async () => { + const input = fields('garbled') + const keys = previousBuildKeys(input, READ) + const count = tokenRequests + const before = Date.now() + + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ + code: 'provider_unavailable', + }) + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ + code: 'provider_unavailable', + }) + + expect(tokenRequests).toBe(count + 1) + const stored = await readStoredRow(keys.failureKey) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(15_000) + }) + it('fails closed when coordination storage fails', async () => { const count = tokenRequests vi.spyOn(db, 'transaction').mockRejectedValueOnce(new Error('database unavailable')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts index 7f9a50862f2..c32462d892f 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts @@ -8,9 +8,9 @@ import { import { fetchProvider, isTransientProviderStatus, - providerRetryAfterMs, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const SCOPES = VANTA_PERMISSION_OPTIONS.map((option) => option.value) @@ -98,14 +98,20 @@ async function exchangeToken( : 'provider_unavailable', response.status, { step: STEP, reason: 'provider rejected token exchange' }, - { retryAfterMs: providerRetryAfterMs(response) } + { retryAfterMs: throttledRetryAfterMs(response) } ) } - const payload = await readResponseJsonWithLimit(response, { - maxBytes: 64 * 1024, - label: 'Vanta authentication response', - signal, - }) + let payload: unknown + try { + payload = await readResponseJsonWithLimit(response, { + maxBytes: 64 * 1024, + label: 'Vanta authentication response', + signal, + }) + } catch { + signal?.throwIfAborted() + throw unavailable('provider returned an unreadable token response') + } if ( !isRecordLike(payload) || typeof payload.access_token !== 'string' || diff --git a/apps/sim/lib/credentials/client-credentials-grant.ts b/apps/sim/lib/credentials/client-credentials-grant.ts index e97e932e9ff..5aefff42d61 100644 --- a/apps/sim/lib/credentials/client-credentials-grant.ts +++ b/apps/sim/lib/credentials/client-credentials-grant.ts @@ -5,9 +5,9 @@ import { fetchProvider, providerFailureCode, providerFailureReason, - providerRetryAfterMs, readProviderErrorSnippet, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const MAX_TOKEN_RESPONSE_BYTES = 64 * 1024 @@ -35,7 +35,7 @@ export interface ClientCredentialsGrantRequest { /** * Maps a refused exchange to a provider-specific error. Without it, a refusal is * `invalid_credentials` for a non-transient 4xx and `provider_unavailable` otherwise. - * `retryAfterMs` is the refusal's `Retry-After` (see {@link providerRetryAfterMs}), which an + * `retryAfterMs` is the refusal's `Retry-After` (see {@link throttledRetryAfterMs}), which an * error the provider would answer differently later should carry. */ classifyRefusal?: ( @@ -86,7 +86,7 @@ export async function exchangeClientCredentialsGrant( if (!response.ok) { const body = await readProviderErrorSnippet(response) signal?.throwIfAborted() - const retryAfterMs = providerRetryAfterMs(response) + const retryAfterMs = throttledRetryAfterMs(response) throw request.classifyRefusal ? request.classifyRefusal(response.status, body, retryAfterMs) : new TokenServiceAccountValidationError( diff --git a/apps/sim/lib/credentials/coordinated-client-token.integration.ts b/apps/sim/lib/credentials/coordinated-client-token.integration.ts index 572fe56ec91..4b648177684 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.integration.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.integration.ts @@ -333,6 +333,22 @@ describe('coordinated token resolution while the provider fails', () => { expect(exchange).toHaveBeenCalledTimes(2) }) + it('keeps refusing for every worker a refusal whose code the shared row cannot store', async () => { + const exchange = issuingOnce('consent-refresh', 4 * 60_000, async () => { + throw new TokenServiceAccountValidationError('consent_required', 400, { step: 'mint' }) + }) + await resolveCoordinatedClientToken(input('consent-refresh', exchange)) + + for (let call = 0; call < 2; call++) { + await expect( + resolveCoordinatedClientToken(input('consent-refresh', exchange)) + ).rejects.toBeInstanceOf(TokenServiceAccountValidationError) + } + expect(exchange).toHaveBeenCalledTimes(2) + const payload: unknown = JSON.parse((await failureRowOf('consent-refresh')).decrypted) + expect(previousBuildDecodesFailure.coordinated(payload)).toBe(true) + }) + it('never serves a still-valid token to a different secret whose refresh fails', async () => { const exchange = issuingOnce('rotated-refresh', 4 * 60_000, failing(429)) await resolveCoordinatedClientToken(input('rotated-refresh', exchange)) diff --git a/apps/sim/lib/credentials/coordinated-client-token.ts b/apps/sim/lib/credentials/coordinated-client-token.ts index 406eb0b8dbf..020a771c96b 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.ts @@ -92,12 +92,14 @@ interface StoredApplicationToken { /** * A remembered exchange failure, replayed until its row's `expiresAt`. Builds before failure * kinds stored `{ code, status }` only and ignore other keys, so `consecutiveFailures` is - * written only for outages and throttling, to grow their backoff while they continue. + * written only for outages and throttling, to grow their backoff while they continue, and + * `refusal` only for a refusal whose code those builds read as `provider_unavailable`. */ interface StoredFailure { code: 'invalid_credentials' | 'grant_not_permitted' | 'site_not_found' | 'provider_unavailable' status: number consecutiveFailures?: number + refusal?: true } const STORED_FAILURE_CODES: ReadonlySet = new Set([ @@ -204,9 +206,15 @@ function decodeFailure(value: string, step: string): StoredFailure { ...(Number.isSafeInteger(parsed.consecutiveFailures) && Number(parsed.consecutiveFailures) > 0 ? { consecutiveFailures: Number(parsed.consecutiveFailures) } : {}), + ...(parsed.refusal === true ? { refusal: true } : {}), } } +/** Whether a remembered failure refused the credentials, so no stored token is served for it. */ +function isRememberedRefusal(failure: StoredFailure): boolean { + return failure.refusal === true || isPermanentTokenFailureCode(failure.code) +} + /** The error a remembered failure is replayed as, with how long it is still replayed. */ function replayedFailure( failure: StoredFailure, @@ -232,7 +240,7 @@ async function consecutiveFailuresBefore( if (!row || Date.now() - row.expiresAt.getTime() > TOKEN_FAILURE_STREAK_RETENTION_MS) return 0 try { const failure = decodeFailure((await decryptSecret(row.encryptedValue)).decrypted, step) - if (isPermanentTokenFailureCode(failure.code)) return 0 + if (isRememberedRefusal(failure)) return 0 return failure.consecutiveFailures ?? 1 } catch { return 0 @@ -332,7 +340,7 @@ export async function resolveCoordinatedClientToken( > => { if (!failureRow || failureRow.expiresAt.getTime() <= Date.now()) return undefined const failure = decodeFailure((await decryptSecret(failureRow.encryptedValue)).decrypted, step) - if (!isPermanentTokenFailureCode(failure.code)) { + if (!isRememberedRefusal(failure)) { const fallback = await fallbackFrom(applicationRow) if (fallback) return fallback } @@ -413,14 +421,17 @@ export async function resolveCoordinatedClientToken( await consecutiveFailuresBefore(failureRow, step) ) const replayMs = Math.max(assessed.replayMs, input.minExchangeIntervalMs ?? 0) + const code = STORED_FAILURE_CODES.has(error.code) + ? (error.code as StoredFailure['code']) + : 'provider_unavailable' const failure: StoredFailure = { - code: STORED_FAILURE_CODES.has(error.code) - ? (error.code as StoredFailure['code']) - : 'provider_unavailable', + code, status: error.status, - ...(assessed.kind === 'permanent' - ? {} - : { consecutiveFailures: assessed.consecutiveFailures }), + ...(assessed.kind !== 'permanent' + ? { consecutiveFailures: assessed.consecutiveFailures } + : isPermanentTokenFailureCode(code) + ? {} + : { refusal: true }), } const { encrypted } = await encryptSecret(JSON.stringify(failure)) const expiresAt = new Date(Date.now() + replayMs) diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.ts b/apps/sim/lib/credentials/token-service-accounts/errors.ts index 1cf8a805568..3364235d8a8 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.ts @@ -157,7 +157,7 @@ export function providerFailureCode( * as delay-seconds or an HTTP-date (RFC 9110 section 10.2.3); `undefined` for any other status or * a missing or unparsable header. Uncapped: the caller that waits decides how long is too long. */ -export function providerRetryAfterMs(response: ProviderResponse): number | undefined { +export function throttledRetryAfterMs(response: ProviderResponse): number | undefined { if (response.status !== 429 && response.status !== 503) return undefined return parseRetryAfter(response.headers.get('retry-after'), Number.POSITIVE_INFINITY) ?? undefined } @@ -307,6 +307,6 @@ export async function throwForProviderResponse( providerFailureCode(res.status), res.status, { step, body, ...context }, - { retryAfterMs: providerRetryAfterMs(res) } + { retryAfterMs: throttledRetryAfterMs(res) } ) } diff --git a/apps/sim/lib/internal/crowdstrike/client.ts b/apps/sim/lib/internal/crowdstrike/client.ts index 92c4974d3ee..2bd2523e8bf 100644 --- a/apps/sim/lib/internal/crowdstrike/client.ts +++ b/apps/sim/lib/internal/crowdstrike/client.ts @@ -2,6 +2,7 @@ import { toNumberOrNull, toStringOrNull } from '@sim/utils/coerce' import { isRecordLike, toRecordOrNull } from '@sim/utils/object' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { throttledRetryAfterMs } from '@/lib/credentials/token-service-accounts/errors' import type { CrowdStrikeCloud, CrowdStrikeInlineCredentials } from '@/tools/crowdstrike/types' export type JsonRecord = Record @@ -160,7 +161,12 @@ export function getFalconErrorMessage(data: unknown, fallback: string): string { export class CrowdStrikeAuthError extends Error { readonly status: number - constructor(message: string, status: number) { + constructor( + message: string, + status: number, + /** The token endpoint's `Retry-After` on a 429 or 503, in milliseconds. */ + readonly retryAfterMs?: number + ) { super(message) this.name = 'CrowdStrikeAuthError' this.status = status >= 400 && status <= 599 ? status : 502 @@ -263,7 +269,8 @@ export async function requestCrowdStrikeToken( if (!response.ok) { throw new CrowdStrikeAuthError( getFalconErrorMessage(data, 'Failed to authenticate with CrowdStrike'), - response.status + response.status, + throttledRetryAfterMs(response) ) } if (!isRecordLike(data) || typeof data.access_token !== 'string' || !data.access_token.trim()) { diff --git a/apps/sim/lib/internal/sailpoint/client.ts b/apps/sim/lib/internal/sailpoint/client.ts index 43d721cbfe5..a32b5e33668 100644 --- a/apps/sim/lib/internal/sailpoint/client.ts +++ b/apps/sim/lib/internal/sailpoint/client.ts @@ -9,6 +9,7 @@ import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' +import { throttledRetryAfterMs } from '@/lib/credentials/token-service-accounts/errors' /** API client credentials typed inline on the legacy block. */ export interface SailPointCredentials { @@ -29,7 +30,9 @@ export type SailPointAuth = SailPointCredentials | SailPointTokenCredentials export class SailPointAuthError extends Error { constructor( message: string, - readonly status: number + readonly status: number, + /** The token endpoint's `Retry-After` on a 429 or 503, in milliseconds. */ + readonly retryAfterMs?: number ) { super(message) this.name = 'SailPointAuthError' @@ -228,7 +231,8 @@ export async function requestSailPointToken( if (!response.ok) { throw new SailPointAuthError( getSailPointErrorMessage(data, 'Failed to authenticate with SailPoint'), - response.status + response.status, + throttledRetryAfterMs(response) ) } if (!isRecordLike(data) || typeof data.access_token !== 'string' || !data.access_token) { diff --git a/apps/sim/lib/internal/zoominfo/client.ts b/apps/sim/lib/internal/zoominfo/client.ts index 223100b23c0..5fc03de3859 100644 --- a/apps/sim/lib/internal/zoominfo/client.ts +++ b/apps/sim/lib/internal/zoominfo/client.ts @@ -5,6 +5,7 @@ import { MAX_JSON_API_RESPONSE_BYTES, secureFetchWithValidation, } from '@/lib/core/security/input-validation.server' +import { throttledRetryAfterMs } from '@/lib/credentials/token-service-accounts/errors' import { assertSafeZoomInfoUrl, ZOOMINFO_API_BASE, @@ -51,7 +52,11 @@ export class ZoomInfoOperationError extends Error { * provider status), and carries that status for the saved-credential minter. */ export class ZoomInfoTokenError extends ZoomInfoOperationError { - constructor(readonly tokenStatus: number) { + constructor( + readonly tokenStatus: number, + /** The token endpoint's `Retry-After` on a 429 or 503, in milliseconds. */ + readonly retryAfterMs?: number + ) { super(`ZoomInfo token request failed: HTTP ${tokenStatus}`, 500) this.name = 'ZoomInfoTokenError' } @@ -129,7 +134,7 @@ export async function requestZoomInfoToken( if (!response.ok) { const text = await response.text().catch(() => '') logger.warn('ZoomInfo token fetch failed', { requestId, status: response.status, error: text }) - throw new ZoomInfoTokenError(response.status) + throw new ZoomInfoTokenError(response.status, throttledRetryAfterMs(response)) } const data: unknown = await response.json() if (!isRecordLike(data) || typeof data.access_token !== 'string' || !data.access_token.trim()) { diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index d9b92d612d0..59cd476cf29 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -226,6 +226,21 @@ describe('client credential token refresh after a failed mint', () => { }) }) + it('does not serve a cached token that a slow failed refresh carried inside its expiry margin', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refresh-slow') + mint.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 15_000) + throw unavailable() + }) + vi.advanceTimersByTime(600_000 - 40_000) + + await expect(resolveFor('ramp-refresh-slow')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + }) + it('stops serving the cached token once the provider refuses the credentials', async () => { const mint = minter() mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 7e5949002e6..055b99b5a5c 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -777,13 +777,15 @@ async function resolveMintedServiceAccountToken( if (cached && cached.expiresAtMs - Date.now() > CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS) { return { ...cached.token } } - const stillValid = + /** The cached token while more than the expiry margin of it is left, checked at each use. */ + const stillValid = () => cached && cached.expiresAtMs - Date.now() > TOKEN_EXPIRY_MARGIN_MS ? cached : undefined const failedEntry = clientCredentialMintFailureCache.get(cacheIdentity) const failed = failedEntry?.secretFingerprint === secretFingerprint ? failedEntry : undefined if (failed && Date.now() < failed.retryAtMs) { - if (stillValid && failed.kind !== 'permanent') return { ...stillValid.token } + const fallback = failed.kind === 'permanent' ? undefined : stillValid() + if (fallback) return { ...fallback.token } throw failed.error } @@ -817,13 +819,14 @@ async function resolveMintedServiceAccountToken( retryAtMs, expiresAtMs: retryAtMs + TOKEN_FAILURE_STREAK_RETENTION_MS, }) - if (stillValid && failure.kind !== 'permanent') { + const fallback = failure.kind === 'permanent' ? undefined : stillValid() + if (fallback) { logger.warn('Serving a still-valid client-credential token after its refresh failed', { credentialId, failure: failure.kind, retryInMs: failure.replayMs, }) - return { ...stillValid.token } + return { ...fallback.token } } throw error } From 351cfa82270be9aee257866ba2b1be3882331033 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 11 Nov 2026 21:31:59 -0800 Subject: [PATCH 6/9] fix(credentials): keep throttling and outage backoff streaks apart and redact the fallback log in selector mode --- .../coordinated-client-token.integration.ts | 38 +++++++++++++++++++ .../credentials/coordinated-client-token.ts | 35 +++++++++++------ .../credentials/token-request-resilience.ts | 12 +++--- .../credential-service-cancellation.test.ts | 29 +++++++++++++- apps/sim/lib/oauth/credential-service.ts | 9 ++--- 5 files changed, 99 insertions(+), 24 deletions(-) diff --git a/apps/sim/lib/credentials/coordinated-client-token.integration.ts b/apps/sim/lib/credentials/coordinated-client-token.integration.ts index 4b648177684..c8777790a06 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.integration.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.integration.ts @@ -414,6 +414,44 @@ describe('coordinated token resolution while the provider fails', () => { ).toBeGreaterThanOrEqual(60_000) }) + it('backs off a throttled answer after an outage from a minute, not by the outage streak', async () => { + const exchange = vi.fn() + for (const status of [502, 502]) { + exchange.mockImplementation(failing(status)) + await expect( + resolveCoordinatedClientToken(input('outage-then-throttled', exchange)) + ).rejects.toThrow() + await expireFailureRowsOf('outage-then-throttled') + } + exchange.mockImplementation(failing(429)) + const before = Date.now() + + await expect( + resolveCoordinatedClientToken(input('outage-then-throttled', exchange)) + ).rejects.toMatchObject({ status: 429 }) + + const replayMs = (await failureRowOf('outage-then-throttled')).expiresAt.getTime() - before + expect(replayMs).toBeGreaterThanOrEqual(60_000) + expect(replayMs).toBeLessThanOrEqual(72_000 + 2_500) + }) + + it('backs off throttling that continues for every worker longer than its first minute', async () => { + const exchange = failing(429) + await expect( + resolveCoordinatedClientToken(input('throttled-twice', exchange)) + ).rejects.toThrow() + await expireFailureRowsOf('throttled-twice') + const before = Date.now() + + await expect( + resolveCoordinatedClientToken(input('throttled-twice', exchange)) + ).rejects.toThrow() + + expect( + (await failureRowOf('throttled-twice')).expiresAt.getTime() - before + ).toBeGreaterThanOrEqual(96_000) + }) + it.each([ [ 'the still-valid token while its refresh is remembered failing', diff --git a/apps/sim/lib/credentials/coordinated-client-token.ts b/apps/sim/lib/credentials/coordinated-client-token.ts index 020a771c96b..0a17ed9d52a 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.ts @@ -11,6 +11,7 @@ import { requestTokenWithRetry, TOKEN_EXPIRY_MARGIN_MS, TOKEN_FAILURE_STREAK_RETENTION_MS, + type TokenRequestFailure, } from '@/lib/credentials/token-request-resilience' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' @@ -92,13 +93,15 @@ interface StoredApplicationToken { /** * A remembered exchange failure, replayed until its row's `expiresAt`. Builds before failure * kinds stored `{ code, status }` only and ignore other keys, so `consecutiveFailures` is - * written only for outages and throttling, to grow their backoff while they continue, and - * `refusal` only for a refusal whose code those builds read as `provider_unavailable`. + * written only for outages and throttling, to grow their backoff while they continue, `throttled` + * only for throttling, so its streak is kept apart from an outage's, and `refusal` only for a + * refusal whose code those builds read as `provider_unavailable`. */ interface StoredFailure { code: 'invalid_credentials' | 'grant_not_permitted' | 'site_not_found' | 'provider_unavailable' status: number consecutiveFailures?: number + throttled?: true refusal?: true } @@ -206,6 +209,7 @@ function decodeFailure(value: string, step: string): StoredFailure { ...(Number.isSafeInteger(parsed.consecutiveFailures) && Number(parsed.consecutiveFailures) > 0 ? { consecutiveFailures: Number(parsed.consecutiveFailures) } : {}), + ...(parsed.throttled === true ? { throttled: true } : {}), ...(parsed.refusal === true ? { refusal: true } : {}), } } @@ -230,20 +234,24 @@ function replayedFailure( } /** - * Outages and throttled answers in a row before this exchange, read from the application's last - * failure memo for this secret once its replay has ended. Anything unreadable starts a new streak. + * The outage or throttling streak before this exchange, read from the application's last failure + * memo for this secret once its replay has ended. A refusal, or anything unreadable, starts a new + * streak. */ -async function consecutiveFailuresBefore( +async function failureStreakBefore( row: StoredRow | undefined, step: string -): Promise { - if (!row || Date.now() - row.expiresAt.getTime() > TOKEN_FAILURE_STREAK_RETENTION_MS) return 0 +): Promise | undefined> { + if (!row || Date.now() - row.expiresAt.getTime() > TOKEN_FAILURE_STREAK_RETENTION_MS) return try { const failure = decodeFailure((await decryptSecret(row.encryptedValue)).decrypted, step) - if (isRememberedRefusal(failure)) return 0 - return failure.consecutiveFailures ?? 1 + if (isRememberedRefusal(failure)) return + return { + kind: failure.throttled ? 'throttled' : 'transient', + consecutiveFailures: failure.consecutiveFailures ?? 1, + } } catch { - return 0 + return } } @@ -418,7 +426,7 @@ export async function resolveCoordinatedClientToken( if (!(error instanceof TokenServiceAccountValidationError)) throw error const assessed = assessTokenRequestFailure( error, - await consecutiveFailuresBefore(failureRow, step) + await failureStreakBefore(failureRow, step) ) const replayMs = Math.max(assessed.replayMs, input.minExchangeIntervalMs ?? 0) const code = STORED_FAILURE_CODES.has(error.code) @@ -428,7 +436,10 @@ export async function resolveCoordinatedClientToken( code, status: error.status, ...(assessed.kind !== 'permanent' - ? { consecutiveFailures: assessed.consecutiveFailures } + ? { + consecutiveFailures: assessed.consecutiveFailures, + ...(assessed.kind === 'throttled' ? { throttled: true } : {}), + } : isPermanentTokenFailureCode(code) ? {} : { refusal: true }), diff --git a/apps/sim/lib/credentials/token-request-resilience.ts b/apps/sim/lib/credentials/token-request-resilience.ts index 0bae5667732..6d8ed310c42 100644 --- a/apps/sim/lib/credentials/token-request-resilience.ts +++ b/apps/sim/lib/credentials/token-request-resilience.ts @@ -13,7 +13,7 @@ export type TokenRequestFailureKind = 'permanent' | 'throttled' | 'transient' export interface TokenRequestFailure { kind: TokenRequestFailureKind - /** Non-permanent failures in a row, this one included; 1 for a permanent failure. */ + /** Failures of this kind in a row, this one included; 1 for a permanent failure. */ consecutiveFailures: number /** How long the failure is replayed before the provider is asked again. */ replayMs: number @@ -87,15 +87,17 @@ function boundedBackoff(attempt: number, bounds: { baseMs: number; maxMs: number * replaying (`replayForMs`) is replayed exactly as long as that layer still does. Otherwise a * provider's `Retry-After` is honored within 1 s to 5 min; without one, a throttled answer backs * off from 1 min to 5 min, an outage from 5 s to 30 s, and a refusal is replayed 30 s. - * `previousConsecutiveFailures` counts the non-permanent failures immediately before this one (0 - * after a success or a permanent failure). + * `previous` is the remembered failure immediately before this one, if any: the backoff grows only + * while failures of the same kind continue, so a throttled answer after an outage starts its own + * streak at a minute instead of inheriting the outage's count. */ export function assessTokenRequestFailure( error: unknown, - previousConsecutiveFailures = 0 + previous?: Pick ): TokenRequestFailure { const kind = classifyTokenRequestFailure(error) - const consecutiveFailures = kind === 'permanent' ? 1 : previousConsecutiveFailures + 1 + const consecutiveFailures = + kind !== 'permanent' && previous?.kind === kind ? previous.consecutiveFailures + 1 : 1 const replay = (replayMs: number): TokenRequestFailure => ({ kind, consecutiveFailures, diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index 59cd476cf29..f392c6a7cbf 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -169,7 +169,10 @@ describe('client credential token refresh after a failed mint', () => { ) /** Resolves with fake timers running, so an in-mint retry pause elapses without real waiting. */ - async function resolveFor(credentialId: string, providerId = RAMP_SERVICE_ACCOUNT_PROVIDER_ID) { + async function resolveFor( + credentialId: string, + providerId: string = RAMP_SERVICE_ACCOUNT_PROVIDER_ID + ) { queueTableRows(credential, [storedCredential]) const outcome = resolveServiceAccountToken(credentialId, providerId).then( (value) => ({ value }), @@ -356,6 +359,30 @@ describe('client credential token refresh after a failed mint', () => { }) }) + it('waits about a minute, not an outage streak longer, for a throttled answer after an outage', async () => { + const mint = minter() + const answers = [unavailable(), unavailable(), unavailable(429)] + mint.mockImplementation(async () => { + throw answers[0] + }) + + await resolveFor('ramp-outage-then-throttled') + answers.shift() + vi.advanceTimersByTime(30_001) + await resolveFor('ramp-outage-then-throttled') + answers.shift() + vi.advanceTimersByTime(30_001) + await expect(resolveFor('ramp-outage-then-throttled')).resolves.toEqual({ + error: expect.objectContaining({ status: 429 }), + }) + mint.mockResolvedValueOnce(token) + + vi.advanceTimersByTime(73_000) + await expect(resolveFor('ramp-outage-then-throttled')).resolves.toEqual({ + value: { accessToken: token.accessToken }, + }) + }) + it('waits at least 5 seconds before asking again after an outage, whatever the jitter', async () => { const lowestJitter = vi .spyOn(globalThis.crypto, 'getRandomValues') diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 055b99b5a5c..eb0902c0b2f 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -639,7 +639,7 @@ interface FailedClientCredentialMint { /** The error the failed mint threw, re-thrown to callers while memoized. */ error: unknown kind: TokenRequestFailureKind - /** Non-permanent failures in a row; the next one backs off longer. */ + /** Failures of this kind in a row; the next one of the same kind backs off longer. */ consecutiveFailures: number secretFingerprint: string /** Until when the failure is replayed instead of minting again. */ @@ -806,10 +806,7 @@ async function resolveMintedServiceAccountToken( return { ...minted.token } } catch (error) { if (error instanceof Error && error.name === 'AbortError') throw error - const failure = assessTokenRequestFailure( - error, - failed && failed.kind !== 'permanent' ? failed.consecutiveFailures : 0 - ) + const failure = assessTokenRequestFailure(error, failed) const retryAtMs = Date.now() + failure.replayMs clientCredentialMintFailureCache.set(cacheIdentity, { error: options?.privacyMode === 'selector' ? new Error('Credential mint failed') : error, @@ -822,7 +819,7 @@ async function resolveMintedServiceAccountToken( const fallback = failure.kind === 'permanent' ? undefined : stillValid() if (fallback) { logger.warn('Serving a still-valid client-credential token after its refresh failed', { - credentialId, + ...(options?.privacyMode === 'selector' ? {} : { credentialId }), failure: failure.kind, retryInMs: failure.replayMs, }) From 23f3ac20837cd035bec5ab4e0229a200441b33a2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 04:03:07 -0800 Subject: [PATCH 7/9] fix(credentials): end a refused token, read legacy refusals as refusals, and keep throttle delays through SailPoint and CrowdStrike - A refusal of the credentials drops the per-instance cached token and expires the shared token minted with that secret, so a later outage cannot serve it again. - A previous build's provider_unavailable memo with a non-transient 4xx status replays as a refusal instead of serving the stored token. - The SailPoint saved-credential minter returns the first 429 with its Retry-After instead of retrying until the exchange timeout swallows it. - A CrowdStrike error whose body cannot be read keeps its status and Retry-After. - Vanta logs a malformed or oversized token response apart from an unreadable stream. --- .../minters/crowdstrike.test.ts | 19 +++++++ .../minters/sailpoint.test.ts | 13 +++++ .../minters/sailpoint.ts | 7 ++- .../client-credential-accounts/vanta-token.ts | 10 ++-- .../coordinated-client-token.integration.ts | 52 +++++++++++++++--- .../credentials/coordinated-client-token.ts | 53 ++++++++++++++----- apps/sim/lib/internal/crowdstrike/client.ts | 7 ++- apps/sim/lib/internal/sailpoint/client.ts | 10 ++-- .../credential-service-cancellation.test.ts | 14 +++++ apps/sim/lib/oauth/credential-service.ts | 9 +++- 10 files changed, 162 insertions(+), 32 deletions(-) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts index 8b636f864c1..795bdc4ab3f 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts @@ -98,6 +98,25 @@ describe('CrowdStrike API client token exchange', () => { }) }) + it("keeps a throttled answer's status and Retry-After when its error body cannot be read", async () => { + mockFetch.mockResolvedValue( + new Response( + new ReadableStream({ + start(controller) { + controller.error(new TypeError('terminated')) + }, + }), + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) + + await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + retryAfterMs: 7_000, + }) + }) + it("reports a redirect to the API client's own cloud as a wrong cloud and never follows it", async () => { mockFetch.mockResolvedValue( new Response(null, { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts index cf7443dc65e..3468240797d 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts @@ -113,6 +113,19 @@ describe('SailPoint API client token exchange', () => { }) }) + it('returns the first throttled answer with its Retry-After instead of retrying it', async () => { + mockFetch.mockImplementation(async () => + jsonResponse({ error: 'rate limited' }, { status: 429, headers: { 'Retry-After': '120' } }) + ) + + await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + retryAfterMs: 120_000, + }) + expect(mockFetch).toHaveBeenCalledOnce() + }) + it('treats a network failure or refused redirect as a provider failure', async () => { mockFetch.mockRejectedValue(new TypeError('fetch failed: unexpected redirect')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts index e4d9e187ba9..9d218df7f61 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts @@ -18,7 +18,7 @@ import { } from '@/lib/internal/sailpoint/client' const STEP = 'sailpoint_token_mint' -/** Bounds the whole exchange, including its 429 backoff retries. */ +/** Bounds the whole exchange. */ const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000 /** @@ -32,6 +32,8 @@ const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000 * from `expires_in`; the credential cache re-mints before it lapses. * * The token response names no identity, so the connect-time check is the mint. + * A 429 is returned at once with its `Retry-After` rather than retried here, so + * the credential resolver's throttling backoff waits as long as SailPoint asks. * * @see https://developer.sailpoint.com/docs/api/authentication */ @@ -57,7 +59,8 @@ export async function mintSailPointServiceAccountToken( tokenUrl, fields.clientId, clientSecret, - options?.signal ? AbortSignal.any([options.signal, timeout]) : timeout + options?.signal ? AbortSignal.any([options.signal, timeout]) : timeout, + { maxThrottleRetries: 0 } ) } catch (error) { options?.signal?.throwIfAborted() diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts index c32462d892f..a3ea31abb02 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts @@ -1,5 +1,5 @@ import { isRecordLike } from '@sim/utils/object' -import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { PayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { VANTA_PERMISSION_OPTIONS } from '@/lib/credentials/client-credential-accounts/descriptors' import { invalidateCoordinatedClientToken, @@ -108,9 +108,13 @@ async function exchangeToken( label: 'Vanta authentication response', signal, }) - } catch { + } catch (error) { signal?.throwIfAborted() - throw unavailable('provider returned an unreadable token response') + throw unavailable( + error instanceof SyntaxError || error instanceof PayloadSizeLimitError + ? 'provider returned an invalid token response' + : 'provider token response could not be read' + ) } if ( !isRecordLike(payload) || diff --git a/apps/sim/lib/credentials/coordinated-client-token.integration.ts b/apps/sim/lib/credentials/coordinated-client-token.integration.ts index c8777790a06..b2454165b52 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.integration.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.integration.ts @@ -333,6 +333,32 @@ describe('coordinated token resolution while the provider fails', () => { expect(exchange).toHaveBeenCalledTimes(2) }) + it('never serves a refused token again once the refusal memo ends and an outage follows', async () => { + const exchange = issuingOnce('refused-then-outage', 4 * 60_000, rejecting()) + await resolveCoordinatedClientToken(input('refused-then-outage', exchange)) + await expect( + resolveCoordinatedClientToken(input('refused-then-outage', exchange)) + ).rejects.toMatchObject({ code: 'invalid_credentials' }) + await expireFailureRowsOf('refused-then-outage') + exchange.mockImplementation(failing(503)) + + await expect( + resolveCoordinatedClientToken(input('refused-then-outage', exchange)) + ).rejects.toMatchObject({ code: 'provider_unavailable' }) + }) + + it('keeps the token another secret minted when a different secret is refused', async () => { + const exchange = issuingOnce('refused-other-secret', 4 * 60_000, failing(503)) + await resolveCoordinatedClientToken(input('refused-other-secret', exchange)) + await expect( + resolveCoordinatedClientToken(input('refused-other-secret', rejecting(), 'revoked')) + ).rejects.toMatchObject({ code: 'invalid_credentials' }) + + await expect( + resolveCoordinatedClientToken(input('refused-other-secret', exchange)) + ).resolves.toMatchObject({ accessToken: 'refused-other-secret:token-1' }) + }) + it('keeps refusing for every worker a refusal whose code the shared row cannot store', async () => { const exchange = issuingOnce('consent-refresh', 4 * 60_000, async () => { throw new TokenServiceAccountValidationError('consent_required', 400, { step: 'mint' }) @@ -562,16 +588,14 @@ describe('coordinated token resolution while the provider fails', () => { }) describe('default token rows shared with the previous build during a rolling deploy', () => { - it('serves a still-valid token when the previous build remembered an outage', async () => { - const application = 'legacy-outage' + /** Seeds a still-valid token and the previous build's failure memo of `status` for one secret. */ + async function seedPreviousBuildFailure(application: string, status: number) { const appKey = applicationKey(NAMESPACE, [application]) const fingerprint = sha256Json([application, 'secret-1']) const token = await encryptSecret( JSON.stringify({ accessToken: 'legacy-token', secretFingerprint: fingerprint }) ) - const outage = await encryptSecret( - JSON.stringify({ code: 'provider_unavailable', status: 503 }) - ) + const failure = await encryptSecret(JSON.stringify({ code: 'provider_unavailable', status })) await db.insert(clientCredentialToken).values([ { id: appKey, @@ -581,10 +605,15 @@ describe('default token rows shared with the previous build during a rolling dep }, { id: `${appKey}:${sha256Json(fingerprint)}`, - encryptedValue: outage.encrypted, + encryptedValue: failure.encrypted, expiresAt: new Date(Date.now() + 20_000), }, ]) + } + + it('serves a still-valid token when the previous build remembered an outage', async () => { + const application = 'legacy-outage' + await seedPreviousBuildFailure(application, 503) const { exchange } = provider(application) await expect( @@ -593,6 +622,17 @@ describe('default token rows shared with the previous build during a rolling dep expect(exchange).not.toHaveBeenCalled() }) + it('replays a refusal the previous build stored as provider_unavailable without serving the token', async () => { + const application = 'legacy-unmapped-refusal' + await seedPreviousBuildFailure(application, 400) + const { exchange } = provider(application) + + await expect(resolveCoordinatedClientToken(input(application, exchange))).rejects.toMatchObject( + { code: 'provider_unavailable', status: 400 } + ) + expect(exchange).not.toHaveBeenCalled() + }) + it('stores every outage and throttled answer as a failure the previous build reads', async () => { for (const [application, exchange] of [ ['legacy-readable-outage', failing(502)], diff --git a/apps/sim/lib/credentials/coordinated-client-token.ts b/apps/sim/lib/credentials/coordinated-client-token.ts index 0a17ed9d52a..aa6dd77b900 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.ts @@ -13,7 +13,10 @@ import { TOKEN_FAILURE_STREAK_RETENTION_MS, type TokenRequestFailure, } from '@/lib/credentials/token-request-resilience' -import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' +import { + providerFailureCode, + TokenServiceAccountValidationError, +} from '@/lib/credentials/token-service-accounts/errors' import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' const logger = createLogger('CoordinatedClientToken') @@ -214,9 +217,18 @@ function decodeFailure(value: string, step: string): StoredFailure { } } -/** Whether a remembered failure refused the credentials, so no stored token is served for it. */ +/** + * Whether a remembered failure refused the credentials, so no stored token is served for it. A + * build before failure kinds stored a refusal whose code it could not store as + * `provider_unavailable` with the provider's status and no streak count, so such a row whose + * status names a refusal (a non-transient 4xx) is one too. + */ function isRememberedRefusal(failure: StoredFailure): boolean { - return failure.refusal === true || isPermanentTokenFailureCode(failure.code) + if (failure.refusal === true || isPermanentTokenFailureCode(failure.code)) return true + return ( + failure.consecutiveFailures === undefined && + isPermanentTokenFailureCode(providerFailureCode(failure.status)) + ) } /** The error a remembered failure is replayed as, with how long it is still replayed. */ @@ -279,7 +291,8 @@ function isServable(expiresAt: Date, minRemainingMs: number): boolean { * for a minute growing to five without one, and an outage for 5 seconds growing to 30 while it * continues, so a hot workflow neither hammers the token endpoint with a revoked secret nor floods * a struggling one. While an outage or throttling is remembered, the stored token for the same - * secret is still served until shortly before it actually expires; a refusal is reported at once. + * secret is still served until shortly before it actually expires; a refusal is reported at once + * and expires that secret's stored token, so no later outage serves it again. * A remembered failure, and that fallback, are served from committed rows without taking the * application lock. Tokens are encrypted at rest. */ @@ -335,6 +348,13 @@ export async function resolveCoordinatedClientToken( const fallbackFrom = (row: StoredRow | undefined) => serveStored(row, fallbackRemainingMs).catch(() => undefined) + /** + * Whether `row` holds a token minted with this secret, expired or not, so a refusal of the + * secret ends that token too and a token another secret minted stays in service. + */ + const holdsTokenForSecret = async (row: StoredRow | undefined) => + (await serveStored(row, Number.NEGATIVE_INFINITY).catch(() => undefined)) !== undefined + /** * Replays the application's unexpired failure memo for this secret: the still-valid token for a * remembered outage or throttling, the failure otherwise. `undefined` when nothing is remembered. @@ -453,16 +473,23 @@ export async function resolveCoordinatedClientToken( target: clientCredentialToken.id, set: { encryptedValue: encrypted, expiresAt, updatedAt: new Date() }, }) - if (assessed.kind !== 'permanent') { - const fallback = await fallbackFrom(applicationRow) - if (fallback) { - logger.warn('Serving a still-valid client-credentials token after its refresh failed', { - namespace, - failure: assessed.kind, - retryInMs: replayMs, - }) - return fallback + if (assessed.kind === 'permanent') { + if (await holdsTokenForSecret(applicationRow)) { + await tx + .update(clientCredentialToken) + .set({ expiresAt: new Date(0), updatedAt: new Date() }) + .where(eq(clientCredentialToken.id, applicationKey)) } + return { failure: replayedFailure(failure, step, replayMs) } + } + const fallback = await fallbackFrom(applicationRow) + if (fallback) { + logger.warn('Serving a still-valid client-credentials token after its refresh failed', { + namespace, + failure: assessed.kind, + retryInMs: replayMs, + }) + return fallback } return { failure: replayedFailure(failure, step, replayMs) } } diff --git a/apps/sim/lib/internal/crowdstrike/client.ts b/apps/sim/lib/internal/crowdstrike/client.ts index 2bd2523e8bf..4ab2896711e 100644 --- a/apps/sim/lib/internal/crowdstrike/client.ts +++ b/apps/sim/lib/internal/crowdstrike/client.ts @@ -226,7 +226,8 @@ export class CrowdStrikeTokenRedirectError extends Error { * With `followRedirects: false` a redirect is never followed and throws * {@link CrowdStrikeTokenRedirectError}, so the secret reaches no other host; * the legacy block's inline path keeps following one as it always has. A - * refusal throws {@link CrowdStrikeAuthError} with Falcon's status. + * refusal throws {@link CrowdStrikeAuthError} with Falcon's status and + * `Retry-After`, even when its error body cannot be read. * * @see https://developer.crowdstrike.com/api-reference/collections/oauth2.md */ @@ -264,7 +265,9 @@ export async function requestCrowdStrikeToken( ) } - const data = await readFalconJson(response, signal) + const data = response.ok + ? await readFalconJson(response, signal) + : await readFalconJson(response, signal).catch(() => null) signal?.throwIfAborted() if (!response.ok) { throw new CrowdStrikeAuthError( diff --git a/apps/sim/lib/internal/sailpoint/client.ts b/apps/sim/lib/internal/sailpoint/client.ts index a32b5e33668..dc218e7f05c 100644 --- a/apps/sim/lib/internal/sailpoint/client.ts +++ b/apps/sim/lib/internal/sailpoint/client.ts @@ -183,8 +183,9 @@ async function readBoundedBody( /** * Exchanges API client credentials for an access token at a tenant's token * endpoint (`client_credentials` grant, credentials in the form body), with - * redirects refused and up to four backoff retries on 429. A refusal throws - * {@link SailPointAuthError} with the provider's status and message. + * redirects refused and up to four backoff retries on 429 (`maxThrottleRetries`). + * A refusal throws {@link SailPointAuthError} with the provider's status, + * message, and `Retry-After`. * * @see https://developer.sailpoint.com/docs/api/authentication */ @@ -192,7 +193,8 @@ export async function requestSailPointToken( tokenUrl: string, clientId: string, clientSecret: string, - signal: AbortSignal + signal: AbortSignal, + { maxThrottleRetries = MAX_FETCH_RETRIES }: { maxThrottleRetries?: number } = {} ): Promise<{ token: string; expiresInSeconds: number }> { let attempt = 0 @@ -214,7 +216,7 @@ export async function requestSailPointToken( signal, }) - if (response.status === 429 && attempt < MAX_FETCH_RETRIES) { + if (response.status === 429 && attempt < maxThrottleRetries) { const retryAfterMs = parseRetryAfter(response.headers.get('retry-after')) await consumeOrCancelBody(response, DEFAULT_MAX_ERROR_BODY_BYTES) attempt += 1 diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index f392c6a7cbf..856da73ff1f 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -257,6 +257,20 @@ describe('client credential token refresh after a failed mint', () => { expect(mint).toHaveBeenCalledTimes(2) }) + it('never serves a refused token again when a later refresh fails with an outage', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refused-then-outage') + mint.mockRejectedValueOnce(refused()).mockRejectedValue(unavailable()) + vi.advanceTimersByTime(6 * 60_000) + await resolveFor('ramp-refused-then-outage') + vi.advanceTimersByTime(31_000) + + await expect(resolveFor('ramp-refused-then-outage')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + }) + it('retries a quick transient failure once inside the mint', async () => { const mint = minter() mint.mockRejectedValueOnce(unavailable(503)).mockResolvedValueOnce(token) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index eb0902c0b2f..2d69c7133eb 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -733,7 +733,8 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string { * per-instance cache: re-reads the credential row, serves a cached token minted * from the same ciphertext while enough of it remains, replays a recent mint * failure, and otherwise decrypts and mints, falling back to the cached token - * while it is still valid when the refresh fails with an outage. Wrapped in + * while it is still valid when the refresh fails with an outage and dropping it + * when the refresh is refused, so no later outage serves it again. Wrapped in * `coalesceLocally` so concurrent block executions on one instance share a * single mint. `sharedToken` providers coordinate one token per application * themselves (including the in-request retry and the fallback), so their tokens @@ -816,7 +817,11 @@ async function resolveMintedServiceAccountToken( retryAtMs, expiresAtMs: retryAtMs + TOKEN_FAILURE_STREAK_RETENTION_MS, }) - const fallback = failure.kind === 'permanent' ? undefined : stillValid() + if (failure.kind === 'permanent') { + clientCredentialTokenCache.delete(cacheIdentity) + throw error + } + const fallback = stillValid() if (fallback) { logger.warn('Serving a still-valid client-credential token after its refresh failed', { ...(options?.privacyMode === 'selector' ? {} : { credentialId }), From 2a94a6291c91831d338d2666248cda3a81934eec Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 05:17:38 -0800 Subject: [PATCH 8/9] fix(credentials): keep SailPoint throttle delays through unreadable error bodies and replay only coordinated failures locally - A SailPoint token error whose body cannot be read keeps its status and Retry-After, as CrowdStrike's does; caller cancellation still propagates. - A shared-token (Vanta) mint failure is replayed per instance only when the coordinated layer remembers it (replayForMs); a failure it does not remember asks again at once. --- .../minters/sailpoint.test.ts | 19 +++++++++++++++ apps/sim/lib/internal/sailpoint/client.ts | 11 ++++----- .../credential-service-cancellation.test.ts | 23 +++++++++++++++++++ apps/sim/lib/oauth/credential-service.ts | 19 +++++++++++++-- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts index 3468240797d..4ac73a2e329 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts @@ -126,6 +126,25 @@ describe('SailPoint API client token exchange', () => { expect(mockFetch).toHaveBeenCalledOnce() }) + it("keeps a throttled answer's status and Retry-After when its error body cannot be read", async () => { + mockFetch.mockResolvedValue( + new Response( + new ReadableStream({ + start(controller) { + controller.error(new TypeError('terminated')) + }, + }), + { status: 429, headers: { 'Retry-After': '120' } } + ) + ) + + await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + retryAfterMs: 120_000, + }) + }) + it('treats a network failure or refused redirect as a provider failure', async () => { mockFetch.mockRejectedValue(new TypeError('fetch failed: unexpected redirect')) diff --git a/apps/sim/lib/internal/sailpoint/client.ts b/apps/sim/lib/internal/sailpoint/client.ts index dc218e7f05c..910b1273dec 100644 --- a/apps/sim/lib/internal/sailpoint/client.ts +++ b/apps/sim/lib/internal/sailpoint/client.ts @@ -185,7 +185,7 @@ async function readBoundedBody( * endpoint (`client_credentials` grant, credentials in the form body), with * redirects refused and up to four backoff retries on 429 (`maxThrottleRetries`). * A refusal throws {@link SailPointAuthError} with the provider's status, - * message, and `Retry-After`. + * message, and `Retry-After`, even when its error body cannot be read. * * @see https://developer.sailpoint.com/docs/api/authentication */ @@ -225,11 +225,10 @@ export async function requestSailPointToken( continue } - const data = await readBoundedBody( - response, - response.ok ? MAX_TOKEN_RESPONSE_BYTES : DEFAULT_MAX_ERROR_BODY_BYTES, - signal - ) + const data = response.ok + ? await readBoundedBody(response, MAX_TOKEN_RESPONSE_BYTES, signal) + : await readBoundedBody(response, DEFAULT_MAX_ERROR_BODY_BYTES, signal).catch(() => null) + signal?.throwIfAborted() if (!response.ok) { throw new SailPointAuthError( getSailPointErrorMessage(data, 'Failed to authenticate with SailPoint'), diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index 856da73ff1f..cf8e4ceb793 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -446,4 +446,27 @@ describe('client credential token refresh after a failed mint', () => { resolveFor('vanta-coordinated-replay', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) ).resolves.toEqual({ value: { accessToken: token.accessToken } }) }) + + it('asks the coordinated layer again at once after a failure it does not remember', async () => { + encryptionMockFns.mockDecryptSecret.mockResolvedValue({ + decrypted: JSON.stringify({ + type: CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, + providerId: VANTA_SERVICE_ACCOUNT_PROVIDER_ID, + clientId: 'vanta-client', + clientSecret: 'vanta-client-secret', + }), + }) + const mint = minter() + mint + .mockRejectedValueOnce( + new TokenServiceAccountValidationError('provider_unavailable', 503, { step: 'mint' }) + ) + .mockResolvedValueOnce(token) + + await resolveFor('vanta-coordination-failed', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) + + await expect( + resolveFor('vanta-coordination-failed', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) + ).resolves.toEqual({ value: { accessToken: token.accessToken } }) + }) }) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 2d69c7133eb..b335af6956e 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -39,6 +39,7 @@ import { isTokenServiceAccountProviderId, SNOWFLAKE_SERVICE_ACCOUNT_PROVIDER_ID, } from '@/lib/credentials/token-service-accounts/descriptors' +import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { parseStoredSecretMode, parseTokenServiceAccountSecret, @@ -728,6 +729,16 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string { return encryptedServiceAccountKey.slice(0, 32) } +/** + * Whether a `sharedToken` mint failed with a failure the coordinated layer remembers for every + * worker. Only those are replayed here, for exactly as long as that layer still replays them; a + * failure it does not remember (a lock timeout, a database error) may already be over for + * another worker, so the next resolution asks again. + */ +function isCoordinatedReplay(error: unknown): boolean { + return error instanceof TokenServiceAccountValidationError && error.replayForMs !== undefined +} + /** * Resolves a stored secret that mints short-lived tokens, read-through the * per-instance cache: re-reads the credential row, serves a cached token minted @@ -738,8 +749,8 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string { * `coalesceLocally` so concurrent block executions on one instance share a * single mint. `sharedToken` providers coordinate one token per application * themselves (including the in-request retry and the fallback), so their tokens - * are never cached here and a failure is replayed only as long as their own - * replay reports; a caller that already read the secret to choose its mode + * are never cached here and only a failure their own layer remembers is + * replayed, for as long as it reports; a caller that already read the secret to choose its mode * passes it as `stored` rather than reading it twice. */ async function resolveMintedServiceAccountToken( @@ -807,6 +818,10 @@ async function resolveMintedServiceAccountToken( return { ...minted.token } } catch (error) { if (error instanceof Error && error.name === 'AbortError') throw error + if (sharedToken && !isCoordinatedReplay(error)) { + clientCredentialMintFailureCache.delete(cacheIdentity) + throw error + } const failure = assessTokenRequestFailure(error, failed) const retryAtMs = Date.now() + failure.replayMs clientCredentialMintFailureCache.set(cacheIdentity, { From 9067a85478d26e00977e50ea3194cfc36c31b05e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 06:48:06 -0800 Subject: [PATCH 9/9] fix(credentials): skip the fallback token once a provider refused it during the failed refresh --- .../oauth/credential-service-cancellation.test.ts | 15 +++++++++++++++ apps/sim/lib/oauth/credential-service.ts | 11 +++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index cf8e4ceb793..3fe7db1a82f 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -244,6 +244,21 @@ describe('client credential token refresh after a failed mint', () => { }) }) + it('does not serve a cached token the provider refused while its failed refresh was in flight', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refresh-rejected-midway') + mint.mockImplementation(async () => { + forgetRejectedClientCredentialToken('current-token') + throw unavailable() + }) + vi.advanceTimersByTime(6 * 60_000) + + await expect(resolveFor('ramp-refresh-rejected-midway')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + }) + it('stops serving the cached token once the provider refuses the credentials', async () => { const mint = minter() mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index b335af6956e..7c8ec3ed57b 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -789,9 +789,16 @@ async function resolveMintedServiceAccountToken( if (cached && cached.expiresAtMs - Date.now() > CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS) { return { ...cached.token } } - /** The cached token while more than the expiry margin of it is left, checked at each use. */ + /** + * The cached token while it is still the one in the cache (a provider may refuse it while a + * refresh is in flight) and more than the expiry margin of it is left, checked at each use. + */ const stillValid = () => - cached && cached.expiresAtMs - Date.now() > TOKEN_EXPIRY_MARGIN_MS ? cached : undefined + cached && + clientCredentialTokenCache.get(cacheIdentity) === cached && + cached.expiresAtMs - Date.now() > TOKEN_EXPIRY_MARGIN_MS + ? cached + : undefined const failedEntry = clientCredentialMintFailureCache.get(cacheIdentity) const failed = failedEntry?.secretFingerprint === secretFingerprint ? failedEntry : undefined