diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts index 4a46a16d2bb..e7ff2e4c558 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts @@ -99,12 +99,18 @@ describe('mintBoxServiceAccountToken', () => { }) }) - it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'rate_limit_exceeded' }, 429)) + it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse( + { error: 'rate_limit_exceeded' }, + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) await expect(mintBoxServiceAccountToken(FIELDS)).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, + retryAfterMs: 7_000, }) }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/box.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/box.ts index 58a17d48fc3..0cf59991317 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/box.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/box.ts @@ -14,6 +14,7 @@ import { readProviderErrorSnippet, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const logger = createLogger('BoxServiceAccountMinter') @@ -176,10 +177,12 @@ export async function mintBoxServiceAccountToken( ...(hint ? { hint } : {}), }) } - throw new TokenServiceAccountValidationError('provider_unavailable', res.status, { - step: 'box_token_mint', - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + res.status, + { step: 'box_token_mint', body }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } const payload = await parseProviderJson(res, 'box_token_mint') diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts index 8a6b2bbf57a..795bdc4ab3f 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts @@ -84,6 +84,39 @@ describe('CrowdStrike API client token exchange', () => { await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({ code }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValue( + jsonResponse( + { errors: [{ code: 429, message: 'rate limited' }] }, + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) + + await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + + it("keeps a throttled answer's status and Retry-After when its error body cannot be read", async () => { + mockFetch.mockResolvedValue( + new Response( + new ReadableStream({ + start(controller) { + controller.error(new TypeError('terminated')) + }, + }), + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) + + await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + retryAfterMs: 7_000, + }) + }) + it("reports a redirect to the API client's own cloud as a wrong cloud and never follows it", async () => { mockFetch.mockResolvedValue( new Response(null, { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts index 6bf77c4c8fb..85474bf2121 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts @@ -82,7 +82,8 @@ export async function mintCrowdStrikeServiceAccountToken( throw new TokenServiceAccountValidationError( providerFailureCode(error.status), error.status, - { step: STEP, cloud } + { step: STEP, cloud }, + { retryAfterMs: error.retryAfterMs } ) } throw new TokenServiceAccountValidationError('provider_unavailable', 502, { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts index 51912c73c5a..ed68e865831 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.test.ts @@ -254,6 +254,17 @@ describe('mintDocuSignServiceAccountToken', () => { }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintDocuSignServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('keeps the key, assertion, and access token out of every failure', async () => { const failures: unknown[] = [] mockFetch.mockResolvedValueOnce( diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts index 2a0aee7ee39..f7e7e300541 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/docusign.ts @@ -19,6 +19,7 @@ import { parseProviderJson, readProviderErrorSnippet, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const TOKEN_STEP = 'docusign_token_mint' @@ -266,11 +267,12 @@ export async function mintDocuSignServiceAccountToken( { step: TOKEN_STEP, environment, body, ...(hint ? { hint } : {}) } ) } - throw new TokenServiceAccountValidationError('provider_unavailable', tokenRes.status, { - step: TOKEN_STEP, - environment, - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + tokenRes.status, + { step: TOKEN_STEP, environment, body }, + { retryAfterMs: throttledRetryAfterMs(tokenRes) } + ) } const payload = await parseProviderJson(tokenRes, TOKEN_STEP) @@ -297,11 +299,12 @@ export async function mintDocuSignServiceAccountToken( ) if (!userinfoRes.ok) { const body = await readProviderErrorSnippet(userinfoRes) - throw new TokenServiceAccountValidationError('provider_unavailable', userinfoRes.status, { - step: USERINFO_STEP, - environment, - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + userinfoRes.status, + { step: USERINFO_STEP, environment, body }, + { retryAfterMs: throttledRetryAfterMs(userinfoRes) } + ) } const userinfo = await parseProviderJson(userinfoRes, USERINFO_STEP) const accounts = (Array.isArray(userinfo.accounts) ? userinfo.accounts : []).filter( diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.test.ts new file mode 100644 index 00000000000..9266a12175c --- /dev/null +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.test.ts @@ -0,0 +1,24 @@ +import { jsonResponse } from '@sim/testing/helpers/http' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mintHelpScoutServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/helpscout' + +const FIELDS = { clientId: 'helpscout-client', clientSecret: 'helpscout-secret', orgId: '' } + +const mockFetch = vi.fn() + +describe('Help Scout client credential exchange', () => { + beforeEach(() => { + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) + }) + + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect( + mintHelpScoutServiceAccountToken(FIELDS, { skipIdentity: true }) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, retryAfterMs: 7_000 }) + }) +}) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts index 3ae63521c17..8b256d8dad7 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/helpscout.ts @@ -11,6 +11,7 @@ import { isTransientProviderStatus, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const MAX_HELPSCOUT_AUTH_RESPONSE_BYTES = 64 * 1024 @@ -28,7 +29,8 @@ async function readHelpScoutAuthResponse( ? 'invalid_credentials' : 'provider_unavailable', response.status, - { step } + { step }, + { retryAfterMs: throttledRetryAfterMs(response) } ) } diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts index 66592bf3efa..e7163b7dda9 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/jama.test.ts @@ -30,4 +30,18 @@ describe('Jama credential validation', () => { ).rejects.toMatchObject({ code: expectedCode, status }) } ) + + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + inputValidationMockFns.mockSecureFetchWithValidation.mockResolvedValueOnce( + Response.json({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect( + mintJamaServiceAccountToken({ + orgId: 'https://example.jamacloud.com', + clientId: 'client-id', + clientSecret: 'client-secret', + }) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, retryAfterMs: 7_000 }) + }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts index 22038795a26..463b9371dea 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/jama.ts @@ -10,6 +10,7 @@ import { isTransientProviderStatus, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' import { normalizeJamaInstanceUrl } from '@/lib/jama/client' @@ -49,7 +50,9 @@ export async function mintJamaServiceAccountToken( !isTransientProviderStatus(response.status) ? 'invalid_credentials' : 'provider_unavailable', - response.status + response.status, + undefined, + { retryAfterMs: throttledRetryAfterMs(response) } ) } return toRecord(await response.json()) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts index d211bde2cf0..57ffd1443f7 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts @@ -148,6 +148,22 @@ describe('mintNetSuiteServiceAccountToken', () => { await expect(mintNetSuiteServiceAccountToken(FIELDS)).rejects.toMatchObject({ code, status }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValue( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + ) + + await expect(mintNetSuiteServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('redacts credentials and a reflected assertion from provider error metadata', async () => { let assertion = '' vi.stubGlobal( diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts index d0737e1d68a..9e3738e5970 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts @@ -17,6 +17,7 @@ import { tenantPrincipal } from '@/lib/credentials/principal' import { isTransientProviderStatus, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const TOKEN_PATH = '/services/rest/auth/oauth2/v1/token' @@ -141,10 +142,12 @@ async function exchangeNetSuiteToken( response.status >= 400 && response.status < 500 && !isTransientProviderStatus(response.status) ? 'invalid_credentials' : 'provider_unavailable' - throw new TokenServiceAccountValidationError(code, response.status, { - step: TOKEN_MINT_STEP, - body: sanitizeNetSuiteTokenError(body, fields, assertion), - }) + throw new TokenServiceAccountValidationError( + code, + response.status, + { step: TOKEN_MINT_STEP, body: sanitizeNetSuiteTokenError(body, fields, assertion) }, + { retryAfterMs: throttledRetryAfterMs(response) } + ) } let payload: NetSuiteTokenResponse diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.test.ts index f620f90fd59..4a32ae86b6a 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.test.ts @@ -207,6 +207,16 @@ describe('Ramp client credential exchange', () => { } ) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintRampServiceAccountToken(FIELDS, { skipIdentity: true })).rejects.toMatchObject( + { code: 'provider_unavailable', retryAfterMs: 7_000 } + ) + }) + it.each([400, 401, 403, 408, 429, 500])( 'classifies provider failure %s without retaining its secret-bearing body', async (status) => { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.ts index 65921857245..994c2b6bfdd 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/ramp.ts @@ -11,6 +11,7 @@ import { isTransientProviderStatus, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' import { getScopesForService } from '@/lib/oauth/utils' @@ -29,7 +30,8 @@ async function readRampAuthResponse( ? 'invalid_credentials' : 'provider_unavailable', response.status, - { step } + { step }, + { retryAfterMs: throttledRetryAfterMs(response) } ) } diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts index 186612668a7..4ac73a2e329 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.test.ts @@ -102,6 +102,49 @@ describe('SailPoint API client token exchange', () => { await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({ code }) }) + it("carries the token endpoint's Retry-After on an unavailable answer", async () => { + mockFetch.mockResolvedValue( + jsonResponse({ error: 'unavailable' }, { status: 503, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + + it('returns the first throttled answer with its Retry-After instead of retrying it', async () => { + mockFetch.mockImplementation(async () => + jsonResponse({ error: 'rate limited' }, { status: 429, headers: { 'Retry-After': '120' } }) + ) + + await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + retryAfterMs: 120_000, + }) + expect(mockFetch).toHaveBeenCalledOnce() + }) + + it("keeps a throttled answer's status and Retry-After when its error body cannot be read", async () => { + mockFetch.mockResolvedValue( + new Response( + new ReadableStream({ + start(controller) { + controller.error(new TypeError('terminated')) + }, + }), + { status: 429, headers: { 'Retry-After': '120' } } + ) + ) + + await expect(mintSailPointServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + retryAfterMs: 120_000, + }) + }) + it('treats a network failure or refused redirect as a provider failure', async () => { mockFetch.mockRejectedValue(new TypeError('fetch failed: unexpected redirect')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts index 9a8683ae38a..9d218df7f61 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/sailpoint.ts @@ -18,7 +18,7 @@ import { } from '@/lib/internal/sailpoint/client' const STEP = 'sailpoint_token_mint' -/** Bounds the whole exchange, including its 429 backoff retries. */ +/** Bounds the whole exchange. */ const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000 /** @@ -32,6 +32,8 @@ const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000 * from `expires_in`; the credential cache re-mints before it lapses. * * The token response names no identity, so the connect-time check is the mint. + * A 429 is returned at once with its `Retry-After` rather than retried here, so + * the credential resolver's throttling backoff waits as long as SailPoint asks. * * @see https://developer.sailpoint.com/docs/api/authentication */ @@ -57,7 +59,8 @@ export async function mintSailPointServiceAccountToken( tokenUrl, fields.clientId, clientSecret, - options?.signal ? AbortSignal.any([options.signal, timeout]) : timeout + options?.signal ? AbortSignal.any([options.signal, timeout]) : timeout, + { maxThrottleRetries: 0 } ) } catch (error) { options?.signal?.throwIfAborted() @@ -68,7 +71,8 @@ export async function mintSailPointServiceAccountToken( { step: STEP, tenant: host, - } + }, + { retryAfterMs: error.retryAfterMs } ) } if (isHostNotFound(error)) { diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts index d918dac7a6c..7516fe0cf35 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts @@ -130,12 +130,18 @@ describe('mintSalesforceServiceAccountToken', () => { }) }) - it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'rate_limit_exceeded' }, 429)) + it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse( + { error: 'rate_limit_exceeded' }, + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) await expect(mintSalesforceServiceAccountToken(FIELDS)).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, + retryAfterMs: 7_000, }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.ts index 495dbb48310..87cc11ad645 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.ts @@ -21,6 +21,7 @@ import { providerFailureReason, readProviderErrorSnippet, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' /** @@ -463,11 +464,12 @@ export async function mintSalesforceServiceAccountToken( ...(hint ? { hint } : {}), }) } - throw new TokenServiceAccountValidationError('provider_unavailable', res.status, { - step: TOKEN_MINT_STEP, - host, - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + res.status, + { step: TOKEN_MINT_STEP, host, body }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } const payload = await parseProviderJson(res, TOKEN_MINT_STEP) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.test.ts index 00ccdccd258..a22b744cdd5 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.test.ts @@ -153,6 +153,17 @@ describe('Zendesk credential minting', () => { await expect(mintZendeskServiceAccountToken(CLIENT)).rejects.toMatchObject({ code }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintZendeskServiceAccountToken(CLIENT)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('refuses a token response without an access token', async () => { mockFetch.mockResolvedValueOnce(jsonResponse({ token_type: 'bearer' })) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts index ad7bae47c3a..062ad14b05d 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zendesk.ts @@ -18,6 +18,7 @@ import { providerFailureCode, readProviderErrorSnippet, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' import { zendeskApiTokenCredential } from '@/tools/zendesk/types' @@ -64,7 +65,8 @@ async function throwForZendeskResponse(response: Response, step: string): Promis throw new TokenServiceAccountValidationError( response.status === 404 ? 'site_not_found' : providerFailureCode(response.status), response.status, - { step, body } + { step, body }, + { retryAfterMs: throttledRetryAfterMs(response) } ) } diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts index a0d87d76ee7..a5342b83433 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts @@ -187,12 +187,15 @@ describe('mintZohoDeskServiceAccountToken', () => { }) }) - it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'too_many_requests' }, 429)) + it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ error: 'too_many_requests' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) await expect(mintZohoDeskServiceAccountToken(FIELDS)).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, + retryAfterMs: 7_000, }) }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts index 4113ce36120..6960d0583f4 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.ts @@ -20,6 +20,7 @@ import { readProviderErrorSnippet, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' import { tryDeriveZohoDeskBaseFromApiDomain } from '@/tools/zoho_desk/host-allowlist' @@ -199,12 +200,17 @@ export async function mintZohoDeskServiceAccountToken( if (res.status >= 400 && res.status < 500 && !isTransientProviderStatus(res.status)) { throw invalidCredentials(res.status, soid, dataCenter.id, body, zohoBodyError(body)) } - throw new TokenServiceAccountValidationError('provider_unavailable', res.status, { - step: STEP, - soid, - dataCenter: dataCenter.id, - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + res.status, + { + step: STEP, + soid, + dataCenter: dataCenter.id, + body, + }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } const payload = await parseProviderJson(res, STEP) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts index 63cf63dd9a8..6421913c8d1 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts @@ -69,12 +69,18 @@ describe('mintZoomServiceAccountToken', () => { }) }) - it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ error: 'rate_limit_exceeded' }, 429)) + it('throws provider_unavailable (not invalid_credentials) on a 429 rate limit, with its Retry-After', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse( + { error: 'rate_limit_exceeded' }, + { status: 429, headers: { 'Retry-After': '7' } } + ) + ) await expect(mintZoomServiceAccountToken(FIELDS)).rejects.toMatchObject({ code: 'provider_unavailable', status: 429, + retryAfterMs: 7_000, }) }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.ts index 4d86f95da32..89daf45c2a7 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.ts @@ -11,6 +11,7 @@ import { readProviderErrorSnippet, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const ZOOM_TOKEN_URL = 'https://zoom.us/oauth/token' @@ -88,10 +89,12 @@ export async function mintZoomServiceAccountToken( ...(hint ? { hint } : {}), }) } - throw new TokenServiceAccountValidationError('provider_unavailable', res.status, { - step: 'zoom_token_mint', - body, - }) + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + res.status, + { step: 'zoom_token_mint', body }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } const payload = await parseProviderJson(res, 'zoom_token_mint') diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.test.ts index 446f1ec8a7e..ddd9baf0210 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.test.ts @@ -64,6 +64,17 @@ describe('ZoomInfo client credentials exchange', () => { await expect(mintZoomInfoServiceAccountToken(FIELDS)).rejects.toMatchObject({ code }) }) + it("carries the token endpoint's Retry-After on a throttled answer", async () => { + mockSecureFetchWithValidation.mockResolvedValue( + Response.json({ error: 'busy' }, { status: 429, headers: { 'Retry-After': '7' } }) + ) + + await expect(mintZoomInfoServiceAccountToken(FIELDS)).rejects.toMatchObject({ + code: 'provider_unavailable', + retryAfterMs: 7_000, + }) + }) + it('treats a refused redirect or network failure as a provider failure', async () => { mockSecureFetchWithValidation.mockRejectedValue(new Error('Too many redirects (max: 0)')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.ts index c7bda0f1c68..9b72f30d82c 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoominfo.ts @@ -56,7 +56,8 @@ export async function mintZoomInfoServiceAccountToken( throw new TokenServiceAccountValidationError( providerFailureCode(error.tokenStatus), error.tokenStatus, - { step: STEP } + { step: STEP }, + { retryAfterMs: error.retryAfterMs } ) } throw new TokenServiceAccountValidationError('provider_unavailable', 502, { diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts index 2ba5cd6f308..f015a7bf441 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.integration.ts @@ -137,6 +137,25 @@ beforeAll(async () => { ) return } + if (parsed.client_id.endsWith('-throttled')) { + response.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '90' }) + response.end(JSON.stringify({ error: 'rate_limited' })) + return + } + if (parsed.client_id.endsWith('-rate-limited')) { + response.writeHead(429, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ error: 'rate_limited' })) + return + } + if (parsed.client_id.endsWith('-outage')) { + response.writeHead(502).end() + return + } + if (parsed.client_id.endsWith('-garbled')) { + response.writeHead(200, { 'Content-Type': 'text/html' }) + response.end('upstream proxy error') + return + } await sleep(15) const identity = `${request.headers['x-provider-origin']}:${parsed.client_id}` const issued = (applications.get(identity)?.issued ?? 0) + 1 @@ -321,6 +340,69 @@ describe('Vanta application token lifecycle against shared storage', () => { expect(await isActive(next.accessToken)).toBe(true) }) + it('waits as long as a throttled Vanta asks before any worker asks again', async () => { + const input = fields('throttled') + const keys = previousBuildKeys(input, READ) + const count = tokenRequests + const before = Date.now() + + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + }) + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ status: 429 }) + + expect(tokenRequests).toBe(count + 1) + const stored = await readStoredRow(keys.failureKey) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(90_000) + expect(stored.expiresAt.getTime() - Date.now()).toBeLessThanOrEqual(90_000) + }) + + it('waits at least a minute after a Vanta 429 that names no Retry-After', async () => { + const input = fields('rate-limited') + const keys = previousBuildKeys(input, READ) + const before = Date.now() + + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ status: 429 }) + + const stored = await readStoredRow(keys.failureKey) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(60_000) + }) + + it('asks a failing Vanta for a token at most once per 15 seconds, within its 5 tokens a minute', async () => { + const input = fields('outage') + const keys = previousBuildKeys(input, READ) + const count = tokenRequests + const before = Date.now() + + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 502, + }) + + expect(tokenRequests).toBe(count + 1) + const stored = await readStoredRow(keys.failureKey) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(15_000) + }) + + it('paces a Vanta that answers 200 with an unreadable body like any other failing exchange', async () => { + const input = fields('garbled') + const keys = previousBuildKeys(input, READ) + const count = tokenRequests + const before = Date.now() + + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ + code: 'provider_unavailable', + }) + await expect(resolveVantaClientToken(input)).rejects.toMatchObject({ + code: 'provider_unavailable', + }) + + expect(tokenRequests).toBe(count + 1) + const stored = await readStoredRow(keys.failureKey) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(15_000) + }) + it('fails closed when coordination storage fails', async () => { const count = tokenRequests vi.spyOn(db, 'transaction').mockRejectedValueOnce(new Error('database unavailable')) diff --git a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts index 2d1bc48ec70..a3ea31abb02 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/vanta-token.ts @@ -1,5 +1,5 @@ import { isRecordLike } from '@sim/utils/object' -import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { PayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { VANTA_PERMISSION_OPTIONS } from '@/lib/credentials/client-credential-accounts/descriptors' import { invalidateCoordinatedClientToken, @@ -10,10 +10,16 @@ import { isTransientProviderStatus, requireClientSecret, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const SCOPES = VANTA_PERMISSION_OPTIONS.map((option) => option.value) const EXPIRY_BUFFER_MS = 30_000 +/** + * Vanta answers 429 when an application mints more than 5 tokens a minute, and every token it + * issues revokes the previous one, so a failing exchange is retried at most every 15 s (4 a minute). + */ +const MIN_EXCHANGE_INTERVAL_MS = 15_000 const STEP = 'vanta_token_mint' export interface VantaClientTokenFields { @@ -91,14 +97,25 @@ async function exchangeToken( ? 'invalid_credentials' : 'provider_unavailable', response.status, - { step: STEP, reason: 'provider rejected token exchange' } + { step: STEP, reason: 'provider rejected token exchange' }, + { retryAfterMs: throttledRetryAfterMs(response) } + ) + } + let payload: unknown + try { + payload = await readResponseJsonWithLimit(response, { + maxBytes: 64 * 1024, + label: 'Vanta authentication response', + signal, + }) + } catch (error) { + signal?.throwIfAborted() + throw unavailable( + error instanceof SyntaxError || error instanceof PayloadSizeLimitError + ? 'provider returned an invalid token response' + : 'provider token response could not be read' ) } - const payload = await readResponseJsonWithLimit(response, { - maxBytes: 64 * 1024, - label: 'Vanta authentication response', - signal, - }) if ( !isRecordLike(payload) || typeof payload.access_token !== 'string' || @@ -150,6 +167,7 @@ export async function resolveVantaClientToken( step: STEP, signal: options.signal, minRemainingMs: EXPIRY_BUFFER_MS, + minExchangeIntervalMs: MIN_EXCHANGE_INTERVAL_MS, failureKeyQualifier: scope, accessTokenDigestTag: apiDomain, pinnedState: { diff --git a/apps/sim/lib/credentials/client-credentials-grant.test.ts b/apps/sim/lib/credentials/client-credentials-grant.test.ts index 9cbaf6432c1..53c099ea3a0 100644 --- a/apps/sim/lib/credentials/client-credentials-grant.test.ts +++ b/apps/sim/lib/credentials/client-credentials-grant.test.ts @@ -140,7 +140,11 @@ describe('exchangeClientCredentialsGrant', () => { await expect( exchangeClientCredentialsGrant({ ...REQUEST, classifyRefusal }) ).rejects.toMatchObject({ code: 'grant_not_permitted', status: 400 }) - expect(classifyRefusal).toHaveBeenCalledWith(400, expect.stringContaining('org_mismatch')) + expect(classifyRefusal).toHaveBeenCalledWith( + 400, + expect.stringContaining('org_mismatch'), + undefined + ) }) it.each([ @@ -158,6 +162,30 @@ describe('exchangeClientCredentialsGrant', () => { }) }) + it('carries the Retry-After of a throttled exchange, also to a provider classification', async () => { + mockFetch.mockImplementation(async () => + jsonResponse({ error: 'slow_down' }, { status: 429, headers: { 'Retry-After': '45' } }) + ) + + await expect(exchangeClientCredentialsGrant(REQUEST)).rejects.toMatchObject({ + code: 'provider_unavailable', + status: 429, + retryAfterMs: 45_000, + }) + const classifyRefusal = vi.fn( + (status: number, _body: string, retryAfterMs?: number) => + new TokenServiceAccountValidationError( + 'provider_unavailable', + status, + { step: REQUEST.step }, + { retryAfterMs } + ) + ) + await expect( + exchangeClientCredentialsGrant({ ...REQUEST, classifyRefusal }) + ).rejects.toMatchObject({ retryAfterMs: 45_000 }) + }) + it('refuses a token response that is not JSON', async () => { mockFetch.mockResolvedValue(new Response('gateway', { status: 200 })) diff --git a/apps/sim/lib/credentials/client-credentials-grant.ts b/apps/sim/lib/credentials/client-credentials-grant.ts index 93522121d78..5aefff42d61 100644 --- a/apps/sim/lib/credentials/client-credentials-grant.ts +++ b/apps/sim/lib/credentials/client-credentials-grant.ts @@ -7,6 +7,7 @@ import { providerFailureReason, readProviderErrorSnippet, TokenServiceAccountValidationError, + throttledRetryAfterMs, } from '@/lib/credentials/token-service-accounts/errors' const MAX_TOKEN_RESPONSE_BYTES = 64 * 1024 @@ -34,8 +35,14 @@ export interface ClientCredentialsGrantRequest { /** * Maps a refused exchange to a provider-specific error. Without it, a refusal is * `invalid_credentials` for a non-transient 4xx and `provider_unavailable` otherwise. + * `retryAfterMs` is the refusal's `Retry-After` (see {@link throttledRetryAfterMs}), which an + * error the provider would answer differently later should carry. */ - classifyRefusal?: (status: number, body: string) => TokenServiceAccountValidationError + classifyRefusal?: ( + status: number, + body: string, + retryAfterMs: number | undefined + ) => TokenServiceAccountValidationError signal?: AbortSignal } @@ -44,7 +51,7 @@ export interface ClientCredentialsGrantRequest { * with the client ID and secret in the body, redirects refused so the secret reaches no other host. * A response without a non-blank `access_token` and a positive integer `expires_in` is * `provider_unavailable`, and the token expires `min(expires_in, maxLifetimeSeconds)` after the - * request started. + * request started. A throttled or unavailable refusal carries its `Retry-After` as `retryAfterMs`. * * @see https://datatracker.ietf.org/doc/html/rfc6749#section-4.4 */ @@ -79,12 +86,14 @@ export async function exchangeClientCredentialsGrant( if (!response.ok) { const body = await readProviderErrorSnippet(response) signal?.throwIfAborted() + const retryAfterMs = throttledRetryAfterMs(response) throw request.classifyRefusal - ? request.classifyRefusal(response.status, body) + ? request.classifyRefusal(response.status, body, retryAfterMs) : new TokenServiceAccountValidationError( providerFailureCode(response.status), response.status, - { step, body } + { step, body }, + { retryAfterMs } ) } let payload: unknown diff --git a/apps/sim/lib/credentials/coordinated-client-token.integration.ts b/apps/sim/lib/credentials/coordinated-client-token.integration.ts index 714466025d9..b2454165b52 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.integration.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.integration.ts @@ -1,11 +1,13 @@ import { createHash } from 'node:crypto' import { db } from '@sim/db' import { clientCredentialToken } from '@sim/db/schema' +import { createDeferred } from '@sim/testing/helpers/deferred' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' import { eq, like, or } from 'drizzle-orm' import { afterAll, afterEach, describe, expect, it, vi } from 'vitest' -import { decryptSecret } from '@/lib/core/security/encryption' +import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { type CoordinatedClientTokenInput, invalidateCoordinatedClientToken, @@ -13,6 +15,7 @@ import { } from '@/lib/credentials/coordinated-client-token' import { verifyAndBuildServiceAccountSecret } from '@/lib/credentials/service-account-secret' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' +import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' const NAMESPACE = `fixture${generateId().replaceAll('-', '')}` const ownedKeyPrefixes = new Set([`${NAMESPACE}:`]) @@ -63,6 +66,68 @@ function rejecting() { }) } +function failing(status: number, retryAfterMs?: number) { + return vi.fn(async (): Promise<{ accessToken: string; expiresAt: Date }> => { + throw new TokenServiceAccountValidationError( + 'provider_unavailable', + status, + { step: 'mint' }, + { + retryAfterMs, + } + ) + }) +} + +/** An exchange that issues one token valid for `lifetimeMs`, then fails with `then`. */ +function issuingOnce( + application: string, + lifetimeMs: number, + then: CoordinatedClientTokenInput['exchange'] +) { + return vi + .fn() + .mockResolvedValueOnce({ + accessToken: `${application}:token-1`, + expiresAt: new Date(Date.now() + lifetimeMs), + }) + .mockImplementation(then) +} + +async function failureRowOf(application: string) { + const rows = await db + .select() + .from(clientCredentialToken) + .where(like(clientCredentialToken.id, `${applicationKey(NAMESPACE, [application])}:%`)) + if (rows.length !== 1) throw new Error(`Expected one failure row, found ${rows.length}`) + return readStoredRow(rows[0].id) +} + +async function expireFailureRowsOf(application: string) { + await db + .update(clientCredentialToken) + .set({ expiresAt: new Date(Date.now() - 1000) }) + .where(like(clientCredentialToken.id, `${applicationKey(NAMESPACE, [application])}:%`)) +} + +/** The previous build's failure decoders, which must keep accepting every failure this build stores. */ +const previousBuildDecodesFailure = { + coordinated: (value: unknown) => + isRecordLike(value) && + typeof value.code === 'string' && + [ + 'invalid_credentials', + 'grant_not_permitted', + 'site_not_found', + 'provider_unavailable', + ].includes(value.code) && + typeof value.status === 'number', + vantaBeforeCoordination: (value: unknown) => + isRecordLike(value) && + (value.code === 'invalid_credentials' || value.code === 'provider_unavailable') && + typeof value.status === 'number', +} + afterEach(() => { vi.unstubAllGlobals() }) @@ -118,9 +183,11 @@ describe('coordinated client-credentials token against shared storage', () => { await expect( resolveCoordinatedClientToken(input('memo', exchange, 'revoked')) ).rejects.toMatchObject({ code: 'invalid_credentials' }) - await expect( - resolveCoordinatedClientToken(input('memo', exchange, 'revoked')) - ).rejects.toMatchObject({ code: 'invalid_credentials' }) + const replayed = await resolveCoordinatedClientToken(input('memo', exchange, 'revoked')).catch( + (error: unknown) => error + ) + expect(replayed).toMatchObject({ code: 'invalid_credentials', replayForMs: expect.any(Number) }) + expect(replayed).not.toHaveProperty('retryAfterMs', expect.anything()) expect(exchange).toHaveBeenCalledOnce() await db @@ -204,27 +271,383 @@ describe('coordinated client-credentials token against shared storage', () => { ).resolves.toMatchObject({ accessToken: 'fresh-rejection:token-2' }) }) - it('sweeps expired failure memos', async () => { + it('sweeps failure memos once they can no longer extend a backoff', async () => { + for (const application of ['swept', 'retained']) { + await expect( + resolveCoordinatedClientToken(input(application, rejecting(), 'revoked')) + ).rejects.toMatchObject({ code: 'invalid_credentials' }) + } + const failurePrefix = (application: string) => `${applicationKey(NAMESPACE, [application])}:` + const expire = (application: string, agoMs: number) => + db + .update(clientCredentialToken) + .set({ expiresAt: new Date(Date.now() - agoMs) }) + .where(like(clientCredentialToken.id, `${failurePrefix(application)}%`)) + await expire('swept', 6 * 60_000) + await expire('retained', 60_000) + + await resolveCoordinatedClientToken(input('sweeper', provider('sweeper').exchange)) + + const remaining = async (application: string) => + db + .select({ id: clientCredentialToken.id }) + .from(clientCredentialToken) + .where(like(clientCredentialToken.id, `${failurePrefix(application)}%`)) + expect(await remaining('swept')).toEqual([]) + expect(await remaining('retained')).toHaveLength(1) + }) +}) + +describe('coordinated token resolution while the provider fails', () => { + it('serves the stored token while it is still valid when its early refresh is throttled', async () => { + const exchange = issuingOnce('throttled-refresh', 4 * 60_000, failing(429)) + await resolveCoordinatedClientToken(input('throttled-refresh', exchange)) + + for (let call = 0; call < 3; call++) { + await expect( + resolveCoordinatedClientToken(input('throttled-refresh', exchange)) + ).resolves.toMatchObject({ accessToken: 'throttled-refresh:token-1' }) + } + expect(exchange).toHaveBeenCalledTimes(2) + }) + + it('reports the failed refresh once the stored token has run out', async () => { + const exchange = issuingOnce('expired-refresh', 20_000, failing(429)) + await resolveCoordinatedClientToken(input('expired-refresh', exchange)) + await expect( - resolveCoordinatedClientToken(input('swept', rejecting(), 'revoked')) + resolveCoordinatedClientToken(input('expired-refresh', exchange)) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429 }) + }) + + it('stops serving the stored token once the provider refuses the credentials', async () => { + const exchange = issuingOnce('refused-refresh', 4 * 60_000, rejecting()) + await resolveCoordinatedClientToken(input('refused-refresh', exchange)) + + await expect( + resolveCoordinatedClientToken(input('refused-refresh', exchange)) ).rejects.toMatchObject({ code: 'invalid_credentials' }) - const failurePrefix = `${applicationKey(NAMESPACE, ['swept'])}:` - await db - .update(clientCredentialToken) - .set({ expiresAt: new Date(Date.now() - 1000) }) - .where(like(clientCredentialToken.id, `${failurePrefix}%`)) + await expect( + resolveCoordinatedClientToken(input('refused-refresh', exchange)) + ).rejects.toMatchObject({ code: 'invalid_credentials' }) + expect(exchange).toHaveBeenCalledTimes(2) + }) - await resolveCoordinatedClientToken(input('sweeper', provider('sweeper').exchange)) + it('never serves a refused token again once the refusal memo ends and an outage follows', async () => { + const exchange = issuingOnce('refused-then-outage', 4 * 60_000, rejecting()) + await resolveCoordinatedClientToken(input('refused-then-outage', exchange)) + await expect( + resolveCoordinatedClientToken(input('refused-then-outage', exchange)) + ).rejects.toMatchObject({ code: 'invalid_credentials' }) + await expireFailureRowsOf('refused-then-outage') + exchange.mockImplementation(failing(503)) - const rows = await db - .select({ id: clientCredentialToken.id }) - .from(clientCredentialToken) - .where(like(clientCredentialToken.id, `${failurePrefix}%`)) - expect(rows).toEqual([]) + await expect( + resolveCoordinatedClientToken(input('refused-then-outage', exchange)) + ).rejects.toMatchObject({ code: 'provider_unavailable' }) + }) + + it('keeps the token another secret minted when a different secret is refused', async () => { + const exchange = issuingOnce('refused-other-secret', 4 * 60_000, failing(503)) + await resolveCoordinatedClientToken(input('refused-other-secret', exchange)) + await expect( + resolveCoordinatedClientToken(input('refused-other-secret', rejecting(), 'revoked')) + ).rejects.toMatchObject({ code: 'invalid_credentials' }) + + await expect( + resolveCoordinatedClientToken(input('refused-other-secret', exchange)) + ).resolves.toMatchObject({ accessToken: 'refused-other-secret:token-1' }) + }) + + it('keeps refusing for every worker a refusal whose code the shared row cannot store', async () => { + const exchange = issuingOnce('consent-refresh', 4 * 60_000, async () => { + throw new TokenServiceAccountValidationError('consent_required', 400, { step: 'mint' }) + }) + await resolveCoordinatedClientToken(input('consent-refresh', exchange)) + + for (let call = 0; call < 2; call++) { + await expect( + resolveCoordinatedClientToken(input('consent-refresh', exchange)) + ).rejects.toBeInstanceOf(TokenServiceAccountValidationError) + } + expect(exchange).toHaveBeenCalledTimes(2) + const payload: unknown = JSON.parse((await failureRowOf('consent-refresh')).decrypted) + expect(previousBuildDecodesFailure.coordinated(payload)).toBe(true) + }) + + it('never serves a still-valid token to a different secret whose refresh fails', async () => { + const exchange = issuingOnce('rotated-refresh', 4 * 60_000, failing(429)) + await resolveCoordinatedClientToken(input('rotated-refresh', exchange)) + + await expect( + resolveCoordinatedClientToken(input('rotated-refresh', exchange, 'secret-2')) + ).rejects.toMatchObject({ code: 'provider_unavailable' }) + }) + + it('retries a quick outage once under the application lock, for every waiting resolution', async () => { + const { exchange: issue } = provider('retried') + const exchange = vi + .fn() + .mockRejectedValueOnce( + new TokenServiceAccountValidationError('provider_unavailable', 503, { step: 'mint' }) + ) + .mockImplementation(issue) + + const results = await Promise.all( + Array.from({ length: 5 }, () => resolveCoordinatedClientToken(input('retried', exchange))) + ) + + expect(new Set(results.map((result) => result.accessToken))).toEqual( + new Set(['retried:token-1']) + ) + expect(exchange).toHaveBeenCalledTimes(2) + }) + + it.each([ + ['as long as the provider asks', 'throttled-90s', 90_000, 90_000], + ['for at most five minutes', 'throttled-1h', 3_600_000, 300_000], + ])( + "shares a throttled exchange's Retry-After with every worker %s", + async (_case, application, retryAfterMs, replayMs) => { + const exchange = failing(429, retryAfterMs) + const before = Date.now() + + await expect( + resolveCoordinatedClientToken(input(application, exchange)) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429 }) + await expect( + resolveCoordinatedClientToken(input(application, exchange)) + ).rejects.toMatchObject({ status: 429, replayForMs: expect.any(Number) }) + + expect(exchange).toHaveBeenCalledOnce() + const stored = await failureRowOf(application) + expect(stored.expiresAt.getTime() - before).toBeGreaterThanOrEqual(replayMs) + expect(stored.expiresAt.getTime() - Date.now()).toBeLessThanOrEqual(replayMs) + } + ) + + it('replays a throttled exchange that names no Retry-After for at least a minute', async () => { + const exchange = failing(429) + const before = Date.now() + + await expect( + resolveCoordinatedClientToken(input('throttled-bare', exchange)) + ).rejects.toMatchObject({ code: 'provider_unavailable', status: 429 }) + + expect(exchange).toHaveBeenCalledOnce() + expect( + (await failureRowOf('throttled-bare')).expiresAt.getTime() - before + ).toBeGreaterThanOrEqual(60_000) + }) + + it('backs off a throttled answer after an outage from a minute, not by the outage streak', async () => { + const exchange = vi.fn() + for (const status of [502, 502]) { + exchange.mockImplementation(failing(status)) + await expect( + resolveCoordinatedClientToken(input('outage-then-throttled', exchange)) + ).rejects.toThrow() + await expireFailureRowsOf('outage-then-throttled') + } + exchange.mockImplementation(failing(429)) + const before = Date.now() + + await expect( + resolveCoordinatedClientToken(input('outage-then-throttled', exchange)) + ).rejects.toMatchObject({ status: 429 }) + + const replayMs = (await failureRowOf('outage-then-throttled')).expiresAt.getTime() - before + expect(replayMs).toBeGreaterThanOrEqual(60_000) + expect(replayMs).toBeLessThanOrEqual(72_000 + 2_500) + }) + + it('backs off throttling that continues for every worker longer than its first minute', async () => { + const exchange = failing(429) + await expect( + resolveCoordinatedClientToken(input('throttled-twice', exchange)) + ).rejects.toThrow() + await expireFailureRowsOf('throttled-twice') + const before = Date.now() + + await expect( + resolveCoordinatedClientToken(input('throttled-twice', exchange)) + ).rejects.toThrow() + + expect( + (await failureRowOf('throttled-twice')).expiresAt.getTime() - before + ).toBeGreaterThanOrEqual(96_000) + }) + + it.each([ + [ + 'the still-valid token while its refresh is remembered failing', + 'unlocked-fallback', + 'outage', + ], + ['a remembered refusal', 'unlocked-refusal', 'refusal'], + ] as const)( + 'serves %s without waiting for the application lock', + async (_case, application, remembered) => { + const appKey = applicationKey(NAMESPACE, [application]) + const fingerprint = sha256Json([application, 'secret-1']) + const token = await encryptSecret( + JSON.stringify({ accessToken: `${application}:token-1`, secretFingerprint: fingerprint }) + ) + const failure = await encryptSecret( + JSON.stringify( + remembered === 'outage' + ? { code: 'provider_unavailable', status: 503, consecutiveFailures: 1 } + : { code: 'invalid_credentials', status: 401 } + ) + ) + await db.insert(clientCredentialToken).values([ + { + id: appKey, + encryptedValue: token.encrypted, + accessTokenDigest: sha256Json(['coordinated_client_token', `${application}:token-1`]), + expiresAt: new Date(Date.now() + 4 * 60_000), + }, + { + id: `${appKey}:${sha256Json(fingerprint)}`, + encryptedValue: failure.encrypted, + expiresAt: new Date(Date.now() + 60_000), + }, + ]) + const { exchange } = provider(application) + const lockHeld = createDeferred() + const release = createDeferred() + const holder = db.transaction(async (tx) => { + await acquireAdvisoryXactLock(tx, `${NAMESPACE}_client_token`, appKey) + lockHeld.resolve() + await release.promise + }) + + try { + await lockHeld.promise + const outcome = await Promise.race([ + resolveCoordinatedClientToken(input(application, exchange)).then( + (value) => ({ value }), + (error: unknown) => ({ error }) + ), + sleep(3_000).then(() => 'waited on the lock' as const), + ]) + + expect(outcome).toEqual( + remembered === 'outage' + ? { value: expect.objectContaining({ accessToken: `${application}:token-1` }) } + : { error: expect.objectContaining({ code: 'invalid_credentials' }) } + ) + expect(exchange).not.toHaveBeenCalled() + } finally { + release.resolve() + await holder + } + } + ) + + it('backs off an outage from a few seconds, longer while it continues, never past 30 seconds', async () => { + const exchange = failing(502) + + const replays: number[] = [] + for (let attempt = 0; attempt < 5; attempt++) { + const before = Date.now() + await expect(resolveCoordinatedClientToken(input('outage', exchange))).rejects.toMatchObject({ + code: 'provider_unavailable', + }) + replays.push((await failureRowOf('outage')).expiresAt.getTime() - before) + await expireFailureRowsOf('outage') + } + + expect(replays[0]).toBeGreaterThanOrEqual(5_000) + expect(replays[0]).toBeLessThanOrEqual(6_000 + 2_500) + expect(replays[1]).toBeGreaterThanOrEqual(8_000) + expect(Math.max(...replays)).toBeLessThanOrEqual(30_000 + 2_500) + }) + + it('starts the backoff over after the provider recovers', async () => { + const { exchange: issue } = provider('recovered') + const exchange = vi + .fn() + .mockImplementation(failing(502)) + for (let attempt = 0; attempt < 3; attempt++) { + await expect(resolveCoordinatedClientToken(input('recovered', exchange))).rejects.toThrow() + await expireFailureRowsOf('recovered') + } + exchange.mockImplementationOnce(issue) + const { accessToken } = await resolveCoordinatedClientToken(input('recovered', exchange)) + await invalidateCoordinatedClientToken(accessToken) + exchange.mockImplementation(failing(502)) + + const before = Date.now() + await expect(resolveCoordinatedClientToken(input('recovered', exchange))).rejects.toThrow() + + expect((await failureRowOf('recovered')).expiresAt.getTime() - before).toBeLessThanOrEqual( + 6_000 + 2_500 + ) }) }) describe('default token rows shared with the previous build during a rolling deploy', () => { + /** Seeds a still-valid token and the previous build's failure memo of `status` for one secret. */ + async function seedPreviousBuildFailure(application: string, status: number) { + const appKey = applicationKey(NAMESPACE, [application]) + const fingerprint = sha256Json([application, 'secret-1']) + const token = await encryptSecret( + JSON.stringify({ accessToken: 'legacy-token', secretFingerprint: fingerprint }) + ) + const failure = await encryptSecret(JSON.stringify({ code: 'provider_unavailable', status })) + await db.insert(clientCredentialToken).values([ + { + id: appKey, + encryptedValue: token.encrypted, + accessTokenDigest: sha256Json(['coordinated_client_token', 'legacy-token']), + expiresAt: new Date(Date.now() + 4 * 60_000), + }, + { + id: `${appKey}:${sha256Json(fingerprint)}`, + encryptedValue: failure.encrypted, + expiresAt: new Date(Date.now() + 20_000), + }, + ]) + } + + it('serves a still-valid token when the previous build remembered an outage', async () => { + const application = 'legacy-outage' + await seedPreviousBuildFailure(application, 503) + const { exchange } = provider(application) + + await expect( + resolveCoordinatedClientToken(input(application, exchange)) + ).resolves.toMatchObject({ accessToken: 'legacy-token' }) + expect(exchange).not.toHaveBeenCalled() + }) + + it('replays a refusal the previous build stored as provider_unavailable without serving the token', async () => { + const application = 'legacy-unmapped-refusal' + await seedPreviousBuildFailure(application, 400) + const { exchange } = provider(application) + + await expect(resolveCoordinatedClientToken(input(application, exchange))).rejects.toMatchObject( + { code: 'provider_unavailable', status: 400 } + ) + expect(exchange).not.toHaveBeenCalled() + }) + + it('stores every outage and throttled answer as a failure the previous build reads', async () => { + for (const [application, exchange] of [ + ['legacy-readable-outage', failing(502)], + ['legacy-readable-throttle', failing(429, 60_000)], + ] as const) { + await expect(resolveCoordinatedClientToken(input(application, exchange))).rejects.toThrow() + + const stored = await failureRowOf(application) + const payload: unknown = JSON.parse(stored.decrypted) + expect(previousBuildDecodesFailure.coordinated(payload)).toBe(true) + expect(previousBuildDecodesFailure.vantaBeforeCoordination(payload)).toBe(true) + expect(stored.accessTokenDigest).toBeNull() + } + }) + it('stores tokens and refusals under the ids, digest, and payloads the previous build used', async () => { const application = ['legacy.myshopify.com', 'legacy-client'] const appKey = applicationKey(NAMESPACE, application) diff --git a/apps/sim/lib/credentials/coordinated-client-token.ts b/apps/sim/lib/credentials/coordinated-client-token.ts index 0fea2e04b00..aa6dd77b900 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.ts @@ -1,15 +1,28 @@ import { createHash } from 'node:crypto' import { db } from '@sim/db' import { clientCredentialToken } from '@sim/db/schema' +import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { and, eq, inArray, lt, sql } from 'drizzle-orm' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' -import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' +import { + assessTokenRequestFailure, + isPermanentTokenFailureCode, + requestTokenWithRetry, + TOKEN_EXPIRY_MARGIN_MS, + TOKEN_FAILURE_STREAK_RETENTION_MS, + type TokenRequestFailure, +} from '@/lib/credentials/token-request-resilience' +import { + providerFailureCode, + TokenServiceAccountValidationError, +} from '@/lib/credentials/token-service-accounts/errors' import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' -/** A stored token is only handed out while at least this much of its lifetime remains. */ +const logger = createLogger('CoordinatedClientToken') + +/** A stored token is refreshed once less than this much of its lifetime remains. */ const MIN_REMAINING_MS = 5 * 60 * 1000 -const FAILURE_TTL_MS = 30_000 const DEFAULT_ACCESS_TOKEN_DIGEST_TAG = 'coordinated_client_token' /** The provider token a coordinated exchange produced. */ @@ -33,8 +46,19 @@ export interface CoordinatedClientTokenInput { /** Performs the provider exchange. Throws `TokenServiceAccountValidationError` on refusal. */ exchange: (signal?: AbortSignal) => Promise signal?: AbortSignal - /** Shortest remaining lifetime a stored token is still served with. Defaults to five minutes. */ + /** + * Remaining lifetime below which a stored token is refreshed. Defaults to five minutes. If the + * refresh fails with an outage or throttling, the token is still served until 30 seconds (or + * this value, when smaller) before it expires. + */ minRemainingMs?: number + /** + * The provider is asked for a token at most once per this interval while its exchanges fail: + * a quick outage is not retried inside the exchange, and every failure is replayed at least this + * long. For providers that rate-limit token minting itself (Vanta: 5 tokens a minute), where + * every exchange the provider processes also revokes the application's previous token. + */ + minExchangeIntervalMs?: number /** * A request setting beyond the secret that a remembered exchange failure is specific to, so a * refusal for one setting does not block another. @@ -61,15 +85,27 @@ export interface CoordinatedClientToken { expiresInSeconds: number } +type StoredRow = typeof clientCredentialToken.$inferSelect + interface StoredApplicationToken { accessToken: string secretFingerprint: string record: Readonly> } +/** + * A remembered exchange failure, replayed until its row's `expiresAt`. Builds before failure + * kinds stored `{ code, status }` only and ignore other keys, so `consecutiveFailures` is + * written only for outages and throttling, to grow their backoff while they continue, `throttled` + * only for throttling, so its streak is kept apart from an outage's, and `refusal` only for a + * refusal whose code those builds read as `provider_unavailable`. + */ interface StoredFailure { code: 'invalid_credentials' | 'grant_not_permitted' | 'site_not_found' | 'provider_unavailable' status: number + consecutiveFailures?: number + throttled?: true + refusal?: true } const STORED_FAILURE_CODES: ReadonlySet = new Set([ @@ -89,16 +125,18 @@ function accessTokenDigestOf(accessToken: string, tag = DEFAULT_ACCESS_TOKEN_DIG } /** - * Deletes a bounded batch of expired failure memos (rows without a token digest). Run it outside - * the transaction that takes an application lock, so its row locks are released before any - * provider call. + * Deletes a bounded batch of failure memos (rows without a token digest) that expired more than + * {@link TOKEN_FAILURE_STREAK_RETENTION_MS} ago; a more recently expired one is kept so the next + * failure continues its backoff. Run it outside the transaction that takes an application lock, + * so its row locks are released before any provider call. */ async function deleteExpiredClientTokenFailures(): Promise { + const retentionSeconds = TOKEN_FAILURE_STREAK_RETENTION_MS / 1000 await db.execute(sql` DELETE FROM ${clientCredentialToken} WHERE ${clientCredentialToken.id} IN ( SELECT ${clientCredentialToken.id} FROM ${clientCredentialToken} WHERE ${clientCredentialToken.accessTokenDigest} IS NULL - AND ${clientCredentialToken.expiresAt} < now() + AND ${clientCredentialToken.expiresAt} < now() - make_interval(secs => ${retentionSeconds}) ORDER BY ${clientCredentialToken.expiresAt} LIMIT 100 FOR UPDATE SKIP LOCKED )`) } @@ -168,7 +206,65 @@ function decodeFailure(value: string, step: string): StoredFailure { typeof parsed.status !== 'number' ) throw unavailable(step, 'stored token failure is malformed') - return { code: parsed.code as StoredFailure['code'], status: parsed.status } + return { + code: parsed.code as StoredFailure['code'], + status: parsed.status, + ...(Number.isSafeInteger(parsed.consecutiveFailures) && Number(parsed.consecutiveFailures) > 0 + ? { consecutiveFailures: Number(parsed.consecutiveFailures) } + : {}), + ...(parsed.throttled === true ? { throttled: true } : {}), + ...(parsed.refusal === true ? { refusal: true } : {}), + } +} + +/** + * Whether a remembered failure refused the credentials, so no stored token is served for it. A + * build before failure kinds stored a refusal whose code it could not store as + * `provider_unavailable` with the provider's status and no streak count, so such a row whose + * status names a refusal (a non-transient 4xx) is one too. + */ +function isRememberedRefusal(failure: StoredFailure): boolean { + if (failure.refusal === true || isPermanentTokenFailureCode(failure.code)) return true + return ( + failure.consecutiveFailures === undefined && + isPermanentTokenFailureCode(providerFailureCode(failure.status)) + ) +} + +/** The error a remembered failure is replayed as, with how long it is still replayed. */ +function replayedFailure( + failure: StoredFailure, + step: string, + replayMs: number +): TokenServiceAccountValidationError { + return new TokenServiceAccountValidationError( + failure.code, + failure.status, + { step }, + { replayForMs: Math.max(replayMs, 0) } + ) +} + +/** + * The outage or throttling streak before this exchange, read from the application's last failure + * memo for this secret once its replay has ended. A refusal, or anything unreadable, starts a new + * streak. + */ +async function failureStreakBefore( + row: StoredRow | undefined, + step: string +): Promise | undefined> { + if (!row || Date.now() - row.expiresAt.getTime() > TOKEN_FAILURE_STREAK_RETENTION_MS) return + try { + const failure = decodeFailure((await decryptSecret(row.encryptedValue)).decrypted, step) + if (isRememberedRefusal(failure)) return + return { + kind: failure.throttled ? 'throttled' : 'transient', + consecutiveFailures: failure.consecutiveFailures ?? 1, + } + } catch { + return + } } function remainingSeconds(expiresAt: Date): number { @@ -186,11 +282,19 @@ function isServable(expiresAt: Date, minRemainingMs: number): boolean { * to reuse one token rather than accumulate them. Minting happens only under a * transaction-scoped advisory lock keyed by the application, so two workers never race two * tokens into existence; a token minted with a different secret is never served, and a token the - * provider rejects is expired through {@link invalidateCoordinatedClientToken}. An exchange that - * fails validation, whether refused for the credentials or answered with an outage, is remembered - * for 30 seconds, the same backoff the per-instance mint cache applies, so a hot workflow neither - * hammers the token endpoint with a revoked secret nor floods a struggling one. Tokens are - * encrypted at rest. + * provider rejects is expired through {@link invalidateCoordinatedClientToken}. + * + * A quick outage is retried once inside the exchange (unless `minExchangeIntervalMs` is set). An + * exchange that still fails validation is remembered for every worker, with the policy the + * per-instance mint cache applies (see `assessTokenRequestFailure`): a refusal of the credentials + * for 30 seconds, a throttled answer as long as its `Retry-After` asks (at most five minutes) or + * for a minute growing to five without one, and an outage for 5 seconds growing to 30 while it + * continues, so a hot workflow neither hammers the token endpoint with a revoked secret nor floods + * a struggling one. While an outage or throttling is remembered, the stored token for the same + * secret is still served until shortly before it actually expires; a refusal is reported at once + * and expires that secret's stored token, so no later outage serves it again. + * A remembered failure, and that fallback, are served from committed rows without taking the + * application lock. Tokens are encrypted at rest. */ export async function resolveCoordinatedClientToken( input: CoordinatedClientTokenInput @@ -206,107 +310,191 @@ export async function resolveCoordinatedClientToken( : [secretFingerprint, input.failureKeyQualifier] )}` - const serveIfFresh = async ( - row: typeof clientCredentialToken.$inferSelect | undefined, + const fallbackRemainingMs = Math.min(TOKEN_EXPIRY_MARGIN_MS, minRemainingMs) + const decodedRows = new WeakMap>() + + /** + * The token `row` holds for this secret while more than `remainingMs` of it is left. The record + * is decoded and its pinned state verified (throwing on a malformed record or a mismatch) once + * per row, whenever it could be served, and for an expired record too when `verifyExpired` is + * set and settings are pinned. + */ + const serveStored = async ( + row: StoredRow | undefined, + remainingMs: number, verifyExpired = false - ) => { + ): Promise => { if (!row) return undefined - const servable = isServable(row.expiresAt, minRemainingMs) + const servable = isServable(row.expiresAt, remainingMs) if (!servable && !(verifyExpired && pinnedState)) return undefined - const stored = decodeApplicationToken((await decryptSecret(row.encryptedValue)).decrypted, step) - pinnedState?.verify(stored.record) + let decoded = decodedRows.get(row) + if (!decoded) { + decoded = decryptSecret(row.encryptedValue).then(({ decrypted }) => { + const stored = decodeApplicationToken(decrypted, step) + pinnedState?.verify(stored.record) + return stored + }) + decodedRows.set(row, decoded) + } + const stored = await decoded if (!servable || stored.secretFingerprint !== secretFingerprint) return undefined return { accessToken: stored.accessToken, expiresInSeconds: remainingSeconds(row.expiresAt) } } + /** + * The stored token for this secret while it has not yet run out, served while its refresh fails + * with an outage or throttling. A record that does not decode or verify is never a fallback. + */ + const fallbackFrom = (row: StoredRow | undefined) => + serveStored(row, fallbackRemainingMs).catch(() => undefined) + + /** + * Whether `row` holds a token minted with this secret, expired or not, so a refusal of the + * secret ends that token too and a token another secret minted stays in service. + */ + const holdsTokenForSecret = async (row: StoredRow | undefined) => + (await serveStored(row, Number.NEGATIVE_INFINITY).catch(() => undefined)) !== undefined + + /** + * Replays the application's unexpired failure memo for this secret: the still-valid token for a + * remembered outage or throttling, the failure otherwise. `undefined` when nothing is remembered. + * Reading committed rows needs no lock, so this also runs before the application lock is taken. + */ + const replayRemembered = async ( + applicationRow: StoredRow | undefined, + failureRow: StoredRow | undefined + ): Promise< + CoordinatedClientToken | { failure: TokenServiceAccountValidationError } | undefined + > => { + if (!failureRow || failureRow.expiresAt.getTime() <= Date.now()) return undefined + const failure = decodeFailure((await decryptSecret(failureRow.encryptedValue)).decrypted, step) + if (!isRememberedRefusal(failure)) { + const fallback = await fallbackFrom(applicationRow) + if (fallback) return fallback + } + return { failure: replayedFailure(failure, step, failureRow.expiresAt.getTime() - Date.now()) } + } + + const settle = ( + outcome: CoordinatedClientToken | { failure: TokenServiceAccountValidationError } + ): CoordinatedClientToken => { + signal?.throwIfAborted() + if ('failure' in outcome) throw outcome.failure + return outcome + } + try { - const [current] = await db + const current = await db .select() .from(clientCredentialToken) - .where(eq(clientCredentialToken.id, applicationKey)) - .limit(1) - const fresh = await serveIfFresh(current) + .where(inArray(clientCredentialToken.id, [applicationKey, failureKey])) + const currentApplication = current.find((entry) => entry.id === applicationKey) + const fresh = await serveStored(currentApplication, minRemainingMs, true) if (fresh) return fresh + const remembered = await replayRemembered( + currentApplication, + current.find((entry) => entry.id === failureKey) + ) + if (remembered) return settle(remembered) await deleteExpiredClientTokenFailures() - const outcome = await db.transaction( - async (tx) => { - await tx.execute(sql`SET LOCAL lock_timeout = '15s'`) - await acquireAdvisoryXactLock(tx, `${namespace}_client_token`, applicationKey) - signal?.throwIfAborted() - const entries = await tx - .select() - .from(clientCredentialToken) - .where(inArray(clientCredentialToken.id, [applicationKey, failureKey])) - const minted = await serveIfFresh( - entries.find((entry) => entry.id === applicationKey), - true + const outcome = await db.transaction< + CoordinatedClientToken | { failure: TokenServiceAccountValidationError } + >(async (tx) => { + await tx.execute(sql`SET LOCAL lock_timeout = '15s'`) + await acquireAdvisoryXactLock(tx, `${namespace}_client_token`, applicationKey) + signal?.throwIfAborted() + const entries = await tx + .select() + .from(clientCredentialToken) + .where(inArray(clientCredentialToken.id, [applicationKey, failureKey])) + const applicationRow = entries.find((entry) => entry.id === applicationKey) + const minted = await serveStored(applicationRow, minRemainingMs, true) + if (minted) return minted + const failureRow = entries.find((entry) => entry.id === failureKey) + const rememberedUnderLock = await replayRemembered(applicationRow, failureRow) + if (rememberedUnderLock) return rememberedUnderLock + signal?.throwIfAborted() + try { + const exchanged = + input.minExchangeIntervalMs === undefined + ? await requestTokenWithRetry(() => input.exchange(signal), signal) + : await input.exchange(signal) + const { encrypted } = await encryptSecret( + JSON.stringify({ + accessToken: exchanged.accessToken, + secretFingerprint, + ...pinnedState?.values, + }) ) - if (minted) return minted - const failure = entries.find( - (entry) => entry.id === failureKey && entry.expiresAt.getTime() > Date.now() + const values = { + encryptedValue: encrypted, + accessTokenDigest: accessTokenDigestOf(exchanged.accessToken, input.accessTokenDigestTag), + expiresAt: exchanged.expiresAt, + updatedAt: new Date(), + } + await tx + .insert(clientCredentialToken) + .values({ id: applicationKey, ...values }) + .onConflictDoUpdate({ target: clientCredentialToken.id, set: values }) + await tx.delete(clientCredentialToken).where(eq(clientCredentialToken.id, failureKey)) + return { + accessToken: exchanged.accessToken, + expiresInSeconds: remainingSeconds(exchanged.expiresAt), + } + } catch (error) { + if (!(error instanceof TokenServiceAccountValidationError)) throw error + const assessed = assessTokenRequestFailure( + error, + await failureStreakBefore(failureRow, step) ) - if (failure) { - return { - failure: decodeFailure((await decryptSecret(failure.encryptedValue)).decrypted, step), - } + const replayMs = Math.max(assessed.replayMs, input.minExchangeIntervalMs ?? 0) + const code = STORED_FAILURE_CODES.has(error.code) + ? (error.code as StoredFailure['code']) + : 'provider_unavailable' + const failure: StoredFailure = { + code, + status: error.status, + ...(assessed.kind !== 'permanent' + ? { + consecutiveFailures: assessed.consecutiveFailures, + ...(assessed.kind === 'throttled' ? { throttled: true } : {}), + } + : isPermanentTokenFailureCode(code) + ? {} + : { refusal: true }), } - signal?.throwIfAborted() - try { - const exchanged = await input.exchange(signal) - const { encrypted } = await encryptSecret( - JSON.stringify({ - accessToken: exchanged.accessToken, - secretFingerprint, - ...pinnedState?.values, - }) - ) - const values = { - encryptedValue: encrypted, - accessTokenDigest: accessTokenDigestOf( - exchanged.accessToken, - input.accessTokenDigestTag - ), - expiresAt: exchanged.expiresAt, - updatedAt: new Date(), - } - await tx - .insert(clientCredentialToken) - .values({ id: applicationKey, ...values }) - .onConflictDoUpdate({ target: clientCredentialToken.id, set: values }) - await tx.delete(clientCredentialToken).where(eq(clientCredentialToken.id, failureKey)) - return { - accessToken: exchanged.accessToken, - expiresInSeconds: remainingSeconds(exchanged.expiresAt), + const { encrypted } = await encryptSecret(JSON.stringify(failure)) + const expiresAt = new Date(Date.now() + replayMs) + await tx + .insert(clientCredentialToken) + .values({ id: failureKey, encryptedValue: encrypted, expiresAt }) + .onConflictDoUpdate({ + target: clientCredentialToken.id, + set: { encryptedValue: encrypted, expiresAt, updatedAt: new Date() }, + }) + if (assessed.kind === 'permanent') { + if (await holdsTokenForSecret(applicationRow)) { + await tx + .update(clientCredentialToken) + .set({ expiresAt: new Date(0), updatedAt: new Date() }) + .where(eq(clientCredentialToken.id, applicationKey)) } - } catch (error) { - if (!(error instanceof TokenServiceAccountValidationError)) throw error - const failure: StoredFailure = { - code: STORED_FAILURE_CODES.has(error.code) - ? (error.code as StoredFailure['code']) - : 'provider_unavailable', - status: error.status, - } - const { encrypted } = await encryptSecret(JSON.stringify(failure)) - const expiresAt = new Date(Date.now() + FAILURE_TTL_MS) - await tx - .insert(clientCredentialToken) - .values({ id: failureKey, encryptedValue: encrypted, expiresAt }) - .onConflictDoUpdate({ - target: clientCredentialToken.id, - set: { encryptedValue: encrypted, expiresAt, updatedAt: new Date() }, - }) - return { failure } + return { failure: replayedFailure(failure, step, replayMs) } + } + const fallback = await fallbackFrom(applicationRow) + if (fallback) { + logger.warn('Serving a still-valid client-credentials token after its refresh failed', { + namespace, + failure: assessed.kind, + retryInMs: replayMs, + }) + return fallback } + return { failure: replayedFailure(failure, step, replayMs) } } - ) - signal?.throwIfAborted() - if ('failure' in outcome) { - throw new TokenServiceAccountValidationError(outcome.failure.code, outcome.failure.status, { - step, - }) - } - return outcome + }) + return settle(outcome) } catch (error) { signal?.throwIfAborted() if (error instanceof TokenServiceAccountValidationError) throw error diff --git a/apps/sim/lib/credentials/token-request-resilience.ts b/apps/sim/lib/credentials/token-request-resilience.ts new file mode 100644 index 00000000000..6d8ed310c42 --- /dev/null +++ b/apps/sim/lib/credentials/token-request-resilience.ts @@ -0,0 +1,141 @@ +import { interruptibleSleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' +import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' + +/** + * How a failed token request is retried: + * - `permanent`: the provider refused the credentials or the account (or the failure is not a + * provider answer at all); asking again cannot help until the secret changes. + * - `throttled`: a 429, or a 503 with a `Retry-After`; the provider is rate limiting the caller. + * - `transient`: any other outage (5xx, timeout, network failure, malformed token response). + */ +export type TokenRequestFailureKind = 'permanent' | 'throttled' | 'transient' + +export interface TokenRequestFailure { + kind: TokenRequestFailureKind + /** Failures of this kind in a row, this one included; 1 for a permanent failure. */ + consecutiveFailures: number + /** How long the failure is replayed before the provider is asked again. */ + replayMs: number +} + +/** + * A token with less than this left is treated as expired and never served, so clock skew and a + * call already in flight cannot carry it past its expiry. + */ +export const TOKEN_EXPIRY_MARGIN_MS = 30_000 + +/** + * How long a remembered failure outlives its replay, so the next failure continues the streak + * and backs off longer. A failure after a quieter gap starts a new streak. + */ +export const TOKEN_FAILURE_STREAK_RETENTION_MS = 5 * 60_000 + +/** A refusal of the credentials is replayed this long, so a revoked secret does not hammer the provider. */ +const PERMANENT_FAILURE_REPLAY_MS = 30_000 +/** + * A provider's `Retry-After` is honored for at most five minutes. Microsoft Entra ID, the most + * explicit documented case, throttles client-credentials token requests with `Retry-After: 60`; + * five minutes leaves room for per-minute provider windows while a proxy or a misbehaving provider + * answering with hours cannot lock every workflow on an application out for that long. + */ +const RETRY_AFTER_BOUNDS = { baseMs: 1_000, maxMs: 5 * 60_000 } as const +/** + * A throttled answer that names no `Retry-After` is replayed for at least a minute, doubling while + * throttling continues, up to the five-minute `Retry-After` cap. Token endpoints document their + * limits per minute (Vanta: 5 tokens and 25 token requests a minute), so asking again sooner lands + * in the window that was just exhausted. + */ +const THROTTLED_BACKOFF = { baseMs: 60_000, maxMs: RETRY_AFTER_BOUNDS.maxMs } as const +/** + * An outage is retried no sooner than 5 s after it (Entra's resilience guidance: without a + * `Retry-After`, the first back-off retry at least 5 s after the response), doubling with jitter + * while it continues, up to 30 s: the flat replay every failure used to get, so no outage is + * replayed longer than before. + */ +const OUTAGE_BACKOFF = { baseMs: 5_000, maxMs: 30_000 } as const +/** The pause before the one in-request retry: MSAL's documented one-second retry-once. */ +const IN_REQUEST_RETRY_DELAY = { baseMs: 1_000, maxMs: 1_000 } as const +/** + * A failed attempt is retried only when it failed this quickly. A timed-out attempt (10 s) is + * never retried in-request, and a retried exchange holding a coordinated application lock stays + * within the 15 s its waiters allow: 2 s, the 1.2 s pause, and one more 10 s attempt. + */ +const IN_REQUEST_RETRY_BUDGET_MS = 2_000 + +/** Whether a token request failure with this code is a refusal that asking again cannot fix. */ +export function isPermanentTokenFailureCode(code: string): boolean { + return code !== 'provider_unavailable' +} + +/** Classifies a failed token request; see {@link TokenRequestFailureKind}. */ +function classifyTokenRequestFailure(error: unknown): TokenRequestFailureKind { + if (!(error instanceof TokenServiceAccountValidationError)) return 'permanent' + if (isPermanentTokenFailureCode(error.code)) return 'permanent' + return error.status === 429 || (error.status === 503 && error.retryAfterMs !== undefined) + ? 'throttled' + : 'transient' +} + +/** Exponential backoff with ±20% jitter, kept within `bounds` so jitter never undercuts the floor. */ +function boundedBackoff(attempt: number, bounds: { baseMs: number; maxMs: number }): number { + return Math.min(Math.max(backoffWithJitter(attempt, null, bounds), bounds.baseMs), bounds.maxMs) +} + +/** + * Decides how long a failed token request is replayed. A remembered failure another layer is + * replaying (`replayForMs`) is replayed exactly as long as that layer still does. Otherwise a + * provider's `Retry-After` is honored within 1 s to 5 min; without one, a throttled answer backs + * off from 1 min to 5 min, an outage from 5 s to 30 s, and a refusal is replayed 30 s. + * `previous` is the remembered failure immediately before this one, if any: the backoff grows only + * while failures of the same kind continue, so a throttled answer after an outage starts its own + * streak at a minute instead of inheriting the outage's count. + */ +export function assessTokenRequestFailure( + error: unknown, + previous?: Pick +): TokenRequestFailure { + const kind = classifyTokenRequestFailure(error) + const consecutiveFailures = + kind !== 'permanent' && previous?.kind === kind ? previous.consecutiveFailures + 1 : 1 + const replay = (replayMs: number): TokenRequestFailure => ({ + kind, + consecutiveFailures, + replayMs, + }) + if (error instanceof TokenServiceAccountValidationError) { + if (error.replayForMs !== undefined) return replay(Math.max(error.replayForMs, 0)) + if (error.retryAfterMs !== undefined && kind !== 'permanent') { + return replay(backoffWithJitter(1, error.retryAfterMs, RETRY_AFTER_BOUNDS)) + } + } + if (kind === 'permanent') return replay(PERMANENT_FAILURE_REPLAY_MS) + if (kind === 'throttled') return replay(boundedBackoff(consecutiveFailures, THROTTLED_BACKOFF)) + return replay(boundedBackoff(consecutiveFailures, OUTAGE_BACKOFF)) +} + +/** + * Runs a token request, retrying a transient failure once after about a second when the failed + * attempt was quick (see {@link IN_REQUEST_RETRY_BUDGET_MS}). Throttled and permanent failures, + * and an aborted `signal`, are never retried. + */ +export async function requestTokenWithRetry( + request: () => Promise, + signal?: AbortSignal +): Promise { + const startedAt = Date.now() + try { + return await request() + } catch (error) { + signal?.throwIfAborted() + if ( + classifyTokenRequestFailure(error) !== 'transient' || + Date.now() - startedAt > IN_REQUEST_RETRY_BUDGET_MS + ) { + throw error + } + await interruptibleSleep(backoffWithJitter(1, null, IN_REQUEST_RETRY_DELAY), signal) + signal?.throwIfAborted() + return request() + } +} diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts index f12b7bae6e6..5c5f9dc205d 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts @@ -133,6 +133,50 @@ describe('token service-account error helpers', () => { expect(error.code).toBe('provider_unavailable') }) + + it.each([ + [429, '120', 120_000], + [503, '7', 7_000], + ])( + 'carries the Retry-After a %i asked for in seconds', + async (status, retryAfter, retryAfterMs) => { + const res = new Response('slow down', { status, headers: { 'Retry-After': retryAfter } }) + + const error = await expectValidationError(throwForProviderResponse(res, 'self')) + + expect(error.code).toBe('provider_unavailable') + expect(error.retryAfterMs).toBe(retryAfterMs) + } + ) + + it('carries the Retry-After a 429 asked for as an HTTP date', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + try { + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')) + const res = new Response('slow down', { + status: 429, + headers: { 'Retry-After': 'Thu, 01 Jan 2026 00:01:30 GMT' }, + }) + + const error = await expectValidationError(throwForProviderResponse(res, 'self')) + + expect(error.retryAfterMs).toBe(90_000) + } finally { + vi.useRealTimers() + } + }) + + it.each([ + [500, '30'], + [401, '30'], + [429, 'soon'], + ])('carries no Retry-After for a %i with Retry-After %s', async (status, retryAfter) => { + const res = new Response('nope', { status, headers: { 'Retry-After': retryAfter } }) + + const error = await expectValidationError(throwForProviderResponse(res, 'self')) + + expect(error.retryAfterMs).toBeUndefined() + }) }) describe('readProviderErrorSnippet', () => { diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.ts b/apps/sim/lib/credentials/token-service-accounts/errors.ts index 0dff3b071b1..3364235d8a8 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '@sim/utils/errors' +import { parseRetryAfter } from '@sim/utils/retry' import { truncate } from '@sim/utils/string' import { DEFAULT_MAX_ERROR_BODY_BYTES, @@ -33,14 +34,38 @@ export type TokenServiceAccountValidationCode = */ | 'grant_not_permitted' +export interface TokenServiceAccountValidationErrorOptions { + /** See {@link TokenServiceAccountValidationError.retryAfterMs}. */ + retryAfterMs?: number + /** See {@link TokenServiceAccountValidationError.replayForMs}. */ + replayForMs?: number +} + export class TokenServiceAccountValidationError extends Error { + /** + * The provider's `Retry-After` on a 429 or 503 answer, in milliseconds, uncapped as the provider + * sent it. Absent when the provider named none. Only ever set from a provider's own answer. + */ + public readonly retryAfterMs?: number + + /** + * Set when this is a remembered failure being replayed rather than a fresh provider answer: how + * much longer, in milliseconds, it is replayed before the provider is asked again. It says when + * the provider will next be asked, not that asking will then succeed (a replayed refusal of the + * credentials carries it too). + */ + public readonly replayForMs?: number + constructor( public readonly code: TokenServiceAccountValidationCode, public readonly status: number, - public readonly logDetail?: Record + public readonly logDetail?: Record, + options: TokenServiceAccountValidationErrorOptions = {} ) { super(code) this.name = 'TokenServiceAccountValidationError' + if (options.retryAfterMs !== undefined) this.retryAfterMs = options.retryAfterMs + if (options.replayForMs !== undefined) this.replayForMs = options.replayForMs } } @@ -127,6 +152,16 @@ export function providerFailureCode( : 'provider_unavailable' } +/** + * The `Retry-After` of a 429 (RFC 6585) or 503 (RFC 9110 section 15.6.4) answer in milliseconds, + * as delay-seconds or an HTTP-date (RFC 9110 section 10.2.3); `undefined` for any other status or + * a missing or unparsable header. Uncapped: the caller that waits decides how long is too long. + */ +export function throttledRetryAfterMs(response: ProviderResponse): number | undefined { + if (response.status !== 429 && response.status !== 503) return undefined + return parseRetryAfter(response.headers.get('retry-after'), Number.POSITIVE_INFINITY) ?? undefined +} + export interface FetchProviderOptions { /** * Validation code thrown when the host does not resolve (`ENOTFOUND` only — @@ -254,7 +289,7 @@ export async function readProviderErrorSnippet(res: ProviderResponse): Promise { if (res.ok) return const body = await readProviderErrorSnippet(res) - throw new TokenServiceAccountValidationError(providerFailureCode(res.status), res.status, { - step, - body, - ...context, - }) + throw new TokenServiceAccountValidationError( + providerFailureCode(res.status), + res.status, + { step, body, ...context }, + { retryAfterMs: throttledRetryAfterMs(res) } + ) } diff --git a/apps/sim/lib/credentials/token-service-accounts/grants/shopify.ts b/apps/sim/lib/credentials/token-service-accounts/grants/shopify.ts index 440cdb0e4da..3672a6878b8 100644 --- a/apps/sim/lib/credentials/token-service-accounts/grants/shopify.ts +++ b/apps/sim/lib/credentials/token-service-accounts/grants/shopify.ts @@ -21,7 +21,12 @@ const MAX_TOKEN_LIFETIME_SECONDS = 86_400 * Shopify answers `shop_not_permitted` when the app and store are in different organizations, * which no secret fixes, and 404 for a store domain that does not exist. */ -function classifyShopifyRefusal(shop: string, status: number, body: string) { +function classifyShopifyRefusal( + shop: string, + status: number, + body: string, + retryAfterMs: number | undefined +) { if (/shop_not_permitted/i.test(body)) { return new TokenServiceAccountValidationError('grant_not_permitted', status, { step: STEP, @@ -32,7 +37,8 @@ function classifyShopifyRefusal(shop: string, status: number, body: string) { return new TokenServiceAccountValidationError( status === 404 ? 'site_not_found' : providerFailureCode(status), status, - { step: STEP, domain: shop, reason: 'provider rejected token exchange' } + { step: STEP, domain: shop, reason: 'provider rejected token exchange' }, + { retryAfterMs } ) } @@ -82,7 +88,8 @@ export async function mintShopifyClientCredentialsToken( dnsFailureCode: 'site_not_found', dnsFailureReason: 'store domain does not resolve', }, - classifyRefusal: (status, body) => classifyShopifyRefusal(shop, status, body), + classifyRefusal: (status, body, retryAfterMs) => + classifyShopifyRefusal(shop, status, body, retryAfterMs), signal, }), signal: options?.signal, diff --git a/apps/sim/lib/internal/crowdstrike/client.ts b/apps/sim/lib/internal/crowdstrike/client.ts index 92c4974d3ee..4ab2896711e 100644 --- a/apps/sim/lib/internal/crowdstrike/client.ts +++ b/apps/sim/lib/internal/crowdstrike/client.ts @@ -2,6 +2,7 @@ import { toNumberOrNull, toStringOrNull } from '@sim/utils/coerce' import { isRecordLike, toRecordOrNull } from '@sim/utils/object' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { throttledRetryAfterMs } from '@/lib/credentials/token-service-accounts/errors' import type { CrowdStrikeCloud, CrowdStrikeInlineCredentials } from '@/tools/crowdstrike/types' export type JsonRecord = Record @@ -160,7 +161,12 @@ export function getFalconErrorMessage(data: unknown, fallback: string): string { export class CrowdStrikeAuthError extends Error { readonly status: number - constructor(message: string, status: number) { + constructor( + message: string, + status: number, + /** The token endpoint's `Retry-After` on a 429 or 503, in milliseconds. */ + readonly retryAfterMs?: number + ) { super(message) this.name = 'CrowdStrikeAuthError' this.status = status >= 400 && status <= 599 ? status : 502 @@ -220,7 +226,8 @@ export class CrowdStrikeTokenRedirectError extends Error { * With `followRedirects: false` a redirect is never followed and throws * {@link CrowdStrikeTokenRedirectError}, so the secret reaches no other host; * the legacy block's inline path keeps following one as it always has. A - * refusal throws {@link CrowdStrikeAuthError} with Falcon's status. + * refusal throws {@link CrowdStrikeAuthError} with Falcon's status and + * `Retry-After`, even when its error body cannot be read. * * @see https://developer.crowdstrike.com/api-reference/collections/oauth2.md */ @@ -258,12 +265,15 @@ export async function requestCrowdStrikeToken( ) } - const data = await readFalconJson(response, signal) + const data = response.ok + ? await readFalconJson(response, signal) + : await readFalconJson(response, signal).catch(() => null) signal?.throwIfAborted() if (!response.ok) { throw new CrowdStrikeAuthError( getFalconErrorMessage(data, 'Failed to authenticate with CrowdStrike'), - response.status + response.status, + throttledRetryAfterMs(response) ) } if (!isRecordLike(data) || typeof data.access_token !== 'string' || !data.access_token.trim()) { diff --git a/apps/sim/lib/internal/sailpoint/client.ts b/apps/sim/lib/internal/sailpoint/client.ts index 43d721cbfe5..910b1273dec 100644 --- a/apps/sim/lib/internal/sailpoint/client.ts +++ b/apps/sim/lib/internal/sailpoint/client.ts @@ -9,6 +9,7 @@ import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' +import { throttledRetryAfterMs } from '@/lib/credentials/token-service-accounts/errors' /** API client credentials typed inline on the legacy block. */ export interface SailPointCredentials { @@ -29,7 +30,9 @@ export type SailPointAuth = SailPointCredentials | SailPointTokenCredentials export class SailPointAuthError extends Error { constructor( message: string, - readonly status: number + readonly status: number, + /** The token endpoint's `Retry-After` on a 429 or 503, in milliseconds. */ + readonly retryAfterMs?: number ) { super(message) this.name = 'SailPointAuthError' @@ -180,8 +183,9 @@ async function readBoundedBody( /** * Exchanges API client credentials for an access token at a tenant's token * endpoint (`client_credentials` grant, credentials in the form body), with - * redirects refused and up to four backoff retries on 429. A refusal throws - * {@link SailPointAuthError} with the provider's status and message. + * redirects refused and up to four backoff retries on 429 (`maxThrottleRetries`). + * A refusal throws {@link SailPointAuthError} with the provider's status, + * message, and `Retry-After`, even when its error body cannot be read. * * @see https://developer.sailpoint.com/docs/api/authentication */ @@ -189,7 +193,8 @@ export async function requestSailPointToken( tokenUrl: string, clientId: string, clientSecret: string, - signal: AbortSignal + signal: AbortSignal, + { maxThrottleRetries = MAX_FETCH_RETRIES }: { maxThrottleRetries?: number } = {} ): Promise<{ token: string; expiresInSeconds: number }> { let attempt = 0 @@ -211,7 +216,7 @@ export async function requestSailPointToken( signal, }) - if (response.status === 429 && attempt < MAX_FETCH_RETRIES) { + if (response.status === 429 && attempt < maxThrottleRetries) { const retryAfterMs = parseRetryAfter(response.headers.get('retry-after')) await consumeOrCancelBody(response, DEFAULT_MAX_ERROR_BODY_BYTES) attempt += 1 @@ -220,15 +225,15 @@ export async function requestSailPointToken( continue } - const data = await readBoundedBody( - response, - response.ok ? MAX_TOKEN_RESPONSE_BYTES : DEFAULT_MAX_ERROR_BODY_BYTES, - signal - ) + const data = response.ok + ? await readBoundedBody(response, MAX_TOKEN_RESPONSE_BYTES, signal) + : await readBoundedBody(response, DEFAULT_MAX_ERROR_BODY_BYTES, signal).catch(() => null) + signal?.throwIfAborted() if (!response.ok) { throw new SailPointAuthError( getSailPointErrorMessage(data, 'Failed to authenticate with SailPoint'), - response.status + response.status, + throttledRetryAfterMs(response) ) } if (!isRecordLike(data) || typeof data.access_token !== 'string' || !data.access_token) { diff --git a/apps/sim/lib/internal/zoominfo/client.ts b/apps/sim/lib/internal/zoominfo/client.ts index 223100b23c0..5fc03de3859 100644 --- a/apps/sim/lib/internal/zoominfo/client.ts +++ b/apps/sim/lib/internal/zoominfo/client.ts @@ -5,6 +5,7 @@ import { MAX_JSON_API_RESPONSE_BYTES, secureFetchWithValidation, } from '@/lib/core/security/input-validation.server' +import { throttledRetryAfterMs } from '@/lib/credentials/token-service-accounts/errors' import { assertSafeZoomInfoUrl, ZOOMINFO_API_BASE, @@ -51,7 +52,11 @@ export class ZoomInfoOperationError extends Error { * provider status), and carries that status for the saved-credential minter. */ export class ZoomInfoTokenError extends ZoomInfoOperationError { - constructor(readonly tokenStatus: number) { + constructor( + readonly tokenStatus: number, + /** The token endpoint's `Retry-After` on a 429 or 503, in milliseconds. */ + readonly retryAfterMs?: number + ) { super(`ZoomInfo token request failed: HTTP ${tokenStatus}`, 500) this.name = 'ZoomInfoTokenError' } @@ -129,7 +134,7 @@ export async function requestZoomInfoToken( if (!response.ok) { const text = await response.text().catch(() => '') logger.warn('ZoomInfo token fetch failed', { requestId, status: response.status, error: text }) - throw new ZoomInfoTokenError(response.status) + throw new ZoomInfoTokenError(response.status, throttledRetryAfterMs(response)) } const data: unknown = await response.json() if (!isRecordLike(data) || typeof data.access_token !== 'string' || !data.access_token.trim()) { diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index a3ee4f787d7..3fe7db1a82f 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -2,12 +2,14 @@ import { credential } from '@sim/db/schema' import { createDeferred } from '@sim/testing/helpers/deferred' import { queueTableRows, resetDbChainMock } from '@sim/testing/mocks/database.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, RAMP_SERVICE_ACCOUNT_PROVIDER_ID, + VANTA_SERVICE_ACCOUNT_PROVIDER_ID, } from '@/lib/credentials/client-credential-accounts/descriptors' import * as clientAccounts from '@/lib/credentials/client-credential-accounts/server' +import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { forgetRejectedClientCredentialToken, resolveServiceAccountToken, @@ -152,3 +154,334 @@ describe('client credential token cache', () => { } }) }) + +describe('client credential token refresh after a failed mint', () => { + const refused = () => + new TokenServiceAccountValidationError('invalid_credentials', 401, { step: 'mint' }) + const unavailable = (status = 502, retryAfterMs?: number) => + new TokenServiceAccountValidationError( + 'provider_unavailable', + status, + { step: 'mint' }, + { + retryAfterMs, + } + ) + + /** Resolves with fake timers running, so an in-mint retry pause elapses without real waiting. */ + async function resolveFor( + credentialId: string, + providerId: string = RAMP_SERVICE_ACCOUNT_PROVIDER_ID + ) { + queueTableRows(credential, [storedCredential]) + const outcome = resolveServiceAccountToken(credentialId, providerId).then( + (value) => ({ value }), + (error: unknown) => ({ error }) + ) + await vi.runAllTimersAsync() + return outcome + } + + function minter() { + const mint = vi.fn() + vi.spyOn(clientAccounts, 'getClientCredentialAccountMinter').mockReturnValue(mint) + return mint + } + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('keeps serving a still-valid token while its early refresh fails transiently', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await expect(resolveFor('ramp-refresh-outage')).resolves.toEqual({ + value: { accessToken: 'current-token' }, + }) + mint.mockRejectedValue(unavailable()) + vi.advanceTimersByTime(6 * 60_000) + + await expect(resolveFor('ramp-refresh-outage')).resolves.toEqual({ + value: { accessToken: 'current-token' }, + }) + const askedDuringRefresh = mint.mock.calls.length + await expect(resolveFor('ramp-refresh-outage')).resolves.toEqual({ + value: { accessToken: 'current-token' }, + }) + expect(askedDuringRefresh).toBeGreaterThan(1) + expect(mint).toHaveBeenCalledTimes(askedDuringRefresh) + }) + + it('reports the failed refresh once the cached token has run out', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refresh-expired') + mint.mockRejectedValue(unavailable()) + vi.advanceTimersByTime(600_000 - 20_000) + + await expect(resolveFor('ramp-refresh-expired')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + }) + + it('does not serve a cached token that a slow failed refresh carried inside its expiry margin', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refresh-slow') + mint.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 15_000) + throw unavailable() + }) + vi.advanceTimersByTime(600_000 - 40_000) + + await expect(resolveFor('ramp-refresh-slow')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + }) + + it('does not serve a cached token the provider refused while its failed refresh was in flight', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refresh-rejected-midway') + mint.mockImplementation(async () => { + forgetRejectedClientCredentialToken('current-token') + throw unavailable() + }) + vi.advanceTimersByTime(6 * 60_000) + + await expect(resolveFor('ramp-refresh-rejected-midway')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + }) + + it('stops serving the cached token once the provider refuses the credentials', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refresh-refused') + mint.mockRejectedValue(refused()) + vi.advanceTimersByTime(6 * 60_000) + + await expect(resolveFor('ramp-refresh-refused')).resolves.toEqual({ + error: expect.objectContaining({ code: 'invalid_credentials' }), + }) + expect(mint).toHaveBeenCalledTimes(2) + }) + + it('never serves a refused token again when a later refresh fails with an outage', async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-refused-then-outage') + mint.mockRejectedValueOnce(refused()).mockRejectedValue(unavailable()) + vi.advanceTimersByTime(6 * 60_000) + await resolveFor('ramp-refused-then-outage') + vi.advanceTimersByTime(31_000) + + await expect(resolveFor('ramp-refused-then-outage')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + }) + + it('retries a quick transient failure once inside the mint', async () => { + const mint = minter() + mint.mockRejectedValueOnce(unavailable(503)).mockResolvedValueOnce(token) + + await expect(resolveFor('ramp-retry-once')).resolves.toEqual({ + value: { accessToken: token.accessToken }, + }) + expect(mint).toHaveBeenCalledTimes(2) + }) + + it('does not retry an attempt that took as long as a provider timeout', async () => { + const mint = minter() + mint.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 10_000) + throw unavailable() + }) + + await expect(resolveFor('ramp-timed-out')).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable' }), + }) + expect(mint).toHaveBeenCalledOnce() + }) + + it('replays a refusal of the credentials for 30 seconds without retrying it', async () => { + const mint = minter() + mint.mockRejectedValue(refused()) + + await resolveFor('ramp-refused') + vi.advanceTimersByTime(29_000) + await expect(resolveFor('ramp-refused')).resolves.toEqual({ + error: expect.objectContaining({ code: 'invalid_credentials' }), + }) + expect(mint).toHaveBeenCalledOnce() + + vi.advanceTimersByTime(2_000) + await resolveFor('ramp-refused') + expect(mint).toHaveBeenCalledTimes(2) + }) + + it.each([ + ['as long as a throttled provider asks', 120_000, 120_000], + ['no longer than five minutes, whatever the provider asks', 3_600_000, 300_000], + ])('waits %s before asking again', async (_case, retryAfterMs, waitMs) => { + const mint = minter() + const credentialId = `ramp-throttled-${retryAfterMs}` + mint.mockRejectedValueOnce(unavailable(429, retryAfterMs)).mockResolvedValueOnce(token) + + await expect(resolveFor(credentialId)).resolves.toEqual({ + error: expect.objectContaining({ code: 'provider_unavailable', status: 429 }), + }) + vi.advanceTimersByTime(waitMs - 1_000) + await expect(resolveFor(credentialId)).resolves.toEqual({ + error: expect.objectContaining({ status: 429 }), + }) + expect(mint).toHaveBeenCalledOnce() + + vi.advanceTimersByTime(2_000) + await expect(resolveFor(credentialId)).resolves.toEqual({ + value: { accessToken: token.accessToken }, + }) + }) + + it('backs off an outage briefly at first, longer while it lasts, and never past 30 seconds', async () => { + const mint = minter() + mint.mockRejectedValue(unavailable()) + + await resolveFor('ramp-outage') + const firstAttempt = mint.mock.calls.length + vi.advanceTimersByTime(6_001) + await resolveFor('ramp-outage') + const secondAttempt = mint.mock.calls.length + expect(secondAttempt).toBeGreaterThan(firstAttempt) + + vi.advanceTimersByTime(6_001) + await resolveFor('ramp-outage') + expect(mint).toHaveBeenCalledTimes(secondAttempt) + + for (let attempt = 0; attempt < 6; attempt++) { + const asked = mint.mock.calls.length + vi.advanceTimersByTime(30_001) + await resolveFor('ramp-outage') + expect(mint.mock.calls.length).toBeGreaterThan(asked) + } + }) + + it('waits at least a minute after a throttled answer that gave no Retry-After', async () => { + const mint = minter() + mint.mockRejectedValueOnce(unavailable(429)).mockResolvedValueOnce(token) + + await resolveFor('ramp-throttled-bare') + vi.advanceTimersByTime(59_000) + await expect(resolveFor('ramp-throttled-bare')).resolves.toEqual({ + error: expect.objectContaining({ status: 429 }), + }) + expect(mint).toHaveBeenCalledOnce() + + vi.advanceTimersByTime(14_000) + await expect(resolveFor('ramp-throttled-bare')).resolves.toEqual({ + value: { accessToken: token.accessToken }, + }) + }) + + it('waits about a minute, not an outage streak longer, for a throttled answer after an outage', async () => { + const mint = minter() + const answers = [unavailable(), unavailable(), unavailable(429)] + mint.mockImplementation(async () => { + throw answers[0] + }) + + await resolveFor('ramp-outage-then-throttled') + answers.shift() + vi.advanceTimersByTime(30_001) + await resolveFor('ramp-outage-then-throttled') + answers.shift() + vi.advanceTimersByTime(30_001) + await expect(resolveFor('ramp-outage-then-throttled')).resolves.toEqual({ + error: expect.objectContaining({ status: 429 }), + }) + mint.mockResolvedValueOnce(token) + + vi.advanceTimersByTime(73_000) + await expect(resolveFor('ramp-outage-then-throttled')).resolves.toEqual({ + value: { accessToken: token.accessToken }, + }) + }) + + it('waits at least 5 seconds before asking again after an outage, whatever the jitter', async () => { + const lowestJitter = vi + .spyOn(globalThis.crypto, 'getRandomValues') + .mockImplementation((array) => array) + try { + const mint = minter() + mint.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 10_000) + throw unavailable() + }) + + await resolveFor('ramp-outage-floor') + vi.advanceTimersByTime(4_900) + await resolveFor('ramp-outage-floor') + + expect(mint).toHaveBeenCalledOnce() + } finally { + lowestJitter.mockRestore() + } + }) + + it('replays a coordinated failure exactly as long as the coordinated layer still replays it', async () => { + encryptionMockFns.mockDecryptSecret.mockResolvedValue({ + decrypted: JSON.stringify({ + type: CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, + providerId: VANTA_SERVICE_ACCOUNT_PROVIDER_ID, + clientId: 'vanta-client', + clientSecret: 'vanta-client-secret', + }), + }) + const mint = minter() + mint + .mockRejectedValueOnce( + new TokenServiceAccountValidationError( + 'provider_unavailable', + 502, + { step: 'mint' }, + { replayForMs: 300 } + ) + ) + .mockResolvedValueOnce(token) + + await resolveFor('vanta-coordinated-replay', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) + vi.advanceTimersByTime(301) + + await expect( + resolveFor('vanta-coordinated-replay', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) + ).resolves.toEqual({ value: { accessToken: token.accessToken } }) + }) + + it('asks the coordinated layer again at once after a failure it does not remember', async () => { + encryptionMockFns.mockDecryptSecret.mockResolvedValue({ + decrypted: JSON.stringify({ + type: CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, + providerId: VANTA_SERVICE_ACCOUNT_PROVIDER_ID, + clientId: 'vanta-client', + clientSecret: 'vanta-client-secret', + }), + }) + const mint = minter() + mint + .mockRejectedValueOnce( + new TokenServiceAccountValidationError('provider_unavailable', 503, { step: 'mint' }) + ) + .mockResolvedValueOnce(token) + + await resolveFor('vanta-coordination-failed', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) + + await expect( + resolveFor('vanta-coordination-failed', VANTA_SERVICE_ACCOUNT_PROVIDER_ID) + ).resolves.toEqual({ value: { accessToken: token.accessToken } }) + }) +}) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 93d428ac8b4..7c8ec3ed57b 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -24,6 +24,13 @@ import { GOOGLE_SERVICE_ACCOUNT_EXCLUDED_SCOPES, isGoogleWorkloadIdentitySecret, } from '@/lib/credentials/google-workload-identity' +import { + assessTokenRequestFailure, + requestTokenWithRetry, + TOKEN_EXPIRY_MARGIN_MS, + TOKEN_FAILURE_STREAK_RETENTION_MS, + type TokenRequestFailureKind, +} from '@/lib/credentials/token-request-resilience' import { getTokenClientCredentialMinter } from '@/lib/credentials/token-service-accounts/client-credentials' import { type CLIENT_CREDENTIALS_AUTH_METHOD, @@ -32,6 +39,7 @@ import { isTokenServiceAccountProviderId, SNOWFLAKE_SERVICE_ACCOUNT_PROVIDER_ID, } from '@/lib/credentials/token-service-accounts/descriptors' +import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { parseStoredSecretMode, parseTokenServiceAccountSecret, @@ -631,7 +639,13 @@ interface MintedServiceAccountToken { interface FailedClientCredentialMint { /** The error the failed mint threw, re-thrown to callers while memoized. */ error: unknown + kind: TokenRequestFailureKind + /** Failures of this kind in a row; the next one of the same kind backs off longer. */ + consecutiveFailures: number secretFingerprint: string + /** Until when the failure is replayed instead of minting again. */ + retryAtMs: number + /** When the entry is dropped: after its replay, once a later failure can no longer extend the streak. */ expiresAtMs: number } @@ -652,10 +666,19 @@ interface FailedClientCredentialMint { * multiple concurrently valid tokens, so each instance minting its own token * is correct. * - * Failed mints are never cached as tokens; instead they are memoized for - * {@link CLIENT_CREDENTIAL_MINT_FAILURE_TTL_MS} so a hot workflow on a - * revoked/invalid secret doesn't hammer the provider's token endpoint once - * per block execution. + * A token inside its last {@link CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS} is refreshed + * early, and a refresh that fails with an outage or throttling (not a refusal of + * the credentials) keeps the still-valid token in service until + * {@link TOKEN_EXPIRY_MARGIN_MS} before it expires, as MSAL, Azure.Core and + * botocore do with a failed proactive refresh. + * + * Failed mints are never cached as tokens; instead they are replayed (see + * {@link assessTokenRequestFailure}) so a hot workflow neither hammers the + * provider's token endpoint with a revoked secret nor floods a struggling one: + * a refusal for 30 s, a throttled answer as long as its `Retry-After` asks (at + * most five minutes) or for 1 min growing to 5 min without one, and an outage + * for 5 s growing to 30 s while it lasts. A quick outage is first retried once + * inside the mint. * * Both maps are pruned of expired entries on each resolution * ({@link pruneExpiredClientCredentialCaches}), so their size is bounded by the @@ -665,7 +688,6 @@ interface FailedClientCredentialMint { const clientCredentialTokenCache = new Map() const clientCredentialMintFailureCache = new Map() const CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS = 5 * 60 * 1000 -const CLIENT_CREDENTIAL_MINT_FAILURE_TTL_MS = 30 * 1000 /** Drops fully-expired token and failure entries so the maps stay bounded. */ function pruneExpiredClientCredentialCaches(nowMs: number): void { @@ -707,15 +729,29 @@ function secretFingerprintOf(encryptedServiceAccountKey: string): string { return encryptedServiceAccountKey.slice(0, 32) } +/** + * Whether a `sharedToken` mint failed with a failure the coordinated layer remembers for every + * worker. Only those are replayed here, for exactly as long as that layer still replays them; a + * failure it does not remember (a lock timeout, a database error) may already be over for + * another worker, so the next resolution asks again. + */ +function isCoordinatedReplay(error: unknown): boolean { + return error instanceof TokenServiceAccountValidationError && error.replayForMs !== undefined +} + /** * Resolves a stored secret that mints short-lived tokens, read-through the * per-instance cache: re-reads the credential row, serves a cached token minted * from the same ciphertext while enough of it remains, replays a recent mint - * failure, and otherwise decrypts and mints. Wrapped in `coalesceLocally` so - * concurrent block executions on one instance share a single mint. - * `sharedToken` providers coordinate one token per application themselves, so - * their tokens are never cached here, and a caller that already read the - * secret to choose its mode passes it as `stored` rather than reading it twice. + * failure, and otherwise decrypts and mints, falling back to the cached token + * while it is still valid when the refresh fails with an outage and dropping it + * when the refresh is refused, so no later outage serves it again. Wrapped in + * `coalesceLocally` so concurrent block executions on one instance share a + * single mint. `sharedToken` providers coordinate one token per application + * themselves (including the in-request retry and the fallback), so their tokens + * are never cached here and only a failure their own layer remembers is + * replayed, for as long as it reports; a caller that already read the secret to choose its mode + * passes it as `stored` rather than reading it twice. */ async function resolveMintedServiceAccountToken( credentialId: string, @@ -748,29 +784,37 @@ async function resolveMintedServiceAccountToken( } const secretFingerprint = secretFingerprintOf(encryptedServiceAccountKey) - const cached = sharedToken ? undefined : clientCredentialTokenCache.get(cacheIdentity) - if ( - cached && - cached.secretFingerprint === secretFingerprint && - cached.expiresAtMs - Date.now() > CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS - ) { + const cachedEntry = sharedToken ? undefined : clientCredentialTokenCache.get(cacheIdentity) + const cached = cachedEntry?.secretFingerprint === secretFingerprint ? cachedEntry : undefined + if (cached && cached.expiresAtMs - Date.now() > CLIENT_CREDENTIAL_TOKEN_MIN_TTL_MS) { return { ...cached.token } } - - const failed = clientCredentialMintFailureCache.get(cacheIdentity) - if ( - failed && - failed.secretFingerprint === secretFingerprint && - Date.now() < failed.expiresAtMs - ) { + /** + * The cached token while it is still the one in the cache (a provider may refuse it while a + * refresh is in flight) and more than the expiry margin of it is left, checked at each use. + */ + const stillValid = () => + cached && + clientCredentialTokenCache.get(cacheIdentity) === cached && + cached.expiresAtMs - Date.now() > TOKEN_EXPIRY_MARGIN_MS + ? cached + : undefined + + const failedEntry = clientCredentialMintFailureCache.get(cacheIdentity) + const failed = failedEntry?.secretFingerprint === secretFingerprint ? failedEntry : undefined + if (failed && Date.now() < failed.retryAtMs) { + const fallback = failed.kind === 'permanent' ? undefined : stillValid() + if (fallback) return { ...fallback.token } throw failed.error } - clientCredentialMintFailureCache.delete(cacheIdentity) try { const decrypted = stored?.decrypted ?? (await decryptSecret(encryptedServiceAccountKey)).decrypted - const minted = await mint(decrypted) + const minted = sharedToken + ? await mint(decrypted) + : await requestTokenWithRetry(() => mint(decrypted)) + clientCredentialMintFailureCache.delete(cacheIdentity) if (!sharedToken) clientCredentialTokenCache.set(cacheIdentity, { token: minted.token, @@ -780,12 +824,33 @@ async function resolveMintedServiceAccountToken( }) return { ...minted.token } } catch (error) { - if (!(error instanceof Error && error.name === 'AbortError')) { - clientCredentialMintFailureCache.set(cacheIdentity, { - error: options?.privacyMode === 'selector' ? new Error('Credential mint failed') : error, - secretFingerprint, - expiresAtMs: Date.now() + CLIENT_CREDENTIAL_MINT_FAILURE_TTL_MS, + if (error instanceof Error && error.name === 'AbortError') throw error + if (sharedToken && !isCoordinatedReplay(error)) { + clientCredentialMintFailureCache.delete(cacheIdentity) + throw error + } + const failure = assessTokenRequestFailure(error, failed) + const retryAtMs = Date.now() + failure.replayMs + clientCredentialMintFailureCache.set(cacheIdentity, { + error: options?.privacyMode === 'selector' ? new Error('Credential mint failed') : error, + kind: failure.kind, + consecutiveFailures: failure.consecutiveFailures, + secretFingerprint, + retryAtMs, + expiresAtMs: retryAtMs + TOKEN_FAILURE_STREAK_RETENTION_MS, + }) + if (failure.kind === 'permanent') { + clientCredentialTokenCache.delete(cacheIdentity) + throw error + } + const fallback = stillValid() + if (fallback) { + logger.warn('Serving a still-valid client-credential token after its refresh failed', { + ...(options?.privacyMode === 'selector' ? {} : { credentialId }), + failure: failure.kind, + retryInMs: failure.replayMs, }) + return { ...fallback.token } } throw error }