From d786e331309ec138a452ee8de893632ea7f17b0a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 19:52:42 -0800 Subject: [PATCH 01/32] improvement(egress): add outboundFetch for organization-routed requests --- .../lib/core/network/gateway.server.test.ts | 21 +++++++++ .../lib/core/network/outbound-fetch.server.ts | 47 +++++++++++++++++++ .../documents/secure-fetch.server.ts | 16 ++----- 3 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 apps/sim/lib/core/network/outbound-fetch.server.ts diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts index 28ba87ab155..93ef4a92dce 100644 --- a/apps/sim/lib/core/network/gateway.server.test.ts +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -12,9 +12,15 @@ import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import type { OutboundGateway, ResolvedOutboundRoute } from '@/lib/core/network/config.server' vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +/** The synthetic origin's name resolves to the one address the local gateway admits. */ +vi.mock('@sim/security/dns', async (importOriginal) => ({ + ...(await importOriginal()), + resolveHostAddresses: vi.fn(async () => ({ addresses: ['1.1.1.1'], preferred: '1.1.1.1' })), +})) import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { openGatewayTunnel, secureOutboundTunnel } from '@/lib/core/network/gateway.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createPinnedFetchWithDispatcher, secureFetchWithPinnedIP, @@ -296,6 +302,21 @@ describe('organization gateways over real TLS CONNECT sockets', () => { await transport.dispatcher.destroy() } }) + it("sends a credential request through the organization's gateway", async () => { + const start = admissions.length + await runWithOutboundOrganization('org_b', async () => { + const response = await outboundFetch(url(), { + method: 'POST', + body: 'grant_type=client_credentials', + redirect: 'error', + }) + expect(response.headers.get('x-via-proxy')).toBe('yes') + expect(await response.text()).toBe('grant_type=client_credentials') + }) + expect(admissions.slice(start)).toEqual([ + { token: 'Bearer bravo', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, + ]) + }) it('fails closed on denied proxy credentials without contacting the origin', async () => { state.gateways.set('denied', { ...state.gateways.get('org_a')!, diff --git a/apps/sim/lib/core/network/outbound-fetch.server.ts b/apps/sim/lib/core/network/outbound-fetch.server.ts new file mode 100644 index 00000000000..a87ae320b45 --- /dev/null +++ b/apps/sim/lib/core/network/outbound-fetch.server.ts @@ -0,0 +1,47 @@ +import { createLogger } from '@sim/logger' +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' + +const logger = createLogger('OutboundFetch') + +let gatewayTransport: ReturnType | undefined + +/** + * `fetch` for a request Sim makes on an organization's behalf to a destination outside Sim: a + * credential's token endpoint, a customer's tenant host, or a vendor API. When the current + * outbound scope belongs to an organization with a dedicated egress gateway, the request leaves + * through that gateway, so it comes from the organization's static IPs and passes the IP + * allowlists its destinations enforce. Every other route is the global `fetch`, unchanged. + * + * A call made outside any outbound scope goes direct and logs a warning instead of failing with + * `MISSING_SCOPE` the way the SSRF-guarded helpers do: these requests always went direct, and + * failing closed would turn a code path that never established a scope into an outage for every + * organization. A blocked route or an unavailable gateway still fails closed. + * + * Never use it for Sim's own infrastructure (its storage, queues, or internal services), which + * must not leave through a customer's gateway. + */ +export const outboundFetch: typeof fetch = async (input, init) => { + let route: Awaited> + try { + route = await resolveCurrentOutboundRoute() + } catch (error) { + if (!(error instanceof OutboundRoutingError) || error.code !== 'MISSING_SCOPE') throw error + logger.warn('Outbound request made without an organization scope', { + host: requestHost(input), + }) + return fetch(input, init) + } + if (route.kind === 'direct') return fetch(input, init) + gatewayTransport ??= createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + return gatewayTransport.fetch(input, init) +} + +function requestHost(input: RequestInfo | URL): string | undefined { + try { + return new URL(input instanceof Request ? input.url : input.toString()).host + } catch { + return undefined + } +} diff --git a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts index 27302ee9442..6a04079cfc6 100644 --- a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts +++ b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts @@ -1,6 +1,5 @@ -import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { - createSsrfGuardedFetchWithDispatcher, type SecureFetchOptions, type SecureFetchResponse, secureFetchWithValidation, @@ -61,7 +60,6 @@ export async function secureFetchWithRetry( } const DEFAULT_FETCH_RETRY_BUDGET_MS = 150_000 -let connectorTransport: ReturnType | undefined /** * Bounds requests and response bodies within one retry budget. @@ -84,18 +82,10 @@ export async function fetchWithRetry( signal, AbortSignal.timeout(Math.max(0, Math.ceil(deadlineAt - Date.now()))), ]) - const transport: typeof fetch = async (input, init) => { - const route = await resolveCurrentOutboundRoute() - if (route.kind === 'direct') return fetch(input, init) - connectorTransport ??= createSsrfGuardedFetchWithDispatcher({ - profile: 'configuredEndpoint', - }) - return connectorTransport.fetch(input, init) - } const init = { ...options, signal: requestSignal } const response = retryOptions.fetcher - ? await retryOptions.fetcher(url, init, transport) - : await transport(url, init) + ? await retryOptions.fetcher(url, init, outboundFetch) + : await outboundFetch(url, init) if ( !response.ok && From f01595b3dc43e63d310613b6d538b89a9ba8764c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:03:30 -0800 Subject: [PATCH 02/32] improvement(egress): route credential token mints and verifications through the organization egress gateway --- .../application/slack-managed-users.test.ts | 21 +++++++++++++++++++ .../application/slack-managed-users.ts | 8 +++++-- .../credential-groups/slack-managed-users.ts | 7 ++++--- .../standard-oauth-provider.ts | 3 ++- .../credentials/atlassian-service-account.ts | 5 +++-- .../minters/netsuite.ts | 3 ++- .../databricks-service-account.server.ts | 11 +++++----- .../google-workload-identity.server.ts | 5 +++-- .../microsoft-app-service-account.server.ts | 5 +++-- .../token-service-accounts/errors.ts | 12 ++++++++--- apps/sim/lib/internal/crowdstrike/client.ts | 5 +++-- .../sim/lib/internal/sailpoint/client.test.ts | 4 ++-- apps/sim/lib/internal/sailpoint/client.ts | 5 +++-- 13 files changed, 67 insertions(+), 27 deletions(-) diff --git a/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts b/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts index ab422cead40..9c2fc51f4a9 100644 --- a/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts +++ b/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts @@ -6,6 +6,7 @@ import { credentialGroupsAvailabilityMockFns, } from '@sim/testing/mocks/credential-groups-availability.mock' import { credentialGroupsOrganizationSetupMock } from '@sim/testing/mocks/credential-groups-organization-setup.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { organizationAuthorizationMock, organizationAuthorizationMockFns, @@ -29,6 +30,7 @@ const hoisted = vi.hoisted(() => ({ vi.mock('@/lib/credential-groups/organization-setup', () => credentialGroupsOrganizationSetupMock) vi.mock('@sim/audit', () => auditMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@/lib/core/application/organization-authorization', () => organizationAuthorizationMock) vi.mock('@/lib/core/application/workspace-authorization', () => workspaceAuthorizationMock) vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) @@ -40,6 +42,7 @@ vi.mock('@/lib/credential-groups/slack-managed-users', () => ({ exchangeAndConfigureSlackManagedUsers: hoisted.exchange, })) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { completeSlackCredentialGroupConfiguration, startSlackCredentialGroupConfiguration, @@ -73,6 +76,7 @@ const attempt = { beforeEach(() => { resetDbChainMock() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) mocks.available.mockResolvedValue(true) organizationAccess.mockResolvedValue({ organizationId: 'organization-1', @@ -194,4 +198,21 @@ describe('scoped Slack setup', () => { }) ) }) + + it("exchanges the Slack grant through the owning organization's egress route", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + queueTableRows(credentialGroup, [group]) + mocks.exchange.mockImplementation(async () => { + await resolveCurrentOutboundRoute() + return { + credentialGroupId: 'group-1', + credentialGroupName: 'Connected accounts', + appId: 'app-1', + teamId: 'team-1', + requiredScopes: ['search:read'], + } + }) + await expect(finish()).resolves.toMatchObject({ ok: true }) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('organization-1') + }) }) diff --git a/apps/sim/lib/credential-groups/application/slack-managed-users.ts b/apps/sim/lib/credential-groups/application/slack-managed-users.ts index 698b7e8d640..9d8a5bc8597 100644 --- a/apps/sim/lib/credential-groups/application/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/application/slack-managed-users.ts @@ -7,6 +7,7 @@ import { recordProjectedUseCaseAuditEntries } from '@/lib/core/application/autho import { assertOperationPrincipal, type OperationUseCase } from '@/lib/core/application/operation' import { requireOrganizationMembership } from '@/lib/core/application/organization-authorization' import { authorizeWorkspaceOperation } from '@/lib/core/application/workspace-authorization' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceScope, @@ -181,8 +182,11 @@ export const completeSlackCredentialGroupConfiguration: OperationUseCase< throw new OrchestrationError('validation', 'Authorization state is invalid or expired') } if (input.providerError) return { ok: false as const, reason: 'provider_error' as const } - if (!input.code) throw new OrchestrationError('validation', 'Authorization code is missing') - const result = await exchangeAndConfigureSlackManagedUsers({ attempt, code: input.code }) + const code = input.code + if (!code) throw new OrchestrationError('validation', 'Authorization code is missing') + const result = await withResourceOutboundScope(resourceScopeFields(scope), () => + exchangeAndConfigureSlackManagedUsers({ attempt, code }) + ) recordProjectedUseCaseAuditEntries( credentialGroupOperations.completeSlackConfiguration, scope.kind === 'workspace' ? scope.workspaceId : undefined, diff --git a/apps/sim/lib/credential-groups/slack-managed-users.ts b/apps/sim/lib/credential-groups/slack-managed-users.ts index 4aefdb93f45..d46d8d1abd8 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.ts @@ -16,6 +16,7 @@ import { isRecordLike } from '@sim/utils/object' import { compareStrings } from '@sim/utils/string' import { and, eq, inArray, isNull, or } from 'drizzle-orm' import { getRedisClient } from '@/lib/core/config/redis' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' @@ -333,7 +334,7 @@ function parseSlackOAuthResponse(value: unknown): SlackOAuthSuccess { } export async function revokeSlackToken(token: string): Promise { - const response = await fetch('https://slack.com/api/auth.revoke', { + const response = await outboundFetch('https://slack.com/api/auth.revoke', { method: 'POST', headers: { Authorization: `Bearer ${token}`, @@ -359,7 +360,7 @@ export async function revokeSlackToken(token: string): Promise { async function callSlackApi(method: string, accessToken: string, body?: URLSearchParams) { let response: Response try { - response = await fetch(`https://slack.com/api/${method}`, { + response = await outboundFetch(`https://slack.com/api/${method}`, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -475,7 +476,7 @@ export async function exchangeSlackUserAuthorization(params: { const body = new URLSearchParams({ code: params.code, redirect_uri: params.redirectUri }) let response: Response try { - response = await fetch('https://slack.com/api/oauth.v2.access', { + response = await outboundFetch('https://slack.com/api/oauth.v2.access', { method: 'POST', headers: { Authorization: `Basic ${basicAuth}`, diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index 575aef31263..1be033d4597 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -12,6 +12,7 @@ import { getManagedOAuthConnectorProviderConfig, type ManagedOAuthConnectorConfig, } from '@/lib/auth/connectors/managed-oauth' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { credentialGroupOAuthNonceMatches } from '@/lib/credential-groups/oauth-state' import type { @@ -81,7 +82,7 @@ async function resolveOAuthEndpoints( if (connector.discoveryUrl) { let response: Response try { - response = await fetch(connector.discoveryUrl, { + response = await outboundFetch(connector.discoveryUrl, { headers: connector.discoveryHeaders, signal: AbortSignal.timeout(OAUTH_DISCOVERY_TIMEOUT_MS), }) diff --git a/apps/sim/lib/credentials/atlassian-service-account.ts b/apps/sim/lib/credentials/atlassian-service-account.ts index e5a737cf531..97fdbd524ab 100644 --- a/apps/sim/lib/credentials/atlassian-service-account.ts +++ b/apps/sim/lib/credentials/atlassian-service-account.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { AtlassianProduct } from '@/lib/credentials/service-account-fields' import { parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -94,7 +95,7 @@ export async function validateAtlassianServiceAccount( }> { assertAtlassianCloudHost(domain) - const tenantInfoRes = await fetch(`https://${domain}/_edge/tenant_info`, { + const tenantInfoRes = await outboundFetch(`https://${domain}/_edge/tenant_info`, { headers: { Accept: 'application/json' }, }) if (tenantInfoRes.status === 404) { @@ -112,7 +113,7 @@ export async function validateAtlassianServiceAccount( const cloudId = tenantInfo.cloudId const identityPath = product === 'confluence' ? 'wiki/rest/api/user/current' : 'rest/api/3/myself' - const myselfRes = await fetch( + const myselfRes = await outboundFetch( `https://api.atlassian.com/ex/${product}/${cloudId}/${identityPath}`, { headers: { Authorization: `Bearer ${apiToken}`, Accept: 'application/json' }, diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts index 9e3738e5970..5f122915936 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts @@ -2,6 +2,7 @@ import { createPrivateKey, type KeyObject } from 'node:crypto' import { generateId } from '@sim/utils/id' import { truncate } from '@sim/utils/string' import { SignJWT } from 'jose' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -109,7 +110,7 @@ async function exchangeNetSuiteToken( const signal = AbortSignal.timeout(TOKEN_EXCHANGE_TIMEOUT_MS) let response: Response try { - response = await fetch(tokenUrl, { + response = await outboundFetch(tokenUrl, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ diff --git a/apps/sim/lib/credentials/databricks-service-account.server.ts b/apps/sim/lib/credentials/databricks-service-account.server.ts index be27b8670d2..70d25eae18d 100644 --- a/apps/sim/lib/credentials/databricks-service-account.server.ts +++ b/apps/sim/lib/credentials/databricks-service-account.server.ts @@ -3,6 +3,7 @@ import { credential } from '@sim/db/schema' import { getErrorMessage } from '@sim/utils/errors' import { eq } from 'drizzle-orm' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { EgressRefusedError } from '@/lib/core/network/transport.server' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation' @@ -162,7 +163,7 @@ async function requestWorkspaceToken( const url = `${workspaceOrigin(host)}/oidc/v1/token` let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', ...headers }, body, @@ -308,10 +309,10 @@ function databricksRefusalReason(body: string): string | undefined { } /** - * Proves a configuration by calling the workspace's SCIM `Me` with the resolved - * token. The call goes through the egress guard, which also routes it through - * the organization's outbound gateway, so a workspace whose IP access list - * allows only the organization's dedicated egress addresses accepts it. + * Proves a configuration by minting a workspace token and calling the workspace's + * SCIM `Me` with it. Both requests leave through the organization's outbound + * gateway when it has one, so a workspace whose IP access list allows only the + * organization's dedicated egress addresses accepts them. */ async function verifyWorkspaceIdentity( config: DatabricksCredentialConfig, diff --git a/apps/sim/lib/credentials/google-workload-identity.server.ts b/apps/sim/lib/credentials/google-workload-identity.server.ts index 31af85ea02f..41deb3dd685 100644 --- a/apps/sim/lib/credentials/google-workload-identity.server.ts +++ b/apps/sim/lib/credentials/google-workload-identity.server.ts @@ -3,6 +3,7 @@ import { toRecord } from '@sim/utils/object' import { LRUCache } from 'lru-cache' import { z } from 'zod' import { coalesceLocally } from '@/lib/concurrency/singleflight' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { encryptSecret } from '@/lib/core/security/encryption' import { GOOGLE_DELEGATION_SERVICE_NAMES, @@ -273,7 +274,7 @@ async function postJson( ): Promise<{ status: number; payload: Record }> { let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json', ...headers }, body: JSON.stringify(body), @@ -295,7 +296,7 @@ async function postJson( */ async function getStatus(url: string, bearer: string): Promise { try { - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${bearer}`, Accept: 'application/json' }, redirect: 'error', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), diff --git a/apps/sim/lib/credentials/microsoft-app-service-account.server.ts b/apps/sim/lib/credentials/microsoft-app-service-account.server.ts index 1a37df45dcd..f5f478ad836 100644 --- a/apps/sim/lib/credentials/microsoft-app-service-account.server.ts +++ b/apps/sim/lib/credentials/microsoft-app-service-account.server.ts @@ -11,6 +11,7 @@ import { SignJWT } from 'jose' import { LRUCache } from 'lru-cache' import { z } from 'zod' import { coalesceLocally } from '@/lib/concurrency/singleflight' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { MICROSOFT_CLOUD_IDS, @@ -414,7 +415,7 @@ async function requestEntraToken( const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS) let response: Response try { - response = await fetch(tokenEndpoint(config), { + response = await outboundFetch(tokenEndpoint(config), { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', Accept: 'application/json' }, body, @@ -453,7 +454,7 @@ async function requestEntraToken( } async function getJson(url: string, token: string): Promise<{ status: number; body: unknown }> { - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${token}`, Accept: 'application/json', diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.ts b/apps/sim/lib/credentials/token-service-accounts/errors.ts index a4ff91cfc10..47cbc8faef8 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.ts @@ -1,6 +1,8 @@ import { getErrorMessage } from '@sim/utils/errors' import { parseRetryAfter } from '@sim/utils/retry' import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -206,7 +208,8 @@ export function isHostNotFound(error: unknown): boolean { * (DNS, TLS, connection reset) to `provider_unavailable` so they never escape * as raw undici errors — whose `cause` can carry connection details — and are * never blamed on the pasted token. DNS-resolution failures can optionally be - * mapped to a different code via {@link FetchProviderOptions}. + * mapped to a different code via {@link FetchProviderOptions}. A blocked or unavailable + * organization egress route is also `provider_unavailable`. * * Redirects are refused (`redirect: 'error'`) unless `init` names another mode: * every caller sends a secret, and fetch replays custom headers and 307/308 @@ -222,7 +225,7 @@ export async function fetchProvider( try { const timeout = AbortSignal.timeout(PROVIDER_FETCH_TIMEOUT_MS) const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout - return await fetch(url, { ...init, redirect: init.redirect ?? 'error', signal }) + return await outboundFetch(url, { ...init, redirect: init.redirect ?? 'error', signal }) } catch (error) { init.signal?.throwIfAborted() // Only ENOTFOUND proves the host doesn't exist; EAI_AGAIN is a transient @@ -235,7 +238,10 @@ export async function fetchProvider( } throw new TokenServiceAccountValidationError('provider_unavailable', 502, { step, - reason: 'network error reaching provider', + reason: + error instanceof OutboundRoutingError + ? 'organization egress route unavailable' + : 'network error reaching provider', }) } } diff --git a/apps/sim/lib/internal/crowdstrike/client.ts b/apps/sim/lib/internal/crowdstrike/client.ts index 4ab2896711e..8acd635f5a0 100644 --- a/apps/sim/lib/internal/crowdstrike/client.ts +++ b/apps/sim/lib/internal/crowdstrike/client.ts @@ -1,5 +1,6 @@ import { toNumberOrNull, toStringOrNull } from '@sim/utils/coerce' import { isRecordLike, toRecordOrNull } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { throttledRetryAfterMs } from '@/lib/credentials/token-service-accounts/errors' @@ -244,7 +245,7 @@ export async function requestCrowdStrikeToken( }) if (request.memberCid) body.set('member_cid', request.memberCid) - const response = await fetch(`${getCloudBaseUrl(request.cloud)}/oauth2/token`, { + const response = await outboundFetch(`${getCloudBaseUrl(request.cloud)}/oauth2/token`, { method: 'POST', headers: { Accept: 'application/json', @@ -347,7 +348,7 @@ export async function callCrowdStrike( headers['Content-Type'] = 'application/json' } - const response = await fetch(buildUrl(baseUrl, options), { + const response = await outboundFetch(buildUrl(baseUrl, options), { method: options.method, headers, body: options.body === undefined ? undefined : JSON.stringify(options.body), diff --git a/apps/sim/lib/internal/sailpoint/client.test.ts b/apps/sim/lib/internal/sailpoint/client.test.ts index a7b3adac435..c955e7fcdaa 100644 --- a/apps/sim/lib/internal/sailpoint/client.test.ts +++ b/apps/sim/lib/internal/sailpoint/client.test.ts @@ -58,7 +58,7 @@ describe('SailPoint client', () => { const credentials = credentialsFor('single-flight') const first = getSailPointAccessToken(credentials) const second = getSailPointAccessToken(credentials) - expect(mockFetch).toHaveBeenCalledTimes(1) + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(1)) release?.(tokenResponse('shared')) await expect(Promise.all([first, second])).resolves.toEqual(['shared', 'shared']) }) @@ -98,7 +98,7 @@ describe('SailPoint client', () => { await expect(getSailPointAccessToken(credentialsFor('in-flight-overflow'))).rejects.toThrow( 'Too many concurrent SailPoint token exchanges' ) - expect(mockFetch).toHaveBeenCalledTimes(100) + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(100)) for (const release of releases) release(tokenResponse('token')) await Promise.all(pending) diff --git a/apps/sim/lib/internal/sailpoint/client.ts b/apps/sim/lib/internal/sailpoint/client.ts index ec0840142c7..56f4e4c98e7 100644 --- a/apps/sim/lib/internal/sailpoint/client.ts +++ b/apps/sim/lib/internal/sailpoint/client.ts @@ -3,6 +3,7 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { LRUCache } from 'lru-cache' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { consumeOrCancelBody, @@ -205,7 +206,7 @@ export async function requestSailPointToken( while (true) { signal?.throwIfAborted() - const response = await fetch(tokenUrl, { + const response = await outboundFetch(tokenUrl, { method: 'POST', headers: { Accept: 'application/json', @@ -317,7 +318,7 @@ export async function sailpointFetch( headers.set('Authorization', `Bearer ${token}`) if (!headers.has('Accept')) headers.set('Accept', 'application/json') - const response = await fetch(url, { + const response = await outboundFetch(url, { ...init, cache: 'no-store', headers, From f4a91d3593b58123ad43fd4a3bf9dae0ac71b7dc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 19:59:26 -0800 Subject: [PATCH 03/32] improvement(egress): route a-l internal tool clients through outboundFetch --- apps/sim/lib/internal/asana/client.ts | 3 ++- apps/sim/lib/internal/ashby/operations.ts | 3 ++- apps/sim/lib/internal/box/client.ts | 3 ++- apps/sim/lib/internal/brex/client.ts | 3 ++- .../browser-use/operations/run-task.ts | 3 ++- apps/sim/lib/internal/buffer/operations.ts | 3 ++- apps/sim/lib/internal/clickup/client.ts | 3 ++- .../operations/get-zone-settings.ts | 3 ++- apps/sim/lib/internal/confluence/client.ts | 3 ++- apps/sim/lib/internal/cursor/operations.ts | 3 ++- .../internal/datadog/operations/update-slo.ts | 5 +++-- apps/sim/lib/internal/daytona/operations.ts | 3 ++- apps/sim/lib/internal/discord/client.ts | 22 +++++++++++-------- apps/sim/lib/internal/docusign/client.ts | 3 ++- apps/sim/lib/internal/dropbox/client.ts | 3 ++- apps/sim/lib/internal/elevenlabs/client.ts | 3 ++- apps/sim/lib/internal/firecrawl/operations.ts | 3 ++- apps/sim/lib/internal/fireflies/operations.ts | 3 ++- apps/sim/lib/internal/gmail/client.ts | 3 ++- apps/sim/lib/internal/image/operations.ts | 11 +++++----- apps/sim/lib/internal/instagram/operations.ts | 3 ++- apps/sim/lib/internal/instagram/publishing.ts | 16 +++++++++----- apps/sim/lib/internal/jira/client.ts | 3 ++- apps/sim/lib/internal/jsm/client.ts | 3 ++- apps/sim/lib/internal/latex/operations.ts | 3 ++- apps/sim/lib/internal/linq/client.ts | 3 ++- 26 files changed, 76 insertions(+), 44 deletions(-) diff --git a/apps/sim/lib/internal/asana/client.ts b/apps/sim/lib/internal/asana/client.ts index d5b8e9b3f74..818f4983d8c 100644 --- a/apps/sim/lib/internal/asana/client.ts +++ b/apps/sim/lib/internal/asana/client.ts @@ -1,4 +1,5 @@ import { toArray, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { AsanaOperationError } from '@/lib/internal/asana/errors' @@ -33,7 +34,7 @@ export class AsanaClient { private async fetch(path: string, init: RequestInit, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(this.url(path), { + return outboundFetch(this.url(path), { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/ashby/operations.ts b/apps/sim/lib/internal/ashby/operations.ts index b0547b52309..60e1129761f 100644 --- a/apps/sim/lib/internal/ashby/operations.ts +++ b/apps/sim/lib/internal/ashby/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -40,7 +41,7 @@ async function ashbyPost( onBehalfOfUserId: string | null | undefined, signal?: AbortSignal ): Promise { - const response = await fetch(`https://api.ashbyhq.com/${path}`, { + const response = await outboundFetch(`https://api.ashbyhq.com/${path}`, { method: 'POST', headers: ashbyAuthHeaders(apiKey, onBehalfOfUserId ?? undefined), body: JSON.stringify(body), diff --git a/apps/sim/lib/internal/box/client.ts b/apps/sim/lib/internal/box/client.ts index be7cc2150f1..735aba86019 100644 --- a/apps/sim/lib/internal/box/client.ts +++ b/apps/sim/lib/internal/box/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -46,7 +47,7 @@ export class BoxClient { new Blob([new Uint8Array(buffer)], { type: 'application/octet-stream' }), fileName ) - const response = await fetch('https://upload.box.com/api/2.0/files/content', { + const response = await outboundFetch('https://upload.box.com/api/2.0/files/content', { method: 'POST', headers: { Authorization: `Bearer ${this.accessToken}` }, body: formData, diff --git a/apps/sim/lib/internal/brex/client.ts b/apps/sim/lib/internal/brex/client.ts index 82f1b0fdca9..7a7b9fa9fce 100644 --- a/apps/sim/lib/internal/brex/client.ts +++ b/apps/sim/lib/internal/brex/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -49,7 +50,7 @@ export class BrexReceiptClient { const endpoint = expenseId ? `${BREX_API_BASE}/v1/expenses/card/${encodeURIComponent(expenseId)}/receipt_upload` : `${BREX_API_BASE}/v1/expenses/card/receipt_match` - const response = await fetch(endpoint, { + const response = await outboundFetch(endpoint, { method: 'POST', headers: buildBrexHeaders(this.apiKey), body: JSON.stringify({ receipt_name: receiptName }), diff --git a/apps/sim/lib/internal/browser-use/operations/run-task.ts b/apps/sim/lib/internal/browser-use/operations/run-task.ts index 4b9bf78b10a..4146c3a5425 100644 --- a/apps/sim/lib/internal/browser-use/operations/run-task.ts +++ b/apps/sim/lib/internal/browser-use/operations/run-task.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { z } from 'zod' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { BrowserUseRunTaskParams, @@ -86,7 +87,7 @@ async function fetchBrowserUse( ): Promise { options.signal?.throwIfAborted() const hasBody = options.body !== undefined - const response = await fetch(`${API_BASE}${path}`, { + const response = await outboundFetch(`${API_BASE}${path}`, { method: options.method ?? 'GET', headers: { ...(hasBody ? { 'Content-Type': 'application/json' } : {}), diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts index 8c9aac638cf..aaff6075ea6 100644 --- a/apps/sim/lib/internal/buffer/operations.ts +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, @@ -160,7 +161,7 @@ async function executePostMutation(args: { const { apiKey, mutation, input, context } = args let result: Record try { - const response = await fetch(BUFFER_API_URL, { + const response = await outboundFetch(BUFFER_API_URL, { method: 'POST', headers: bufferHeaders(apiKey), body: JSON.stringify({ query: mutation, variables: { input } }), diff --git a/apps/sim/lib/internal/clickup/client.ts b/apps/sim/lib/internal/clickup/client.ts index 78da954d554..60c391154db 100644 --- a/apps/sim/lib/internal/clickup/client.ts +++ b/apps/sim/lib/internal/clickup/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ClickUpOperationError } from '@/lib/internal/clickup/errors' @@ -14,7 +15,7 @@ export async function uploadClickUpAttachment( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch( + const response = await outboundFetch( `${CLICKUP_API_BASE_URL}/task/${encodeURIComponent(taskId)}/attachment`, { method: 'POST', diff --git a/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts b/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts index 4722472c8eb..9dd1c7b5713 100644 --- a/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts +++ b/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import { mapZoneSetting, zoneSettingUrl } from '@/tools/cloudflare/get_zone_settings' @@ -32,7 +33,7 @@ export const executeGetZoneSettingsOperation: InternalToolOperationImplementatio const reads = await Promise.all( settingIds.map(async (settingId) => { try { - const response = await fetch(zoneSettingUrl(zoneId, settingId), { + const response = await outboundFetch(zoneSettingUrl(zoneId, settingId), { method: 'GET', headers, signal, diff --git a/apps/sim/lib/internal/confluence/client.ts b/apps/sim/lib/internal/confluence/client.ts index fb9d5791443..ce148d6f823 100644 --- a/apps/sim/lib/internal/confluence/client.ts +++ b/apps/sim/lib/internal/confluence/client.ts @@ -1,4 +1,5 @@ import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { @@ -46,7 +47,7 @@ export class ConfluenceClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return outboundFetch(path, { ...init, headers: { Accept: 'application/json', diff --git a/apps/sim/lib/internal/cursor/operations.ts b/apps/sim/lib/internal/cursor/operations.ts index 132cb3bdbd1..5d72ddbde7e 100644 --- a/apps/sim/lib/internal/cursor/operations.ts +++ b/apps/sim/lib/internal/cursor/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -57,7 +58,7 @@ export async function downloadCursorArtifact( ): Promise { context.signal?.throwIfAborted() const authHeader = `Basic ${Buffer.from(`${input.apiKey}:`).toString('base64')}` - const artifactResponse = await fetch( + const artifactResponse = await outboundFetch( `https://api.cursor.com/v0/agents/${encodeURIComponent(input.agentId)}/artifacts/download?path=${encodeURIComponent(input.path)}`, { method: 'GET', diff --git a/apps/sim/lib/internal/datadog/operations/update-slo.ts b/apps/sim/lib/internal/datadog/operations/update-slo.ts index 074fb4c3f5e..eb33cdc5eda 100644 --- a/apps/sim/lib/internal/datadog/operations/update-slo.ts +++ b/apps/sim/lib/internal/datadog/operations/update-slo.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { UpdateSloParams } from '@/tools/datadog/types' import { @@ -14,7 +15,7 @@ export const executeUpdateSloOperation: InternalToolOperationImplementation< const url = datadogApiUrl(params.site, `/api/v1/slo/${datadogPathSegment(params.sloId)}`) const headers = datadogHeaders(params) - const existingResponse = await fetch(url, { method: 'GET', headers, signal }) + const existingResponse = await outboundFetch(url, { method: 'GET', headers, signal }) if (!existingResponse.ok) { return { success: false, @@ -33,7 +34,7 @@ export const executeUpdateSloOperation: InternalToolOperationImplementation< } } - const response = await fetch(url, { + const response = await outboundFetch(url, { method: 'PUT', headers, body: JSON.stringify(mergeSloUpdatePayload(stored, params)), diff --git a/apps/sim/lib/internal/daytona/operations.ts b/apps/sim/lib/internal/daytona/operations.ts index 9338689c3a0..04ebe7019dd 100644 --- a/apps/sim/lib/internal/daytona/operations.ts +++ b/apps/sim/lib/internal/daytona/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { DaytonaOperationError } from '@/lib/internal/daytona/errors' import { isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' @@ -96,7 +97,7 @@ export async function uploadDaytonaFile( new Blob([new Uint8Array(fileBuffer)], { type: 'application/octet-stream' }), fileName ) - const response = await fetch( + const response = await outboundFetch( daytonaToolboxUrl( input.sandboxId, `/files/upload-v2?path=${encodeURIComponent(destinationPath)}` diff --git a/apps/sim/lib/internal/discord/client.ts b/apps/sim/lib/internal/discord/client.ts index 30959f99873..c24c14f3f35 100644 --- a/apps/sim/lib/internal/discord/client.ts +++ b/apps/sim/lib/internal/discord/client.ts @@ -1,4 +1,5 @@ import { isRecordLike, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { DiscordOperationError } from '@/lib/internal/discord/errors' @@ -11,15 +12,18 @@ export async function sendDiscordMessage( signal?: AbortSignal ): Promise> { signal?.throwIfAborted() - const response = await fetch(`https://discord.com/api/v10/channels/${channelId}/messages`, { - method: 'POST', - headers: { - Authorization: `Bot ${botToken}`, - ...(contentType === 'json' ? { 'Content-Type': 'application/json' } : {}), - }, - body, - signal, - }) + const response = await outboundFetch( + `https://discord.com/api/v10/channels/${channelId}/messages`, + { + method: 'POST', + headers: { + Authorization: `Bot ${botToken}`, + ...(contentType === 'json' ? { 'Content-Type': 'application/json' } : {}), + }, + body, + signal, + } + ) let data: unknown try { data = await readResponseJsonWithLimit(response, { diff --git a/apps/sim/lib/internal/docusign/client.ts b/apps/sim/lib/internal/docusign/client.ts index 372762da549..48a3f3941ca 100644 --- a/apps/sim/lib/internal/docusign/client.ts +++ b/apps/sim/lib/internal/docusign/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -35,7 +36,7 @@ async function fetchDocusign( const abort = () => controller.abort(parentSignal?.reason ?? new Error('Request aborted')) parentSignal?.addEventListener('abort', abort, { once: true }) try { - return await fetch(input, { ...init, signal: controller.signal }) + return await outboundFetch(input, { ...init, signal: controller.signal }) } finally { clearTimeout(timeout) parentSignal?.removeEventListener('abort', abort) diff --git a/apps/sim/lib/internal/dropbox/client.ts b/apps/sim/lib/internal/dropbox/client.ts index 5e0f66d6a0a..da87a9c6fa5 100644 --- a/apps/sim/lib/internal/dropbox/client.ts +++ b/apps/sim/lib/internal/dropbox/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -35,7 +36,7 @@ export class DropboxClient { } ): Promise> { this.signal?.throwIfAborted() - const response = await fetch('https://content.dropboxapi.com/2/files/upload', { + const response = await outboundFetch('https://content.dropboxapi.com/2/files/upload', { method: 'POST', headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/elevenlabs/client.ts b/apps/sim/lib/internal/elevenlabs/client.ts index 9fc0e862db8..47331db61ac 100644 --- a/apps/sim/lib/internal/elevenlabs/client.ts +++ b/apps/sim/lib/internal/elevenlabs/client.ts @@ -2,6 +2,7 @@ import { createTimeoutAbortController, DEFAULT_EXECUTION_TIMEOUT_MS, } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { consumeOrCancelBody, readResponseToBufferWithLimit } from '@/lib/core/utils/stream-limits' import { ElevenLabsOperationError } from '@/lib/internal/elevenlabs/errors' import type { @@ -90,7 +91,7 @@ export async function generateElevenLabsAudio( const timeout = createTimeoutAbortController(DEFAULT_EXECUTION_TIMEOUT_MS, signal) try { const { url, init } = buildRequest(args) - const response = await fetch(url, { ...init, signal: timeout.signal }) + const response = await outboundFetch(url, { ...init, signal: timeout.signal }) timeout.signal.throwIfAborted() if (!response.ok) { await consumeOrCancelBody(response) diff --git a/apps/sim/lib/internal/firecrawl/operations.ts b/apps/sim/lib/internal/firecrawl/operations.ts index 3b4faa306e0..efde8ac77f9 100644 --- a/apps/sim/lib/internal/firecrawl/operations.ts +++ b/apps/sim/lib/internal/firecrawl/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, isPayloadSizeLimitError, @@ -85,7 +86,7 @@ export async function executeFirecrawlParse( formData.append('options', JSON.stringify(input.options)) } - const response = await fetch('https://api.firecrawl.dev/v2/parse', { + const response = await outboundFetch('https://api.firecrawl.dev/v2/parse', { method: 'POST', headers: { Authorization: `Bearer ${input.apiKey}` }, body: formData, diff --git a/apps/sim/lib/internal/fireflies/operations.ts b/apps/sim/lib/internal/fireflies/operations.ts index 8b0dff6a229..40464b27222 100644 --- a/apps/sim/lib/internal/fireflies/operations.ts +++ b/apps/sim/lib/internal/fireflies/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' import type { FirefliesUploadAudioInput } from '@/lib/internal/fireflies/schema' @@ -96,7 +97,7 @@ export async function executeFirefliesUploadAudio( if (body.clientReferenceId) input.client_reference_id = body.clientReferenceId if (body.attendees !== undefined) input.attendees = body.attendees - const response = await fetch(FIREFLIES_API_URL, { + const response = await outboundFetch(FIREFLIES_API_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/gmail/client.ts b/apps/sim/lib/internal/gmail/client.ts index 9d77de93ddd..94b8678f6f2 100644 --- a/apps/sim/lib/internal/gmail/client.ts +++ b/apps/sim/lib/internal/gmail/client.ts @@ -1,4 +1,5 @@ import { toArray, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type ReadResponseWithLimitOptions, readResponseJsonWithLimit, @@ -29,7 +30,7 @@ export class GmailClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return outboundFetch(path, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/image/operations.ts b/apps/sim/lib/internal/image/operations.ts index bc3071472a9..3707791eaa0 100644 --- a/apps/sim/lib/internal/image/operations.ts +++ b/apps/sim/lib/internal/image/operations.ts @@ -4,6 +4,7 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike, omit } from '@sim/utils/object' import { getUserFacingErrorMessage } from '@/lib/core/errors/database-query-error' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -264,7 +265,7 @@ async function generateWithOpenAI( const requestBody = prepared.body const outputFormat = typeof requestBody.output_format === 'string' ? requestBody.output_format : 'png' - const openaiResponse = await fetch(prepared.endpoint, { + const openaiResponse = await outboundFetch(prepared.endpoint, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -331,7 +332,7 @@ async function generateWithGemini( ): Promise { const model = prepared.model.id const requestBody = prepared.body - const geminiResponse = await fetch(prepared.endpoint, { + const geminiResponse = await outboundFetch(prepared.endpoint, { method: 'POST', headers: { 'x-goog-api-key': apiKey, @@ -437,7 +438,7 @@ async function generateWithFalAI( const endpoint = prepared.model.endpoint if (!endpoint) throw new Error('Missing Fal.ai image endpoint') const requestBody = prepared.body - const createResponse = await fetch(prepared.endpoint, { + const createResponse = await outboundFetch(prepared.endpoint, { redirect: 'error', method: 'POST', headers: { @@ -648,7 +649,7 @@ async function generateWithHiggsfield( const model = prepared.model.id const headers = higgsfieldAuthHeaders(apiKey) let data = await readHiggsfieldRequest( - await fetch(prepared.endpoint, { + await outboundFetch(prepared.endpoint, { method: 'POST', redirect: 'error', headers: { ...headers, 'Content-Type': 'application/json' }, @@ -687,7 +688,7 @@ async function generateWithHiggsfield( await interruptibleSleep(pollIntervalMs, signal) signal?.throwIfAborted() data = await readHiggsfieldRequest( - await fetch(higgsfieldStatusUrl(higgsfieldRequestId), { + await outboundFetch(higgsfieldStatusUrl(higgsfieldRequestId), { redirect: 'error', headers, signal, diff --git a/apps/sim/lib/internal/instagram/operations.ts b/apps/sim/lib/internal/instagram/operations.ts index 07e71152dad..a4e6e1f11e7 100644 --- a/apps/sim/lib/internal/instagram/operations.ts +++ b/apps/sim/lib/internal/instagram/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { createMediaContainer, @@ -123,7 +124,7 @@ async function fetchMediaMetadata({ fields: string signal?: AbortSignal }): Promise { - const response = await fetch(graphUrl(`/${encodeURIComponent(mediaId)}`, { fields }), { + const response = await outboundFetch(graphUrl(`/${encodeURIComponent(mediaId)}`, { fields }), { headers: bearerHeaders(accessToken), signal, }) diff --git a/apps/sim/lib/internal/instagram/publishing.ts b/apps/sim/lib/internal/instagram/publishing.ts index cb00babd95b..f82c7ae152b 100644 --- a/apps/sim/lib/internal/instagram/publishing.ts +++ b/apps/sim/lib/internal/instagram/publishing.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { hasCloudStorage } from '@/lib/uploads/core/storage-service' import { getFileExtension, @@ -291,7 +292,7 @@ export async function resolveIgUserId( ): Promise { if (igUserId?.trim()) return igUserId.trim() - const response = await fetch(graphUrl('/me', { fields: 'user_id' }), { + const response = await outboundFetch(graphUrl('/me', { fields: 'user_id' }), { headers: bearerHeaders(accessToken), signal, }) @@ -316,10 +317,13 @@ async function getContainerStatus( containerId: string, signal?: AbortSignal ): Promise<{ statusCode: ContainerStatusCode | null; status: string | null }> { - const response = await fetch(graphUrl(`/${containerId}`, { fields: 'status_code,status' }), { - headers: bearerHeaders(accessToken), - signal, - }) + const response = await outboundFetch( + graphUrl(`/${containerId}`, { fields: 'status_code,status' }), + { + headers: bearerHeaders(accessToken), + signal, + } + ) if (!response.ok) { throw new Error(`Failed to get container status: ${await readGraphError(response)}`) } @@ -393,7 +397,7 @@ async function postGraphForm( if (value !== undefined && value !== null) form.set(key, String(value)) } - return fetch(graphUrl(path), { + return outboundFetch(graphUrl(path), { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/internal/jira/client.ts b/apps/sim/lib/internal/jira/client.ts index 62243e95e08..a9ac47b1cb5 100644 --- a/apps/sim/lib/internal/jira/client.ts +++ b/apps/sim/lib/internal/jira/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' @@ -56,7 +57,7 @@ export class JiraClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(url, { + const response = await outboundFetch(url, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/jsm/client.ts b/apps/sim/lib/internal/jsm/client.ts index 39ea6730377..404b5dd61b4 100644 --- a/apps/sim/lib/internal/jsm/client.ts +++ b/apps/sim/lib/internal/jsm/client.ts @@ -1,4 +1,5 @@ import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { JsmOperationError } from '@/lib/internal/jsm/errors' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -67,7 +68,7 @@ export class JsmClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return outboundFetch(path, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/latex/operations.ts b/apps/sim/lib/internal/latex/operations.ts index 6baae21f1e3..1f21974790d 100644 --- a/apps/sim/lib/internal/latex/operations.ts +++ b/apps/sim/lib/internal/latex/operations.ts @@ -1,4 +1,5 @@ import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseToBufferWithLimit, @@ -80,7 +81,7 @@ export async function compileLatexDocument( const signal = context.signal ? AbortSignal.any([context.signal, timeoutSignal]) : timeoutSignal let response: Response try { - response = await fetch(LATEX_COMPILE_URL, { + response = await outboundFetch(LATEX_COMPILE_URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ diff --git a/apps/sim/lib/internal/linq/client.ts b/apps/sim/lib/internal/linq/client.ts index 7ab0fd5e45f..f47a3ca86e1 100644 --- a/apps/sim/lib/internal/linq/client.ts +++ b/apps/sim/lib/internal/linq/client.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES, secureFetchWithPinnedIP, @@ -38,7 +39,7 @@ export async function registerLinqAttachment( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${LINQ_API_BASE}/attachments`, { + const response = await outboundFetch(`${LINQ_API_BASE}/attachments`, { method: 'POST', headers: linqHeaders(input.apiKey), body: JSON.stringify({ From 23bf80fb8f77897d41545f6959f77f0b05a136d6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 19:57:37 -0800 Subject: [PATCH 04/32] improvement(egress): route m-z internal vendor clients through outboundFetch --- .../add-user-app-role-assignment.ts | 3 +- .../lib/internal/microsoft-teams/client.ts | 3 +- .../internal/microsoft-teams/operations.ts | 5 ++-- .../internal/okta/operations/update-group.ts | 5 ++-- apps/sim/lib/internal/outlook/client.ts | 7 +++-- apps/sim/lib/internal/persona/operations.ts | 3 +- .../sim/lib/internal/quickbooks/operations.ts | 9 +++--- .../quickbooks/provider-operations.ts | 11 ++++---- apps/sim/lib/internal/quiver/client.ts | 3 +- apps/sim/lib/internal/resend/client.ts | 3 +- apps/sim/lib/internal/sendgrid/client.ts | 3 +- apps/sim/lib/internal/slack/client.ts | 3 +- apps/sim/lib/internal/slack/oauth.ts | 3 +- apps/sim/lib/internal/square/operations.ts | 3 +- apps/sim/lib/internal/stagehand/operations.ts | 14 ++++++---- apps/sim/lib/internal/stt/operations.ts | 15 +++++----- apps/sim/lib/internal/supabase/operations.ts | 3 +- .../operations/storage-update-bucket.ts | 5 ++-- apps/sim/lib/internal/telegram/operations.ts | 3 +- apps/sim/lib/internal/tiktok/operations.ts | 3 +- apps/sim/lib/internal/tiktok/upload.ts | 3 +- apps/sim/lib/internal/tts/client.ts | 5 ++-- apps/sim/lib/internal/uptimerobot/client.ts | 3 +- apps/sim/lib/internal/vanta/operations.ts | 7 +++-- apps/sim/lib/internal/video/client.ts | 28 +++++++++++-------- apps/sim/lib/internal/vision/client.ts | 3 +- apps/sim/lib/internal/whatsapp/operations.ts | 14 ++++++---- apps/sim/lib/internal/whatsapp/upload.ts | 3 +- apps/sim/lib/internal/wordpress/operations.ts | 3 +- apps/sim/lib/internal/workday/client.ts | 3 +- 30 files changed, 110 insertions(+), 69 deletions(-) diff --git a/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts b/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts index fd0e7ff4d45..d00266b2907 100644 --- a/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts +++ b/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import { mapAppRoleAssignment, @@ -17,7 +18,7 @@ export const executeAddUserAppRoleAssignmentOperation: InternalToolOperationImpl const { userId, resourceId, appRoleId } = readIdentifiers(params) const principalId = await resolveGraphUserObjectId(userId, params.accessToken, signal) - const response = await fetch( + const response = await outboundFetch( `https://graph.microsoft.com/v1.0/users/${safeUrlPathSegment(userId, 'userId')}/appRoleAssignments`, { method: 'POST', diff --git a/apps/sim/lib/internal/microsoft-teams/client.ts b/apps/sim/lib/internal/microsoft-teams/client.ts index e2eed2148f3..6ef48da97c2 100644 --- a/apps/sim/lib/internal/microsoft-teams/client.ts +++ b/apps/sim/lib/internal/microsoft-teams/client.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { MicrosoftTeamsOperationError } from '@/lib/internal/microsoft-teams/errors' @@ -23,7 +24,7 @@ export class MicrosoftTeamsClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${MICROSOFT_GRAPH_BASE_URL}${path}`, { + const response = await outboundFetch(`${MICROSOFT_GRAPH_BASE_URL}${path}`, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/microsoft-teams/operations.ts b/apps/sim/lib/internal/microsoft-teams/operations.ts index 7dad3cef30f..a64b540543a 100644 --- a/apps/sim/lib/internal/microsoft-teams/operations.ts +++ b/apps/sim/lib/internal/microsoft-teams/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { MicrosoftTeamsClient, @@ -465,7 +466,7 @@ export async function deleteMicrosoftTeamsChatMessage( throw new MicrosoftTeamsOperationError('Chat ID and Message ID are required', 400) } - const meResponse = await fetch('https://graph.microsoft.com/v1.0/me', { + const meResponse = await outboundFetch('https://graph.microsoft.com/v1.0/me', { headers: { Authorization: `Bearer ${input.accessToken}` }, signal: context.signal, }) @@ -477,7 +478,7 @@ export async function deleteMicrosoftTeamsChatMessage( ) } - const response = await fetch( + const response = await outboundFetch( `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(me.id)}/chats/${encodeURIComponent(chatId)}/messages/${encodeURIComponent(messageId)}/softDelete`, { method: 'POST', diff --git a/apps/sim/lib/internal/okta/operations/update-group.ts b/apps/sim/lib/internal/okta/operations/update-group.ts index dfcf829dbf5..ae2f63ef237 100644 --- a/apps/sim/lib/internal/okta/operations/update-group.ts +++ b/apps/sim/lib/internal/okta/operations/update-group.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateOktaDomain } from '@/lib/core/security/input-validation' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { OktaGroup, OktaUpdateGroupParams, OktaUpdateGroupResponse } from '@/tools/okta/types' @@ -34,13 +35,13 @@ export const executeOktaUpdateGroupOperation: InternalToolOperationImplementatio const url = `https://${domain}/api/v1/groups/${encodeURIComponent(params.groupId.trim())}` const headers = oktaHeaders(params.apiKey) - const readResponse = await fetch(url, { headers, signal }) + const readResponse = await outboundFetch(url, { headers, signal }) if (!readResponse.ok) { await throwOktaError(readResponse, logger, 'Failed to load group for update in Okta') } const existing: OktaGroup = await readResponse.json() - const writeResponse = await fetch(url, { + const writeResponse = await outboundFetch(url, { method: 'PUT', headers, body: JSON.stringify({ profile: mergeOktaGroupProfile(existing.profile, params) }), diff --git a/apps/sim/lib/internal/outlook/client.ts b/apps/sim/lib/internal/outlook/client.ts index 22427779c73..4a0f6c9173c 100644 --- a/apps/sim/lib/internal/outlook/client.ts +++ b/apps/sim/lib/internal/outlook/client.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -36,7 +37,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await outboundFetch(this.url(path), { ...init, headers: { 'Content-Type': 'application/json', @@ -72,7 +73,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await outboundFetch(this.url(path), { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, @@ -105,7 +106,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise<{ buffer: Buffer; contentType: string | null }> { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await outboundFetch(this.url(path), { method: 'GET', headers: { Authorization: `Bearer ${this.accessToken}` }, signal, diff --git a/apps/sim/lib/internal/persona/operations.ts b/apps/sim/lib/internal/persona/operations.ts index b29d19ef1da..4446b4da701 100644 --- a/apps/sim/lib/internal/persona/operations.ts +++ b/apps/sim/lib/internal/persona/operations.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { PersonaOperationError } from '@/lib/internal/persona/errors' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' @@ -46,7 +47,7 @@ export async function importPersonaAccounts( }) const buffer = resolved.buffer context.signal?.throwIfAborted() - const response = await fetch(`${PERSONA_API_BASE}/importer/accounts`, { + const response = await outboundFetch(`${PERSONA_API_BASE}/importer/accounts`, { method: 'POST', headers: buildPersonaHeaders(input.apiKey), body: JSON.stringify({ diff --git a/apps/sim/lib/internal/quickbooks/operations.ts b/apps/sim/lib/internal/quickbooks/operations.ts index 2629edf413c..3e5af8a459c 100644 --- a/apps/sim/lib/internal/quickbooks/operations.ts +++ b/apps/sim/lib/internal/quickbooks/operations.ts @@ -4,6 +4,7 @@ import type { QuickBooksAddAttachmentBody, QuickBooksDownloadDocumentBody, } from '@/lib/api/contracts/tools/quickbooks' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { assertContentLengthWithinLimit, @@ -93,7 +94,7 @@ async function downloadQuickBooksAttachment( body.quickBooksEnvironment ) const metadataSignal = quickBooksDocumentSignal(signal, QUICKBOOKS_DOCUMENT_METADATA_TIMEOUT_MS) - const downloadUrlResponse = await fetch(downloadUrl, { + const downloadUrlResponse = await outboundFetch(downloadUrl, { method: 'GET', headers: { ...buildQuickBooksHeaders(body.accessToken), Accept: '*/*' }, signal: metadataSignal, @@ -183,7 +184,7 @@ async function downloadQuickBooksTransactionPdf( body.quickBooksEnvironment ) const transferSignal = quickBooksDocumentSignal(signal, QUICKBOOKS_DOCUMENT_TRANSFER_TIMEOUT_MS) - const response = await fetch(url, { + const response = await outboundFetch(url, { method: 'GET', headers: { ...buildQuickBooksHeaders(body.accessToken), Accept: 'application/pdf' }, signal: transferSignal, @@ -221,7 +222,7 @@ export async function executeQuickBooksAddAttachment( const metadata = buildQuickBooksAttachableMetadata(data.targetType, data.targetId, { note: data.note!, }) - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { ...buildQuickBooksHeaders(data.accessToken), @@ -301,7 +302,7 @@ export async function executeQuickBooksAddAttachment( ), resolvedName ) - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: buildQuickBooksHeaders(data.accessToken), body: formData, diff --git a/apps/sim/lib/internal/quickbooks/provider-operations.ts b/apps/sim/lib/internal/quickbooks/provider-operations.ts index c6a2a644904..1e3b2341367 100644 --- a/apps/sim/lib/internal/quickbooks/provider-operations.ts +++ b/apps/sim/lib/internal/quickbooks/provider-operations.ts @@ -1,4 +1,5 @@ import { filterUndefined } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { buildQuickBooksCreateBillPaymentBody, buildQuickBooksUpdateBillBody, @@ -178,7 +179,7 @@ export async function executeQuickBooksCreateBillPaymentOperation( const paymentAccountId = params.paymentAccountId.trim() if (!paymentAccountId) throw new Error('paymentAccountId is required') - const accountResponse = await fetch( + const accountResponse = await outboundFetch( buildQuickBooksEntityUrl(params, 'account', paymentAccountId), { method: 'GET', @@ -197,7 +198,7 @@ export async function executeQuickBooksCreateBillPaymentOperation( assertCompatiblePaymentAccount(account, params.paymentType, paymentAccountId) signal?.throwIfAborted() - const paymentResponse = await fetch( + const paymentResponse = await outboundFetch( addQuickBooksRequestId(buildQuickBooksEntityUrl(params, 'billpayment'), params.requestId), { method: 'POST', @@ -273,7 +274,7 @@ export async function executeQuickBooksUpdateRefundReceiptOperation( params: QuickBooksUpdateRefundReceiptParams, signal?: AbortSignal ) { - const response = await fetch(buildQuickBooksEntityUrl(params, 'refundreceipt'), { + const response = await outboundFetch(buildQuickBooksEntityUrl(params, 'refundreceipt'), { method: 'POST', headers: getQuickBooksToolHeaders(params.accessToken, 'application/json'), body: JSON.stringify(buildQuickBooksUpdateSalesDocumentBody(params)), @@ -301,7 +302,7 @@ export async function executeQuickBooksUpdateCustomerPaymentOperation( const syncToken = params.syncToken?.trim() if (!syncToken) throw new Error('syncToken is required') - const readResponse = await fetch(buildQuickBooksEntityUrl(params, 'payment', paymentId), { + const readResponse = await outboundFetch(buildQuickBooksEntityUrl(params, 'payment', paymentId), { method: 'GET', headers: getQuickBooksToolHeaders(params.accessToken), signal, @@ -335,7 +336,7 @@ export async function executeQuickBooksUpdateCustomerPaymentOperation( paymentId, syncToken ) - const updateResponse = await fetch(buildQuickBooksEntityUrl(params, 'payment'), { + const updateResponse = await outboundFetch(buildQuickBooksEntityUrl(params, 'payment'), { method: 'POST', headers: getQuickBooksToolHeaders(params.accessToken, 'application/json'), body: JSON.stringify(fullBody), diff --git a/apps/sim/lib/internal/quiver/client.ts b/apps/sim/lib/internal/quiver/client.ts index acb7d512020..d52223e0750 100644 --- a/apps/sim/lib/internal/quiver/client.ts +++ b/apps/sim/lib/internal/quiver/client.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit, @@ -19,7 +20,7 @@ export async function requestQuiverSvg( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${QUIVER_API_BASE_URL}/${path}`, { + const response = await outboundFetch(`${QUIVER_API_BASE_URL}/${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/resend/client.ts b/apps/sim/lib/internal/resend/client.ts index 2527689b487..4ab78b519a2 100644 --- a/apps/sim/lib/internal/resend/client.ts +++ b/apps/sim/lib/internal/resend/client.ts @@ -1,4 +1,5 @@ import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ResendOperationError } from '@/lib/internal/resend/errors' @@ -20,7 +21,7 @@ export async function sendResendEmail( signal?: AbortSignal ): Promise> { signal?.throwIfAborted() - const response = await fetch('https://api.resend.com/emails', { + const response = await outboundFetch('https://api.resend.com/emails', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/internal/sendgrid/client.ts b/apps/sim/lib/internal/sendgrid/client.ts index 7075e1d41a2..f910b446485 100644 --- a/apps/sim/lib/internal/sendgrid/client.ts +++ b/apps/sim/lib/internal/sendgrid/client.ts @@ -1,4 +1,5 @@ import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { SendGridOperationError } from '@/lib/internal/sendgrid/errors' import { SENDGRID_REGIONS } from '@/tools/sendgrid/utils' @@ -26,7 +27,7 @@ export async function sendSendGridMail( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${SENDGRID_REGIONS.apiBase(region)}/mail/send`, { + const response = await outboundFetch(`${SENDGRID_REGIONS.apiBase(region)}/mail/send`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/internal/slack/client.ts b/apps/sim/lib/internal/slack/client.ts index 23700aec6f8..95a2226067a 100644 --- a/apps/sim/lib/internal/slack/client.ts +++ b/apps/sim/lib/internal/slack/client.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' const MAX_SLACK_JSON_BYTES = 2 * 1024 * 1024 @@ -72,7 +73,7 @@ export async function requestSlackApi({ } const isForm = body instanceof URLSearchParams - const response = await fetch(url, { + const response = await outboundFetch(url, { method: httpMethod, headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/internal/slack/oauth.ts b/apps/sim/lib/internal/slack/oauth.ts index 8d1f1dcccb1..bcdfb2b95d9 100644 --- a/apps/sim/lib/internal/slack/oauth.ts +++ b/apps/sim/lib/internal/slack/oauth.ts @@ -1,5 +1,6 @@ import { Buffer } from 'node:buffer' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { requestSlackApi } from '@/lib/internal/slack/client' @@ -25,7 +26,7 @@ export async function exchangeSlackBotAuthorization(input: { code: string redirectUri?: string }) { - const response = await fetch('https://slack.com/api/oauth.v2.access', { + const response = await outboundFetch('https://slack.com/api/oauth.v2.access', { method: 'POST', headers: { Authorization: `Basic ${Buffer.from(`${input.clientId}:${input.clientSecret}`).toString('base64')}`, diff --git a/apps/sim/lib/internal/square/operations.ts b/apps/sim/lib/internal/square/operations.ts index 46fad0ec68b..03f6c84faac 100644 --- a/apps/sim/lib/internal/square/operations.ts +++ b/apps/sim/lib/internal/square/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -71,7 +72,7 @@ export async function executeSquareCreateCatalogImage( input.fileName || userFile.name ) - const response = await fetch( + const response = await outboundFetch( `${SQUARE_ENVIRONMENTS.apiBase(input.environment)}/v2/catalog/images`, { method: 'POST', diff --git a/apps/sim/lib/internal/stagehand/operations.ts b/apps/sim/lib/internal/stagehand/operations.ts index 7cafab86303..5f04c231a25 100644 --- a/apps/sim/lib/internal/stagehand/operations.ts +++ b/apps/sim/lib/internal/stagehand/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { isSensitiveKey, REDACTED_MARKER } from '@/lib/core/security/redaction' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' @@ -98,11 +99,14 @@ async function getLiveViewUrl(sessionId: string, signal?: AbortSignal): Promise< const browserbaseApiKey = getBrowserbaseApiKey() if (!browserbaseApiKey) return null try { - const response = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/debug`, { - method: 'GET', - headers: { 'X-BB-API-Key': browserbaseApiKey }, - signal, - }) + const response = await outboundFetch( + `https://api.browserbase.com/v1/sessions/${sessionId}/debug`, + { + method: 'GET', + headers: { 'X-BB-API-Key': browserbaseApiKey }, + signal, + } + ) if (!response.ok) return null const data = await readResponseJsonWithLimit(response, { maxBytes: MAX_BROWSERBASE_DEBUG_RESPONSE_BYTES, diff --git a/apps/sim/lib/internal/stt/operations.ts b/apps/sim/lib/internal/stt/operations.ts index a68c15750ea..0f7f297fb36 100644 --- a/apps/sim/lib/internal/stt/operations.ts +++ b/apps/sim/lib/internal/stt/operations.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, @@ -492,7 +493,7 @@ async function transcribeWithWhisper( } const endpoint = translate ? 'translations' : 'transcriptions' - const response = await fetch(`https://api.openai.com/v1/audio/${endpoint}`, { + const response = await outboundFetch(`https://api.openai.com/v1/audio/${endpoint}`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -562,7 +563,7 @@ async function transcribeWithDeepgram( params.append('diarize', 'true') } - const response = await fetch(`https://api.deepgram.com/v1/listen?${params.toString()}`, { + const response = await outboundFetch(`https://api.deepgram.com/v1/listen?${params.toString()}`, { method: 'POST', headers: { Authorization: `Token ${apiKey}`, @@ -645,7 +646,7 @@ async function transcribeWithElevenLabs( formData.append('timestamps_granularity', 'word') } - const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', { + const response = await outboundFetch('https://api.elevenlabs.io/v1/speech-to-text', { method: 'POST', headers: { 'xi-api-key': apiKey, @@ -705,7 +706,7 @@ async function transcribeWithAssemblyAI( entities?: Record[] summary?: string }> { - const uploadResponse = await fetch('https://api.assemblyai.com/v2/upload', { + const uploadResponse = await outboundFetch('https://api.assemblyai.com/v2/upload', { method: 'POST', headers: { authorization: apiKey, @@ -764,7 +765,7 @@ async function transcribeWithAssemblyAI( transcriptRequest.summary_type = 'bullets' } - const transcriptResponse = await fetch('https://api.assemblyai.com/v2/transcript', { + const transcriptResponse = await outboundFetch('https://api.assemblyai.com/v2/transcript', { method: 'POST', headers: { authorization: apiKey, @@ -788,7 +789,7 @@ async function transcribeWithAssemblyAI( const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) while (attempts < maxAttempts) { - const statusResponse = await fetch(`https://api.assemblyai.com/v2/transcript/${id}`, { + const statusResponse = await outboundFetch(`https://api.assemblyai.com/v2/transcript/${id}`, { headers: { authorization: apiKey, }, @@ -902,7 +903,7 @@ async function transcribeWithGemini( ], } - const response = await fetch( + const response = await outboundFetch( `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:generateContent?key=${apiKey}`, { method: 'POST', diff --git a/apps/sim/lib/internal/supabase/operations.ts b/apps/sim/lib/internal/supabase/operations.ts index 05f95de0b83..4f4e80d8dbf 100644 --- a/apps/sim/lib/internal/supabase/operations.ts +++ b/apps/sim/lib/internal/supabase/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateSupabaseProjectId } from '@/lib/core/security/input-validation' import { assertKnownSizeWithinLimit, @@ -136,7 +137,7 @@ export async function executeSupabaseStorageUpload( } if (input.upsert) headers['x-upsert'] = 'true' - const response = await fetch(`${baseUrl}/${encodedBucket}/${encodedPath}`, { + const response = await outboundFetch(`${baseUrl}/${encodedBucket}/${encodedPath}`, { method: 'POST', headers, body: new Uint8Array(upload.body), diff --git a/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts b/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts index f2f6734d8d4..97eba56c257 100644 --- a/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts +++ b/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { filterUndefined } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { SupabaseStorageUpdateBucketParams, @@ -43,7 +44,7 @@ export const executeStorageUpdateBucketOperation: InternalToolOperationImplement }) if (Object.keys(payload).length === 0) { - const currentResponse = await fetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { + const currentResponse = await outboundFetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { method: 'GET', headers, redirect: 'error', @@ -65,7 +66,7 @@ export const executeStorageUpdateBucketOperation: InternalToolOperationImplement } } - const updateResponse = await fetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { + const updateResponse = await outboundFetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { method: 'PUT', headers, body: JSON.stringify(payload), diff --git a/apps/sim/lib/internal/telegram/operations.ts b/apps/sim/lib/internal/telegram/operations.ts index 4fea2e14764..7caf93653ba 100644 --- a/apps/sim/lib/internal/telegram/operations.ts +++ b/apps/sim/lib/internal/telegram/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { TelegramOperationError } from '@/lib/internal/telegram/errors' import { @@ -100,7 +101,7 @@ export async function sendTelegramDocument( form.append('parse_mode', 'HTML') } - const response = await fetch( + const response = await outboundFetch( `https://api.telegram.org/bot${encodeURIComponent(input.botToken)}/sendDocument`, { method: 'POST', body: form, signal: context.signal } ) diff --git a/apps/sim/lib/internal/tiktok/operations.ts b/apps/sim/lib/internal/tiktok/operations.ts index 0ecdd7eb1a1..44a49389807 100644 --- a/apps/sim/lib/internal/tiktok/operations.ts +++ b/apps/sim/lib/internal/tiktok/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import type { TikTokUploadVideoDraftInput } from '@/lib/internal/tiktok/schema' import { @@ -77,7 +78,7 @@ export async function executeTikTokUploadVideoDraft( if (videoSize === 0) return failureResponse('The video file is empty.', 400) const { chunkSize, totalChunkCount } = computeTikTokChunkPlan(videoSize) - const initResponse = await fetch( + const initResponse = await outboundFetch( 'https://open.tiktokapis.com/v2/post/publish/inbox/video/init/', { method: 'POST', diff --git a/apps/sim/lib/internal/tiktok/upload.ts b/apps/sim/lib/internal/tiktok/upload.ts index 4fc1c72cb23..b3da1d9525f 100644 --- a/apps/sim/lib/internal/tiktok/upload.ts +++ b/apps/sim/lib/internal/tiktok/upload.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { assertKnownSizeWithinLimit, readResponseTextWithLimit, @@ -235,7 +236,7 @@ async function uploadChunk(options: { for (let attempt = 1; attempt <= MAX_UPLOAD_ATTEMPTS; attempt++) { throwIfAborted(options.signal) - const response = await fetch(options.uploadUrl, { + const response = await outboundFetch(options.uploadUrl, { method: 'PUT', headers: { 'Content-Type': options.mimeType, diff --git a/apps/sim/lib/internal/tts/client.ts b/apps/sim/lib/internal/tts/client.ts index c88a9b4b23c..f5240a8e160 100644 --- a/apps/sim/lib/internal/tts/client.ts +++ b/apps/sim/lib/internal/tts/client.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { assertKnownSizeWithinLimit, @@ -51,14 +52,14 @@ async function providerFetch( timeoutMs?: number ): Promise { signal?.throwIfAborted() - if (!timeoutMs) return fetch(input, { ...init, signal }) + if (!timeoutMs) return outboundFetch(input, { ...init, signal }) const controller = new AbortController() const timeout = setTimeout(() => controller.abort(new Error('TTS request timed out')), timeoutMs) const abort = () => controller.abort(signal?.reason ?? new Error('Request aborted')) signal?.addEventListener('abort', abort, { once: true }) try { - return await fetch(input, { ...init, signal: controller.signal }) + return await outboundFetch(input, { ...init, signal: controller.signal }) } finally { clearTimeout(timeout) signal?.removeEventListener('abort', abort) diff --git a/apps/sim/lib/internal/uptimerobot/client.ts b/apps/sim/lib/internal/uptimerobot/client.ts index cfa44d61505..9f299281e89 100644 --- a/apps/sim/lib/internal/uptimerobot/client.ts +++ b/apps/sim/lib/internal/uptimerobot/client.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { UptimeRobotOperationError } from '@/lib/internal/uptimerobot/errors' @@ -24,7 +25,7 @@ export async function requestUptimeRobotPsp(args: { }): Promise { const { apiKey, method, path, form, signal } = args signal?.throwIfAborted() - const response = await fetch(`${UPTIMEROBOT_API_BASE}${path}`, { + const response = await outboundFetch(`${UPTIMEROBOT_API_BASE}${path}`, { method, headers: { Authorization: `Bearer ${apiKey}`, Accept: 'application/json' }, body: form, diff --git a/apps/sim/lib/internal/vanta/operations.ts b/apps/sim/lib/internal/vanta/operations.ts index 3aa546769a7..67de8842f49 100644 --- a/apps/sim/lib/internal/vanta/operations.ts +++ b/apps/sim/lib/internal/vanta/operations.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, @@ -415,7 +416,7 @@ export async function executeVantaQuery( const response = await fetchVantaWithAuth( params, (accessToken) => - fetch(apiRequest.url, { + outboundFetch(apiRequest.url, { method: apiRequest.method, headers: { Accept: 'application/json', @@ -471,7 +472,7 @@ export async function executeVantaUploadDocumentFile( const response = await fetchVantaWithAuth( input, (accessToken) => - fetch(uploadUrl, { + outboundFetch(uploadUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}` }, body: formData, @@ -508,7 +509,7 @@ export async function executeVantaDownloadDocumentFile( const response = await fetchVantaWithAuth( input, (accessToken) => - fetch(mediaUrl, { + outboundFetch(mediaUrl, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, cache: 'no-store', diff --git a/apps/sim/lib/internal/video/client.ts b/apps/sim/lib/internal/video/client.ts index 1972d909937..be728beb479 100644 --- a/apps/sim/lib/internal/video/client.ts +++ b/apps/sim/lib/internal/video/client.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -179,7 +180,7 @@ async function generateWithDirectProvider( return value } let data = await readProviderResponse( - await fetch(prepared.url, { + await outboundFetch(prepared.url, { method: 'POST', redirect: 'error', headers, @@ -232,7 +233,7 @@ async function generateWithDirectProvider( const fileId = getStringProperty(task, 'file_id') if (!fileId) throw new Error('MiniMax response missing file_id') const fileData = await readProviderResponse( - await fetch( + await outboundFetch( `https://api.minimax.io/v1/files/retrieve?file_id=${encodeURIComponent(fileId)}`, { headers, redirect: 'error', signal } ) @@ -274,7 +275,7 @@ async function generateWithDirectProvider( const metadataUrl = `https://generativelanguage.googleapis.com/v1beta/files/${fileId}` for (let fileAttempt = 0; ; fileAttempt++) { const file = await readProviderResponse( - await fetch(metadataUrl, { headers, redirect: 'error', signal }) + await outboundFetch(metadataUrl, { headers, redirect: 'error', signal }) ) const state = getStringProperty(file, 'state') if (state === 'ACTIVE') break @@ -359,7 +360,9 @@ async function generateWithDirectProvider( throw new Error('Google synchronous video generation did not complete') if (attempt === maxAttempts) break await waitForProvider(context) - data = await readProviderResponse(await fetch(pollUrl, { headers, redirect: 'error', signal })) + data = await readProviderResponse( + await outboundFetch(pollUrl, { headers, redirect: 'error', signal }) + ) } throw new Error(`${input.provider} video generation timed out`) } @@ -410,7 +413,7 @@ async function generateWithFalAI( ) signal?.throwIfAborted() } - const createResponse = await fetch(`https://queue.fal.run/${endpoint}`, { + const createResponse = await outboundFetch(`https://queue.fal.run/${endpoint}`, { method: 'POST', redirect: 'error', headers: { @@ -456,7 +459,7 @@ async function generateWithFalAI( while (attempts < maxAttempts) { await waitForProvider(context) - const statusResponse = await fetch(statusUrl, { + const statusResponse = await outboundFetch(statusUrl, { redirect: 'error', headers: { Authorization: `Key ${apiKey}`, @@ -486,11 +489,14 @@ async function generateWithFalAI( onCost: context.onFalCost, signal, fetchResponse: (resultSignal) => - fetch(falQueueUrl(getStringProperty(statusData, 'response_url') || responseUrl), { - redirect: 'error', - headers: { Authorization: `Key ${apiKey}` }, - signal: resultSignal, - }), + outboundFetch( + falQueueUrl(getStringProperty(statusData, 'response_url') || responseUrl), + { + redirect: 'error', + headers: { Authorization: `Key ${apiKey}` }, + signal: resultSignal, + } + ), }) if (!isRecordLike(resultData)) { diff --git a/apps/sim/lib/internal/vision/client.ts b/apps/sim/lib/internal/vision/client.ts index 0a09ae97322..9eaefc201e8 100644 --- a/apps/sim/lib/internal/vision/client.ts +++ b/apps/sim/lib/internal/vision/client.ts @@ -1,6 +1,7 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { MAX_JSON_API_RESPONSE_BYTES, @@ -197,7 +198,7 @@ async function analyzeWithHttpProvider( } signal?.throwIfAborted() - const response = await fetch(apiUrl, { + const response = await outboundFetch(apiUrl, { method: 'POST', headers, body: JSON.stringify(isClaude ? anthropicRequest(input) : openAiRequest(input)), diff --git a/apps/sim/lib/internal/whatsapp/operations.ts b/apps/sim/lib/internal/whatsapp/operations.ts index e9f68bc86e7..e9c0d87af0e 100644 --- a/apps/sim/lib/internal/whatsapp/operations.ts +++ b/apps/sim/lib/internal/whatsapp/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -149,7 +150,7 @@ export async function executeWhatsAppSendMedia( caption: input.caption ?? undefined, filename, }) - const response = await fetch(buildMessagesUrl(input.phoneNumberId), { + const response = await outboundFetch(buildMessagesUrl(input.phoneNumberId), { method: 'POST', headers: buildAuthHeaders(input.accessToken), body: JSON.stringify(messageBody), @@ -183,10 +184,13 @@ export async function executeWhatsAppGetMedia( context.signal?.throwIfAborted() const authorization = `Bearer ${input.accessToken.trim()}` try { - const metadataResponse = await fetch(buildMediaUrl(input.mediaId, input.phoneNumberId), { - headers: { Authorization: authorization }, - signal: context.signal, - }) + const metadataResponse = await outboundFetch( + buildMediaUrl(input.mediaId, input.phoneNumberId), + { + headers: { Authorization: authorization }, + signal: context.signal, + } + ) const metadataBody = await readWhatsAppGraphResponse( metadataResponse, `WhatsApp media ${input.mediaId} metadata`, diff --git a/apps/sim/lib/internal/whatsapp/upload.ts b/apps/sim/lib/internal/whatsapp/upload.ts index 9c9f733bb38..580036c13dd 100644 --- a/apps/sim/lib/internal/whatsapp/upload.ts +++ b/apps/sim/lib/internal/whatsapp/upload.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readWhatsAppGraphResponse } from '@/lib/internal/whatsapp/client' import type { RawFileInput } from '@/lib/uploads/utils/file' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file' @@ -93,7 +94,7 @@ export async function uploadWhatsAppMedia({ userFile.name ) - const response = await fetch(buildMediaUploadUrl(phoneNumberId), { + const response = await outboundFetch(buildMediaUploadUrl(phoneNumberId), { method: 'POST', headers: { Authorization: `Bearer ${accessToken.trim()}` }, body: formData, diff --git a/apps/sim/lib/internal/wordpress/operations.ts b/apps/sim/lib/internal/wordpress/operations.ts index 06559b77abc..8f2840bed43 100644 --- a/apps/sim/lib/internal/wordpress/operations.ts +++ b/apps/sim/lib/internal/wordpress/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -96,7 +97,7 @@ export async function uploadWordPressMedia( if (input.altText) formData.append('alt_text', input.altText) if (input.description) formData.append('description', input.description) - const response = await fetch(`${WORDPRESS_COM_API_BASE}/${input.siteId}/media`, { + const response = await outboundFetch(`${WORDPRESS_COM_API_BASE}/${input.siteId}/media`, { method: 'POST', headers: { Authorization: `Bearer ${input.accessToken}` }, body: formData, diff --git a/apps/sim/lib/internal/workday/client.ts b/apps/sim/lib/internal/workday/client.ts index 2e578c8f658..43021343cde 100644 --- a/apps/sim/lib/internal/workday/client.ts +++ b/apps/sim/lib/internal/workday/client.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateWorkdayTenantUrl } from '@/lib/core/security/input-validation' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' @@ -566,7 +567,7 @@ async function callOperation( signal?.throwIfAborted() const envelope = buildEnvelope(operation, args, username, password) - const response = await fetch(endpoint, { + const response = await outboundFetch(endpoint, { method: 'POST', headers: { 'Content-Type': 'text/xml; charset=utf-8', From 482d344090be7a5b0ef9bbb1ce25ddebceab1a0c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:08:23 -0800 Subject: [PATCH 05/32] improvement(egress): route workspace vendor calls in tools, webhooks, providers, and runtime libs through outboundFetch Vendor API calls made for a workspace (tool post-processing, webhook subscription and polling, LLM provider SDK clients, embeddings, OCR, reranking, data drains, KB connectors and access checks, selectors, managed agents, media) now use outboundFetch so an organization with a dedicated egress gateway sends them from its static IPs. Sim-internal, operator-local (Ollama, vLLM/LiteLLM env endpoints), and platform calls stay on the global fetch. Slack Search deliveries (home publish, stop, message response, onboarding sender check) now run in the installation's organization scope; the public marketplace install exchange runs as a platform request. --- apps/sim/connectors/fireflies/fireflies.ts | 3 +- .../google-drive/google-drive-errors.ts | 3 +- .../executor/handlers/pi/cloud/github-pr.ts | 5 ++- .../lib/data-drains/destinations/bigquery.ts | 5 ++- .../lib/data-drains/destinations/datadog.ts | 3 +- apps/sim/lib/data-drains/destinations/gcs.ts | 3 +- .../lib/data-drains/destinations/snowflake.ts | 5 ++- apps/sim/lib/embeddings/client.ts | 5 ++- .../lib/knowledge/access/confluence-site.ts | 3 +- .../knowledge/access/github-installation.ts | 3 +- .../application/slack-search/authorization.ts | 23 ++++++++++ .../application/slack-search/home.test.ts | 16 +++++++ .../application/slack-search/home.ts | 5 ++- .../application/slack-search/onboarding.ts | 13 +++--- .../slack-search/process-message.test.ts | 1 + .../slack-search/process-message.ts | 5 ++- .../application/slack-search/stop.test.ts | 1 + .../application/slack-search/stop.ts | 5 ++- .../documents/image-extraction-gemini.ts | 3 +- apps/sim/lib/knowledge/documents/processor.ts | 3 +- apps/sim/lib/knowledge/reranker.ts | 3 +- .../lib/managed-agents/session-client.test.ts | 1 + apps/sim/lib/managed-agents/session-client.ts | 45 +++++++++++-------- apps/sim/lib/media/falai.ts | 18 +++++--- .../selectors/server/providers/harmonic.ts | 3 +- .../lib/selectors/server/providers/http.ts | 3 +- .../selectors/server/providers/snowflake.ts | 3 +- .../lib/slack-search/public-install-auth.ts | 10 ++--- apps/sim/lib/tools/falai-pricing.ts | 5 ++- apps/sim/lib/webhooks/polling/gmail.ts | 11 ++--- .../lib/webhooks/polling/google-calendar.ts | 3 +- apps/sim/lib/webhooks/polling/google-drive.ts | 5 ++- .../sim/lib/webhooks/polling/google-sheets.ts | 7 +-- apps/sim/lib/webhooks/polling/hubspot.ts | 7 ++- apps/sim/lib/webhooks/providers/airtable.ts | 9 ++-- apps/sim/lib/webhooks/providers/ashby.ts | 5 ++- apps/sim/lib/webhooks/providers/attio.ts | 5 ++- apps/sim/lib/webhooks/providers/bitbucket.ts | 3 +- apps/sim/lib/webhooks/providers/calendly.ts | 5 ++- apps/sim/lib/webhooks/providers/checkr.ts | 16 ++++--- apps/sim/lib/webhooks/providers/clickup.ts | 5 ++- apps/sim/lib/webhooks/providers/fathom.ts | 5 ++- apps/sim/lib/webhooks/providers/fivetran.ts | 3 +- apps/sim/lib/webhooks/providers/grain.ts | 24 +++++----- apps/sim/lib/webhooks/providers/granola.ts | 5 ++- apps/sim/lib/webhooks/providers/helpscout.ts | 5 ++- apps/sim/lib/webhooks/providers/instantly.ts | 5 ++- apps/sim/lib/webhooks/providers/jotform.ts | 7 +-- apps/sim/lib/webhooks/providers/lemlist.ts | 7 +-- apps/sim/lib/webhooks/providers/linear.ts | 5 ++- apps/sim/lib/webhooks/providers/linq.ts | 5 ++- .../lib/webhooks/providers/microsoft-teams.ts | 17 ++++--- apps/sim/lib/webhooks/providers/monday.ts | 5 ++- apps/sim/lib/webhooks/providers/pagerduty.ts | 9 ++-- apps/sim/lib/webhooks/providers/resend.ts | 5 ++- apps/sim/lib/webhooks/providers/revenuecat.ts | 5 ++- apps/sim/lib/webhooks/providers/rootly.ts | 20 +++++---- apps/sim/lib/webhooks/providers/slack.ts | 7 +-- apps/sim/lib/webhooks/providers/telegram.ts | 5 ++- apps/sim/lib/webhooks/providers/typeform.ts | 5 ++- apps/sim/lib/webhooks/providers/vercel.ts | 5 ++- apps/sim/lib/webhooks/providers/webflow.ts | 5 ++- apps/sim/lib/webhooks/providers/zendesk.ts | 9 ++-- apps/sim/lib/webhooks/providers/zoho-desk.ts | 5 ++- apps/sim/lib/webhooks/slack-agent-api.ts | 3 +- apps/sim/providers/anthropic/index.ts | 3 +- .../providers/azure-anthropic/index.test.ts | 5 ++- apps/sim/providers/azure-anthropic/index.ts | 3 +- apps/sim/providers/azure-openai/index.test.ts | 5 ++- apps/sim/providers/azure-openai/index.ts | 3 +- apps/sim/providers/baseten/index.ts | 2 + apps/sim/providers/cerebras/index.ts | 2 + apps/sim/providers/deepseek/index.ts | 2 + apps/sim/providers/file-attachments.server.ts | 3 +- apps/sim/providers/fireworks/index.ts | 2 + apps/sim/providers/groq/index.ts | 7 ++- apps/sim/providers/kie/index.ts | 3 +- apps/sim/providers/kimi/index.ts | 2 + apps/sim/providers/meta/index.ts | 2 + apps/sim/providers/mistral/index.ts | 2 + apps/sim/providers/nebius/index.ts | 2 + apps/sim/providers/nvidia/index.ts | 2 + apps/sim/providers/ollama-cloud/index.ts | 2 + apps/sim/providers/openai/core.ts | 5 ++- apps/sim/providers/openrouter/index.ts | 2 + apps/sim/providers/sakana/index.ts | 2 + apps/sim/providers/thinkingmachines/index.ts | 2 + apps/sim/providers/together/index.ts | 2 + apps/sim/providers/typesafe/transport.ts | 3 +- apps/sim/providers/xai/index.ts | 2 + apps/sim/providers/zai/index.ts | 2 + apps/sim/tools/apify/run_actor_async.ts | 5 ++- apps/sim/tools/azure_devops/utils.ts | 3 +- apps/sim/tools/brightdata/discover.ts | 3 +- apps/sim/tools/cbinsights/utils.ts | 5 ++- apps/sim/tools/dropbox/download.ts | 20 +++++---- apps/sim/tools/dropcontact/enrich_contact.ts | 3 +- apps/sim/tools/enrow/find_email.ts | 3 +- apps/sim/tools/enrow/verify_email.ts | 3 +- apps/sim/tools/exa/agent.ts | 3 +- apps/sim/tools/firecrawl/agent.ts | 3 +- apps/sim/tools/firecrawl/batch-scrape.ts | 18 +++++--- apps/sim/tools/firecrawl/crawl.ts | 3 +- apps/sim/tools/firecrawl/extract.ts | 18 +++++--- apps/sim/tools/github/pr.ts | 3 +- apps/sim/tools/gmail/read.ts | 5 ++- apps/sim/tools/gmail/search.ts | 3 +- apps/sim/tools/gmail/utils.ts | 5 ++- apps/sim/tools/google_calendar/invite.ts | 3 +- apps/sim/tools/google_calendar/quick_add.ts | 5 ++- apps/sim/tools/google_calendar/respond.ts | 3 +- apps/sim/tools/google_drive/create_folder.ts | 3 +- apps/sim/tools/google_drive/get_content.ts | 7 +-- apps/sim/tools/google_slides/write.ts | 5 ++- apps/sim/tools/icypeas/find_email.ts | 20 +++++---- apps/sim/tools/icypeas/verify_email.ts | 20 +++++---- apps/sim/tools/jira/add_comment.ts | 3 +- apps/sim/tools/jira/add_watcher.ts | 3 +- apps/sim/tools/jira/add_worklog.ts | 3 +- apps/sim/tools/jira/assign_issue.ts | 3 +- apps/sim/tools/jira/bulk_read.ts | 5 ++- apps/sim/tools/jira/create_issue_link.ts | 5 ++- apps/sim/tools/jira/delete_attachment.ts | 3 +- apps/sim/tools/jira/delete_comment.ts | 3 +- apps/sim/tools/jira/delete_issue.ts | 3 +- apps/sim/tools/jira/delete_issue_link.ts | 3 +- apps/sim/tools/jira/delete_worklog.ts | 3 +- apps/sim/tools/jira/get_attachments.ts | 3 +- apps/sim/tools/jira/get_comments.ts | 3 +- apps/sim/tools/jira/get_fields.ts | 3 +- apps/sim/tools/jira/get_project.ts | 3 +- apps/sim/tools/jira/get_transitions.ts | 18 +++++--- apps/sim/tools/jira/get_users.ts | 3 +- apps/sim/tools/jira/get_worklogs.ts | 3 +- apps/sim/tools/jira/list_issue_types.ts | 3 +- apps/sim/tools/jira/list_projects.ts | 3 +- apps/sim/tools/jira/remove_watcher.ts | 3 +- apps/sim/tools/jira/retrieve.ts | 9 ++-- apps/sim/tools/jira/search_issues.ts | 3 +- apps/sim/tools/jira/search_users.ts | 3 +- apps/sim/tools/jira/transition_issue.ts | 7 +-- apps/sim/tools/jira/update_comment.ts | 3 +- apps/sim/tools/jira/update_worklog.ts | 3 +- apps/sim/tools/linkedin/share_post.ts | 3 +- apps/sim/tools/microsoft_ad/utils.ts | 3 +- .../sim/tools/microsoft_excel/format_range.ts | 3 +- apps/sim/tools/microsoft_excel/read.ts | 3 +- apps/sim/tools/microsoft_excel/utils.ts | 3 +- apps/sim/tools/microsoft_teams/utils.ts | 13 +++--- apps/sim/tools/netsuite/utils.ts | 3 +- apps/sim/tools/notion/read.ts | 3 +- apps/sim/tools/outlook/read.ts | 4 +- apps/sim/tools/parallel/deep_research.ts | 3 +- apps/sim/tools/powerbi/listing.ts | 3 +- apps/sim/tools/quickbooks/client.ts | 3 +- apps/sim/tools/quickbooks/utils.ts | 5 ++- apps/sim/tools/sharepoint/read_page.ts | 5 ++- apps/sim/tools/slack/utils.ts | 3 +- apps/sim/tools/twilio_voice/list_calls.ts | 3 +- apps/sim/tools/wiza/individual_reveal.ts | 3 +- apps/sim/tools/x/read.ts | 3 +- 161 files changed, 573 insertions(+), 311 deletions(-) diff --git a/apps/sim/connectors/fireflies/fireflies.ts b/apps/sim/connectors/fireflies/fireflies.ts index 47a7de32aa2..bd077bb53a6 100644 --- a/apps/sim/connectors/fireflies/fireflies.ts +++ b/apps/sim/connectors/fireflies/fireflies.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { isRetryableError, @@ -200,7 +201,7 @@ async function firefliesGraphQL( return retryWithExponentialBackoff( async () => { /** One retry layer owns transport, HTTP, and GraphQL semantic failures. */ - const response = await fetch(FIREFLIES_GRAPHQL_URL, { + const response = await outboundFetch(FIREFLIES_GRAPHQL_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/connectors/google-drive/google-drive-errors.ts b/apps/sim/connectors/google-drive/google-drive-errors.ts index b26749a4155..377a949cc2f 100644 --- a/apps/sim/connectors/google-drive/google-drive-errors.ts +++ b/apps/sim/connectors/google-drive/google-drive-errors.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { attachRetryHeaders, isRetryableError, @@ -169,7 +170,7 @@ export async function fetchGoogleDriveWithRetry( ): Promise { return retryWithExponentialBackoff( async () => { - const response = await fetch(url, options) + const response = await outboundFetch(url, options) if (response.ok) return response const error = await readGoogleDriveApiError(response, operation) diff --git a/apps/sim/executor/handlers/pi/cloud/github-pr.ts b/apps/sim/executor/handlers/pi/cloud/github-pr.ts index 290a38da059..8a9b3566472 100644 --- a/apps/sim/executor/handlers/pi/cloud/github-pr.ts +++ b/apps/sim/executor/handlers/pi/cloud/github-pr.ts @@ -7,6 +7,7 @@ */ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { toolResultError } from '@/executor/handlers/pi/cloud/shared' import { executeTool } from '@/tools' import { GITHUB_GRAPHQL_URL, githubGraphQlHeaders, readGraphQlData } from '@/tools/github/graphql' @@ -227,7 +228,7 @@ export async function setPullRequestDraftState( repo: params.repo, pullNumber: params.pullNumber, } - const queryResponse = await fetch(GITHUB_GRAPHQL_URL, { + const queryResponse = await outboundFetch(GITHUB_GRAPHQL_URL, { method: 'POST', headers: githubGraphQlHeaders(params.githubToken), body: JSON.stringify({ @@ -268,7 +269,7 @@ export async function setPullRequestDraftState( if (isDraft === shouldBeDraft) return const mutationName = shouldBeDraft ? 'convertPullRequestToDraft' : 'markPullRequestReadyForReview' - const mutationResponse = await fetch(GITHUB_GRAPHQL_URL, { + const mutationResponse = await outboundFetch(GITHUB_GRAPHQL_URL, { method: 'POST', headers: githubGraphQlHeaders(params.githubToken), body: JSON.stringify({ diff --git a/apps/sim/lib/data-drains/destinations/bigquery.ts b/apps/sim/lib/data-drains/destinations/bigquery.ts index b4dc41b841a..6e1cf38fe8a 100644 --- a/apps/sim/lib/data-drains/destinations/bigquery.ts +++ b/apps/sim/lib/data-drains/destinations/bigquery.ts @@ -4,6 +4,7 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { JWT } from 'google-auth-library' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type ParsedServiceAccount, parseNdjsonObjects, @@ -110,7 +111,7 @@ async function postInsertAll( const token = await getAccessToken(input.jwt, forceRefresh) const perAttempt = AbortSignal.any([input.signal, AbortSignal.timeout(PER_ATTEMPT_TIMEOUT_MS)]) try { - return await fetch(url, { + return await outboundFetch(url, { method: 'POST', headers: { Authorization: `Bearer ${token}`, @@ -292,7 +293,7 @@ export const bigqueryDestination: DrainDestination< const token = await getAccessToken(jwt) const url = `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(config.projectId)}/datasets/${encodeURIComponent(config.datasetId)}/tables/${encodeURIComponent(config.tableId)}?fields=id` const perAttempt = AbortSignal.any([signal, AbortSignal.timeout(PER_ATTEMPT_TIMEOUT_MS)]) - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${token}` }, signal: perAttempt, }) diff --git a/apps/sim/lib/data-drains/destinations/datadog.ts b/apps/sim/lib/data-drains/destinations/datadog.ts index edb89210d12..d300282fcaa 100644 --- a/apps/sim/lib/data-drains/destinations/datadog.ts +++ b/apps/sim/lib/data-drains/destinations/datadog.ts @@ -4,6 +4,7 @@ import { toError } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { redactExactSensitiveValues } from '@/lib/core/security/redaction' import { parseNdjsonObjects } from '@/lib/data-drains/destinations/utils' import type { DeliveryMetadata, DrainDestination } from '@/lib/data-drains/types' @@ -160,7 +161,7 @@ async function postWithRetries(input: PostInput): Promise { let retryAfterMs: number | null = null let response: Response | undefined try { - response = await fetch(input.url, { + response = await outboundFetch(input.url, { method: 'POST', // double-cast-allowed: Uint8Array is a valid runtime BodyInit but the DOM lib types only enumerate Blob/FormData/string/etc. body: body as unknown as BodyInit, diff --git a/apps/sim/lib/data-drains/destinations/gcs.ts b/apps/sim/lib/data-drains/destinations/gcs.ts index e7cfe734536..0be0d2e08c1 100644 --- a/apps/sim/lib/data-drains/destinations/gcs.ts +++ b/apps/sim/lib/data-drains/destinations/gcs.ts @@ -4,6 +4,7 @@ import { generateShortId } from '@sim/utils/id' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { JWT } from 'google-auth-library' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { buildObjectKey, normalizePrefix, @@ -139,7 +140,7 @@ async function fetchWithRetry(input: RetryRequestInput): Promise { let response: Response try { const headers = await input.buildHeaders() - response = await fetch(input.url, { + response = await outboundFetch(input.url, { method: input.method, body: input.body as BodyInit | undefined, headers, diff --git a/apps/sim/lib/data-drains/destinations/snowflake.ts b/apps/sim/lib/data-drains/destinations/snowflake.ts index b5550495bec..9dfdbb88501 100644 --- a/apps/sim/lib/data-drains/destinations/snowflake.ts +++ b/apps/sim/lib/data-drains/destinations/snowflake.ts @@ -6,6 +6,7 @@ import { generateId } from '@sim/utils/id' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { importPKCS8, SignJWT } from 'jose' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { DrainDestination } from '@/lib/data-drains/types' const logger = createLogger('DataDrainSnowflakeDestination') @@ -245,7 +246,7 @@ async function executeStatement(input: ExecuteInput): Promise { const url = `${baseUrl}?${params.toString()}` let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { Authorization: `Bearer ${jwt}`, @@ -347,7 +348,7 @@ async function pollStatement(input: PollInput): Promise { const perAttempt = AbortSignal.any([input.signal, AbortSignal.timeout(PER_ATTEMPT_TIMEOUT_MS)]) let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { headers: { Authorization: `Bearer ${jwt}`, Accept: 'application/json', diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index 4596d14750e..f05794d96bf 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -12,6 +12,7 @@ import { getRotatingApiKey } from '@/lib/core/config/api-keys' import { env, envNumber } from '@/lib/core/config/env' import { isHosted } from '@/lib/core/config/env-flags' import { isQuotaExhaustionBody } from '@/lib/core/errors/provider-quota' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { ProviderQuotaExhaustedError, recordProviderCooldown, @@ -581,7 +582,9 @@ async function callEmbeddingAPI( const timeout = setTimeout(() => controller.abort(), EMBEDDING_REQUEST_TIMEOUT_MS) try { - const response = await fetch(request.apiUrl, { + // Ollama is the deployment's own server, never an organization's vendor. + const requestFetch = providerId === 'ollama' ? fetch : outboundFetch + const response = await requestFetch(request.apiUrl, { method: 'POST', headers: request.headers, body: JSON.stringify(request.body), diff --git a/apps/sim/lib/knowledge/access/confluence-site.ts b/apps/sim/lib/knowledge/access/confluence-site.ts index dbb0b65a08d..0fe5aec71c9 100644 --- a/apps/sim/lib/knowledge/access/confluence-site.ts +++ b/apps/sim/lib/knowledge/access/confluence-site.ts @@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' @@ -99,7 +100,7 @@ async function verifySite( signal: AbortSignal ): Promise { signal.throwIfAborted() - const response = await fetch( + const response = await outboundFetch( `https://api.atlassian.com/ex/confluence/${binding.cloudId}/wiki/rest/api/user/current`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, diff --git a/apps/sim/lib/knowledge/access/github-installation.ts b/apps/sim/lib/knowledge/access/github-installation.ts index ac182449158..dea8ffec58a 100644 --- a/apps/sim/lib/knowledge/access/github-installation.ts +++ b/apps/sim/lib/knowledge/access/github-installation.ts @@ -9,6 +9,7 @@ import { import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' @@ -71,7 +72,7 @@ function positiveId(value: unknown): string | null { async function readGitHubJson(path: string, accessToken: string, signal: AbortSignal) { signal.throwIfAborted() - const response = await fetch(`https://api.github.com${path}`, { + const response = await outboundFetch(`https://api.github.com${path}`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/vnd.github+json', diff --git a/apps/sim/lib/knowledge/application/slack-search/authorization.ts b/apps/sim/lib/knowledge/application/slack-search/authorization.ts index e322b5d0299..352c9480708 100644 --- a/apps/sim/lib/knowledge/application/slack-search/authorization.ts +++ b/apps/sim/lib/knowledge/application/slack-search/authorization.ts @@ -1,4 +1,7 @@ import type { Principal, SlackInstallationPrincipal } from '@sim/auth/principal' +import type { ApplicationOperation, OperationUseCase } from '@/lib/core/application/operation' +import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability' import { @@ -25,6 +28,26 @@ export function requireSlackInstallationPrincipal( throw new OrchestrationError('forbidden', 'Slack delivery has expired') } +/** + * Runs a Slack installation use case in the outbound scope of the organization that owns the + * principal's installation, so its Slack API calls leave through that organization's egress. + * An unknown credential runs unscoped; the use case's own authorization rejects it. + */ +export function withSlackSearchOutboundScope( + useCase: OperationUseCase +): OperationUseCase { + return { + ...useCase, + async execute(args) { + requireSlackInstallationPrincipal(args.principal) + if (!isOutboundRoutingEnabled()) return useCase.execute(args) + const installation = await findSlackSearchInstallation(args.principal.credentialId) + if (!installation) return useCase.execute(args) + return runWithOutboundOrganization(installation.organizationId, () => useCase.execute(args)) + }, + } +} + /** Rechecks the binding and exact authenticated secret on every lifecycle boundary. */ export async function authorizeSlackSearchInstallation( principal: Principal, diff --git a/apps/sim/lib/knowledge/application/slack-search/home.test.ts b/apps/sim/lib/knowledge/application/slack-search/home.test.ts index af290e206aa..69dd10e838c 100644 --- a/apps/sim/lib/knowledge/application/slack-search/home.test.ts +++ b/apps/sim/lib/knowledge/application/slack-search/home.test.ts @@ -1,10 +1,12 @@ import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' import { asyncJobsMock, asyncJobsMockFns } from '@sim/testing/mocks/async-jobs.mock' import { knowledgeAvailabilityMock } from '@sim/testing/mocks/knowledge-availability.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { urlsMockFns } from '@sim/testing/mocks/urls.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' const hoisted = vi.hoisted(() => ({ installation: vi.fn(), publish: vi.fn() })) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@/lib/core/async-jobs', () => asyncJobsMock) vi.mock('@/lib/internal/slack/client', () => ({ requestSlackApi: hoisted.publish })) vi.mock('@/lib/knowledge/application/slack-search/repository', () => ({ @@ -13,6 +15,7 @@ vi.mock('@/lib/knowledge/application/slack-search/repository', () => ({ })) vi.mock('@/lib/knowledge/access/availability', () => knowledgeAvailabilityMock) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { publishSlackSearchHome, receiveSlackSearchHome, @@ -131,6 +134,19 @@ describe('static Home delivery', () => { expect(view.blocks).toHaveLength(3) expect(view.blocks[2].elements[0].url).toBe('https://sim.test/o/org1/integrations') }) + it("calls Slack from the installation's organization outbound scope", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (organizationId) => ({ + organizationId, + })) + const routes: unknown[] = [] + mocks.publish.mockImplementationOnce(async () => { + routes.push(await resolveCurrentOutboundRoute()) + return { status: 200, data: { ok: true } } + }) + await publish() + expect(routes).toEqual([{ organizationId: 'org1' }]) + }) it.each([ { enabled: false }, { revision: 'r2' }, diff --git a/apps/sim/lib/knowledge/application/slack-search/home.ts b/apps/sim/lib/knowledge/application/slack-search/home.ts index fdc56d80678..3cec3930ecb 100644 --- a/apps/sim/lib/knowledge/application/slack-search/home.ts +++ b/apps/sim/lib/knowledge/application/slack-search/home.ts @@ -7,6 +7,7 @@ import { requestSlackApi } from '@/lib/internal/slack/client' import { authorizeSlackSearchInstallation, requireSlackInstallationPrincipal, + withSlackSearchOutboundScope, } from '@/lib/knowledge/application/slack-search/authorization' import { organizationRoutes } from '@/lib/navigation/paths' import { @@ -89,7 +90,7 @@ export const publishSlackSearchHome: OperationUseCase< typeof operation, { job: SlackSearchHomeJob; signal: AbortSignal }, void -> = { +> = withSlackSearchOutboundScope({ operation, async execute({ principal, input: { job, signal } }) { requireSlackInstallationPrincipal(principal) @@ -121,4 +122,4 @@ export const publishSlackSearchHome: OperationUseCase< if (response.status !== 200 || response.data.ok !== true) throw new Error('Could not publish the Slack Search Home tab') }, -} +}) diff --git a/apps/sim/lib/knowledge/application/slack-search/onboarding.ts b/apps/sim/lib/knowledge/application/slack-search/onboarding.ts index 84602d24549..ac27128544b 100644 --- a/apps/sim/lib/knowledge/application/slack-search/onboarding.ts +++ b/apps/sim/lib/knowledge/application/slack-search/onboarding.ts @@ -10,6 +10,7 @@ import { type OperationUseCase, } from '@/lib/core/application/operation' import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { postSlackMessage, requestSlackApi, slackString } from '@/lib/internal/slack/client' import { getSlackSearchSender } from '@/lib/internal/slack/search-client' @@ -232,11 +233,13 @@ async function resolveOnboarding(principal: Principal, token: string) { 'forbidden', 'Slack Search is disabled. Ask your administrator to reconnect it.' ) - const sender = await getSlackSearchSender( - context.secret.botToken, - job.message.userId, - job.message.teamId, - AbortSignal.timeout(10_000) + const sender = await runWithOutboundOrganization(context.installation.organizationId, () => + getSlackSearchSender( + context.secret.botToken, + job.message.userId, + job.message.teamId, + AbortSignal.timeout(10_000) + ) ) if (!sender || sender.email !== state.email) return { view: { status: 'identity_conflict' } as OnboardingBlocked } diff --git a/apps/sim/lib/knowledge/application/slack-search/process-message.test.ts b/apps/sim/lib/knowledge/application/slack-search/process-message.test.ts index 9e505352ddf..0e652265f03 100644 --- a/apps/sim/lib/knowledge/application/slack-search/process-message.test.ts +++ b/apps/sim/lib/knowledge/application/slack-search/process-message.test.ts @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ })) vi.mock('@/lib/knowledge/application/slack-search/authorization', () => ({ requireSlackInstallationPrincipal: vi.fn(), + withSlackSearchOutboundScope: (useCase: T) => useCase, authorizeSlackSearchInstallation: mocks.authorize, authorizeSlackSearchRedirect: mocks.redirect, })) diff --git a/apps/sim/lib/knowledge/application/slack-search/process-message.ts b/apps/sim/lib/knowledge/application/slack-search/process-message.ts index 76848dd8729..352226c8922 100644 --- a/apps/sim/lib/knowledge/application/slack-search/process-message.ts +++ b/apps/sim/lib/knowledge/application/slack-search/process-message.ts @@ -7,6 +7,7 @@ import { authorizeSlackSearchInstallation, authorizeSlackSearchRedirect, requireSlackInstallationPrincipal, + withSlackSearchOutboundScope, } from '@/lib/knowledge/application/slack-search/authorization' import { routeSlackSearchMentionToDm } from '@/lib/knowledge/application/slack-search/mention' import { dispatchSlackSearchTurn } from '@/lib/knowledge/application/slack-search/outbox' @@ -98,7 +99,7 @@ export const respondToSlackSearchMessage: OperationUseCase< typeof respondOperation, RespondInput, void -> = { +> = withSlackSearchOutboundScope({ operation: respondOperation, async execute({ principal, input }) { requireSlackInstallationPrincipal(principal) @@ -151,4 +152,4 @@ export const respondToSlackSearchMessage: OperationUseCase< } await runSlackSearchAssistant(principal, { ...input, job }) }, -} +}) diff --git a/apps/sim/lib/knowledge/application/slack-search/stop.test.ts b/apps/sim/lib/knowledge/application/slack-search/stop.test.ts index 835d7403779..f49674cf596 100644 --- a/apps/sim/lib/knowledge/application/slack-search/stop.test.ts +++ b/apps/sim/lib/knowledge/application/slack-search/stop.test.ts @@ -18,6 +18,7 @@ const mocks = vi.hoisted(() => ({ })) vi.mock('@/lib/knowledge/application/slack-search/authorization', () => ({ requireSlackInstallationPrincipal: vi.fn(), + withSlackSearchOutboundScope: (useCase: T) => useCase, authorizeSlackSearchInstallation: mocks.authorize, })) vi.mock('@/lib/internal/slack/search-client', () => ({ getSlackSearchSender: mocks.sender })) diff --git a/apps/sim/lib/knowledge/application/slack-search/stop.ts b/apps/sim/lib/knowledge/application/slack-search/stop.ts index a5284743f30..91572374859 100644 --- a/apps/sim/lib/knowledge/application/slack-search/stop.ts +++ b/apps/sim/lib/knowledge/application/slack-search/stop.ts @@ -9,6 +9,7 @@ import { getSlackSearchSender } from '@/lib/internal/slack/search-client' import { authorizeSlackSearchInstallation, requireSlackInstallationPrincipal, + withSlackSearchOutboundScope, } from '@/lib/knowledge/application/slack-search/authorization' import { resolveSlackSearchChatRecord, @@ -51,7 +52,7 @@ export const stopSlackSearchThread: OperationUseCase< typeof operation, z.infer, void -> = { +> = withSlackSearchOutboundScope({ operation, async execute({ principal, input }) { requireSlackInstallationPrincipal(principal) @@ -168,4 +169,4 @@ export const stopSlackSearchThread: OperationUseCase< AbortSignal.timeout(10_000) ) }, -} +}) diff --git a/apps/sim/lib/knowledge/documents/image-extraction-gemini.ts b/apps/sim/lib/knowledge/documents/image-extraction-gemini.ts index a36c981d1f8..521e2d4c507 100644 --- a/apps/sim/lib/knowledge/documents/image-extraction-gemini.ts +++ b/apps/sim/lib/knowledge/documents/image-extraction-gemini.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { parseRetryAfter } from '@sim/utils/retry' import { z } from 'zod' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { EmbeddingConfigurationError } from '@/lib/embeddings/configuration-error' @@ -219,7 +220,7 @@ export async function requestGeminiImageExtraction(input: { maxOutputTokens: number signal: AbortSignal }): Promise { - const response = await fetch( + const response = await outboundFetch( `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(input.model)}:generateContent`, { method: 'POST', diff --git a/apps/sim/lib/knowledge/documents/processor.ts b/apps/sim/lib/knowledge/documents/processor.ts index ad1c1db6016..8e6db7e35ef 100644 --- a/apps/sim/lib/knowledge/documents/processor.ts +++ b/apps/sim/lib/knowledge/documents/processor.ts @@ -18,6 +18,7 @@ import { } from '@/lib/chunkers' import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { recordProviderCooldown, waitForProviderAdmission, @@ -669,7 +670,7 @@ async function makeOCRRequest( const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal try { - const response = await fetch(endpoint, { + const response = await outboundFetch(endpoint, { method: 'POST', headers, body: typeof body === 'string' ? body : JSON.stringify(body), diff --git a/apps/sim/lib/knowledge/reranker.ts b/apps/sim/lib/knowledge/reranker.ts index 2a28d1a4ce5..9f0085d33ff 100644 --- a/apps/sim/lib/knowledge/reranker.ts +++ b/apps/sim/lib/knowledge/reranker.ts @@ -5,6 +5,7 @@ import { getBYOKKey } from '@/lib/api-key/byok' import { getRotatingApiKey, hasRotatingApiKey } from '@/lib/core/config/api-keys' import { env } from '@/lib/core/config/env' import { isHosted } from '@/lib/core/config/env-flags' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type ProviderIdentity, recordProviderCooldown, @@ -177,7 +178,7 @@ export async function rerank( maxWaitMs: Math.max(0, deadlineAt - Date.now()), }) attempt += 1 - const res = await fetch('https://api.cohere.com/v2/rerank', { + const res = await outboundFetch('https://api.cohere.com/v2/rerank', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/managed-agents/session-client.test.ts b/apps/sim/lib/managed-agents/session-client.test.ts index e26034d5ec6..ae52a346652 100644 --- a/apps/sim/lib/managed-agents/session-client.test.ts +++ b/apps/sim/lib/managed-agents/session-client.test.ts @@ -131,6 +131,7 @@ describe('managedAgentsList — selector collection bounds', () => { global.fetch = vi.fn( (_url: string, init?: RequestInit) => new Promise((_resolve, reject) => { + if (init?.signal?.aborted) reject(cancelled) init?.signal?.addEventListener('abort', () => reject(cancelled), { once: true }) }) ) as unknown as typeof fetch diff --git a/apps/sim/lib/managed-agents/session-client.ts b/apps/sim/lib/managed-agents/session-client.ts index f7166ae847e..9252d9290e4 100644 --- a/apps/sim/lib/managed-agents/session-client.ts +++ b/apps/sim/lib/managed-agents/session-client.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' /** * Provider-neutral HTTP client for the Claude Platform Managed Agents API. * @@ -203,7 +204,7 @@ export function buildSessionCreatePayload(input: CreateSessionInput): Record { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions`, { method: 'POST', headers: managedAgentsHeaders(input.apiKey, { json: true }), body: JSON.stringify(buildSessionCreatePayload(input)), @@ -278,7 +279,7 @@ export async function sendSessionEvents( input: SessionAuth & { sessionId: string; events: OutboundSessionEvent[] } ): Promise { if (input.events.length === 0) return - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/events`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/events`, { method: 'POST', headers: managedAgentsHeaders(input.apiKey, { json: true }), body: JSON.stringify({ events: input.events }), @@ -467,11 +468,14 @@ export async function sendCustomToolResults( export async function openSessionStream( input: SessionAuth & { sessionId: string } ): Promise { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/events/stream`, { - method: 'GET', - headers: managedAgentsHeaders(input.apiKey, { accept: 'text/event-stream' }), - signal: input.signal, - }) + const resp = await outboundFetch( + `${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/events/stream`, + { + method: 'GET', + headers: managedAgentsHeaders(input.apiKey, { accept: 'text/event-stream' }), + signal: input.signal, + } + ) if (!resp.ok) { const detail = await resp.text().catch(() => '') throw new Error(`Anthropic events.stream failed (${resp.status}): ${detail.slice(0, 400)}`) @@ -559,7 +563,7 @@ async function fetchSelectorListPage( let response: Response try { - response = await fetch(url.toString(), { + response = await outboundFetch(url.toString(), { method: 'GET', headers: managedAgentsHeaders(input.apiKey, { beta: input.beta }), redirect: 'error', @@ -621,7 +625,7 @@ async function listPaginated( // `append`, not `set` — `types[]` is repeatable and each value must survive. for (const [key, value] of input.searchParams ?? []) url.searchParams.append(key, value) if (page) url.searchParams.set('page', page) - const resp = await fetch(url.toString(), { + const resp = await outboundFetch(url.toString(), { method: 'GET', headers: managedAgentsHeaders(input.apiKey, { beta: input.beta }), redirect: 'error', @@ -779,11 +783,14 @@ export async function getEnvironmentType( input: SessionAuth & { environmentId: string } ): Promise { try { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/environments/${input.environmentId}`, { - method: 'GET', - headers: managedAgentsHeaders(input.apiKey), - signal: input.signal, - }) + const resp = await outboundFetch( + `${ANTHROPIC_API_BASE}/v1/environments/${input.environmentId}`, + { + method: 'GET', + headers: managedAgentsHeaders(input.apiKey), + signal: input.signal, + } + ) if (!resp.ok) return undefined const body = (await resp.json()) as { config?: { type?: unknown } } const type = body.config?.type @@ -803,7 +810,7 @@ export async function getSession( input: SessionAuth & { sessionId: string } ): Promise { try { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { method: 'GET', headers: managedAgentsHeaders(input.apiKey), signal: input.signal, @@ -878,7 +885,7 @@ export function parseSessionSnapshot(raw: unknown): SessionSnapshot { export async function retrieveSession( input: SessionAuth & { sessionId: string } ): Promise { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { method: 'GET', headers: managedAgentsHeaders(input.apiKey), signal: input.signal, @@ -912,7 +919,7 @@ export async function updateSession( if (Object.keys(payload).length === 0) { throw new Error('Update session requires a title or metadata to change.') } - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { method: 'POST', headers: managedAgentsHeaders(input.apiKey, { json: true }), body: JSON.stringify(payload), @@ -931,7 +938,7 @@ export async function updateSession( * it first), and archiving is NOT reversible. */ export async function archiveSession(input: SessionAuth & { sessionId: string }): Promise { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/archive`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/archive`, { method: 'POST', headers: managedAgentsHeaders(input.apiKey), signal: input.signal, @@ -949,7 +956,7 @@ export async function archiveSession(input: SessionAuth & { sessionId: string }) * independent resources and are NOT affected. */ export async function deleteSession(input: SessionAuth & { sessionId: string }): Promise { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { method: 'DELETE', headers: managedAgentsHeaders(input.apiKey), signal: input.signal, diff --git a/apps/sim/lib/media/falai.ts b/apps/sim/lib/media/falai.ts index 28ed99a0687..51c57da2bf1 100644 --- a/apps/sim/lib/media/falai.ts +++ b/apps/sim/lib/media/falai.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -140,7 +141,7 @@ async function submitAndPollFalQueue( ): Promise { // Only a submit already under way outlives its caller; a stopped call never starts one. queueSignal.throwIfAborted() - const createResponse = await fetch(`https://queue.fal.run/${endpoint}`, { + const createResponse = await outboundFetch(`https://queue.fal.run/${endpoint}`, { method: 'POST', headers: { Authorization: `Key ${apiKey}`, @@ -201,7 +202,7 @@ async function pollFalQueue( await interruptibleSleep(POLL_INTERVAL_MS, signal) signal.throwIfAborted() - const statusResponse = await fetch(statusUrl, { + const statusResponse = await outboundFetch(statusUrl, { headers: { Authorization: `Key ${apiKey}` }, signal, }) @@ -226,10 +227,13 @@ async function pollFalQueue( if (statusData.error) { throw new Error(`Fal.ai generation failed: ${falErrorMessage(statusData.error)}`) } - const resultResponse = await fetch(getStringProp(statusData, 'response_url') || responseUrl, { - headers: { Authorization: `Key ${apiKey}` }, - signal, - }) + const resultResponse = await outboundFetch( + getStringProp(statusData, 'response_url') || responseUrl, + { + headers: { Authorization: `Key ${apiKey}` }, + signal, + } + ) if (!resultResponse.ok) { const body = await readResponseTextWithLimit(resultResponse, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, @@ -260,7 +264,7 @@ async function pollFalQueue( export async function cancelFalRequest(cancelUrl: string, apiKey: string, requestId: string) { try { // boundary-raw-fetch: cancels a request on the external Fal.ai queue - const response = await fetch(cancelUrl, { + const response = await outboundFetch(cancelUrl, { method: 'PUT', headers: { Authorization: `Key ${apiKey}` }, signal: AbortSignal.timeout(FAL_CANCEL_TIMEOUT_MS), diff --git a/apps/sim/lib/selectors/server/providers/harmonic.ts b/apps/sim/lib/selectors/server/providers/harmonic.ts index b07923a554d..678fdcff542 100644 --- a/apps/sim/lib/selectors/server/providers/harmonic.ts +++ b/apps/sim/lib/selectors/server/providers/harmonic.ts @@ -1,5 +1,6 @@ import { isPlainRecord } from '@sim/utils/object' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { ServerSelectorKey } from '@/lib/selectors/manifest' import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' @@ -105,7 +106,7 @@ async function listSavedSearches( let response: Response try { - response = await fetch(HARMONIC_URL, { + response = await outboundFetch(HARMONIC_URL, { headers: { Accept: 'application/json', apikey: accessToken }, redirect: 'error', signal, diff --git a/apps/sim/lib/selectors/server/providers/http.ts b/apps/sim/lib/selectors/server/providers/http.ts index 98f33b27d2a..9f0764bd4f0 100644 --- a/apps/sim/lib/selectors/server/providers/http.ts +++ b/apps/sim/lib/selectors/server/providers/http.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { parseRetryAfter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { SelectorConnectionUnavailableError, SelectorOptionsUnavailableError, @@ -119,7 +120,7 @@ export async function fetchProviderJsonWithStatus( const timeoutSignal = AbortSignal.timeout(PROVIDER_TIMEOUT_MS) const signal = init?.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal try { - response = await fetch(input, { ...init, signal, redirect: init?.redirect ?? 'error' }) + response = await outboundFetch(input, { ...init, signal, redirect: init?.redirect ?? 'error' }) } catch (error) { if (init?.signal?.aborted) throw error logProviderFailure(input, 'network') diff --git a/apps/sim/lib/selectors/server/providers/snowflake.ts b/apps/sim/lib/selectors/server/providers/snowflake.ts index 07e6a96d8b2..845d24579cd 100644 --- a/apps/sim/lib/selectors/server/providers/snowflake.ts +++ b/apps/sim/lib/selectors/server/providers/snowflake.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' import type { ServerSelectorKey } from '@/lib/selectors/manifest' import { @@ -91,7 +92,7 @@ function requireStatementHandle(value: string): string { } async function fetchSnowflakeResponse(url: string, init: RequestInit): Promise { - const response = await fetch(url, { ...init, redirect: 'error' }) + const response = await outboundFetch(url, { ...init, redirect: 'error' }) if (!response.ok) { await response.body?.cancel().catch(() => undefined) throw selectorProviderStatusError(response.status) diff --git a/apps/sim/lib/slack-search/public-install-auth.ts b/apps/sim/lib/slack-search/public-install-auth.ts index 6535c17fd4d..2874bb401b4 100644 --- a/apps/sim/lib/slack-search/public-install-auth.ts +++ b/apps/sim/lib/slack-search/public-install-auth.ts @@ -1,4 +1,5 @@ import { isHosted } from '@/lib/core/config/env-flags' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { exchangeSlackBotAuthorization, @@ -9,16 +10,15 @@ import { getSharedSlackSearchAppConfiguration } from '@/lib/slack-search/shared- /** * Completes installation in Slack without binding it to Sim or retaining tokens. + * No organization owns it yet, so the exchange is a platform request on the direct route. * Organization setup later obtains its own grant through the admin's state-bound OAuth flow. */ export async function authenticateSlackPublicInstallation(code: string) { const app = isHosted ? getSharedSlackSearchAppConfiguration() : null if (!app) throw new OrchestrationError('forbidden', 'The Sim Search app is unavailable') - const grant = await exchangeSlackBotAuthorization({ - clientId: app.clientId, - clientSecret: app.clientSecret, - code, - }) + const grant = await runWithOutboundOrganization(null, () => + exchangeSlackBotAuthorization({ clientId: app.clientId, clientSecret: app.clientSecret, code }) + ) validateSlackBotAuthorization(grant, SLACK_SHARED_SEARCH_BOT_SCOPES) if (grant.app_id !== app.id) throw new OrchestrationError('forbidden', 'Slack returned a different app') diff --git a/apps/sim/lib/tools/falai-pricing.ts b/apps/sim/lib/tools/falai-pricing.ts index 2fdd51f1f2d..312fb376718 100644 --- a/apps/sim/lib/tools/falai-pricing.ts +++ b/apps/sim/lib/tools/falai-pricing.ts @@ -6,6 +6,7 @@ import { isRecordLike } from '@sim/utils/object' import { parseRetryAfter } from '@sim/utils/retry' import { LRUCache } from 'lru-cache' import { coalesceLocally } from '@/lib/concurrency/singleflight' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DeadlineExceededError, withinDeadline } from '@/lib/core/utils/deadline' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' @@ -151,7 +152,7 @@ async function fetchFalAIBillingEvent( let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { headers: { Authorization: `Key ${apiKey}`, }, @@ -199,7 +200,7 @@ async function estimateFalAICallCost( ): Promise<{ costDollars?: number; error?: string }> { let response: Response try { - response = await fetch('https://api.fal.ai/v1/models/pricing/estimate', { + response = await outboundFetch('https://api.fal.ai/v1/models/pricing/estimate', { method: 'POST', headers: { Authorization: `Key ${apiKey}`, diff --git a/apps/sim/lib/webhooks/polling/gmail.ts b/apps/sim/lib/webhooks/polling/gmail.ts index 58b40ba8d4b..0c76e27f6a8 100644 --- a/apps/sim/lib/webhooks/polling/gmail.ts +++ b/apps/sim/lib/webhooks/polling/gmail.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getProviderConfig, type PollingProviderHandler, @@ -149,7 +150,7 @@ async function fetchNewEmails(accessToken: string, config: GmailWebhookConfig, l historyUrl += `&pageToken=${pageToken}` } - const historyResponse = await fetch(historyUrl, { + const historyResponse = await outboundFetch(historyUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -303,7 +304,7 @@ async function searchEmails(accessToken: string, config: GmailWebhookConfig, log const query = `${baseQuery}${timeConstraint}` const searchUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages?q=${encodeURIComponent(query)}&maxResults=${config.maxEmailsPerPoll || 25}` - const searchResponse = await fetch(searchUrl, { + const searchResponse = await outboundFetch(searchUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -386,7 +387,7 @@ async function getGmailProfileHistoryId( logger: Logger ): Promise { try { - const response = await fetch('https://gmail.googleapis.com/gmail/v1/users/me/profile', { + const response = await outboundFetch('https://gmail.googleapis.com/gmail/v1/users/me/profile', { headers: { Authorization: `Bearer ${accessToken}` }, }) if (!response.ok) { @@ -404,7 +405,7 @@ async function getGmailProfileHistoryId( async function getEmailDetails(accessToken: string, messageId: string): Promise { const messageUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}?format=full` - const messageResponse = await fetch(messageUrl, { + const messageResponse = await outboundFetch(messageUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -577,7 +578,7 @@ async function markEmailAsRead(accessToken: string, messageId: string, logger: L const modifyUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}/modify` try { - const response = await fetch(modifyUrl, { + const response = await outboundFetch(modifyUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/polling/google-calendar.ts b/apps/sim/lib/webhooks/polling/google-calendar.ts index 4a4a75d05f3..6aae0073773 100644 --- a/apps/sim/lib/webhooks/polling/google-calendar.ts +++ b/apps/sim/lib/webhooks/polling/google-calendar.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -225,7 +226,7 @@ async function fetchChangedEvents( const encodedCalendarId = encodeURIComponent(calendarId) const url = `${CALENDAR_API_BASE}/calendars/${encodedCalendarId}/events?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) diff --git a/apps/sim/lib/webhooks/polling/google-drive.ts b/apps/sim/lib/webhooks/polling/google-drive.ts index cd98379d2f7..dd2131a8b9c 100644 --- a/apps/sim/lib/webhooks/polling/google-drive.ts +++ b/apps/sim/lib/webhooks/polling/google-drive.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -210,7 +211,7 @@ async function getStartPageToken( } const url = `${DRIVE_API_BASE}/changes/startPageToken?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -257,7 +258,7 @@ async function fetchChanges( } const url = `${DRIVE_API_BASE}/changes?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) diff --git a/apps/sim/lib/webhooks/polling/google-sheets.ts b/apps/sim/lib/webhooks/polling/google-sheets.ts index 111858e41a8..900311a2feb 100644 --- a/apps/sim/lib/webhooks/polling/google-sheets.ts +++ b/apps/sim/lib/webhooks/polling/google-sheets.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -259,7 +260,7 @@ async function getDriveFileModifiedTime( fileId: string ): Promise { try { - const response = await fetch( + const response = await outboundFetch( `https://www.googleapis.com/drive/v3/files/${fileId}?fields=modifiedTime`, { headers: { Authorization: `Bearer ${accessToken}` } } ) @@ -296,7 +297,7 @@ async function fetchSheetState( }) const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!A:Z?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -348,7 +349,7 @@ async function fetchRowRange( }) const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!${startRow}:${endRow}?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) diff --git a/apps/sim/lib/webhooks/polling/hubspot.ts b/apps/sim/lib/webhooks/polling/hubspot.ts index 4eb2c5fe303..543243bb47f 100644 --- a/apps/sim/lib/webhooks/polling/hubspot.ts +++ b/apps/sim/lib/webhooks/polling/hubspot.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getProviderConfig, type PollingProviderHandler, @@ -676,7 +677,7 @@ async function fetchHubSpotChanges(args: FetchArgs): Promise '') diff --git a/apps/sim/lib/webhooks/providers/airtable.ts b/apps/sim/lib/webhooks/providers/airtable.ts index 6d7b88985dc..3fb29a65c9d 100644 --- a/apps/sim/lib/webhooks/providers/airtable.ts +++ b/apps/sim/lib/webhooks/providers/airtable.ts @@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger' import { toStringOrNull } from '@sim/utils/coerce' import { toRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAirtableId } from '@/lib/core/security/input-validation' import { getOAuthToken, @@ -198,7 +199,7 @@ async function fetchAndProcessAirtablePayloads( const fullUrl = `${apiUrl}?${queryParams.toString()}` try { - const response = await fetch(fullUrl, { + const response = await outboundFetch(fullUrl, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -509,7 +510,7 @@ export const airtableHandler: WebhookProviderHandler = { specification: specification, } - const airtableResponse = await fetch(airtableApiUrl, { + const airtableResponse = await outboundFetch(airtableApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -622,7 +623,7 @@ export const airtableHandler: WebhookProviderHandler = { const expectedNotificationUrl = getNotificationUrl(webhookRecord) const listUrl = `https://api.airtable.com/v0/bases/${baseId}/webhooks` - const listResp = await fetch(listUrl, { + const listResp = await outboundFetch(listUrl, { headers: { Authorization: `Bearer ${accessToken}`, }, @@ -694,7 +695,7 @@ export const airtableHandler: WebhookProviderHandler = { } const airtableDeleteUrl = `https://api.airtable.com/v0/bases/${baseId}/webhooks/${resolvedExternalId}` - const airtableResponse = await fetch(airtableDeleteUrl, { + const airtableResponse = await outboundFetch(airtableDeleteUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/ashby.ts b/apps/sim/lib/webhooks/providers/ashby.ts index 10e3a6bb066..a2fa371c2fd 100644 --- a/apps/sim/lib/webhooks/providers/ashby.ts +++ b/apps/sim/lib/webhooks/providers/ashby.ts @@ -4,6 +4,7 @@ import { hmacSha256Hex } from '@sim/security/hmac' import { generateId } from '@sim/utils/id' import { isRecordLike, omit, toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -271,7 +272,7 @@ export const ashbyHandler: WebhookProviderHandler = { secretToken, } - const ashbyResponse = await fetch('https://api.ashbyhq.com/webhook.create', { + const ashbyResponse = await outboundFetch('https://api.ashbyhq.com/webhook.create', { method: 'POST', headers: { Authorization: `Basic ${authString}`, @@ -362,7 +363,7 @@ export const ashbyHandler: WebhookProviderHandler = { const authString = Buffer.from(`${apiKey}:`).toString('base64') - const ashbyResponse = await fetch('https://api.ashbyhq.com/webhook.delete', { + const ashbyResponse = await outboundFetch('https://api.ashbyhq.com/webhook.delete', { method: 'POST', headers: { Authorization: `Basic ${authString}`, diff --git a/apps/sim/lib/webhooks/providers/attio.ts b/apps/sim/lib/webhooks/providers/attio.ts index 83f67aa0b54..d6dbd4b6421 100644 --- a/apps/sim/lib/webhooks/providers/attio.ts +++ b/apps/sim/lib/webhooks/providers/attio.ts @@ -4,6 +4,7 @@ import { hmacSha256Hex } from '@sim/security/hmac' import { toError } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getCredentialAccessToken, getNotificationUrl, @@ -169,7 +170,7 @@ export const attioHandler: WebhookProviderHandler = { }, } - const attioResponse = await fetch('https://api.attio.com/v2/webhooks', { + const attioResponse = await outboundFetch('https://api.attio.com/v2/webhooks', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -273,7 +274,7 @@ export const attioHandler: WebhookProviderHandler = { return } - const attioResponse = await fetch(`https://api.attio.com/v2/webhooks/${externalId}`, { + const attioResponse = await outboundFetch(`https://api.attio.com/v2/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/bitbucket.ts b/apps/sim/lib/webhooks/providers/bitbucket.ts index ae9f8077155..7d063a48b26 100644 --- a/apps/sim/lib/webhooks/providers/bitbucket.ts +++ b/apps/sim/lib/webhooks/providers/bitbucket.ts @@ -6,6 +6,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { toRecord, toRecordOrNull } from '@sim/utils/object' import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getCredentialAccessToken, getNotificationUrl, @@ -71,7 +72,7 @@ function bitbucketHooksUrl(workspaceSlug: string, repoSlug: string): string { } function fetchBitbucketManagement(url: string, init: RequestInit = {}): Promise { - return fetch(url, { + return outboundFetch(url, { ...init, signal: AbortSignal.timeout(BITBUCKET_MANAGEMENT_REQUEST_TIMEOUT_MS), }) diff --git a/apps/sim/lib/webhooks/providers/calendly.ts b/apps/sim/lib/webhooks/providers/calendly.ts index 9bf401df6b9..bde8e0d0cce 100644 --- a/apps/sim/lib/webhooks/providers/calendly.ts +++ b/apps/sim/lib/webhooks/providers/calendly.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -88,7 +89,7 @@ export const calendlyHandler: WebhookProviderHandler = { scope: 'organization', } - const calendlyResponse = await fetch(calendlyApiUrl, { + const calendlyResponse = await outboundFetch(calendlyApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -191,7 +192,7 @@ export const calendlyHandler: WebhookProviderHandler = { const calendlyApiUrl = `https://api.calendly.com/webhook_subscriptions/${externalId}` - const calendlyResponse = await fetch(calendlyApiUrl, { + const calendlyResponse = await outboundFetch(calendlyApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/checkr.ts b/apps/sim/lib/webhooks/providers/checkr.ts index a7618573f5c..5e01656c63f 100644 --- a/apps/sim/lib/webhooks/providers/checkr.ts +++ b/apps/sim/lib/webhooks/providers/checkr.ts @@ -6,6 +6,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toArray, toRecord } from '@sim/utils/object' import { and, eq, isNull, ne } from 'drizzle-orm' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -77,7 +78,7 @@ function isValidCheckrSignature(apiKey: string, signature: string, rawBody: stri * adopts the live registration instead of creating another. */ async function findCheckrWebhookForUrl(apiKey: string, url: string): Promise { - const response = await fetch(CHECKR_WEBHOOKS_URL, { headers: checkrAuthHeaders(apiKey) }) + const response = await outboundFetch(CHECKR_WEBHOOKS_URL, { headers: checkrAuthHeaders(apiKey) }) if (!response.ok) return null const body = await readCheckrResponse(response, 'Checkr webhook list response') const match = toArray(body.data) @@ -219,7 +220,7 @@ export const checkrHandler: WebhookProviderHandler = { triggerId: providerConfig.triggerId, }) - const response = await fetch(CHECKR_WEBHOOKS_URL, { + const response = await outboundFetch(CHECKR_WEBHOOKS_URL, { method: 'POST', headers: checkrAuthHeaders(apiKey), body: JSON.stringify({ @@ -279,10 +280,13 @@ export const checkrHandler: WebhookProviderHandler = { return } - const response = await fetch(`${CHECKR_WEBHOOKS_URL}/${encodeURIComponent(externalId)}`, { - method: 'DELETE', - headers: checkrAuthHeaders(apiKey), - }) + const response = await outboundFetch( + `${CHECKR_WEBHOOKS_URL}/${encodeURIComponent(externalId)}`, + { + method: 'DELETE', + headers: checkrAuthHeaders(apiKey), + } + ) if (response.ok) { logger.info(`Deleted Checkr webhook ${externalId}`, { requestId: ctx.requestId }) diff --git a/apps/sim/lib/webhooks/providers/clickup.ts b/apps/sim/lib/webhooks/providers/clickup.ts index 44593b59d24..6187a7da723 100644 --- a/apps/sim/lib/webhooks/providers/clickup.ts +++ b/apps/sim/lib/webhooks/providers/clickup.ts @@ -4,6 +4,7 @@ import { hmacSha256Hex } from '@sim/security/hmac' import { toError } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getCredentialAccessToken, getNotificationUrl, @@ -87,7 +88,7 @@ function parseOptionalStringId(value: unknown): string | undefined { } async function deleteClickUpWebhook(accessToken: string, externalId: string): Promise { - return fetch(`${CLICKUP_API_BASE_URL}/webhook/${externalId}`, { + return outboundFetch(`${CLICKUP_API_BASE_URL}/webhook/${externalId}`, { method: 'DELETE', headers: { Authorization: clickupAuthorizationHeader(accessToken) }, }) @@ -184,7 +185,7 @@ export const clickupHandler: WebhookProviderHandler = { if (listId !== undefined) requestBody.list_id = listId if (taskId !== undefined) requestBody.task_id = taskId - const clickupResponse = await fetch( + const clickupResponse = await outboundFetch( `${CLICKUP_API_BASE_URL}/team/${encodeURIComponent(workspaceId)}/webhook`, { method: 'POST', diff --git a/apps/sim/lib/webhooks/providers/fathom.ts b/apps/sim/lib/webhooks/providers/fathom.ts index 27d5f6f7534..e927b4d8fc2 100644 --- a/apps/sim/lib/webhooks/providers/fathom.ts +++ b/apps/sim/lib/webhooks/providers/fathom.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -59,7 +60,7 @@ export const fathomHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const fathomResponse = await fetch('https://api.fathom.ai/external/v1/webhooks', { + const fathomResponse = await outboundFetch('https://api.fathom.ai/external/v1/webhooks', { method: 'POST', headers: { 'X-Api-Key': apiKey, @@ -157,7 +158,7 @@ export const fathomHandler: WebhookProviderHandler = { const fathomApiUrl = `https://api.fathom.ai/external/v1/webhooks/${externalId}` - const fathomResponse = await fetch(fathomApiUrl, { + const fathomResponse = await outboundFetch(fathomApiUrl, { method: 'DELETE', headers: { 'X-Api-Key': apiKey, diff --git a/apps/sim/lib/webhooks/providers/fivetran.ts b/apps/sim/lib/webhooks/providers/fivetran.ts index c471257bb47..f786423f0b4 100644 --- a/apps/sim/lib/webhooks/providers/fivetran.ts +++ b/apps/sim/lib/webhooks/providers/fivetran.ts @@ -6,6 +6,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { toRecord, toRecordOrNull } from '@sim/utils/object' import { generateRandomHex } from '@sim/utils/random' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -57,7 +58,7 @@ async function fivetranRequest( method: string, body?: Record ): Promise { - return fetch(url, { + return outboundFetch(url, { method, headers: fivetranAuthHeaders(credentials), body: body === undefined ? undefined : JSON.stringify(body), diff --git a/apps/sim/lib/webhooks/providers/grain.ts b/apps/sim/lib/webhooks/providers/grain.ts index 3741e49a191..19af8940ee0 100644 --- a/apps/sim/lib/webhooks/providers/grain.ts +++ b/apps/sim/lib/webhooks/providers/grain.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -63,7 +64,7 @@ async function createGrainV2Hooks(params: { try { for (const hookType of hookTypes) { - const response = await fetch(`${GRAIN_V2_HOOKS_BASE}/create`, { + const response = await outboundFetch(`${GRAIN_V2_HOOKS_BASE}/create`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -107,7 +108,7 @@ async function deleteGrainV2Hook(params: { hookId: string requestId: string }): Promise { - const response = await fetch(`${GRAIN_V2_HOOKS_BASE}/${params.hookId}`, { + const response = await outboundFetch(`${GRAIN_V2_HOOKS_BASE}/${params.hookId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${params.apiKey}`, @@ -193,7 +194,7 @@ async function createLegacyV1Subscription(params: { requestBody.actions = actions } - const grainResponse = await fetch('https://api.grain.com/_/public-api/hooks', { + const grainResponse = await outboundFetch('https://api.grain.com/_/public-api/hooks', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -238,13 +239,16 @@ async function deleteLegacyV1Hook(params: { hookId: string requestId: string }): Promise { - const response = await fetch(`https://api.grain.com/_/public-api/hooks/${params.hookId}`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${params.apiKey}`, - 'Content-Type': 'application/json', - }, - }) + const response = await outboundFetch( + `https://api.grain.com/_/public-api/hooks/${params.hookId}`, + { + method: 'DELETE', + headers: { + Authorization: `Bearer ${params.apiKey}`, + 'Content-Type': 'application/json', + }, + } + ) if (!response.ok && response.status !== 404 && response.status !== 410) { throw new Error(`Failed to delete Grain webhook ${params.hookId}: ${response.status}`) } diff --git a/apps/sim/lib/webhooks/providers/granola.ts b/apps/sim/lib/webhooks/providers/granola.ts index d44d0273b69..30917a3c633 100644 --- a/apps/sim/lib/webhooks/providers/granola.ts +++ b/apps/sim/lib/webhooks/providers/granola.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toRecordOrNull } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -72,7 +73,7 @@ function granolaUserFacingError(status: number, body: string): string { /** Delete one Granola webhook endpoint. Treats an already-deleted endpoint as success. */ async function deleteGranolaEndpoint(apiKey: string, endpointId: string): Promise { - const response = await fetch( + const response = await outboundFetch( `${GRANOLA_WEBHOOK_ENDPOINTS_URL}/${encodeURIComponent(endpointId)}`, { method: 'DELETE', @@ -244,7 +245,7 @@ export const granolaHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(GRANOLA_WEBHOOK_ENDPOINTS_URL, { + const response = await outboundFetch(GRANOLA_WEBHOOK_ENDPOINTS_URL, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/helpscout.ts b/apps/sim/lib/webhooks/providers/helpscout.ts index 89fcccd4ca2..05b33b0c6c2 100644 --- a/apps/sim/lib/webhooks/providers/helpscout.ts +++ b/apps/sim/lib/webhooks/providers/helpscout.ts @@ -5,6 +5,7 @@ import { toNumberOrNull, toStringOrNull } from '@sim/utils/coerce' import { toRecord } from '@sim/utils/object' import { generateRandomHex } from '@sim/utils/random' import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -131,7 +132,7 @@ export const helpScoutHandler: WebhookProviderHandler = { const accessToken = await requireAccessToken(config, requestId) const webhookSecret = generateRandomHex(WEBHOOK_SECRET_LENGTH) - const response = await fetch(`${HELPSCOUT_API_BASE}/webhooks`, { + const response = await outboundFetch(`${HELPSCOUT_API_BASE}/webhooks`, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -191,7 +192,7 @@ export const helpScoutHandler: WebhookProviderHandler = { if (strict) throw new Error('Missing Help Scout token for webhook deletion') return } - const response = await fetch(`${HELPSCOUT_API_BASE}/webhooks/${externalId}`, { + const response = await outboundFetch(`${HELPSCOUT_API_BASE}/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}` }, redirect: 'error', diff --git a/apps/sim/lib/webhooks/providers/instantly.ts b/apps/sim/lib/webhooks/providers/instantly.ts index 142096bc963..e62c21bb671 100644 --- a/apps/sim/lib/webhooks/providers/instantly.ts +++ b/apps/sim/lib/webhooks/providers/instantly.ts @@ -4,6 +4,7 @@ import { toError } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' import { isRecordLike, toRecord, toRecordOrNull } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -170,7 +171,7 @@ export const instantlyHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(instantlyUrl('/api/v2/webhooks'), { + const response = await outboundFetch(instantlyUrl('/api/v2/webhooks'), { method: 'POST', headers: { Authorization: `Bearer ${apiKey.trim()}`, @@ -235,7 +236,7 @@ export const instantlyHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await outboundFetch( instantlyUrl(`/api/v2/webhooks/${encodeURIComponent(externalId.trim())}`), { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/jotform.ts b/apps/sim/lib/webhooks/providers/jotform.ts index 4cb0011cbf3..3e0fb62dace 100644 --- a/apps/sim/lib/webhooks/providers/jotform.ts +++ b/apps/sim/lib/webhooks/providers/jotform.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' import { and, eq, isNull, ne } from 'drizzle-orm' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -68,7 +69,7 @@ async function listWebhookIdForUrl( credentials: JotformSubscriptionCredentials, notificationUrl: string ): Promise { - const response = await fetch( + const response = await outboundFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks` @@ -186,7 +187,7 @@ export const jotformHandler: WebhookProviderHandler = { return {} } - const response = await fetch( + const response = await outboundFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks` @@ -260,7 +261,7 @@ export const jotformHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await outboundFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks/${encodeURIComponent(webhookId)}` diff --git a/apps/sim/lib/webhooks/providers/lemlist.ts b/apps/sim/lib/webhooks/providers/lemlist.ts index 7c2befa3c85..f34cb57cc6b 100644 --- a/apps/sim/lib/webhooks/providers/lemlist.ts +++ b/apps/sim/lib/webhooks/providers/lemlist.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -68,7 +69,7 @@ export const lemlistHandler: WebhookProviderHandler = { requestBody.campaignId = campaignId } - const lemlistResponse = await fetch(lemlistApiUrl, { + const lemlistResponse = await outboundFetch(lemlistApiUrl, { method: 'POST', headers: { Authorization: `Basic ${authString}`, @@ -149,7 +150,7 @@ export const lemlistHandler: WebhookProviderHandler = { } const lemlistApiUrl = `https://api.lemlist.com/api/hooks/${id}` - const lemlistResponse = await fetch(lemlistApiUrl, { + const lemlistResponse = await outboundFetch(lemlistApiUrl, { method: 'DELETE', headers: { Authorization: `Basic ${authString}`, @@ -184,7 +185,7 @@ export const lemlistHandler: WebhookProviderHandler = { } const notificationUrl = getNotificationUrl(webhook) - const listResponse = await fetch('https://api.lemlist.com/api/hooks', { + const listResponse = await outboundFetch('https://api.lemlist.com/api/hooks', { method: 'GET', headers: { Authorization: `Basic ${authString}`, diff --git a/apps/sim/lib/webhooks/providers/linear.ts b/apps/sim/lib/webhooks/providers/linear.ts index c47e649a2ee..93858bc8b44 100644 --- a/apps/sim/lib/webhooks/providers/linear.ts +++ b/apps/sim/lib/webhooks/providers/linear.ts @@ -5,6 +5,7 @@ import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { isRecordLike, toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -217,7 +218,7 @@ export const linearHandler: WebhookProviderHandler = { } try { - const response = await fetch('https://api.linear.app/graphql', { + const response = await outboundFetch('https://api.linear.app/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -298,7 +299,7 @@ export const linearHandler: WebhookProviderHandler = { } try { - const response = await fetch('https://api.linear.app/graphql', { + const response = await outboundFetch('https://api.linear.app/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/webhooks/providers/linq.ts b/apps/sim/lib/webhooks/providers/linq.ts index 6f05fd820d2..783af98a8a2 100644 --- a/apps/sim/lib/webhooks/providers/linq.ts +++ b/apps/sim/lib/webhooks/providers/linq.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -134,7 +135,7 @@ export const linqHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(`${LINQ_API_BASE}/webhook-subscriptions`, { + const response = await outboundFetch(`${LINQ_API_BASE}/webhook-subscriptions`, { method: 'POST', headers: linqHeaders(apiKey), body: JSON.stringify(requestBody), @@ -196,7 +197,7 @@ export const linqHandler: WebhookProviderHandler = { return } - const response = await fetch(`${LINQ_API_BASE}/webhook-subscriptions/${externalId}`, { + const response = await outboundFetch(`${LINQ_API_BASE}/webhook-subscriptions/${externalId}`, { method: 'DELETE', headers: linqHeaders(apiKey), }) diff --git a/apps/sim/lib/webhooks/providers/microsoft-teams.ts b/apps/sim/lib/webhooks/providers/microsoft-teams.ts index 7bc6ab3a3b7..128c3cb9598 100644 --- a/apps/sim/lib/webhooks/providers/microsoft-teams.ts +++ b/apps/sim/lib/webhooks/providers/microsoft-teams.ts @@ -7,6 +7,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isMicrosoftContentUrl } from '@/lib/core/security/input-validation' import { type SecureFetchResponse, @@ -272,7 +273,9 @@ async function formatTeamsGraphNotification( if (accessToken) { const msgUrl = `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(resolvedChatId)}/messages/${encodeURIComponent(resolvedMessageId)}` - const res = await fetch(msgUrl, { headers: { Authorization: `Bearer ${accessToken}` } }) + const res = await outboundFetch(msgUrl, { + headers: { Authorization: `Bearer ${accessToken}` }, + }) if (res.ok) { message = await readResponseJsonWithLimit>(res, { maxBytes: MAX_TEAMS_MESSAGE_BYTES, @@ -336,7 +339,7 @@ async function formatTeamsGraphNotification( .replace(/=+$/, '') const graphUrl = `https://graph.microsoft.com/v1.0/shares/u!${encodedUrl}/driveItem/content` - const graphRes = await fetch(graphUrl, { + const graphRes = await outboundFetch(graphUrl, { headers: { Authorization: `Bearer ${accessToken}` }, redirect: 'follow', }) @@ -386,7 +389,7 @@ async function formatTeamsGraphNotification( } const metadataUrl = `https://graph.microsoft.com/v1.0/shares/${shareToken}/driveItem` - const metadataRes = await fetch(metadataUrl, { + const metadataRes = await outboundFetch(metadataUrl, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -395,7 +398,7 @@ async function formatTeamsGraphNotification( if (!metadataRes.ok) { const directUrl = `https://graph.microsoft.com/v1.0/shares/${shareToken}/driveItem/content` - const directRes = await fetch(directUrl, { + const directRes = await outboundFetch(directUrl, { headers: { Authorization: `Bearer ${accessToken}` }, redirect: 'follow', }) @@ -714,7 +717,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { const existingSubscriptionId = config.externalSubscriptionId as string | undefined if (existingSubscriptionId) { try { - const checkRes = await fetch( + const checkRes = await outboundFetch( `https://graph.microsoft.com/v1.0/subscriptions/${existingSubscriptionId}`, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` } } ) @@ -753,7 +756,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { } try { - const res = await fetch('https://graph.microsoft.com/v1.0/subscriptions', { + const res = await outboundFetch('https://graph.microsoft.com/v1.0/subscriptions', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -846,7 +849,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { return } - const res = await fetch( + const res = await outboundFetch( `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/monday.ts b/apps/sim/lib/webhooks/providers/monday.ts index a70174d3253..d613d4471d9 100644 --- a/apps/sim/lib/webhooks/providers/monday.ts +++ b/apps/sim/lib/webhooks/providers/monday.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateMondayNumericId } from '@/lib/core/security/input-validation' import { getOAuthToken } from '@/lib/oauth/credential-service' import { @@ -106,7 +107,7 @@ export const mondayHandler: WebhookProviderHandler = { const notificationUrl = getNotificationUrl(ctx.webhook) try { - const response = await fetch(MONDAY_API_URL, { + const response = await outboundFetch(MONDAY_API_URL, { method: 'POST', headers: mondayHeaders(accessToken), body: JSON.stringify({ @@ -216,7 +217,7 @@ export const mondayHandler: WebhookProviderHandler = { } try { - const response = await fetch(MONDAY_API_URL, { + const response = await outboundFetch(MONDAY_API_URL, { method: 'POST', headers: mondayHeaders(accessToken), body: JSON.stringify({ diff --git a/apps/sim/lib/webhooks/providers/pagerduty.ts b/apps/sim/lib/webhooks/providers/pagerduty.ts index abb9017a387..de2f82838dd 100644 --- a/apps/sim/lib/webhooks/providers/pagerduty.ts +++ b/apps/sim/lib/webhooks/providers/pagerduty.ts @@ -2,6 +2,7 @@ import crypto from 'crypto' import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -56,7 +57,7 @@ async function cleanupPagerDutySubscription( ): Promise { let id = subscriptionId if (!id) { - const listRes = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { + const listRes = await outboundFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { headers: pagerdutyHeaders(apiKey), }).catch(() => null) if (!listRes || !listRes.ok) return @@ -66,7 +67,7 @@ async function cleanupPagerDutySubscription( id = body?.webhook_subscriptions?.find((sub) => sub.delivery_method?.url === url)?.id } if (!id) return - await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${id}`, { + await outboundFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${id}`, { method: 'DELETE', headers: pagerdutyHeaders(apiKey), }).catch(() => null) @@ -153,7 +154,7 @@ export const pagerdutyHandler: WebhookProviderHandler = { } const { getPagerDutyEvents } = await import('@/triggers/pagerduty/utils') - const res = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { + const res = await outboundFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { method: 'POST', headers: pagerdutyHeaders(apiKey), body: JSON.stringify({ @@ -213,7 +214,7 @@ export const pagerdutyHandler: WebhookProviderHandler = { return } - const res = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${externalId}`, { + const res = await outboundFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${externalId}`, { method: 'DELETE', headers: pagerdutyHeaders(apiKey), }) diff --git a/apps/sim/lib/webhooks/providers/resend.ts b/apps/sim/lib/webhooks/providers/resend.ts index 935911c6604..b200cd1b853 100644 --- a/apps/sim/lib/webhooks/providers/resend.ts +++ b/apps/sim/lib/webhooks/providers/resend.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -145,7 +146,7 @@ export const resendHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const resendResponse = await fetch('https://api.resend.com/webhooks', { + const resendResponse = await outboundFetch('https://api.resend.com/webhooks', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -232,7 +233,7 @@ export const resendHandler: WebhookProviderHandler = { return } - const resendResponse = await fetch(`https://api.resend.com/webhooks/${externalId}`, { + const resendResponse = await outboundFetch(`https://api.resend.com/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/revenuecat.ts b/apps/sim/lib/webhooks/providers/revenuecat.ts index ffbca9b15ec..2f0e424aba4 100644 --- a/apps/sim/lib/webhooks/providers/revenuecat.ts +++ b/apps/sim/lib/webhooks/providers/revenuecat.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -109,7 +110,7 @@ export const revenueCatHandler: WebhookProviderHandler = { requestBody.environment = environment } - const response = await fetch( + const response = await outboundFetch( `${REVENUECAT_API_BASE}/projects/${encodeURIComponent(projectId)}/integrations/webhooks`, { method: 'POST', @@ -187,7 +188,7 @@ export const revenueCatHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await outboundFetch( `${REVENUECAT_API_BASE}/projects/${encodeURIComponent(projectId)}/integrations/webhooks/${encodeURIComponent(externalId)}`, { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/rootly.ts b/apps/sim/lib/webhooks/providers/rootly.ts index cd1ea420d80..2f5e0059355 100644 --- a/apps/sim/lib/webhooks/providers/rootly.ts +++ b/apps/sim/lib/webhooks/providers/rootly.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -177,7 +178,7 @@ export const rootlyHandler: WebhookProviderHandler = { }, } - const response = await fetch('https://api.rootly.com/v1/webhooks/endpoints', { + const response = await outboundFetch('https://api.rootly.com/v1/webhooks/endpoints', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -260,13 +261,16 @@ export const rootlyHandler: WebhookProviderHandler = { return } - const response = await fetch(`https://api.rootly.com/v1/webhooks/endpoints/${externalId}`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${apiKey}`, - Accept: 'application/vnd.api+json', - }, - }) + const response = await outboundFetch( + `https://api.rootly.com/v1/webhooks/endpoints/${externalId}`, + { + method: 'DELETE', + headers: { + Authorization: `Bearer ${apiKey}`, + Accept: 'application/vnd.api+json', + }, + } + ) if (response.ok || response.status === 404) { await response.body?.cancel() diff --git a/apps/sim/lib/webhooks/providers/slack.ts b/apps/sim/lib/webhooks/providers/slack.ts index b103879edc0..a66861f7398 100644 --- a/apps/sim/lib/webhooks/providers/slack.ts +++ b/apps/sim/lib/webhooks/providers/slack.ts @@ -9,6 +9,7 @@ import { isRecordLike, toRecord, toRecordOrNull } from '@sim/utils/object' import { eq } from 'drizzle-orm' import { LRUCache } from 'lru-cache' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -287,7 +288,7 @@ async function resolveSlackFileInfo( botToken: string ): Promise<{ url_private?: string; name?: string; mimetype?: string; size?: number } | null> { try { - const response = await fetch( + const response = await outboundFetch( `https://slack.com/api/files.info?file=${encodeURIComponent(fileId)}`, { headers: { Authorization: `Bearer ${botToken}` } } ) @@ -432,7 +433,7 @@ async function fetchSlackMessageText( ): Promise { try { const params = new URLSearchParams({ channel, timestamp: messageTs }) - const response = await fetch(`https://slack.com/api/reactions.get?${params}`, { + const response = await outboundFetch(`https://slack.com/api/reactions.get?${params}`, { headers: { Authorization: `Bearer ${botToken}` }, }) const data = (await response.json()) as { @@ -486,7 +487,7 @@ export async function fetchSlackTeamId(botToken: string): Promise<{ userId: string | undefined teamName: string | undefined }> { - const response = await fetch('https://slack.com/api/auth.test', { + const response = await outboundFetch('https://slack.com/api/auth.test', { headers: { Authorization: `Bearer ${botToken}` }, }) const data = (await response.json()) as { diff --git a/apps/sim/lib/webhooks/providers/telegram.ts b/apps/sim/lib/webhooks/providers/telegram.ts index 778ec28330f..ab5242698f8 100644 --- a/apps/sim/lib/webhooks/providers/telegram.ts +++ b/apps/sim/lib/webhooks/providers/telegram.ts @@ -4,6 +4,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' import { and, eq, isNull, ne } from 'drizzle-orm' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getEffectiveDecryptedEnv } from '@/lib/environment/utils' import { resolveBackgroundWebhookEnv } from '@/lib/webhooks/env-resolver' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' @@ -160,7 +161,7 @@ export const telegramHandler: WebhookProviderHandler = { const secretToken = await resolveSubscriptionSecretToken(ctx, config, botToken) try { - const telegramResponse = await fetch(telegramApiUrl, { + const telegramResponse = await outboundFetch(telegramApiUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -236,7 +237,7 @@ export const telegramHandler: WebhookProviderHandler = { } const telegramApiUrl = `https://api.telegram.org/bot${botToken}/deleteWebhook` - const telegramResponse = await fetch(telegramApiUrl, { + const telegramResponse = await outboundFetch(telegramApiUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, }) diff --git a/apps/sim/lib/webhooks/providers/typeform.ts b/apps/sim/lib/webhooks/providers/typeform.ts index 3db80586d01..d1aa10a3df7 100644 --- a/apps/sim/lib/webhooks/providers/typeform.ts +++ b/apps/sim/lib/webhooks/providers/typeform.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Base64 } from '@sim/security/hmac' import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -110,7 +111,7 @@ export const typeformHandler: WebhookProviderHandler = { requestBody.secret = secret } - const typeformResponse = await fetch(typeformApiUrl, { + const typeformResponse = await outboundFetch(typeformApiUrl, { method: 'PUT', headers: { Authorization: `Bearer ${apiKey}`, @@ -199,7 +200,7 @@ export const typeformHandler: WebhookProviderHandler = { const tag = webhookTag || `sim-${(ctx.webhook.id as string).substring(0, 8)}` const typeformApiUrl = `https://api.typeform.com/forms/${formId}/webhooks/${tag}` - const typeformResponse = await fetch(typeformApiUrl, { + const typeformResponse = await outboundFetch(typeformApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/vercel.ts b/apps/sim/lib/webhooks/providers/vercel.ts index 96961a17bea..c45998b1115 100644 --- a/apps/sim/lib/webhooks/providers/vercel.ts +++ b/apps/sim/lib/webhooks/providers/vercel.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { isRecordLike } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -146,7 +147,7 @@ export const vercelHandler: WebhookProviderHandler = { ? `https://api.vercel.com/v1/webhooks?teamId=${encodeURIComponent(teamId)}` : 'https://api.vercel.com/v1/webhooks' - const vercelResponse = await fetch(apiUrl, { + const vercelResponse = await outboundFetch(apiUrl, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -234,7 +235,7 @@ export const vercelHandler: WebhookProviderHandler = { ? `https://api.vercel.com/v1/webhooks/${encodeURIComponent(externalId)}?teamId=${encodeURIComponent(teamId)}` : `https://api.vercel.com/v1/webhooks/${encodeURIComponent(externalId)}` - const response = await fetch(apiUrl, { + const response = await outboundFetch(apiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/webflow.ts b/apps/sim/lib/webhooks/providers/webflow.ts index 693afd5cbc5..8979b83c29c 100644 --- a/apps/sim/lib/webhooks/providers/webflow.ts +++ b/apps/sim/lib/webhooks/providers/webflow.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { getOAuthToken } from '@/lib/oauth/credential-service' import { @@ -104,7 +105,7 @@ export const webflowHandler: WebhookProviderHandler = { } } - const webflowResponse = await fetch(webflowApiUrl, { + const webflowResponse = await outboundFetch(webflowApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -213,7 +214,7 @@ export const webflowHandler: WebhookProviderHandler = { const webflowApiUrl = `https://api.webflow.com/v2/sites/${siteId}/webhooks/${externalId}` - const webflowResponse = await fetch(webflowApiUrl, { + const webflowResponse = await outboundFetch(webflowApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/zendesk.ts b/apps/sim/lib/webhooks/providers/zendesk.ts index aecba884097..977052f01a9 100644 --- a/apps/sim/lib/webhooks/providers/zendesk.ts +++ b/apps/sim/lib/webhooks/providers/zendesk.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -46,7 +47,7 @@ async function deleteZendeskWebhookQuietly( authHeader: string, webhookId: string ): Promise { - await fetch(`${apiBase}/webhooks/${webhookId}`, { + await outboundFetch(`${apiBase}/webhooks/${webhookId}`, { method: 'DELETE', headers: { Authorization: authHeader }, }).catch(() => {}) @@ -198,7 +199,7 @@ export const zendeskHandler: WebhookProviderHandler = { const apiBase = zendeskApiBase(subdomain) const authHeader = zendeskAuthHeader(email, apiToken) - const createRes = await fetch(`${apiBase}/webhooks`, { + const createRes = await outboundFetch(`${apiBase}/webhooks`, { method: 'POST', headers: { Authorization: authHeader, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -231,7 +232,7 @@ export const zendeskHandler: WebhookProviderHandler = { const externalId = toRecord(created.webhook).id as string | undefined if (!externalId) throw new Error('Zendesk webhook created but no webhook ID was returned.') - const secretRes = await fetch(`${apiBase}/webhooks/${externalId}/signing_secret`, { + const secretRes = await outboundFetch(`${apiBase}/webhooks/${externalId}/signing_secret`, { headers: { Authorization: authHeader }, }) if (!secretRes.ok) { @@ -278,7 +279,7 @@ export const zendeskHandler: WebhookProviderHandler = { return } - const res = await fetch(`${zendeskApiBase(subdomain)}/webhooks/${externalId}`, { + const res = await outboundFetch(`${zendeskApiBase(subdomain)}/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: zendeskAuthHeader(email, apiToken) }, }) diff --git a/apps/sim/lib/webhooks/providers/zoho-desk.ts b/apps/sim/lib/webhooks/providers/zoho-desk.ts index 3c94d389c90..294e0f323ff 100644 --- a/apps/sim/lib/webhooks/providers/zoho-desk.ts +++ b/apps/sim/lib/webhooks/providers/zoho-desk.ts @@ -7,6 +7,7 @@ import { eq } from 'drizzle-orm' import * as jose from 'jose' import { LRUCache } from 'lru-cache' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getNotificationUrl } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -271,7 +272,7 @@ export const zohoDeskHandler: WebhookProviderHandler = { if (direction === 'in' || direction === 'out') filter.direction = direction } - const response = await fetch(`${apiDomain}/api/v1/webhooks`, { + const response = await outboundFetch(`${apiDomain}/api/v1/webhooks`, { method: 'POST', headers: { Authorization: `Zoho-oauthtoken ${accessToken}`, @@ -364,7 +365,7 @@ export const zohoDeskHandler: WebhookProviderHandler = { : await resolveZohoDeskApiDomain(owner.accountId) const webhookPath = safeUrlPathSegment(externalId, 'externalId') - const response = await fetch(`${apiDomain}/api/v1/webhooks/${webhookPath}`, { + const response = await outboundFetch(`${apiDomain}/api/v1/webhooks/${webhookPath}`, { method: 'DELETE', headers: { Authorization: `Zoho-oauthtoken ${accessToken}`, diff --git a/apps/sim/lib/webhooks/slack-agent-api.ts b/apps/sim/lib/webhooks/slack-agent-api.ts index 7beaf370ce5..6259a4be51a 100644 --- a/apps/sim/lib/webhooks/slack-agent-api.ts +++ b/apps/sim/lib/webhooks/slack-agent-api.ts @@ -1,5 +1,6 @@ import { toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { SlackDeliveryError } from '@/lib/webhooks/slack-delivery-error' @@ -43,7 +44,7 @@ async function callSlackAgentApi( ): Promise { let response: Response try { - response = await fetch(`https://slack.com/api/${method}`, { + response = await outboundFetch(`https://slack.com/api/${method}`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, diff --git a/apps/sim/providers/anthropic/index.ts b/apps/sim/providers/anthropic/index.ts index 7beb53c79e5..e554e5f92aa 100644 --- a/apps/sim/providers/anthropic/index.ts +++ b/apps/sim/providers/anthropic/index.ts @@ -1,5 +1,6 @@ import Anthropic from '@anthropic-ai/sdk' import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { executeAnthropicProviderRequest } from '@/providers/anthropic/core' import { getCachedProviderClient } from '@/providers/client-cache' @@ -27,7 +28,7 @@ export const anthropicProvider: ProviderConfig = { const cacheKey = `anthropic::${apiKey}` return getCachedProviderClient( cacheKey, - () => new Anthropic({ apiKey, maxRetries: PROVIDER_MAX_RETRIES }) + () => new Anthropic({ apiKey, maxRetries: PROVIDER_MAX_RETRIES, fetch: outboundFetch }) ) }, logger, diff --git a/apps/sim/providers/azure-anthropic/index.test.ts b/apps/sim/providers/azure-anthropic/index.test.ts index a2bc29ecbc2..5c25b45a424 100644 --- a/apps/sim/providers/azure-anthropic/index.test.ts +++ b/apps/sim/providers/azure-anthropic/index.test.ts @@ -5,6 +5,7 @@ import { } from '@sim/testing/mocks/input-validation.mock' import { providersModelsMock } from '@sim/testing/mocks/providers-models.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { ProviderRequest } from '@/providers/types' const { mockAnthropic, anthropicArgs, mockExecuteAnthropic, sentinelFetch } = vi.hoisted(() => { @@ -74,14 +75,14 @@ describe('azureAnthropicProvider — SSRF pinning', () => { expect(buildClientOptions()).toMatchObject({ fetch: sentinelFetch }) }) - it('does not pin when the endpoint comes from trusted server env', async () => { + it('sends a trusted server env endpoint through the unpinned outbound fetch', async () => { setEnv({ AZURE_ANTHROPIC_ENDPOINT: 'https://trusted.services.ai.azure.com' }) await azureAnthropicProvider.executeRequest(request({ azureEndpoint: undefined })) expect(mockValidate).not.toHaveBeenCalled() expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(buildClientOptions()).not.toHaveProperty('fetch') + expect(buildClientOptions().fetch).toBe(outboundFetch) }) it('keeps the registry model in core and resolves a separate Azure wire model', async () => { diff --git a/apps/sim/providers/azure-anthropic/index.ts b/apps/sim/providers/azure-anthropic/index.ts index 7737dcd4abc..c6d814c8782 100644 --- a/apps/sim/providers/azure-anthropic/index.ts +++ b/apps/sim/providers/azure-anthropic/index.ts @@ -1,6 +1,7 @@ import Anthropic from '@anthropic-ai/sdk' import { createLogger } from '@sim/logger' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { executeAnthropicProviderRequest } from '@/providers/anthropic/core' @@ -81,7 +82,7 @@ export const azureAnthropicProvider: ProviderConfig = { baseURL, apiKey, maxRetries: PROVIDER_MAX_RETRIES, - ...(pinnedFetch ? { fetch: pinnedFetch } : {}), + fetch: pinnedFetch ?? outboundFetch, defaultHeaders: { 'anthropic-version': anthropicVersion, }, diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index d68dbc3a97d..1a0c513b632 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -11,6 +11,7 @@ import { providersTraceEnrichmentMock } from '@sim/testing/mocks/providers-trace import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock' import { toolsMock, toolsMockFns } from '@sim/testing/mocks/tools.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { AgentStreamEvent } from '@/providers/stream-events' import type { ProviderRequest, ProviderToolConfig } from '@/providers/types' @@ -180,7 +181,7 @@ describe('azureOpenAIProvider — SSRF pinning', () => { expect(azureOpenAIArgs()[0]).toMatchObject({ fetch: sentinelFetch }) }) - it('constructs the AzureOpenAI client without a custom fetch for a trusted env endpoint', async () => { + it('constructs the AzureOpenAI client with the unpinned outbound fetch for a trusted env endpoint', async () => { mockIsChatCompletionsEndpoint.mockReturnValue(true) setEnv({ AZURE_OPENAI_ENDPOINT: @@ -194,7 +195,7 @@ describe('azureOpenAIProvider — SSRF pinning', () => { await azureOpenAIProvider.executeRequest(request({ azureEndpoint: undefined })) expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(azureOpenAIArgs()[0]).not.toHaveProperty('fetch') + expect(azureOpenAIArgs()[0].fetch).toBe(outboundFetch) }) it('preserves a custom deployment name through Chat Completions routing', async () => { diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index 37db6e20761..2fc9791f4df 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -14,6 +14,7 @@ import type { } from 'openai/resources/chat/completions' import type { ReasoningEffort } from 'openai/resources/shared' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' @@ -104,7 +105,7 @@ async function executeChatCompletionsRequest( apiVersion: azureApiVersion, endpoint: azureEndpoint, ...openAICompatTransport(), - ...(pinnedFetch ? { fetch: pinnedFetch } : {}), + fetch: pinnedFetch ?? outboundFetch, }) const allMessages: ChatCompletionMessageParam[] = [] diff --git a/apps/sim/providers/baseten/index.ts b/apps/sim/providers/baseten/index.ts index 3d6ffacfe78..de30620d661 100644 --- a/apps/sim/providers/baseten/index.ts +++ b/apps/sim/providers/baseten/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { @@ -44,6 +45,7 @@ export const basetenProvider: ProviderConfig = { const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://inference.baseten.co/v1', }) diff --git a/apps/sim/providers/cerebras/index.ts b/apps/sim/providers/cerebras/index.ts index 1a6d94f0db8..f0b42f99819 100644 --- a/apps/sim/providers/cerebras/index.ts +++ b/apps/sim/providers/cerebras/index.ts @@ -2,6 +2,7 @@ import { Cerebras } from '@cerebras/cerebras_cloud_sdk' import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -70,6 +71,7 @@ export const cerebrasProvider: ProviderConfig = { const client = new Cerebras({ apiKey: request.apiKey, ...openAICompatTransport(), + fetch: outboundFetch, }) const allMessages = [] diff --git a/apps/sim/providers/deepseek/index.ts b/apps/sim/providers/deepseek/index.ts index b2750a8090c..b47f791d83e 100644 --- a/apps/sim/providers/deepseek/index.ts +++ b/apps/sim/providers/deepseek/index.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -65,6 +66,7 @@ export const deepseekProvider: ProviderConfig = { try { const deepseek = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.deepseek.com', }) diff --git a/apps/sim/providers/file-attachments.server.ts b/apps/sim/providers/file-attachments.server.ts index 9940801c1a5..9a91605569d 100644 --- a/apps/sim/providers/file-attachments.server.ts +++ b/apps/sim/providers/file-attachments.server.ts @@ -2,6 +2,7 @@ import { FileState, GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { assertUserFileContentAccess } from '@/lib/execution/payloads/materialization.server' import { resolveExecutorFileMaterializationContext } from '@/lib/internal/file/materialization-context' import { StorageService } from '@/lib/uploads' @@ -250,7 +251,7 @@ async function uploadOpenAIFile( form.append('expires_after[seconds]', String(OPENAI_FILE_EXPIRY_SECONDS)) form.append('file', blob, file.name) - const response = await fetch(OPENAI_FILES_ENDPOINT, { + const response = await outboundFetch(OPENAI_FILES_ENDPOINT, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}` }, body: form, diff --git a/apps/sim/providers/fireworks/index.ts b/apps/sim/providers/fireworks/index.ts index 53c340ce485..1be08ed305f 100644 --- a/apps/sim/providers/fireworks/index.ts +++ b/apps/sim/providers/fireworks/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { resolveFireworksWireModel } from '@/providers/fireworks/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' @@ -45,6 +46,7 @@ export const fireworksProvider: ProviderConfig = { const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.fireworks.ai/inference/v1', }) diff --git a/apps/sim/providers/groq/index.ts b/apps/sim/providers/groq/index.ts index 1357a763d18..3b28a975717 100644 --- a/apps/sim/providers/groq/index.ts +++ b/apps/sim/providers/groq/index.ts @@ -7,6 +7,7 @@ import type { ChatCompletionChunk, ChatCompletionMessageParam, } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -66,7 +67,11 @@ export const groqProvider: ProviderConfig = { throw new Error('API key is required for Groq') } - const groq = new Groq({ apiKey: request.apiKey, ...openAICompatTransport() }) + const groq = new Groq({ + apiKey: request.apiKey, + ...openAICompatTransport(), + fetch: outboundFetch, + }) const allMessages = [] diff --git a/apps/sim/providers/kie/index.ts b/apps/sim/providers/kie/index.ts index b3440ac2809..0a0b8521560 100644 --- a/apps/sim/providers/kie/index.ts +++ b/apps/sim/providers/kie/index.ts @@ -1,5 +1,6 @@ import Anthropic from '@anthropic-ai/sdk' import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { executeAnthropicProviderRequest } from '@/providers/anthropic/core' import { getCachedProviderClient } from '@/providers/client-cache' @@ -16,7 +17,7 @@ import { PROVIDER_MAX_RETRIES } from '@/providers/transport' import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types' const logger = createLogger('KieProvider') -const kieFetch = createKieFetch() +const kieFetch = createKieFetch(outboundFetch) export const kieProvider: ProviderConfig = { id: 'kie', diff --git a/apps/sim/providers/kimi/index.ts b/apps/sim/providers/kimi/index.ts index 0434dc047d5..37d9e8f86dd 100644 --- a/apps/sim/providers/kimi/index.ts +++ b/apps/sim/providers/kimi/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -125,6 +126,7 @@ export const kimiProvider: ProviderConfig = { try { const kimi = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: KIMI_BASE_URL, }) diff --git a/apps/sim/providers/meta/index.ts b/apps/sim/providers/meta/index.ts index 25e33098d4f..d29213658f3 100644 --- a/apps/sim/providers/meta/index.ts +++ b/apps/sim/providers/meta/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -69,6 +70,7 @@ export const metaProvider: ProviderConfig = { try { const meta = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: META_BASE_URL, }) diff --git a/apps/sim/providers/mistral/index.ts b/apps/sim/providers/mistral/index.ts index 03d225be391..a9657bb262c 100644 --- a/apps/sim/providers/mistral/index.ts +++ b/apps/sim/providers/mistral/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -78,6 +79,7 @@ export const mistralProvider: ProviderConfig = { const mistral = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.mistral.ai/v1', }) diff --git a/apps/sim/providers/nebius/index.ts b/apps/sim/providers/nebius/index.ts index 193b756bc09..10f6e167ef6 100644 --- a/apps/sim/providers/nebius/index.ts +++ b/apps/sim/providers/nebius/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getModelCapabilities, @@ -52,6 +53,7 @@ export const nebiusProvider: ProviderConfig = { if (!request.apiKey) throw new Error('API key is required for Nebius') const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.tokenfactory.nebius.com/v1', }) diff --git a/apps/sim/providers/nvidia/index.ts b/apps/sim/providers/nvidia/index.ts index 956433308b9..8865e8c6b5e 100644 --- a/apps/sim/providers/nvidia/index.ts +++ b/apps/sim/providers/nvidia/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -81,6 +82,7 @@ export const nvidiaProvider: ProviderConfig = { try { const nvidia = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: NVIDIA_BASE_URL, }) diff --git a/apps/sim/providers/ollama-cloud/index.ts b/apps/sim/providers/ollama-cloud/index.ts index 350769082aa..20d11e5e8a7 100644 --- a/apps/sim/providers/ollama-cloud/index.ts +++ b/apps/sim/providers/ollama-cloud/index.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { inheritConversationGenerationContext } from '@/providers/conversation-generation' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' @@ -38,6 +39,7 @@ export const ollamaCloudProvider: ProviderConfig = { createClient: () => new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey, baseURL: OLLAMA_CLOUD_BASE_URL, }), diff --git a/apps/sim/providers/openai/core.ts b/apps/sim/providers/openai/core.ts index 85d53ec8efe..9691445bace 100644 --- a/apps/sim/providers/openai/core.ts +++ b/apps/sim/providers/openai/core.ts @@ -4,6 +4,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import type OpenAI from 'openai' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { @@ -143,7 +144,7 @@ export interface ResponsesProviderConfig { /** * Optional fetch implementation. Used to pin the connection to a pre-validated * IP (DNS-rebinding/SSRF protection) when the endpoint is user-supplied. - * Defaults to the global fetch. + * Defaults to {@link outboundFetch}. */ fetch?: typeof fetch } @@ -156,7 +157,7 @@ export async function executeResponsesProviderRequest( config: ResponsesProviderConfig ): Promise { const { logger } = config - const fetchImpl = config.fetch ?? fetch + const fetchImpl = config.fetch ?? outboundFetch logger.info(`Preparing ${config.providerLabel} request`, { model: request.model, diff --git a/apps/sim/providers/openrouter/index.ts b/apps/sim/providers/openrouter/index.ts index 75fdaa11f69..0d0fa9e609f 100644 --- a/apps/sim/providers/openrouter/index.ts +++ b/apps/sim/providers/openrouter/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { bindConversationGenerationContextWindow } from '@/providers/conversation-generation' import { getConversationRequestContext } from '@/providers/conversation-history' @@ -69,6 +70,7 @@ export const openRouterProvider: ProviderConfig = { const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://openrouter.ai/api/v1', }) diff --git a/apps/sim/providers/sakana/index.ts b/apps/sim/providers/sakana/index.ts index 5ab637f4fde..91d8155e891 100644 --- a/apps/sim/providers/sakana/index.ts +++ b/apps/sim/providers/sakana/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -71,6 +72,7 @@ export const sakanaProvider: ProviderConfig = { try { const sakana = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: SAKANA_BASE_URL, }) diff --git a/apps/sim/providers/thinkingmachines/index.ts b/apps/sim/providers/thinkingmachines/index.ts index 305ce26f07d..8eb07ba1aba 100644 --- a/apps/sim/providers/thinkingmachines/index.ts +++ b/apps/sim/providers/thinkingmachines/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, @@ -65,6 +66,7 @@ export const thinkingMachinesProvider: ProviderConfig = { if (!request.apiKey) throw new Error('API key is required for Thinking Machines') const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: TINKER_BASE_URL, }) diff --git a/apps/sim/providers/together/index.ts b/apps/sim/providers/together/index.ts index a3746e57a19..8d00508d79e 100644 --- a/apps/sim/providers/together/index.ts +++ b/apps/sim/providers/together/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { @@ -44,6 +45,7 @@ export const togetherProvider: ProviderConfig = { const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.together.ai/v1', }) diff --git a/apps/sim/providers/typesafe/transport.ts b/apps/sim/providers/typesafe/transport.ts index 658267367f0..b2a17363fd7 100644 --- a/apps/sim/providers/typesafe/transport.ts +++ b/apps/sim/providers/typesafe/transport.ts @@ -1,5 +1,6 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { @@ -39,7 +40,7 @@ export async function requestJevEvaluation( let response: Response | undefined try { signal.throwIfAborted() - response = await fetch('https://api.typesafe.ai/v1/systemone', { + response = await outboundFetch('https://api.typesafe.ai/v1/systemone', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, body: payload, diff --git a/apps/sim/providers/xai/index.ts b/apps/sim/providers/xai/index.ts index bf708f25058..0ca496b8d40 100644 --- a/apps/sim/providers/xai/index.ts +++ b/apps/sim/providers/xai/index.ts @@ -4,6 +4,7 @@ import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' import type { CompletionUsage } from 'openai/resources/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -91,6 +92,7 @@ export const xAIProvider: ProviderConfig = { const xai = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.x.ai/v1', }) diff --git a/apps/sim/providers/zai/index.ts b/apps/sim/providers/zai/index.ts index 10ff0aec053..b461150344f 100644 --- a/apps/sim/providers/zai/index.ts +++ b/apps/sim/providers/zai/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -98,6 +99,7 @@ export const zaiProvider: ProviderConfig = { try { const zai = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: ZAI_BASE_URL, }) diff --git a/apps/sim/tools/apify/run_actor_async.ts b/apps/sim/tools/apify/run_actor_async.ts index 1cc3c33c449..2e0e548124e 100644 --- a/apps/sim/tools/apify/run_actor_async.ts +++ b/apps/sim/tools/apify/run_actor_async.ts @@ -1,5 +1,6 @@ import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { ApifyRun, RunActorParams, RunActorResult } from '@/tools/apify/types' import { APIFY_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' @@ -148,7 +149,7 @@ export const apifyRunActorAsyncTool: ToolConfig await sleep(POLL_INTERVAL_MS) elapsedTime += POLL_INTERVAL_MS - const statusResponse = await fetch(`https://api.apify.com/v2/actor-runs/${runId}`, { + const statusResponse = await outboundFetch(`https://api.apify.com/v2/actor-runs/${runId}`, { headers: { Authorization: `Bearer ${params.apiKey}`, }, @@ -173,7 +174,7 @@ export const apifyRunActorAsyncTool: ToolConfig ) { if (run.status === 'SUCCEEDED' && run.defaultDatasetId) { const limit = Math.max(1, Math.min(params.itemLimit || 100, 250000)) - const itemsResponse = await fetch( + const itemsResponse = await outboundFetch( `https://api.apify.com/v2/datasets/${run.defaultDatasetId}/items?limit=${limit}`, { headers: { diff --git a/apps/sim/tools/azure_devops/utils.ts b/apps/sim/tools/azure_devops/utils.ts index 568ea5a9bd7..496215af603 100644 --- a/apps/sim/tools/azure_devops/utils.ts +++ b/apps/sim/tools/azure_devops/utils.ts @@ -1,5 +1,6 @@ import { toStringOrNull } from '@sim/utils/coerce' import { toRecordOrNull } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -47,7 +48,7 @@ export async function fetchWorkItemsChunk( url.searchParams.set('$expand', 'all') url.searchParams.set('api-version', '7.2-preview.3') - const response = await fetch(url.toString(), { + const response = await outboundFetch(url.toString(), { method: 'GET', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/tools/brightdata/discover.ts b/apps/sim/tools/brightdata/discover.ts index f0cebce36cd..8e5295939ac 100644 --- a/apps/sim/tools/brightdata/discover.ts +++ b/apps/sim/tools/brightdata/discover.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { BrightDataDiscoverParams, BrightDataDiscoverResponse } from '@/tools/brightdata/types' import { BRIGHTDATA_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' @@ -147,7 +148,7 @@ export const brightDataDiscoverTool: ToolConfig< while (elapsedTime < MAX_POLL_TIME_MS) { try { - const pollResponse = await fetch( + const pollResponse = await outboundFetch( `https://api.brightdata.com/discover?task_id=${encodeURIComponent(taskId)}`, { method: 'GET', diff --git a/apps/sim/tools/cbinsights/utils.ts b/apps/sim/tools/cbinsights/utils.ts index fc0c85d323a..c2d6e8c77fc 100644 --- a/apps/sim/tools/cbinsights/utils.ts +++ b/apps/sim/tools/cbinsights/utils.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { toArray } from '@sim/utils/object' import { LRUCache } from 'lru-cache' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -125,7 +126,7 @@ async function authorize( clientSecret: string, signal?: AbortSignal ): Promise { - const response = await fetch(`${CBINSIGHTS_API_BASE}/v2/authorize`, { + const response = await outboundFetch(`${CBINSIGHTS_API_BASE}/v2/authorize`, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify({ clientId, clientSecret }), @@ -203,7 +204,7 @@ export async function cbInsightsRequest( const effectiveSignal = signal ? AbortSignal.any([signal, timeout]) : timeout const send = async (token: string) => - fetch(`${CBINSIGHTS_API_BASE}${spec.path}`, { + outboundFetch(`${CBINSIGHTS_API_BASE}${spec.path}`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, diff --git a/apps/sim/tools/dropbox/download.ts b/apps/sim/tools/dropbox/download.ts index dbd57e8894c..69614bcff8a 100644 --- a/apps/sim/tools/dropbox/download.ts +++ b/apps/sim/tools/dropbox/download.ts @@ -1,4 +1,5 @@ import { omit } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { httpHeaderSafeJson } from '@/lib/core/utils/validation' import type { DropboxDownloadParams, @@ -28,14 +29,17 @@ async function transformDownloadResponse(response: Response, params?: DropboxDow let temporaryLink: string | undefined if (params?.accessToken) { try { - const linkResponse = await fetch('https://api.dropboxapi.com/2/files/get_temporary_link', { - method: 'POST', - headers: { - Authorization: `Bearer ${params.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ path: params.path.trim() }), - }) + const linkResponse = await outboundFetch( + 'https://api.dropboxapi.com/2/files/get_temporary_link', + { + method: 'POST', + headers: { + Authorization: `Bearer ${params.accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ path: params.path.trim() }), + } + ) if (linkResponse.ok) { const linkData = await linkResponse.json() temporaryLink = linkData.link diff --git a/apps/sim/tools/dropcontact/enrich_contact.ts b/apps/sim/tools/dropcontact/enrich_contact.ts index f3682d196fc..da123efe99b 100644 --- a/apps/sim/tools/dropcontact/enrich_contact.ts +++ b/apps/sim/tools/dropcontact/enrich_contact.ts @@ -1,4 +1,5 @@ import { sleep } from '@sim/utils/helpers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { dropcontactHosting } from '@/tools/dropcontact/hosting' import type { DropcontactEmailEntry, @@ -254,7 +255,7 @@ export const dropcontactEnrichContactTool: ToolConfig< // Poll endpoint: GET https://api.dropcontact.com/v1/enrich/all/{request_id} // Source: https://developer.dropcontact.com (retrieved 2026-05) - const pollResponse = await fetch( + const pollResponse = await outboundFetch( `https://api.dropcontact.com/v1/enrich/all/${encodeURIComponent(requestId)}`, { headers: { diff --git a/apps/sim/tools/enrow/find_email.ts b/apps/sim/tools/enrow/find_email.ts index bcead35e5fa..3cc5cea05a6 100644 --- a/apps/sim/tools/enrow/find_email.ts +++ b/apps/sim/tools/enrow/find_email.ts @@ -1,4 +1,5 @@ import { sleep } from '@sim/utils/helpers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { enrowHosting } from '@/tools/enrow/hosting' import type { EnrowFindEmailParams, @@ -142,7 +143,7 @@ export const enrowFindEmailTool: ToolConfig = { elapsedTime += POLL_INTERVAL_MS try { - const statusResponse = await fetch(`https://api.exa.ai/agent/runs/${runId}`, { + const statusResponse = await outboundFetch(`https://api.exa.ai/agent/runs/${runId}`, { method: 'GET', headers: { 'x-api-key': params.apiKey, diff --git a/apps/sim/tools/firecrawl/agent.ts b/apps/sim/tools/firecrawl/agent.ts index 13c77ffda28..964ed100450 100644 --- a/apps/sim/tools/firecrawl/agent.ts +++ b/apps/sim/tools/firecrawl/agent.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { AgentParams, AgentResponse } from '@/tools/firecrawl/types' import type { ToolConfig } from '@/tools/types' @@ -119,7 +120,7 @@ export const agentTool: ToolConfig = { while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, { + const statusResponse = await outboundFetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, { method: 'GET', headers: { Authorization: `Bearer ${params.apiKey}`, diff --git a/apps/sim/tools/firecrawl/batch-scrape.ts b/apps/sim/tools/firecrawl/batch-scrape.ts index f2d23d82044..26f4d655bdd 100644 --- a/apps/sim/tools/firecrawl/batch-scrape.ts +++ b/apps/sim/tools/firecrawl/batch-scrape.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlFormatModelInput, @@ -196,13 +197,16 @@ export const batchScrapeTool: ToolConfig while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/crawl/${jobId}`, { + const statusResponse = await outboundFetch(`https://api.firecrawl.dev/v2/crawl/${jobId}`, { method: 'GET', headers: { Authorization: `Bearer ${params.apiKey}`, diff --git a/apps/sim/tools/firecrawl/extract.ts b/apps/sim/tools/firecrawl/extract.ts index b6c06b6000d..2d2d2a31020 100644 --- a/apps/sim/tools/firecrawl/extract.ts +++ b/apps/sim/tools/firecrawl/extract.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlScrapeOptionsModelInput, @@ -168,13 +169,16 @@ export const extractTool: ToolConfig = { while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/extract/${jobId}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${params.apiKey}`, - 'Content-Type': 'application/json', - }, - }) + const statusResponse = await outboundFetch( + `https://api.firecrawl.dev/v2/extract/${jobId}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${params.apiKey}`, + 'Content-Type': 'application/json', + }, + } + ) if (!statusResponse.ok) { throw new Error(`Failed to get extract status: ${statusResponse.statusText}`) diff --git a/apps/sim/tools/github/pr.ts b/apps/sim/tools/github/pr.ts index 8c2525bfe87..1a2bfca5bda 100644 --- a/apps/sim/tools/github/pr.ts +++ b/apps/sim/tools/github/pr.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { nullableBoolean, nullableString, @@ -150,7 +151,7 @@ async function fetchPullRequestFiles( const maxPages = MAX_PULL_REQUEST_FILES / PULL_REQUEST_FILES_PER_PAGE for (let page = 1; page <= maxPages; page += 1) { - const response = await fetch( + const response = await outboundFetch( `https://api.github.com/repos/${params.owner}/${params.repo}/pulls/${pullNumber}/files?per_page=${PULL_REQUEST_FILES_PER_PAGE}&page=${page}`, { headers: { diff --git a/apps/sim/tools/gmail/read.ts b/apps/sim/tools/gmail/read.ts index 860dfa2d2d2..a75329696fe 100644 --- a/apps/sim/tools/gmail/read.ts +++ b/apps/sim/tools/gmail/read.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -163,7 +164,7 @@ export const gmailReadTool: ToolConfig = { try { // Get the first message details const messageId = data.messages[0].id - const messageResponse = await fetch( + const messageResponse = await outboundFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}?format=full`, { headers: { @@ -210,7 +211,7 @@ export const gmailReadTool: ToolConfig = { const messages: GmailMessage[] = [] for (const msg of data.messages.slice(0, maxResults)) { context?.signal?.throwIfAborted() - const messageResponse = await fetch( + const messageResponse = await outboundFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(msg.id)}?format=full`, { headers: { Authorization: `Bearer ${params?.accessToken || ''}` }, diff --git a/apps/sim/tools/gmail/search.ts b/apps/sim/tools/gmail/search.ts index 6aef77d9876..acfdc0ae042 100644 --- a/apps/sim/tools/gmail/search.ts +++ b/apps/sim/tools/gmail/search.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { GmailMessage, GmailSearchParams, GmailToolResponse } from '@/tools/gmail/types' import { createMessagesSummary, @@ -97,7 +98,7 @@ export const gmailSearchTool: ToolConfig = context?.signal?.throwIfAborted() const batch = await Promise.all( listed.slice(i, i + MESSAGE_FETCH_CONCURRENCY).map(async (msg) => { - const messageResponse = await fetch( + const messageResponse = await outboundFetch( `${GMAIL_API_BASE}/messages/${safeUrlPathSegment(msg.id, 'messageId')}?format=full`, { headers: { diff --git a/apps/sim/tools/gmail/utils.ts b/apps/sim/tools/gmail/utils.ts index f4002bbcbdd..dcb828fbca6 100644 --- a/apps/sim/tools/gmail/utils.ts +++ b/apps/sim/tools/gmail/utils.ts @@ -1,5 +1,6 @@ import { generateRandomString } from '@sim/utils/random' import { convert } from 'html-to-text' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -32,7 +33,7 @@ export async function fetchThreadingHeaders( subject?: string }> { try { - const messageResponse = await fetch( + const messageResponse = await outboundFetch( `${GMAIL_API_BASE}/messages/${safeUrlPathSegment(messageId, 'messageId')}?format=metadata&metadataHeaders=Message-ID&metadataHeaders=References&metadataHeaders=Subject`, { headers: { @@ -276,7 +277,7 @@ export async function downloadAttachments( for (const attachment of attachmentInfo) { try { budget.assertSize(attachment.size, 'Gmail attachments') - const response = await fetch( + const response = await outboundFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}/attachments/${encodeURIComponent(attachment.attachmentId)}`, { headers: { Authorization: `Bearer ${accessToken}` }, diff --git a/apps/sim/tools/google_calendar/invite.ts b/apps/sim/tools/google_calendar/invite.ts index 32f7a3e3d97..6d44e4e9793 100644 --- a/apps/sim/tools/google_calendar/invite.ts +++ b/apps/sim/tools/google_calendar/invite.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { CALENDAR_API_BASE, type CalendarAttendee, @@ -80,7 +81,7 @@ async function inviteAttendees( queryParams.append('sendUpdates', params?.sendUpdates ?? 'all') const putUrl = `${CALENDAR_API_BASE}/calendars/${safeUrlPathSegment(calendarId, 'calendarId')}/events/${safeUrlPathSegment(params?.eventId ?? '', 'eventId')}?${queryParams.toString()}` - const putResponse = await fetch(putUrl, { + const putResponse = await outboundFetch(putUrl, { method: 'PUT', headers: { Authorization: `Bearer ${params?.accessToken}`, diff --git a/apps/sim/tools/google_calendar/quick_add.ts b/apps/sim/tools/google_calendar/quick_add.ts index cb430425700..1d51a4e7861 100644 --- a/apps/sim/tools/google_calendar/quick_add.ts +++ b/apps/sim/tools/google_calendar/quick_add.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { CALENDAR_API_BASE, type GoogleCalendarApiEventResponse, @@ -113,7 +114,7 @@ export const quickAddTool: ToolConfig< const updateUrl = `${CALENDAR_API_BASE}/calendars/${safeUrlPathSegment(calendarId, 'calendarId')}/events/${safeUrlPathSegment(eventId, 'eventId')}${updateQueryParams.toString() ? `?${updateQueryParams.toString()}` : ''}` - const updateResponse = await fetch(updateUrl, { + const updateResponse = await outboundFetch(updateUrl, { method: 'PATCH', headers: { Authorization: `Bearer ${params.accessToken}`, @@ -244,7 +245,7 @@ export const quickAddV2Tool: ToolConfig< const updateUrl = `${CALENDAR_API_BASE}/calendars/${safeUrlPathSegment(calendarId, 'calendarId')}/events/${safeUrlPathSegment(eventId, 'eventId')}${updateQueryParams.toString() ? `?${updateQueryParams.toString()}` : ''}` - const updateResponse = await fetch(updateUrl, { + const updateResponse = await outboundFetch(updateUrl, { method: 'PATCH', headers: { Authorization: `Bearer ${params.accessToken}`, diff --git a/apps/sim/tools/google_calendar/respond.ts b/apps/sim/tools/google_calendar/respond.ts index 64918e92502..c95f3baebf9 100644 --- a/apps/sim/tools/google_calendar/respond.ts +++ b/apps/sim/tools/google_calendar/respond.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { CALENDAR_API_BASE, type CalendarAttendee, @@ -67,7 +68,7 @@ async function respondToEvent( const patchUrl = `${CALENDAR_API_BASE}/calendars/${safeUrlPathSegment(calendarId, 'calendarId')}/events/${safeUrlPathSegment(params?.eventId ?? '', 'eventId')}${queryString ? `?${queryString}` : ''}` signal?.throwIfAborted() - const patchResponse = await fetch(patchUrl, { + const patchResponse = await outboundFetch(patchUrl, { method: 'PATCH', signal, headers: { diff --git a/apps/sim/tools/google_drive/create_folder.ts b/apps/sim/tools/google_drive/create_folder.ts index 67c512a0864..e0dda457231 100644 --- a/apps/sim/tools/google_drive/create_folder.ts +++ b/apps/sim/tools/google_drive/create_folder.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { GoogleDriveFile, @@ -96,7 +97,7 @@ export const createFolderTool: ToolConfig { const commentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey ?? '', 'issueKey')}/comment` - const commentResponse = await fetch(commentUrl, { + const commentResponse = await outboundFetch(commentUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/add_watcher.ts b/apps/sim/tools/jira/add_watcher.ts index 2699df81494..23d2f739a1c 100644 --- a/apps/sim/tools/jira/add_watcher.ts +++ b/apps/sim/tools/jira/add_watcher.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraAddWatcherParams, JiraAddWatcherResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -80,7 +81,7 @@ export const jiraAddWatcherTool: ToolConfig { const worklogUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey ?? '', 'issueKey')}/worklog` - const worklogResponse = await fetch(worklogUrl, { + const worklogResponse = await outboundFetch(worklogUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/assign_issue.ts b/apps/sim/tools/jira/assign_issue.ts index 5b39127afc1..f6dd197d0ba 100644 --- a/apps/sim/tools/jira/assign_issue.ts +++ b/apps/sim/tools/jira/assign_issue.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraAssignIssueParams, JiraAssignIssueResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -89,7 +90,7 @@ export const jiraAssignIssueTool: ToolConfig { const cloudId = params?.cloudId || (await getJiraCloudId(params!.domain, params!.accessToken)) - const typesResp = await fetch( + const typesResp = await outboundFetch( `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issueLinkType`, { method: 'GET', @@ -115,7 +116,7 @@ export const jiraCreateIssueLinkTool: ToolConfig< } const linkUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issueLink` - const linkResponse = await fetch(linkUrl, { + const linkResponse = await outboundFetch(linkUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/delete_attachment.ts b/apps/sim/tools/jira/delete_attachment.ts index 0dbd4846545..f341393aa05 100644 --- a/apps/sim/tools/jira/delete_attachment.ts +++ b/apps/sim/tools/jira/delete_attachment.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraDeleteAttachmentParams, JiraDeleteAttachmentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -67,7 +68,7 @@ export const jiraDeleteAttachmentTool: ToolConfig< const cloudId = await getJiraCloudId(params!.domain, params!.accessToken) // Make the actual request with the resolved cloudId const attachmentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/attachment/${safeUrlPathSegment(params?.attachmentId ?? '', 'attachmentId')}` - const attachmentResponse = await fetch(attachmentUrl, { + const attachmentResponse = await outboundFetch(attachmentUrl, { method: 'DELETE', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/delete_comment.ts b/apps/sim/tools/jira/delete_comment.ts index a8f61ea1a60..19b019c6d7c 100644 --- a/apps/sim/tools/jira/delete_comment.ts +++ b/apps/sim/tools/jira/delete_comment.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraDeleteCommentParams, JiraDeleteCommentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -71,7 +72,7 @@ export const jiraDeleteCommentTool: ToolConfig { const attachmentsUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey ?? '', 'issueKey')}?fields=attachment` - const attachmentsResponse = await fetch(attachmentsUrl, { + const attachmentsResponse = await outboundFetch(attachmentsUrl, { method: 'GET', signal: context?.signal, headers: { diff --git a/apps/sim/tools/jira/get_comments.ts b/apps/sim/tools/jira/get_comments.ts index a0c41c7f569..346b6a8b40b 100644 --- a/apps/sim/tools/jira/get_comments.ts +++ b/apps/sim/tools/jira/get_comments.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraGetCommentsParams, JiraGetCommentsResponse } from '@/tools/jira/types' import { COMMENT_ITEM_PROPERTIES, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { extractAdfText, getJiraCloudId, transformUser } from '@/tools/jira/utils' @@ -106,7 +107,7 @@ export const jiraGetCommentsTool: ToolConfig { const fetchFields = async (cloudId: string) => { - const fieldsResponse = await fetch(buildFieldsUrl(cloudId), { + const fieldsResponse = await outboundFetch(buildFieldsUrl(cloudId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/get_project.ts b/apps/sim/tools/jira/get_project.ts index b7020c6453b..5cb773c37ff 100644 --- a/apps/sim/tools/jira/get_project.ts +++ b/apps/sim/tools/jira/get_project.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraGetProjectParams, JiraGetProjectResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -65,7 +66,7 @@ export const jiraGetProjectTool: ToolConfig { const fetchProject = async (cloudId: string) => { - const projectResponse = await fetch(buildProjectUrl(cloudId, params!.projectId), { + const projectResponse = await outboundFetch(buildProjectUrl(cloudId, params!.projectId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/get_transitions.ts b/apps/sim/tools/jira/get_transitions.ts index 51daaf83b14..dd102f4f20d 100644 --- a/apps/sim/tools/jira/get_transitions.ts +++ b/apps/sim/tools/jira/get_transitions.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraGetTransitionsParams, JiraGetTransitionsResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -67,13 +68,16 @@ export const jiraGetTransitionsTool: ToolConfig< transformResponse: async (response: Response, params?: JiraGetTransitionsParams) => { const fetchTransitions = async (cloudId: string) => { - const transitionsResponse = await fetch(buildTransitionsUrl(cloudId, params!.issueKey), { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${params!.accessToken}`, - }, - }) + const transitionsResponse = await outboundFetch( + buildTransitionsUrl(cloudId, params!.issueKey), + { + method: 'GET', + headers: { + Accept: 'application/json', + Authorization: `Bearer ${params!.accessToken}`, + }, + } + ) if (!transitionsResponse.ok) { const errorText = await transitionsResponse.text() diff --git a/apps/sim/tools/jira/get_users.ts b/apps/sim/tools/jira/get_users.ts index 4fa1219c0d7..204fb1de06d 100644 --- a/apps/sim/tools/jira/get_users.ts +++ b/apps/sim/tools/jira/get_users.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraGetUsersParams, JiraGetUsersResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -109,7 +110,7 @@ export const jiraGetUsersTool: ToolConfig { const fetchIssueTypes = async (cloudId: string) => { - const issueTypesResponse = await fetch(buildIssueTypesUrl(cloudId), { + const issueTypesResponse = await outboundFetch(buildIssueTypesUrl(cloudId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/list_projects.ts b/apps/sim/tools/jira/list_projects.ts index 7e65df9dec1..eedbd862aeb 100644 --- a/apps/sim/tools/jira/list_projects.ts +++ b/apps/sim/tools/jira/list_projects.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraListProjectsParams, JiraListProjectsResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -101,7 +102,7 @@ export const jiraListProjectsTool: ToolConfig { const fetchProjects = async (cloudId: string) => { - const projectsResponse = await fetch(buildSearchUrl(cloudId, params!), { + const projectsResponse = await outboundFetch(buildSearchUrl(cloudId, params!), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/remove_watcher.ts b/apps/sim/tools/jira/remove_watcher.ts index 4b8424cc98c..a3e64232980 100644 --- a/apps/sim/tools/jira/remove_watcher.ts +++ b/apps/sim/tools/jira/remove_watcher.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraRemoveWatcherParams, JiraRemoveWatcherResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -70,7 +71,7 @@ export const jiraRemoveWatcherTool: ToolConfig { const issueUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params.issueKey, 'issueKey')}?expand=renderedFields,names,schema,transitions,operations,editmeta,changelog,versionedRepresentations` - const issueResponse = await fetch(issueUrl, { + const issueResponse = await outboundFetch(issueUrl, { method: 'GET', signal: context?.signal, headers: { @@ -120,15 +121,15 @@ export const jiraRetrieveTool: ToolConfig { const base = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params.issueKey, 'issueKey')}` const [commentsResp, worklogResp, watchersResp] = await Promise.all([ - fetch(`${base}/comment?maxResults=100&orderBy=-created`, { + outboundFetch(`${base}/comment?maxResults=100&orderBy=-created`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), - fetch(`${base}/worklog?maxResults=100`, { + outboundFetch(`${base}/worklog?maxResults=100`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), - fetch(`${base}/watchers`, { + outboundFetch(`${base}/watchers`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), diff --git a/apps/sim/tools/jira/search_issues.ts b/apps/sim/tools/jira/search_issues.ts index 25d9017166c..803a4b533e3 100644 --- a/apps/sim/tools/jira/search_issues.ts +++ b/apps/sim/tools/jira/search_issues.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraSearchIssuesParams, JiraSearchIssuesResponse } from '@/tools/jira/types' import { SEARCH_ISSUE_ITEM_PROPERTIES, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { @@ -212,7 +213,7 @@ export const jiraSearchIssuesTool: ToolConfig { // First, fetch available transitions to get the name and target status const transitionsUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey, 'issueKey')}/transitions` - const transitionsResp = await fetch(transitionsUrl, { + const transitionsResp = await outboundFetch(transitionsUrl, { method: 'GET', headers: { Accept: 'application/json', @@ -122,7 +123,7 @@ export const jiraTransitionIssueTool: ToolConfig< } // Perform the transition - const transitionResponse = await fetch(transitionsUrl, { + const transitionResponse = await outboundFetch(transitionsUrl, { method: 'POST', headers: { Accept: 'application/json', @@ -167,7 +168,7 @@ export const jiraTransitionIssueTool: ToolConfig< // Fetch transition metadata for the response try { const transitionsUrl = `https://api.atlassian.com/ex/jira/${params.cloudId}/rest/api/3/issue/${safeUrlPathSegment(params.issueKey, 'issueKey')}/transitions` - const transitionsResp = await fetch(transitionsUrl, { + const transitionsResp = await outboundFetch(transitionsUrl, { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/update_comment.ts b/apps/sim/tools/jira/update_comment.ts index e2daeac5549..b1b1d8a0e74 100644 --- a/apps/sim/tools/jira/update_comment.ts +++ b/apps/sim/tools/jira/update_comment.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraUpdateCommentParams, JiraUpdateCommentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { extractAdfText, getJiraCloudId, toAdf, transformUser } from '@/tools/jira/utils' @@ -108,7 +109,7 @@ export const jiraUpdateCommentTool: ToolConfig { const commentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey ?? '', 'issueKey')}/comment/${safeUrlPathSegment(params!.commentId ?? '', 'commentId')}` - const commentResponse = await fetch(commentUrl, { + const commentResponse = await outboundFetch(commentUrl, { method: 'PUT', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/update_worklog.ts b/apps/sim/tools/jira/update_worklog.ts index 130e2715296..2347e397c4e 100644 --- a/apps/sim/tools/jira/update_worklog.ts +++ b/apps/sim/tools/jira/update_worklog.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraUpdateWorklogParams, JiraUpdateWorklogResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { @@ -145,7 +146,7 @@ export const jiraUpdateWorklogTool: ToolConfig { const basePath = getItemBasePath(spreadsheetId, driveId) try { - const response = await fetch(`${basePath}?$select=id,webUrl`, { + const response = await outboundFetch(`${basePath}?$select=id,webUrl`, { signal, headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/tools/microsoft_teams/utils.ts b/apps/sim/tools/microsoft_teams/utils.ts index 3e1dfc13e97..1a7c6daecb7 100644 --- a/apps/sim/tools/microsoft_teams/utils.ts +++ b/apps/sim/tools/microsoft_teams/utils.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toStringOrNull } from '@sim/utils/coerce' import { isRecordLike, toRecord, toRecordOrNull } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -220,7 +221,7 @@ async function fetchHostedContents( const results: ToolFileData[] = [] try { budget.signal?.throwIfAborted() - const response = await fetch(path, { + const response = await outboundFetch(path, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -237,7 +238,7 @@ async function fetchHostedContents( for (const item of data.value ?? []) { if (!item.id) continue budget.signal?.throwIfAborted() - const content = await fetch(`${path}/${encodeURIComponent(item.id)}/$value`, { + const content = await outboundFetch(`${path}/${encodeURIComponent(item.id)}/$value`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -298,7 +299,7 @@ async function downloadReferenceAttachment( budget.signal?.throwIfAborted() const shareId = `u!${Buffer.from(attachment.contentUrl).toString('base64url')}` const path = `https://graph.microsoft.com/v1.0/shares/${shareId}/driveItem` - const metadataResponse = await fetch(`${path}?$select=name,size,file`, { + const metadataResponse = await outboundFetch(`${path}?$select=name,size,file`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -313,7 +314,7 @@ async function downloadReferenceAttachment( file?: { mimeType?: string } }>(metadataResponse, 'Teams attachment metadata', budget.signal) if (item.size !== undefined) budget.assertSize(item.size, 'Teams attachments') - const content = await fetch(`${path}/content`, { + const content = await outboundFetch(`${path}/content`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -371,7 +372,7 @@ function parseMentions(content: string): ParsedMention[] { } async function fetchChatMembers(chatId: string, accessToken: string): Promise { - const response = await fetch( + const response = await outboundFetch( `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(chatId)}/members`, { headers: { @@ -398,7 +399,7 @@ async function fetchChannelMembers( channelId: string, accessToken: string ): Promise { - const response = await fetch( + const response = await outboundFetch( `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/members`, { headers: { diff --git a/apps/sim/tools/netsuite/utils.ts b/apps/sim/tools/netsuite/utils.ts index c52c0c435bb..0f405a6bb35 100644 --- a/apps/sim/tools/netsuite/utils.ts +++ b/apps/sim/tools/netsuite/utils.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -447,7 +448,7 @@ async function sendSuiteTalkRequest( headers['X-NetSuite-PropertyNameValidation'] = 'error' } } - return fetch(url, { + return outboundFetch(url, { method: request.method, headers, ...(serializedBody !== undefined ? { body: serializedBody } : {}), diff --git a/apps/sim/tools/notion/read.ts b/apps/sim/tools/notion/read.ts index 47eab0a5fd1..534cf98e009 100644 --- a/apps/sim/tools/notion/read.ts +++ b/apps/sim/tools/notion/read.ts @@ -1,6 +1,7 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { parseRetryAfter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { NotionReadParams, NotionResponse } from '@/tools/notion/types' import { NOTION_READ_MAX_BLOCK_PAGES, @@ -41,7 +42,7 @@ async function fetchPageBlocks( if (cursor) url.searchParams.set('start_cursor', cursor) const request = () => - fetch(url.toString(), { + outboundFetch(url.toString(), { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/tools/outlook/read.ts b/apps/sim/tools/outlook/read.ts index b7e8a8397d1..2c71084b46c 100644 --- a/apps/sim/tools/outlook/read.ts +++ b/apps/sim/tools/outlook/read.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { AttachmentDownloadBudget, @@ -45,7 +46,8 @@ export async function downloadAttachments( budget = new AttachmentDownloadBudget(), options: OutlookAttachmentDownloadOptions = {} ): Promise { - const fetchImpl = options.fetchImpl ?? ((url: string, init: RequestInit) => fetch(url, init)) + const fetchImpl = + options.fetchImpl ?? ((url: string, init: RequestInit) => outboundFetch(url, init)) const attachments: OutlookAttachment[] = [] const path = `https://graph.microsoft.com/v1.0/me/messages/${encodeURIComponent(messageId)}/attachments` const handleError = (error: unknown, attachment?: OutlookDownloadAttachmentMetadata) => { diff --git a/apps/sim/tools/parallel/deep_research.ts b/apps/sim/tools/parallel/deep_research.ts index cbc0f086820..bbe92fda299 100644 --- a/apps/sim/tools/parallel/deep_research.ts +++ b/apps/sim/tools/parallel/deep_research.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { PlatformEvents } from '@/lib/core/telemetry' import { toList } from '@/tools/parallel/search' import type { ParallelDeepResearchParams } from '@/tools/parallel/types' @@ -199,7 +200,7 @@ export const deepResearchTool: ToolConfig undefined) diff --git a/apps/sim/tools/quickbooks/client.ts b/apps/sim/tools/quickbooks/client.ts index 9888ec5392d..a3cd2c9943c 100644 --- a/apps/sim/tools/quickbooks/client.ts +++ b/apps/sim/tools/quickbooks/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -125,7 +126,7 @@ export async function fetchValidatedQuickBooksCompanyInfo( environment: QuickBooksEnvironment ): Promise { const normalizedRealmId = normalizeQuickBooksRealmId(realmId) - const response = await fetch( + const response = await outboundFetch( buildQuickBooksCompanyUrl( normalizedRealmId, `companyinfo/${encodeURIComponent(normalizedRealmId)}`, diff --git a/apps/sim/tools/quickbooks/utils.ts b/apps/sim/tools/quickbooks/utils.ts index 647c05c40cb..5ef2ddcd2f2 100644 --- a/apps/sim/tools/quickbooks/utils.ts +++ b/apps/sim/tools/quickbooks/utils.ts @@ -1,4 +1,5 @@ import { omit } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ErrorExtractorId, extractErrorMessage } from '@/tools/error-extractors' import { @@ -484,7 +485,7 @@ export async function executeQuickBooksFullUpdate< const recordId = requiredQuickBooksString(options.recordId, 'recordId') const syncToken = requiredQuickBooksString(options.syncToken, 'syncToken') const patch = options.buildPatch(options.params) - const readResponse = await fetch( + const readResponse = await outboundFetch( buildQuickBooksEntityUrl(options.params, options.resource, recordId), { method: 'GET', @@ -525,7 +526,7 @@ export async function executeQuickBooksFullUpdate< * references the item. */ if (options.entity === 'Item') updateUrl.searchParams.set('include', 'donotupdateaccountontxns') - const updateResponse = await fetch(updateUrl, { + const updateResponse = await outboundFetch(updateUrl, { method: 'POST', headers: getQuickBooksToolHeaders(options.params.accessToken, 'application/json'), body: JSON.stringify(fullBody), diff --git a/apps/sim/tools/sharepoint/read_page.ts b/apps/sim/tools/sharepoint/read_page.ts index cd95ae96bcd..8eaa7acb28a 100644 --- a/apps/sim/tools/sharepoint/read_page.ts +++ b/apps/sim/tools/sharepoint/read_page.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { GraphApiPageFields, GraphApiResponse, @@ -246,7 +247,7 @@ export const readPageTool: ToolConfig = { max_results: '100', // Max allowed }) - const repliesResponse = await fetch( + const repliesResponse = await outboundFetch( `https://api.twitter.com/2/tweets/search/recent?${searchParams.toString()}`, { method: 'GET', From 95eb1743c6fd8d58a87054e9366376c87f39467d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:08:39 -0800 Subject: [PATCH 06/32] improvement(egress): route customer-credential AWS SDK clients through the organization's gateway organizationAwsRequestHandler resolves the outbound route per request: a gateway route tunnels through a per-organization keep-alive agent (never shared across organizations or with direct traffic), a direct route keeps awsRequestHandler's behaviour, and a call without a scope goes direct with a warning. With routing disabled it is awsRequestHandler. Used for AWS tool clients (awsClientConfig, unauthenticated STS), saved-credential STS AssumeRole / AssumeRoleWithWebIdentity / GetCallerIdentity, and Bedrock on customer credentials. Sim's own AWS usage keeps awsRequestHandler. The gateway connect sequence is shared with createGatewayDispatcher via connectThroughGateway, and an OutboundRoutingError is reported as unavailable rather than as a rejected credential. --- .../lib/core/network/gateway.server.test.ts | 41 +++++++ apps/sim/lib/core/network/gateway.server.ts | 78 ++++++++----- .../organization-aws-handler.server.ts | 110 ++++++++++++++++++ .../lib/core/network/proxy-agent.server.ts | 11 +- .../credentials/aws-service-account.server.ts | 11 +- apps/sim/lib/internal/aws/credentials.ts | 7 +- apps/sim/lib/internal/aws/failure.ts | 5 +- apps/sim/lib/internal/sts/client.ts | 4 +- apps/sim/providers/bedrock/index.ts | 22 +++- 9 files changed, 240 insertions(+), 49 deletions(-) create mode 100644 apps/sim/lib/core/network/organization-aws-handler.server.ts diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts index 93ef4a92dce..f2543557b3e 100644 --- a/apps/sim/lib/core/network/gateway.server.test.ts +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -7,6 +7,7 @@ import { join } from 'node:path' import { getCACertificates, setDefaultCACertificates } from 'node:tls' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' +import { GetCallerIdentityCommand, STSClient } from '@aws-sdk/client-sts' import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import type { OutboundGateway, ResolvedOutboundRoute } from '@/lib/core/network/config.server' @@ -20,6 +21,7 @@ vi.mock('@sim/security/dns', async (importOriginal) => ({ import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { openGatewayTunnel, secureOutboundTunnel } from '@/lib/core/network/gateway.server' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createPinnedFetchWithDispatcher, @@ -317,6 +319,45 @@ describe('organization gateways over real TLS CONNECT sockets', () => { { token: 'Bearer bravo', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, ]) }) + describe('AWS SDK clients signing with customer credentials', () => { + const sts = new STSClient({ + region: 'us-east-1', + endpoint: url(), + maxAttempts: 1, + credentials: { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'secret' }, + requestHandler: organizationAwsRequestHandler(), + }) + afterAll(() => sts.destroy()) + + /** The origin echoes the signed query body, which STS cannot parse; only the route matters. */ + const viaProxy = async (organizationId: string) => { + const error = await runWithOutboundOrganization(organizationId, () => + sts.send(new GetCallerIdentityCommand({})) + ).then( + () => undefined, + (failure: { $response?: { headers: Record } }) => failure + ) + return error?.$response?.headers['x-via-proxy'] + } + + it("sends a request through the organization's gateway", async () => { + const start = admissions.length + expect(await viaProxy('org_a')).toBe('yes') + expect(admissions.slice(start)).toEqual([ + { token: 'Bearer alpha', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, + ]) + }) + + it("never reuses one organization's pooled socket for another", async () => { + expect(await viaProxy('org_a')).toBe('yes') + const start = admissions.length + expect(await viaProxy('org_b')).toBe('yes') + expect(await viaProxy('org_a')).toBe('yes') + expect(await viaProxy('org_b')).toBe('yes') + expect(admissions.slice(start).map((entry) => entry.token)).toEqual(['Bearer bravo']) + }) + }) + it('fails closed on denied proxy credentials without contacting the origin', async () => { state.gateways.set('denied', { ...state.gateways.get('org_a')!, diff --git a/apps/sim/lib/core/network/gateway.server.ts b/apps/sim/lib/core/network/gateway.server.ts index cbe81a57e55..6fa1819dbcf 100644 --- a/apps/sim/lib/core/network/gateway.server.ts +++ b/apps/sim/lib/core/network/gateway.server.ts @@ -127,6 +127,44 @@ function getErrorCode(error: Error | undefined): string | undefined { return typeof code === 'string' ? code : undefined } +/** + * Opens a TLS connection to `target` through `gateway` after the egress policy approves it: + * HTTPS on 443 only, judged against `profile` (or the already resolved `resolvedIP`). Every + * failure is an {@link OutboundRoutingError}, so no gateway detail reaches the caller. + */ +export async function connectThroughGateway( + gateway: OutboundGateway, + target: { protocol: string; hostname: string; port: number }, + options: { profile: EgressProfile; resolvedIP?: string } +): Promise { + try { + const hostname = target.hostname.replace(/^\[|\]$/g, '') + if (target.protocol !== 'https:' || target.port !== 443) { + throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') + } + const authority = isIP(hostname) === 6 ? `[${hostname}]` : hostname + const url = new URL(`https://${authority}:${target.port}`) + let address = options.resolvedIP + if (address) { + if (!checkResolvedEgress(url, address, options.profile).allowed) { + throw new OutboundRoutingError('ROUTE_BLOCKED') + } + } else { + const result = await validateEgressUrl(url.href, 'url', options.profile, { + logDetails: false, + }) + if (!result.isValid) throw new OutboundRoutingError('ROUTE_BLOCKED') + address = result.resolvedIP + } + const socket = await openGatewayTunnel(gateway, address, target.port) + return await secureOutboundTunnel(socket, hostname, target.port) + } catch (error) { + throw error instanceof OutboundRoutingError + ? error + : new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } +} + /** A dispatcher belongs to one organization and gateway, never a process-wide default. */ export function createGatewayDispatcher( gateway: OutboundGateway, @@ -136,37 +174,17 @@ export function createGatewayDispatcher( allowH2: false, ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), connect(connection, callback) { - const port = Number(connection.port || (connection.protocol === 'https:' ? 443 : 80)) - const hostname = connection.hostname.replace(/^\[|\]$/g, '') - const authority = isIP(hostname) === 6 ? `[${hostname}]` : hostname - const url = new URL(`${connection.protocol}//${authority}:${port}`) - void (async () => { - if (connection.protocol !== 'https:' || port !== 443) { - throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') - } - let address = options.resolvedIP - if (address) { - if (!checkResolvedEgress(url, address, options.profile).allowed) { - throw new OutboundRoutingError('ROUTE_BLOCKED') - } - } else { - const result = await validateEgressUrl(url.href, 'url', options.profile, { - logDetails: false, - }) - if (!result.isValid) throw new OutboundRoutingError('ROUTE_BLOCKED') - address = result.resolvedIP - } - const socket = await openGatewayTunnel(gateway, address, port) - return secureOutboundTunnel(socket, hostname, port) - })().then( + connectThroughGateway( + gateway, + { + protocol: connection.protocol, + hostname: connection.hostname, + port: Number(connection.port || (connection.protocol === 'https:' ? 443 : 80)), + }, + options + ).then( (socket) => callback(null, socket), - (error) => - callback( - error instanceof OutboundRoutingError - ? error - : new OutboundRoutingError('GATEWAY_UNAVAILABLE'), - null - ) + (error: OutboundRoutingError) => callback(error, null) ) }, }) diff --git a/apps/sim/lib/core/network/organization-aws-handler.server.ts b/apps/sim/lib/core/network/organization-aws-handler.server.ts new file mode 100644 index 00000000000..f40bf33fae5 --- /dev/null +++ b/apps/sim/lib/core/network/organization-aws-handler.server.ts @@ -0,0 +1,110 @@ +import type http from 'node:http' +import type { Duplex } from 'node:stream' +import { createLogger } from '@sim/logger' +import { NodeHttpHandler } from '@smithy/node-http-handler' +import { Agent, type AgentConnectOpts } from 'agent-base' +import { isOutboundRoutingEnabled, type OutboundGateway } from '@/lib/core/network/config.server' +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { connectThroughGateway } from '@/lib/core/network/gateway.server' +import { + AGENT_OPTIONS, + type AwsRequestHandlerOptions, + awsRequestHandler, + environmentProxyAgent, +} from '@/lib/core/network/proxy-agent.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' + +const logger = createLogger('OrganizationAwsRequestHandler') + +/** + * Pools connections tunnelled through one organization's gateway. Its sockets carry only that + * organization's requests, so a keep-alive socket is never reused across organizations or + * for a direct request. + */ +class GatewayAgent extends Agent { + constructor(private readonly gateway: OutboundGateway) { + super(AGENT_OPTIONS) + } + + connect(_req: http.ClientRequest, options: AgentConnectOpts): Promise { + return connectThroughGateway( + this.gateway, + { + protocol: options.secureEndpoint ? 'https:' : 'http:', + hostname: options.host ?? '', + port: Number(options.port), + }, + { profile: 'configuredEndpoint' } + ) + } + + /** A replaced gateway finishes its in-flight requests and keeps no idle sockets. */ + retire(): void { + this.on('free', (socket: Duplex) => socket.destroy()) + for (const sockets of Object.values(this.freeSockets)) { + for (const socket of sockets ?? []) socket.destroy() + } + } +} + +const gatewayAgents = new Map() + +function gatewayAgentFor(gateway: OutboundGateway): GatewayAgent { + const current = gatewayAgents.get(gateway.organizationId) + if (current?.gatewayId === gateway.id) return current.agent + current?.agent.retire() + const agent = new GatewayAgent(gateway) + gatewayAgents.set(gateway.organizationId, { gatewayId: gateway.id, agent }) + return agent +} + +/** + * Chooses each connection's agent from the current outbound scope: the organization's + * {@link GatewayAgent} on a gateway route, otherwise the environment proxy agent (a direct + * keep-alive connection when no proxy applies). It never pools a socket itself, and its own + * socket limit is unbounded so a request is never queued here behind another organization's; + * the agent it delegates to applies the SDK's limit. + */ +class OrganizationRoutingAgent extends Agent { + async connect(req: http.ClientRequest, options: AgentConnectOpts): Promise { + const route = await currentRouteOrDirect(options.host) + if (route.kind === 'gateway') return gatewayAgentFor(route.gateway) + return environmentProxyAgent().connect(req, options) + } +} + +/** A request outside any outbound scope goes direct with a warning, as `outboundFetch` does. */ +async function currentRouteOrDirect(host: string | undefined) { + try { + return await resolveCurrentOutboundRoute() + } catch (error) { + if (!(error instanceof OutboundRoutingError) || error.code !== 'MISSING_SCOPE') throw error + logger.warn('AWS request made without an organization scope', { host }) + return { kind: 'direct' } as const + } +} + +let routingAgent: OrganizationRoutingAgent | undefined + +/** + * `requestHandler` for an AWS SDK v3 client that signs with a customer's credentials. When + * outbound routing is enabled, every request is routed by the outbound scope it is sent in, + * like `outboundFetch`: an organization with a dedicated egress gateway reaches AWS from its + * static IPs (passing `aws:SourceIp` conditions), any other route connects as + * {@link awsRequestHandler} would, and a call outside any scope goes direct with a warning. A + * blocked route or an unavailable gateway fails the request with an `OutboundRoutingError`. + * When routing is disabled this is exactly {@link awsRequestHandler}. + * + * Never use it for Sim's own AWS usage (its buckets, queues, email, secrets, metrics, or + * Bedrock on Sim's keys), which must not leave through a customer's gateway. + */ +export function organizationAwsRequestHandler( + options?: AwsRequestHandlerOptions +): ReturnType { + if (!isOutboundRoutingEnabled()) return awsRequestHandler(options) + routingAgent ??= new OrganizationRoutingAgent({ + keepAlive: true, + maxSockets: Number.POSITIVE_INFINITY, + }) + return new NodeHttpHandler({ ...options, httpAgent: routingAgent, httpsAgent: routingAgent }) +} diff --git a/apps/sim/lib/core/network/proxy-agent.server.ts b/apps/sim/lib/core/network/proxy-agent.server.ts index de68381bc39..bed3e5fab6d 100644 --- a/apps/sim/lib/core/network/proxy-agent.server.ts +++ b/apps/sim/lib/core/network/proxy-agent.server.ts @@ -11,7 +11,7 @@ import { HttpProxyAgent } from 'http-proxy-agent' import { HttpsProxyAgent } from 'https-proxy-agent' /** Matches the AWS SDK's own default agent, so swapping it in changes routing and nothing else. */ -const AGENT_OPTIONS: http.AgentOptions = { keepAlive: true, maxSockets: 50 } +export const AGENT_OPTIONS: http.AgentOptions = { keepAlive: true, maxSockets: 50 } /** * A Node agent for SDKs that bring their own HTTP stack (the AWS SDK, the OTLP exporters) and so @@ -46,13 +46,16 @@ class EnvironmentProxyAgent extends Agent { let sharedAgent: EnvironmentProxyAgent | null = null -/** The process-wide {@link EnvironmentProxyAgent}; only meaningful while a proxy is configured. */ -function environmentProxyAgent(): EnvironmentProxyAgent { +/** + * The process-wide {@link EnvironmentProxyAgent}. Without a configured proxy every connection + * takes its direct keep-alive agent. + */ +export function environmentProxyAgent(): EnvironmentProxyAgent { sharedAgent ??= new EnvironmentProxyAgent(AGENT_OPTIONS) return sharedAgent } -interface AwsRequestHandlerOptions { +export interface AwsRequestHandlerOptions { connectionTimeout?: number requestTimeout?: number throwOnRequestTimeout?: boolean diff --git a/apps/sim/lib/credentials/aws-service-account.server.ts b/apps/sim/lib/credentials/aws-service-account.server.ts index 513bd7ab74b..dd8c0c29795 100644 --- a/apps/sim/lib/credentials/aws-service-account.server.ts +++ b/apps/sim/lib/credentials/aws-service-account.server.ts @@ -9,7 +9,8 @@ import { db } from '@sim/db' import { credential } from '@sim/db/schema' import { and, eq, isNull, or } from 'drizzle-orm' import { z } from 'zod' -import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { failureClause, parseServiceAccountConfig } from '@/lib/credentials/service-account-config' import { @@ -195,7 +196,7 @@ export async function resolveAwsSessionCredentials( const client = new STSClient({ region, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), }) const response = await client.send( new AssumeRoleCommand({ @@ -221,7 +222,7 @@ export async function resolveAwsSessionCredentials( const client = new STSClient({ region, credentials: { accessKeyId: 'anonymous', secretAccessKey: 'anonymous' }, - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), }) const response = await client.send( new AssumeRoleWithWebIdentityCommand({ @@ -280,6 +281,8 @@ function bindSubject( function verificationFailure(config: AwsCredentialConfig, error: unknown): Error { // A keyless refusal already names its cause and code; wrapping it would lose both. if (error instanceof KeylessCredentialRefusedError) return error + // A refused or unavailable egress route says nothing about the credential. + if (error instanceof OutboundRoutingError) return error const detail = failureClause(error) if (config.authType === 'oidc_role') { return new AwsCredentialError( @@ -357,7 +360,7 @@ async function verifyCallerArn( const identity = await new STSClient({ region: stsRegionFor(config), credentials: session, - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), }).send(new GetCallerIdentityCommand({}), { abortSignal: signal }) if (!identity.Arn) throw new Error('AWS returned no caller identity') return identity.Arn diff --git a/apps/sim/lib/internal/aws/credentials.ts b/apps/sim/lib/internal/aws/credentials.ts index 1d24f170217..acf68a0c0f1 100644 --- a/apps/sim/lib/internal/aws/credentials.ts +++ b/apps/sim/lib/internal/aws/credentials.ts @@ -1,6 +1,6 @@ import { AsyncLocalStorage } from 'node:async_hooks' import { isRecordLike } from '@sim/utils/object' -import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' import { type AwsSessionCredentials, buildAwsRoleSessionName, @@ -70,13 +70,14 @@ export function awsSessionLifetime( /** * SDK client settings for a client built from an operation's key fields: its region, - * {@link awsClientCredentials}, and the request handler that honors `HTTPS_PROXY`. + * {@link awsClientCredentials}, and the request handler that sends through the organization's + * egress route (and honors `HTTPS_PROXY` on a direct one). */ export function awsClientConfig(region: string, accessKeyId: string, secretAccessKey: string) { return { region, credentials: awsClientCredentials(accessKeyId, secretAccessKey), - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), } } diff --git a/apps/sim/lib/internal/aws/failure.ts b/apps/sim/lib/internal/aws/failure.ts index ce5188ac2fc..a4f4e70f480 100644 --- a/apps/sim/lib/internal/aws/failure.ts +++ b/apps/sim/lib/internal/aws/failure.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { isRetryableNetworkError } from '@/lib/core/errors/retryable-infrastructure' +import { OutboundRoutingError } from '@/lib/core/network/routing' const THROTTLING_ERROR_NAMES = new Set(['Throttling', 'ThrottlingException']) @@ -8,9 +9,11 @@ const THROTTLING_ERROR_NAMES = new Set(['Throttling', 'ThrottlingException']) * Whether resolving a saved AWS credential failed because AWS was briefly unavailable rather * than because it denied the credential: an error STS models as retryable * (IDPCommunicationError, when STS could not reach Sim's OIDC issuer), throttling, an STS 5xx, - * or a network failure. The SDK has already retried these. + * a network failure, or the organization's egress route refusing or failing the connection. + * The SDK has already retried these. */ export function isTransientAwsFailure(error: unknown): boolean { + if (error instanceof OutboundRoutingError) return true if (!isRecordLike(error)) return false const status = isRecordLike(error.$metadata) ? error.$metadata.httpStatusCode : undefined return ( diff --git a/apps/sim/lib/internal/sts/client.ts b/apps/sim/lib/internal/sts/client.ts index aaf30eb5a6d..a3f0a30cd68 100644 --- a/apps/sim/lib/internal/sts/client.ts +++ b/apps/sim/lib/internal/sts/client.ts @@ -9,7 +9,7 @@ import { STSClient, type Tag, } from '@aws-sdk/client-sts' -import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' import { awsClientConfig } from '@/lib/internal/aws/credentials' import type { STSConnectionConfig } from '@/tools/sts/types' @@ -35,7 +35,7 @@ export function createUnauthenticatedSTSClient(region: string): STSClient { return new STSClient({ region, credentials: { accessKeyId: 'anonymous', secretAccessKey: 'anonymous' }, - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), }) } diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index 1cf2f8931b2..282bbdea3a1 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -17,6 +17,7 @@ import { sha256Hex } from '@sim/security/hash' import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { getAwsCredentials } from '@/lib/core/config/aws' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' import { validateAwsRegion } from '@/lib/core/security/input-validation' import type { IterationToolCall, NormalizedBlockOutput, StreamingExecution } from '@/executor/types' @@ -137,12 +138,17 @@ function enrichLastModelSegmentFromBedrockResponse( function resolveBedrockClientCredentials(request: ProviderRequest): { credentials: BedrockRuntimeClientConfig['credentials'] credentialKey: string | null + requestHandler: BedrockRuntimeClientConfig['requestHandler'] } { const session = request.bedrockSession if (session) { const { accessKeyId, secretAccessKey, sessionToken } = session if (sessionToken) { - return { credentials: { accessKeyId, secretAccessKey, sessionToken }, credentialKey: null } + return { + credentials: { accessKeyId, secretAccessKey, sessionToken }, + credentialKey: null, + requestHandler: organizationAwsRequestHandler(), + } } return { credentials: { accessKeyId, secretAccessKey }, @@ -151,6 +157,7 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { session.credentialId, sha256Hex([accessKeyId, secretAccessKey].join('\0')), ].join(':'), + requestHandler: organizationAwsRequestHandler(), } } @@ -172,9 +179,15 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { secretAccessKey: request.bedrockSecretKey, }, credentialKey: `${request.bedrockAccessKeyId}:${request.bedrockSecretKey}`, + requestHandler: organizationAwsRequestHandler(), } } - return { credentials: getAwsCredentials(), credentialKey: 'default-chain' } + // The deployment's own credentials stay on Sim's network, never an organization's route. + return { + credentials: getAwsCredentials(), + credentialKey: 'default-chain', + requestHandler: awsRequestHandler(), + } } export const bedrockProvider: ProviderConfig = { @@ -206,9 +219,8 @@ export const bedrockProvider: ProviderConfig = { region, }) - const { credentials, credentialKey } = resolveBedrockClientCredentials(request) - const buildClient = () => - new BedrockRuntimeClient({ region, credentials, requestHandler: awsRequestHandler() }) + const { credentials, credentialKey, requestHandler } = resolveBedrockClientCredentials(request) + const buildClient = () => new BedrockRuntimeClient({ region, credentials, requestHandler }) const client = credentialKey === null ? buildClient() From 30e1dd694bcd6ca4f4be6bf1e1eb9716c6de51a4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:11:08 -0800 Subject: [PATCH 07/32] improvement(egress): enforce organization-routed requests with check:organization-egress --- .../embeddings/ollama-model-catalog.server.ts | 1 + .../openrouter-model-catalog.server.ts | 1 + apps/sim/providers/litellm/index.ts | 1 + apps/sim/providers/ollama/index.ts | 1 + apps/sim/providers/openrouter/utils.ts | 1 + apps/sim/providers/vllm/index.ts | 1 + package.json | 1 + scripts/check-organization-egress.test.ts | 41 ++++ scripts/check-organization-egress.ts | 188 ++++++++++++++++++ 9 files changed, 236 insertions(+) create mode 100644 scripts/check-organization-egress.test.ts create mode 100644 scripts/check-organization-egress.ts diff --git a/apps/sim/lib/embeddings/ollama-model-catalog.server.ts b/apps/sim/lib/embeddings/ollama-model-catalog.server.ts index 75008e1f4be..6424236a002 100644 --- a/apps/sim/lib/embeddings/ollama-model-catalog.server.ts +++ b/apps/sim/lib/embeddings/ollama-model-catalog.server.ts @@ -70,6 +70,7 @@ export class OllamaEmbeddingWidthUnknownError extends Error { } async function fetchOllamaJson(path: string, init: RequestInit, signal?: AbortSignal) { + // egress-direct: operator-local Ollama server named by the deployment const response = await fetch(`${getOllamaUrl().replace(/\/+$/, '')}${path}`, { headers: { 'Content-Type': 'application/json' }, signal, diff --git a/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts b/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts index b65f499b892..d7f5d93d262 100644 --- a/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts +++ b/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts @@ -24,6 +24,7 @@ export class OpenRouterEmbeddingModelNotFoundError extends Error { export async function fetchOpenRouterEmbeddingModelCatalog( signal?: AbortSignal ): Promise { + // egress-direct: public model catalog, read without a workspace const response = await fetch(OPENROUTER_EMBEDDING_MODELS_URL, { headers: { 'Content-Type': 'application/json' }, next: { revalidate: 300 }, diff --git a/apps/sim/providers/litellm/index.ts b/apps/sim/providers/litellm/index.ts index 913816b5e60..8cca38eff5e 100644 --- a/apps/sim/providers/litellm/index.ts +++ b/apps/sim/providers/litellm/index.ts @@ -81,6 +81,7 @@ export const litellmProvider: ProviderConfig = { headers.Authorization = `Bearer ${env.LITELLM_API_KEY}` } + // egress-direct: operator's own LiteLLM proxy named by the deployment const response = await fetch(`${baseUrl}/v1/models`, { headers }) if (!response.ok) { await response.text().catch(() => {}) diff --git a/apps/sim/providers/ollama/index.ts b/apps/sim/providers/ollama/index.ts index c15f79870a7..36eb784975e 100644 --- a/apps/sim/providers/ollama/index.ts +++ b/apps/sim/providers/ollama/index.ts @@ -28,6 +28,7 @@ export const ollamaProvider: ProviderConfig = { } try { + // egress-direct: operator-local Ollama server named by the deployment const response = await fetch(`${OLLAMA_HOST}/api/tags`) if (!response.ok) { await response.text().catch(() => {}) diff --git a/apps/sim/providers/openrouter/utils.ts b/apps/sim/providers/openrouter/utils.ts index fd65eff65ef..1dd6bc7b47d 100644 --- a/apps/sim/providers/openrouter/utils.ts +++ b/apps/sim/providers/openrouter/utils.ts @@ -23,6 +23,7 @@ const MODEL_CAPABILITIES_TIMEOUT_MS = 5000 async function fetchModelCapabilities(): Promise | undefined> { try { + // egress-direct: public model catalog, read without a workspace const response = await fetch('https://openrouter.ai/api/v1/models', { headers: { 'Content-Type': 'application/json' }, signal: AbortSignal.timeout(MODEL_CAPABILITIES_TIMEOUT_MS), diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index 0fac82e9eaa..c98fdef1dcd 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -86,6 +86,7 @@ export const vllmProvider: ProviderConfig = { headers.Authorization = `Bearer ${env.VLLM_API_KEY}` } + // egress-direct: operator's own vLLM server named by the deployment const response = await fetch(`${apiBaseUrl}/models`, { headers }) if (!response.ok) { await response.text().catch(() => {}) diff --git a/package.json b/package.json index 9bf33d89b40..5bfee8991b3 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "lint:all": "bun run lint && bun run lint:helm", "check": "bun run format:check", "check:egress-boundary": "bun run scripts/check-egress-boundary.ts", + "check:organization-egress": "bun run scripts/check-organization-egress.ts", "check:boundaries": "bun run scripts/check-monorepo-boundaries.ts", "check:api-validation": "bun run scripts/check-api-validation-contracts.ts --check", "check:api-contract-routes": "bun run scripts/check-api-contract-routes.ts", diff --git a/scripts/check-organization-egress.test.ts b/scripts/check-organization-egress.test.ts new file mode 100644 index 00000000000..37780d7a599 --- /dev/null +++ b/scripts/check-organization-egress.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest' +import { findDirectFetches } from './check-organization-egress' + +const lines = (source: string) => findDirectFetches('file.ts', source).map((hit) => hit.line) + +describe('organization egress scan', () => { + it('reports the global fetch and globalThis.fetch', () => { + expect( + lines(`const a = await fetch(url) +const b = await globalThis.fetch(url)`) + ).toEqual([1, 2]) + }) + + it('ignores a client method or a local binding named fetch', () => { + expect( + lines(`class Client { async fetch(path: string) { return this.fetch(path) } } +await client.fetch(url) +async function run(fetch: typeof globalThis.fetch) { return fetch(url) } +import { fetch } from 'undici' +await fetch(url)`) + ).toEqual([]) + }) + + it('accepts an egress-direct annotation within two lines above the call', () => { + expect( + lines(`// egress-direct: operator-local Ollama server +const a = await fetch(url, { +}) +// egress-direct: public catalog + +const b = await fetch(url) + +const c = 1 +const d = await fetch(url)`) + ).toEqual([9]) + }) + + it('ignores fetch named in strings and comments', () => { + expect(lines(`const description = 'fetch() is available'\n// call fetch(url) here`)).toEqual([]) + }) +}) diff --git a/scripts/check-organization-egress.ts b/scripts/check-organization-egress.ts new file mode 100644 index 00000000000..1b7a4cbdd6a --- /dev/null +++ b/scripts/check-organization-egress.ts @@ -0,0 +1,188 @@ +#!/usr/bin/env bun +/** + * Keeps requests made on a workspace's behalf on the organization's egress route. + * + * An organization with a dedicated egress gateway gets static IPs that its vendors and its own + * tenants allowlist. That only holds if Sim's request reaches the network through + * `outboundFetch` (`apps/sim/lib/core/network/outbound-fetch.server.ts`), or one of the + * SSRF-guarded helpers, which pick the gateway from the current outbound scope. The global + * `fetch` never does: a token mint, OAuth refresh or tool call written with it leaves from Sim's + * shared IPs, and a customer who locked their endpoint to their own IPs sees a 403 that no test + * catches, because the code works everywhere else. + * + * So inside the directories whose code acts for a workspace (credentials, OAuth, integration + * clients, tools, webhooks, providers and connectors), a call to the global `fetch` fails this + * check. A call that rightly goes direct (an operator-local server such as Ollama, or a public + * catalog read with no workspace behind it) takes `// egress-direct: ` on one of the two + * lines above it. A method named `fetch` on a client class is not the global and is ignored. + * + * Parsed with the TypeScript AST, so `fetch` inside strings, comments and tool descriptions is + * never reported. The tree was swept when this landed, so there is no baseline. + * + * Usage: bun run check:organization-egress + */ +import type { Dirent } from 'node:fs' +import { readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import ts from '@typescript/typescript6' + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') + +/** Code under these directories makes requests for a workspace or organization. */ +const SCAN_DIRS = [ + 'apps/sim/connectors', + 'apps/sim/providers', + 'apps/sim/tools', + 'apps/sim/lib/auth/connectors', + 'apps/sim/lib/credential-groups', + 'apps/sim/lib/credentials', + 'apps/sim/lib/data-drains', + 'apps/sim/lib/embeddings', + 'apps/sim/lib/internal', + 'apps/sim/lib/knowledge', + 'apps/sim/lib/managed-agents', + 'apps/sim/lib/media', + 'apps/sim/lib/oauth', + 'apps/sim/lib/selectors', + 'apps/sim/lib/webhooks', +] + +const SKIP_DIRS = new Set(['node_modules', 'dist', '.next', '.turbo', 'coverage', 'client']) +const ANNOTATION = '// egress-direct:' +const ANNOTATION_REACH = 2 + +export interface DirectFetch { + line: number + text: string +} + +function walk(dir: string, out: string[] = []): string[] { + let entries: Dirent[] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + throw new Error(`check-organization-egress: SCAN_DIRS entry "${dir}" does not exist`) + } + for (const entry of entries) { + if (SKIP_DIRS.has(entry.name)) continue + const full = path.join(dir, entry.name) + if (entry.isDirectory()) walk(full, out) + else if (/\.ts$/.test(entry.name) && !/\.(test|integration|live\.test)\.ts$/.test(entry.name)) + out.push(full) + } + return out +} + +function bindsName(name: ts.BindingName, target: string): boolean { + if (ts.isIdentifier(name)) return name.text === target + return name.elements.some( + (element) => !ts.isOmittedExpression(element) && bindsName(element.name, target) + ) +} + +/** Whether this scope node itself declares a binding called `target`. */ +function declares(scope: ts.Node, target: string): boolean { + if (ts.isFunctionLike(scope)) { + if (scope.parameters.some((parameter) => bindsName(parameter.name, target))) return true + } + const statements = + ts.isSourceFile(scope) || ts.isBlock(scope) || ts.isModuleBlock(scope) ? scope.statements : [] + return statements.some((statement) => { + if (ts.isVariableStatement(statement)) { + return statement.declarationList.declarations.some((d) => bindsName(d.name, target)) + } + if (ts.isFunctionDeclaration(statement) || ts.isClassDeclaration(statement)) { + return statement.name?.text === target + } + if (ts.isImportDeclaration(statement)) { + const clause = statement.importClause + if (!clause) return false + if (clause.name?.text === target) return true + const bindings = clause.namedBindings + if (!bindings) return false + if (ts.isNamespaceImport(bindings)) return bindings.name.text === target + return bindings.elements.some((element) => element.name.text === target) + } + return false + }) +} + +/** Whether `name` here refers to a local declaration rather than the global. */ +function shadowsGlobal(node: ts.Node, name: string): boolean { + for (let scope: ts.Node | undefined = node.parent; scope; scope = scope.parent) { + if (declares(scope, name)) return true + } + return false +} + +/** + * Every call to the global `fetch` (`fetch(…)`, `globalThis.fetch(…)`) without an + * `egress-direct` annotation on one of the two lines above it. + */ +export function findDirectFetches(file: string, source: string): DirectFetch[] { + if (!source.includes('fetch')) return [] + const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true) + const lines = source.split('\n') + const found: DirectFetch[] = [] + + const visit = (node: ts.Node): void => { + if (ts.isCallExpression(node)) { + const callee = node.expression + const isGlobalFetch = + (ts.isIdentifier(callee) && callee.text === 'fetch' && !shadowsGlobal(callee, 'fetch')) || + (ts.isPropertyAccessExpression(callee) && + callee.name.text === 'fetch' && + ts.isIdentifier(callee.expression) && + callee.expression.text === 'globalThis') + if (isGlobalFetch) { + const line = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line + const annotated = lines + .slice(Math.max(0, line - ANNOTATION_REACH), line) + .some((text) => text.trim().startsWith(ANNOTATION)) + if (!annotated) found.push({ line: line + 1, text: lines[line].trim() }) + } + } + ts.forEachChild(node, visit) + } + + visit(sourceFile) + return found +} + +function main() { + const violations: Array = [] + let scanned = 0 + for (const scanDir of SCAN_DIRS) { + for (const file of walk(path.join(ROOT, scanDir))) { + scanned++ + const rel = path.relative(ROOT, file).split(path.sep).join('/') + for (const hit of findDirectFetches(rel, readFileSync(file, 'utf8'))) { + violations.push({ file: rel, ...hit }) + } + } + } + + if (violations.length === 0) { + console.log( + `✓ check-organization-egress: ${scanned} files, every request is organization-routed` + ) + process.exit(0) + } + + console.error('✗ check-organization-egress: global fetch in code that acts for a workspace\n') + for (const violation of violations) { + console.error(` ${violation.file}:${violation.line}`) + console.error(` ${violation.text}`) + } + console.error( + "\n The global fetch always leaves from Sim's shared IPs, so an organization with a\n" + + ' dedicated egress gateway loses its static IPs for this request. Use outboundFetch from\n' + + ' @/lib/core/network/outbound-fetch.server (same signature as fetch). If the destination\n' + + " is genuinely not on the organization's behalf (an operator-local server, a public\n" + + ' catalog), mark the call with `// egress-direct: ` on the line above it.\n' + ) + process.exit(1) +} + +if (import.meta.main) main() From 79fe874f8df9c001d10f49270a8c72d6422e8357 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:10:35 -0800 Subject: [PATCH 08/32] improvement(egress): route oauth token traffic through the organization gateway - refreshOAuthToken (incl. the Instagram long-lived refresh) and its provider config move to the server-only lib/oauth/refresh.server.ts, so the outboundFetch import never enters client bundles through lib/oauth/oauth.ts - vendor token exchange, refresh, userinfo, revoke, and discovery calls in lib/oauth, lib/auth/connectors, and lib/atlassian use outboundFetch - Instagram, Trello, and Shopify connect callbacks resolve their outbound scope from the caller's connect draft owner - the Teams subscription renewal cron scopes each subscription to its workflow's workspace organization - remove the unused /api/auth/oauth/token route, its contracts, and the delegation-header authenticator only it used --- .../app/api/auth/oauth/token/route.test.ts | 543 --------- apps/sim/app/api/auth/oauth/token/route.ts | 231 ---- apps/sim/app/api/auth/oauth/utils.test.ts | 4 +- .../auth/oauth2/callback/instagram/route.ts | 37 +- apps/sim/app/api/auth/trello/store/route.ts | 11 +- .../cron/renew-subscriptions/route.test.ts | 78 +- .../app/api/cron/renew-subscriptions/route.ts | 208 ++-- apps/sim/executor/utils/credential-token.ts | 7 +- .../lib/api/contracts/oauth-connections.ts | 59 +- apps/sim/lib/atlassian/discovery.ts | 3 +- apps/sim/lib/auth/connectors/managed-oauth.ts | 17 +- apps/sim/lib/auth/connectors/providers.ts | 118 +- .../standard-oauth-provider.ts | 2 +- .../application/managed-oauth-delegation.ts | 40 +- apps/sim/lib/oauth/apollo.ts | 4 +- apps/sim/lib/oauth/atlassian-token.ts | 3 +- .../lib/oauth/connect-scope.server.test.ts | 71 ++ apps/sim/lib/oauth/connect-scope.server.ts | 29 + apps/sim/lib/oauth/credential-service.test.ts | 5 +- apps/sim/lib/oauth/credential-service.ts | 7 +- .../github-installation-credential.test.ts | 2 +- apps/sim/lib/oauth/github-installation.ts | 3 +- .../sim/lib/oauth/github-repositories.test.ts | 2 +- apps/sim/lib/oauth/github-repositories.ts | 5 +- .../oauth/google-service-account-transport.ts | 3 +- apps/sim/lib/oauth/greenhouse.ts | 4 +- apps/sim/lib/oauth/monday.ts | 3 +- apps/sim/lib/oauth/oauth.test.ts | 109 +- apps/sim/lib/oauth/oauth.ts | 1060 +--------------- apps/sim/lib/oauth/quickbooks.ts | 7 +- apps/sim/lib/oauth/refresh.server.ts | 1069 +++++++++++++++++ .../lib/oauth/salesforce-instance.server.ts | 3 +- apps/sim/lib/oauth/shopify-installation.ts | 5 +- apps/sim/lib/oauth/shopify.ts | 8 +- apps/sim/lib/oauth/token-resolution.test.ts | 1 - apps/sim/lib/oauth/token-resolution.ts | 15 +- scripts/check-test-types.baseline.json | 6 - scripts/check-unused-exports.baseline.json | 4 - 38 files changed, 1573 insertions(+), 2213 deletions(-) delete mode 100644 apps/sim/app/api/auth/oauth/token/route.test.ts delete mode 100644 apps/sim/app/api/auth/oauth/token/route.ts create mode 100644 apps/sim/lib/oauth/connect-scope.server.test.ts create mode 100644 apps/sim/lib/oauth/connect-scope.server.ts create mode 100644 apps/sim/lib/oauth/refresh.server.ts diff --git a/apps/sim/app/api/auth/oauth/token/route.test.ts b/apps/sim/app/api/auth/oauth/token/route.test.ts deleted file mode 100644 index 9152d3bfa1d..00000000000 --- a/apps/sim/app/api/auth/oauth/token/route.test.ts +++ /dev/null @@ -1,543 +0,0 @@ -/** - * Tests for OAuth token API routes - */ -import { - authOAuthUtilsMock, - authOAuthUtilsMockFns, - createMockRequest, - hybridAuthMockFns, -} from '@sim/testing' -import { createRouteContext } from '@sim/testing/helpers/http' -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockAuthenticateManagedOAuthDelegation, - mockAuthorizeCredentialUse, - mockResolveManagedOAuthCredentialToken, -} = vi.hoisted(() => ({ - mockAuthenticateManagedOAuthDelegation: vi.fn(), - mockAuthorizeCredentialUse: vi.fn(), - mockResolveManagedOAuthCredentialToken: vi.fn(), -})) - -vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) - -vi.mock('@/lib/auth/credential-access', () => ({ - authorizeCredentialUse: mockAuthorizeCredentialUse, - authorizeCredentialUseForAuth: mockAuthorizeCredentialUse, -})) - -vi.mock('@/lib/credentials/application/managed-oauth-delegation', () => ({ - authenticateManagedOAuthDelegation: mockAuthenticateManagedOAuthDelegation, - InvalidManagedOAuthDelegationError: class InvalidManagedOAuthDelegationError extends Error {}, -})) - -vi.mock('@/lib/credentials/application/resolve-managed-oauth-token', () => ({ - resolveManagedOAuthCredentialToken: { execute: mockResolveManagedOAuthCredentialToken }, -})) - -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { createQuickBooksAccountId } from '@/lib/oauth/quickbooks' -import { GET, POST } from '@/app/api/auth/oauth/token/route' -import { getToolMetadata } from '@/tools/metadata' - -const mockGetToolMetadata = vi.mocked(getToolMetadata) -const mockCheckSessionOrInternalAuth = vi.mocked(checkSessionOrInternalAuth) - -describe('OAuth Token API Routes', () => { - beforeEach(() => { - authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValue(null) - }) - - /** - * POST route tests - */ - describe('POST handler', () => { - it('returns realmId only for QuickBooks credentials', async () => { - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - authType: 'session', - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'owner-user-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accountId: createQuickBooksAccountId( - '123456789', - 'intuit-subject-01234567-89ab-4def-8abc-0123456789ab', - { - clientId: 'client-id', - environment: 'sandbox', - } - ), - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId: 'quickbooks', - }) - authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValueOnce({ - accessToken: 'fresh-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest('POST', { - credentialId: 'credential-id', - }) - ) - - expect(response.status).toBe(200) - expect(await response.json()).toEqual({ - accessToken: 'fresh-token', - credentialType: 'oauth', - realmId: '123456789', - quickBooksEnvironment: 'sandbox', - }) - }) - - it('rejects a malformed QuickBooks company identity with reconnect guidance', async () => { - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - authType: 'session', - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'owner-user-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accountId: 'malformed', - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId: 'quickbooks', - }) - - const response = await POST( - createMockRequest('POST', { - credentialId: 'credential-id', - }) - ) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toMatch(/Reconnect the QuickBooks credential/) - expect(authOAuthUtilsMockFns.mockRefreshTokenIfNeeded).not.toHaveBeenCalled() - }) - - it('does not authenticate managed delegation for an ordinary OAuth credential', async () => { - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - authType: 'internal_jwt', - requesterUserId: 'workflow-owner-id', - credentialOwnerUserId: 'workflow-owner-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId: 'google', - }) - authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValueOnce({ - accessToken: 'fresh-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest( - 'POST', - { credentialId: 'credential-id', workflowId: 'workflow-id' }, - { 'x-sim-managed-oauth-delegation': 'Bearer stale-delegation' } - ) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ accessToken: 'fresh-token' }) - expect(mockAuthenticateManagedOAuthDelegation).not.toHaveBeenCalled() - }) - - describe('managed OAuth path', () => { - const managedCredential = { - accountId: '', - credentialId: 'managed-credential-id', - credentialType: 'managed_oauth', - providerId: 'google-email', - workspaceId: 'workspace-id', - usedCredentialTable: true, - } - - beforeEach(() => { - authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce(managedCredential) - mockGetToolMetadata.mockReturnValue({ - oauth: { - required: true, - provider: 'google-email', - requiredScopes: ['https://www.googleapis.com/auth/gmail.readonly'], - }, - }) - }) - - it('fails closed when workflow delegation is missing', async () => { - const response = await POST( - createMockRequest('POST', { - credentialId: 'managed-credential-id', - toolId: 'gmail_read', - }) - ) - - expect(response.status).toBe(403) - await expect(response.json()).resolves.toMatchObject({ - code: 'MANAGED_CREDENTIAL_DELEGATION_REQUIRED', - }) - expect(mockResolveManagedOAuthCredentialToken).not.toHaveBeenCalled() - }) - - it('resolves a manually supplied managed credential ID with scoped delegation', async () => { - const principal = { - kind: 'delegated' as const, - serviceId: 'executor' as const, - subjectUserId: 'user-id', - workspaceId: 'workspace-id', - delegationId: 'delegation-id', - audience: 'sim:managed-oauth-credentials', - issuedAt: new Date(Date.now() - 1_000), - expiresAt: new Date(Date.now() + 60_000), - resourceScope: { credentialId: 'managed-credential-id' }, - delegationContext: { - kind: 'workflow_execution' as const, - workflowId: 'workflow-id', - }, - } - mockAuthenticateManagedOAuthDelegation.mockResolvedValueOnce(principal) - mockResolveManagedOAuthCredentialToken.mockResolvedValueOnce({ - accessToken: 'managed-access-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest( - 'POST', - { credentialId: 'managed-credential-id', toolId: 'gmail_read' }, - { 'x-sim-managed-oauth-delegation': 'Bearer delegated-token' } - ) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - accessToken: 'managed-access-token', - credentialType: 'managed_oauth', - }) - expect(mockResolveManagedOAuthCredentialToken).toHaveBeenCalledWith({ - principal, - input: { - credentialId: 'managed-credential-id', - expectedProviderId: 'google-email', - requiredScopes: ['https://www.googleapis.com/auth/gmail.readonly'], - toolId: 'gmail_read', - }, - request: expect.any(NextRequest), - }) - }) - - it('uses the trusted provider scope policy when a Slack tool omits narrower scopes', async () => { - mockGetToolMetadata.mockReturnValueOnce({ - oauth: { - required: true, - provider: 'slack', - }, - }) - const principal = { - kind: 'delegated' as const, - serviceId: 'executor' as const, - subjectUserId: 'user-id', - workspaceId: 'workspace-id', - delegationId: 'delegation-id', - audience: 'sim:managed-oauth-credentials', - issuedAt: new Date(Date.now() - 1_000), - expiresAt: new Date(Date.now() + 60_000), - resourceScope: { credentialId: 'managed-credential-id' }, - delegationContext: { - kind: 'workflow_execution' as const, - workflowId: 'workflow-id', - }, - } - mockAuthenticateManagedOAuthDelegation.mockResolvedValueOnce(principal) - mockResolveManagedOAuthCredentialToken.mockResolvedValueOnce({ - accessToken: 'managed-slack-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest( - 'POST', - { credentialId: 'managed-credential-id', toolId: 'slack_message' }, - { 'x-sim-managed-oauth-delegation': 'Bearer delegated-token' } - ) - ) - - expect(response.status).toBe(200) - expect(mockResolveManagedOAuthCredentialToken).toHaveBeenCalledWith({ - principal, - input: { - credentialId: 'managed-credential-id', - expectedProviderId: 'slack', - requiredScopes: expect.arrayContaining([ - 'channels:read', - 'channels:history', - 'chat:write', - ]), - toolId: 'slack_message', - }, - request: expect.any(NextRequest), - }) - }) - }) - - describe('saved service-account credentials', () => { - const authorized = { - ok: true, - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'test-user-id', - workspaceId: 'workspace-id', - } - - beforeEach(() => { - authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce({ - accountId: '', - credentialId: 'saved-credential-id', - credentialType: 'service_account', - providerId: 'vanta-service-account', - workspaceId: 'workspace-id', - usedCredentialTable: true, - }) - authOAuthUtilsMockFns.mockResolveServiceAccountToken.mockResolvedValue({ - accessToken: 'saved-secret', - }) - mockGetToolMetadata.mockReturnValue({ - id: 'vanta_list_tests', - params: {}, - oauth: { required: true, provider: 'vanta', credentialKind: 'service-account' }, - }) - }) - - it('refuses to release the secret to a session caller', async () => { - mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: AuthType.SESSION, - userId: 'test-user-id', - }) - mockAuthorizeCredentialUse.mockResolvedValueOnce(authorized) - - const response = await POST( - createMockRequest('POST', { - credentialId: 'saved-credential-id', - toolId: 'vanta_list_tests', - }), - createRouteContext({}) - ) - - expect(response.status).toBe(403) - const body = await response.json() - expect(body).toMatchObject({ code: 'SERVICE_ACCOUNT_SESSION_FORBIDDEN' }) - expect(JSON.stringify(body)).not.toContain('saved-secret') - expect(authOAuthUtilsMockFns.mockResolveServiceAccountToken).not.toHaveBeenCalled() - }) - - it('resolves the secret for an internal-JWT execution caller', async () => { - mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: AuthType.INTERNAL_JWT, - userId: 'test-user-id', - }) - mockAuthorizeCredentialUse.mockResolvedValueOnce(authorized) - - const response = await POST( - createMockRequest('POST', { - credentialId: 'saved-credential-id', - toolId: 'vanta_list_tests', - workflowId: 'workflow-id', - }), - createRouteContext({}) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - accessToken: 'saved-secret', - credentialType: 'service_account', - }) - }) - }) - - it('still returns an OAuth token to a session caller', async () => { - mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: AuthType.SESSION, - userId: 'test-user-id', - }) - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'test-user-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId: 'google', - }) - authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValueOnce({ - accessToken: 'session-oauth-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest('POST', { credentialId: 'credential-id' }), - createRouteContext({}) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - accessToken: 'session-oauth-token', - credentialType: 'oauth', - }) - }) - - describe('credentialAccountUserId + providerId path', () => { - it('should reject internal JWT authentication', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: 'internal_jwt', - userId: 'test-user-id', - }) - - const req = createMockRequest('POST', { - credentialAccountUserId: 'test-user-id', - providerId: 'google', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data).toHaveProperty('error', 'User not authenticated') - expect(authOAuthUtilsMockFns.mockGetOAuthToken).not.toHaveBeenCalled() - }) - - it('should reject requests for other users credentials', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: 'session', - userId: 'attacker-user-id', - }) - - const req = createMockRequest('POST', { - credentialAccountUserId: 'victim-user-id', - providerId: 'google', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data).toHaveProperty('error', 'Unauthorized') - expect(authOAuthUtilsMockFns.mockGetOAuthToken).not.toHaveBeenCalled() - }) - }) - }) - - /** - * GET route tests - */ - describe('GET handler', () => { - it('rejects a malformed QuickBooks identity before reporting a missing token', async () => { - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - authType: 'session', - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'test-user-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accountId: 'malformed', - accessToken: null, - refreshToken: 'refresh-token', - providerId: 'quickbooks', - }) - - const response = await GET( - createMockRequest( - 'GET', - undefined, - {}, - 'http://localhost:3000/api/auth/oauth/token?credentialId=credential-id' - ) - ) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toMatch(/Reconnect the QuickBooks credential/) - expect(authOAuthUtilsMockFns.mockRefreshTokenIfNeeded).not.toHaveBeenCalled() - }) - }) -}) - -describe('Salesforce instance URL resolution', () => { - const INSTANCE = 'https://acme--sbx.sandbox.my.salesforce.com' - - beforeEach(() => { - authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValue(null) - mockAuthorizeCredentialUse.mockResolvedValue({ - ok: true, - authType: 'session', - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'owner-user-id', - }) - authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValue({ - accessToken: 'fresh-token', - refreshed: false, - }) - }) - - /** - * The org host is smuggled through `scope` because the token response has - * nowhere to put it; the tools read it back as their `instanceUrl` param. - */ - function credentialForProvider(providerId: string) { - return { - id: 'credential-id', - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId, - scope: `__sf_instance__:${INSTANCE} api refresh_token openid`, - } - } - - it.each(['salesforce', 'salesforce-sandbox'])( - 'returns the stored instance URL for a %s credential', - async (providerId) => { - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce( - credentialForProvider(providerId) - ) - - const response = await POST(createMockRequest('POST', { credentialId: 'credential-id' })) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.instanceUrl).toBe(INSTANCE) - } - ) - - it('omits instanceUrl for a non-Salesforce provider carrying a lookalike scope', async () => { - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - ...credentialForProvider('google'), - }) - - const response = await POST(createMockRequest('POST', { credentialId: 'credential-id' })) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.instanceUrl).toBeUndefined() - }) -}) diff --git a/apps/sim/app/api/auth/oauth/token/route.ts b/apps/sim/app/api/auth/oauth/token/route.ts deleted file mode 100644 index 99e7dcc986b..00000000000 --- a/apps/sim/app/api/auth/oauth/token/route.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - MANAGED_OAUTH_DELEGATION_HEADER, - oauthTokenGetContract, - oauthTokenPostContract, -} from '@/lib/api/contracts/oauth-connections' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { authenticateManagedOAuthDelegation } from '@/lib/credentials/application/managed-oauth-delegation' -import { getCredential, getOAuthToken } from '@/lib/oauth/credential-service' -import { - completeOAuthCredentialToken, - resolveCredentialAccessToken, - validateOAuthCredentialContext, -} from '@/lib/oauth/token-resolution' -import { captureServerEvent } from '@/lib/posthog/server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OAuthTokenAPI') - -/** - * Get an access token for a specific credential - * Supports both session-based authentication (for client-side requests) - * and workflow-based authentication (for server-side requests) - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - logger.info('OAuth token API POST request received') - - try { - const parsed = await parseRequest( - oauthTokenPostContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn('Invalid token request', { errors: error.issues }) - return NextResponse.json( - { error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const { - credentialId, - credentialAccountUserId, - providerId, - toolId, - workflowId, - scopes, - impersonateEmail, - } = parsed.data.body - const callerUserId = parsed.data.query.userId - - if (credentialAccountUserId && providerId) { - logger.info('Fetching token by credentialAccountUserId + providerId', { - credentialAccountUserId, - providerId, - }) - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || auth.authType !== AuthType.SESSION || !auth.userId) { - logger.warn('Unauthorized request for credentialAccountUserId path', { - success: auth.success, - authType: auth.authType, - }) - return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) - } - - if (auth.userId !== credentialAccountUserId) { - logger.warn( - `User ${auth.userId} attempted to access credentials for ${credentialAccountUserId}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 403 }) - } - - try { - const accessToken = await getOAuthToken(credentialAccountUserId, providerId) - if (!accessToken) { - return NextResponse.json( - { - error: `No credential found for user ${credentialAccountUserId} and provider ${providerId}`, - }, - { status: 404 } - ) - } - - recordAudit({ - actorId: auth.userId, - action: AuditAction.CREDENTIAL_ACCESSED, - resourceType: AuditResourceType.CREDENTIAL, - resourceId: providerId, - description: `Accessed OAuth credential for provider ${providerId}`, - metadata: { - provider: providerId, - credentialType: 'oauth', - credentialAccountUserId, - }, - request, - }) - captureServerEvent(auth.userId, 'credential_used', { - credential_type: 'oauth', - provider_id: providerId, - }) - - return NextResponse.json({ accessToken }, { status: 200 }) - } catch (error) { - const message = getErrorMessage(error, 'Failed to get OAuth token') - logger.warn(`OAuth token error: ${message}`) - return NextResponse.json({ error: message }, { status: 403 }) - } - } - - const managedOAuthDelegation = parsed.data.headers?.[MANAGED_OAUTH_DELEGATION_HEADER] - const result = await resolveCredentialAccessToken({ - requestId, - credentialId, - workflowId: workflowId ?? undefined, - toolId, - scopes, - impersonateEmail, - callerUserId, - auditRequest: request, - authenticate: () => checkSessionOrInternalAuth(request, { requireWorkflowId: false }), - resolveManagedPrincipal: managedOAuthDelegation - ? (managedCredentialId: string) => - authenticateManagedOAuthDelegation(managedOAuthDelegation, managedCredentialId) - : undefined, - }) - - if (!result.ok) { - return NextResponse.json( - { ...(result.code ? { code: result.code } : {}), error: result.error }, - { status: result.status } - ) - } - - return NextResponse.json(result.token, { status: 200 }) - } catch (error) { - logger.error('Error getting access token', error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } -}) - -/** - * Get the access token for a specific credential - */ -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const parsed = await parseRequest( - oauthTokenGetContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn('Invalid query parameters', { errors: error.issues }) - return NextResponse.json( - { error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const { credentialId } = parsed.data.query - - const authz = await authorizeCredentialUse(request, { - credentialId, - requireWorkflowIdForInternal: false, - }) - if (!authz.ok || authz.authType !== AuthType.SESSION || !authz.credentialOwnerUserId) { - return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) - } - - const resolvedCredentialId = authz.resolvedCredentialId || credentialId - const credential = await getCredential( - requestId, - resolvedCredentialId, - authz.credentialOwnerUserId - ) - - if (!credential) { - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) - } - - const contextValidation = validateOAuthCredentialContext(credential) - if (!contextValidation.ok) { - return NextResponse.json({ error: contextValidation.error }, { status: 401 }) - } - - if (!credential.accessToken) { - logger.warn('No access token available for credential') - return NextResponse.json({ error: 'No access token available' }, { status: 400 }) - } - - const result = await completeOAuthCredentialToken({ - requestId, - credential, - resolvedCredentialId, - actorId: authz.requesterUserId, - workspaceId: authz.workspaceId ?? null, - auditRequest: request, - }) - - if (!result.ok) { - return NextResponse.json( - { ...(result.code ? { code: result.code } : {}), error: result.error }, - { status: result.status } - ) - } - - return NextResponse.json(result.token, { status: 200 }) - } catch (error) { - logger.error('Error fetching access token', error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/auth/oauth/utils.test.ts b/apps/sim/app/api/auth/oauth/utils.test.ts index 7ac0342bf58..37b6ee5755f 100644 --- a/apps/sim/app/api/auth/oauth/utils.test.ts +++ b/apps/sim/app/api/auth/oauth/utils.test.ts @@ -7,10 +7,10 @@ import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/oauth/oauth', () => ({ - refreshOAuthToken: vi.fn(), OAUTH_PROVIDERS: {}, TOKEN_REFRESH_TIMEOUT_MS: 15_000, })) +vi.mock('@/lib/oauth/refresh.server', () => ({ refreshOAuthToken: vi.fn() })) vi.mock('@/lib/core/security/encryption', () => encryptionMock) @@ -23,8 +23,8 @@ vi.mock('@/lib/credentials/client-credential-accounts/server', () => ({ import { db } from '@sim/db' import { __resetCoalesceLocallyForTests } from '@/lib/concurrency/singleflight' import { ZOOM_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/credentials/client-credential-accounts/descriptors' -import { refreshOAuthToken } from '@/lib/oauth' import { refreshTokenIfNeeded, resolveServiceAccountToken } from '@/lib/oauth/credential-service' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' import { getOAuthRefreshCoordinationIdentity } from '@/lib/oauth/refresh-coordination' import { GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' diff --git a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts index 62603299bbc..4c294f427e8 100644 --- a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts +++ b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts @@ -9,6 +9,7 @@ import { instagramCallbackContract } from '@/lib/api/contracts/oauth-connections import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -19,6 +20,7 @@ import { isSameOrigin } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processCredentialDraft } from '@/lib/credentials/draft-processor' import { APP_ENTRY_PATH } from '@/lib/navigation/paths' +import { resolveConnectDraftOutboundScope } from '@/lib/oauth/connect-scope.server' import { safeAccountInsert } from '@/lib/oauth/credential-service' import { parseInstagramLongLivedToken, @@ -108,13 +110,21 @@ export const GET = withRouteHandler(async (request: NextRequest) => { code: authorizationCode, }) - const shortLivedResponse = await fetch('https://api.instagram.com/oauth/access_token', { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: tokenForm.toString(), - signal: request.signal, + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId, + userId: session.user.id, + providerId: 'instagram', }) + const shortLivedResponse = await inDraftScope(() => + outboundFetch('https://api.instagram.com/oauth/access_token', { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: tokenForm.toString(), + signal: request.signal, + }) + ) + if (!shortLivedResponse.ok) { const errorText = await readResponseTextWithLimit(shortLivedResponse, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, @@ -148,10 +158,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { exchangeUrl.searchParams.set('client_secret', clientSecret) exchangeUrl.searchParams.set('access_token', shortLived.access_token) - const longLivedResponse = await fetch(exchangeUrl.toString(), { - method: 'GET', - signal: request.signal, - }) + const longLivedResponse = await inDraftScope(() => + outboundFetch(exchangeUrl.toString(), { method: 'GET', signal: request.signal }) + ) if (!longLivedResponse.ok) { const errorText = await readResponseTextWithLimit(longLivedResponse, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, @@ -186,10 +195,12 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const profileUrl = new URL(`${INSTAGRAM_GRAPH_BASE}/me`) profileUrl.searchParams.set('fields', 'user_id,username,name,account_type,profile_picture_url') - const profileResponse = await fetch(profileUrl, { - headers: { Authorization: `Bearer ${longLivedToken}` }, - signal: request.signal, - }) + const profileResponse = await inDraftScope(() => + outboundFetch(profileUrl, { + headers: { Authorization: `Bearer ${longLivedToken}` }, + signal: request.signal, + }) + ) if (!profileResponse.ok) { const errorText = await readResponseTextWithLimit(profileResponse, { diff --git a/apps/sim/app/api/auth/trello/store/route.ts b/apps/sim/app/api/auth/trello/store/route.ts index 22d9a04aefa..16d5720a385 100644 --- a/apps/sim/app/api/auth/trello/store/route.ts +++ b/apps/sim/app/api/auth/trello/store/route.ts @@ -7,8 +7,10 @@ import { storeTrelloTokenContract } from '@/lib/api/contracts/oauth-connections' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processCredentialDraft } from '@/lib/credentials/draft-processor' +import { resolveConnectDraftOutboundScope } from '@/lib/oauth/connect-scope.server' import { safeAccountInsert } from '@/lib/oauth/credential-service' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' @@ -64,9 +66,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const scope = getCanonicalScopesForProvider('trello').join(',') const validationUrl = `https://api.trello.com/1/members/me?key=${apiKey}&token=${token}&fields=id,username,fullName` - const userResponse = await fetch(validationUrl, { - headers: { Accept: 'application/json' }, + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId, + userId: session.user.id, + providerId: 'trello', }) + const userResponse = await inDraftScope(() => + outboundFetch(validationUrl, { headers: { Accept: 'application/json' } }) + ) if (!userResponse.ok) { const errorText = await userResponse.text() diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts index 4caac82f74d..fd546bc9ca4 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts @@ -3,17 +3,33 @@ */ import { authOAuthUtilsMock, + authOAuthUtilsMockFns, createMockRequest, dbChainMockFns, + queueTableRows, redisConfigMockFns, resetDbChainMock, + schemaMock, } from '@sim/testing' import { authInternalMock, authInternalMockFns } from '@sim/testing/mocks/auth-internal.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -vi.mock('@/lib/auth/internal', () => authInternalMock) +const { mockGetCredentialOwner, mockLoadWorkspaceApplicationContext } = vi.hoisted(() => ({ + mockGetCredentialOwner: vi.fn(), + mockLoadWorkspaceApplicationContext: vi.fn(), +})) +vi.mock('@/lib/auth/internal', () => authInternalMock) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +vi.mock('@/lib/workspaces/application/context', () => ({ + loadWorkspaceApplicationContext: mockLoadWorkspaceApplicationContext, +})) +vi.mock('@/lib/webhooks/provider-subscription-utils', () => ({ + getCredentialOwner: mockGetCredentialOwner, + getNotificationUrl: vi.fn(() => 'https://sim.test/api/webhooks/trigger/hook-1'), +})) import { GET } from './route' @@ -46,4 +62,62 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { expect(data).toMatchObject({ status: 'skip' }) expect(dbChainMockFns.select).not.toHaveBeenCalled() }) + + describe('with outbound routing enabled', () => { + const graphCalls: Array<{ url: string; organizationId: string | null | undefined }> = [] + let routeOrganizationId: string | null | undefined + + beforeEach(() => { + graphCalls.length = 0 + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (organizationId) => { + routeOrganizationId = organizationId + return { kind: 'direct' } + }) + mockLoadWorkspaceApplicationContext.mockResolvedValue({ + workspaceId: 'workspace-1', + workspaceOrganizationId: 'org-1', + }) + mockGetCredentialOwner.mockResolvedValue({ accountId: 'account-1', userId: 'user-1' }) + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockResolvedValue('graph-token') + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => { + graphCalls.push({ url, organizationId: routeOrganizationId }) + return new Response(JSON.stringify({ expirationDateTime: '2030-01-01T00:00:00Z' })) + }) + ) + }) + + afterEach(() => { + vi.unstubAllGlobals() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + + it("renews each subscription through its workspace organization's outbound route", async () => { + queueTableRows(schemaMock.webhook, [ + { + workspaceId: 'workspace-1', + webhook: { + id: 'hook-1', + providerConfig: { + triggerId: 'microsoftteams_chat_subscription', + subscriptionExpiration: new Date(Date.now() + 60_000).toISOString(), + credentialId: 'credential-1', + externalSubscriptionId: 'subscription-1', + }, + }, + }, + ]) + + const response = await GET(createRequest()) + expect(response.status).toBe(202) + + await vi.waitFor(() => expect(graphCalls).toHaveLength(1)) + expect(graphCalls[0]).toEqual({ + url: 'https://graph.microsoft.com/v1.0/subscriptions/subscription-1', + organizationId: 'org-1', + }) + }) + }) }) diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.ts b/apps/sim/app/api/cron/renew-subscriptions/route.ts index 08ba269b241..c54bdd33de6 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.ts @@ -1,11 +1,13 @@ import { db } from '@sim/db' -import { webhook as webhookTable } from '@sim/db/schema' +import { webhook as webhookTable, workflow as workflowTable } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateShortId } from '@sim/utils/id' import { and, eq, or } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { verifyCronAuth } from '@/lib/auth/internal' import { acquireLock, releaseLock } from '@/lib/core/config/redis' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { runDetached } from '@/lib/core/utils/background' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' @@ -42,7 +44,7 @@ async function recreateSubscription( const notificationUrl = getNotificationUrl(webhook) const expirationDateTime = new Date(Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000).toISOString() - const res = await fetch('https://graph.microsoft.com/v1.0/subscriptions', { + const res = await outboundFetch('https://graph.microsoft.com/v1.0/subscriptions', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -72,6 +74,104 @@ async function recreateSubscription( return { id: payload.id as string, expirationDateTime: payload.expirationDateTime as string } } +/** + * Renews (or, once expired on Microsoft's side, recreates) one webhook's Graph subscription. + * Returns whether the subscription is live again. + */ +async function renewSubscription( + webhook: typeof webhookTable.$inferSelect, + config: Record, + expiresAt: Date +): Promise { + const requestId = `renewal-${webhook.id}` + logger.info( + `Renewing Teams subscription for webhook ${webhook.id} (expires: ${expiresAt.toISOString()})` + ) + + const credentialId = config.credentialId as string | undefined + const externalSubscriptionId = config.externalSubscriptionId as string | undefined + + if (!credentialId || !externalSubscriptionId) { + logger.error(`Missing credentialId or externalSubscriptionId for webhook ${webhook.id}`) + return false + } + + const credentialOwner = await getCredentialOwner(credentialId, requestId) + if (!credentialOwner) { + logger.error(`Credential owner not found for credential ${credentialId}`) + return false + } + + const accessToken = await refreshAccessTokenIfNeeded( + credentialOwner.accountId, + credentialOwner.userId, + requestId + ) + + if (!accessToken) { + logger.error(`Failed to get access token for webhook ${webhook.id}`) + return false + } + + const newExpirationDateTime = new Date( + Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000 + ).toISOString() + + const res = await outboundFetch( + `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, + { + method: 'PATCH', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ expirationDateTime: newExpirationDateTime }), + } + ) + + let newSubscriptionId: string | undefined + let newExpiration: string | undefined + + if (!res.ok) { + const error = await res.json() + logger.error( + `Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`, + { status: res.status, error: error.error } + ) + + if (res.status === 404 || res.status === 410) { + const recreated = await recreateSubscription(webhook, config, accessToken) + if (!recreated) return false + newSubscriptionId = recreated.id + newExpiration = recreated.expirationDateTime + logger.info( + `Recreated Teams subscription for webhook ${webhook.id} after the previous one expired (new id: ${newSubscriptionId})` + ) + } else { + return false + } + } else { + const payload = await res.json() + newExpiration = payload.expirationDateTime as string + } + + const updatedConfig = { + ...config, + ...(newSubscriptionId ? { externalSubscriptionId: newSubscriptionId } : {}), + subscriptionExpiration: newExpiration, + } + + await db + .update(webhookTable) + .set({ providerConfig: updatedConfig, updatedAt: new Date() }) + .where(eq(webhookTable.id, webhook.id)) + + logger.info( + `Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${newExpiration}` + ) + return true +} + /** * Renews Microsoft Teams chat subscriptions that are close to expiring. * @@ -93,8 +193,10 @@ async function renewExpiringSubscriptions(): Promise<{ const webhooksWithWorkflows = await db .select({ webhook: webhookTable, + workspaceId: workflowTable.workspaceId, }) .from(webhookTable) + .innerJoin(workflowTable, eq(webhookTable.workflowId, workflowTable.id)) .where( and( deliverableWebhookPredicate(webhookTable, 'active_only'), @@ -112,7 +214,7 @@ async function renewExpiringSubscriptions(): Promise<{ /** Renew any subscription expiring within the next 48 hours. */ const renewalThreshold = new Date(Date.now() + 48 * 60 * 60 * 1000) - for (const { webhook } of webhooksWithWorkflows) { + for (const { webhook, workspaceId } of webhooksWithWorkflows) { const config = (webhook.providerConfig as Record) || {} if (config.triggerId !== 'microsoftteams_chat_subscription') continue @@ -124,102 +226,14 @@ async function renewExpiringSubscriptions(): Promise<{ if (expiresAt > renewalThreshold) continue totalChecked++ - const requestId = `renewal-${webhook.id}` try { - logger.info( - `Renewing Teams subscription for webhook ${webhook.id} (expires: ${expiresAt.toISOString()})` - ) - - const credentialId = config.credentialId as string | undefined - const externalSubscriptionId = config.externalSubscriptionId as string | undefined - - if (!credentialId || !externalSubscriptionId) { - logger.error(`Missing credentialId or externalSubscriptionId for webhook ${webhook.id}`) - totalFailed++ - continue - } - - const credentialOwner = await getCredentialOwner(credentialId, requestId) - if (!credentialOwner) { - logger.error(`Credential owner not found for credential ${credentialId}`) - totalFailed++ - continue - } - - const accessToken = await refreshAccessTokenIfNeeded( - credentialOwner.accountId, - credentialOwner.userId, - requestId - ) - - if (!accessToken) { - logger.error(`Failed to get access token for webhook ${webhook.id}`) - totalFailed++ - continue - } - - const newExpirationDateTime = new Date( - Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000 - ).toISOString() - - const res = await fetch( - `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, - { - method: 'PATCH', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ expirationDateTime: newExpirationDateTime }), - } - ) - - let newSubscriptionId: string | undefined - let newExpiration: string | undefined - - if (!res.ok) { - const error = await res.json() - logger.error( - `Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`, - { status: res.status, error: error.error } - ) - - if (res.status === 404 || res.status === 410) { - const recreated = await recreateSubscription(webhook, config, accessToken) - if (!recreated) { - totalFailed++ - continue - } - newSubscriptionId = recreated.id - newExpiration = recreated.expirationDateTime - logger.info( - `Recreated Teams subscription for webhook ${webhook.id} after the previous one expired (new id: ${newSubscriptionId})` - ) - } else { - totalFailed++ - continue - } - } else { - const payload = await res.json() - newExpiration = payload.expirationDateTime as string - } - - const updatedConfig = { - ...config, - ...(newSubscriptionId ? { externalSubscriptionId: newSubscriptionId } : {}), - subscriptionExpiration: newExpiration, - } - - await db - .update(webhookTable) - .set({ providerConfig: updatedConfig, updatedAt: new Date() }) - .where(eq(webhookTable.id, webhook.id)) - - logger.info( - `Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${newExpiration}` - ) - totalRenewed++ + const renew = () => renewSubscription(webhook, config, expiresAt) + const renewed = workspaceId + ? await withResourceOutboundScope({ workspaceId }, renew, { includeArchived: true }) + : await renew() + if (renewed) totalRenewed++ + else totalFailed++ } catch (error) { logger.error(`Error renewing subscription for webhook ${webhook.id}:`, error) totalFailed++ diff --git a/apps/sim/executor/utils/credential-token.ts b/apps/sim/executor/utils/credential-token.ts index 471c678c1ea..0deff02814b 100644 --- a/apps/sim/executor/utils/credential-token.ts +++ b/apps/sim/executor/utils/credential-token.ts @@ -42,10 +42,9 @@ export interface ResolveExecutorCredentialTokenParams { /** * Resolves a credential's access token in-process for server-side workflow - * execution, through the same authorized application dispatch as - * `POST /api/auth/oauth/token` (`resolveCredentialAccessToken`). Both runtimes - * hold the OAuth client config the refresh branch needs, so authorization, - * refresh, and audit run identically to the route. + * execution through the authorized application dispatch + * (`resolveCredentialAccessToken`), so authorization, refresh, and audit run + * identically to every other server surface. */ export async function resolveExecutorCredentialToken( params: ResolveExecutorCredentialTokenParams diff --git a/apps/sim/lib/api/contracts/oauth-connections.ts b/apps/sim/lib/api/contracts/oauth-connections.ts index b0555c1ec78..79630e0c89a 100644 --- a/apps/sim/lib/api/contracts/oauth-connections.ts +++ b/apps/sim/lib/api/contracts/oauth-connections.ts @@ -7,8 +7,6 @@ import type { } from '@/lib/api/contracts/types' import { defineRouteContract } from '@/lib/api/contracts/types' -export const MANAGED_OAUTH_DELEGATION_HEADER = 'x-sim-managed-oauth-delegation' - export const oauthAccountSummarySchema = z.object({ id: z.string(), name: z.string(), @@ -77,40 +75,9 @@ const trelloCallbackQuerySchema = z }) .passthrough() -/** Google domain-wide-delegation subject. Also applied by in-process credential callers. */ +/** Google domain-wide-delegation subject, validated for in-process credential callers. */ export const impersonateEmailSchema = z.string().email() -export const oauthTokenRequestBodySchema = z - .object({ - credentialId: z.string().min(1).optional(), - credentialAccountUserId: z.string().min(1).optional(), - providerId: z.string().min(1).optional(), - toolId: z.string().min(1).optional(), - workflowId: z.string().min(1).nullish(), - scopes: z.array(z.string()).optional(), - impersonateEmail: impersonateEmailSchema.optional(), - }) - .refine( - (data) => data.credentialId || (data.credentialAccountUserId && data.providerId), - 'Either credentialId or (credentialAccountUserId + providerId) is required' - ) - -export const oauthTokenGetQuerySchema = z.object({ - credentialId: z - .string({ - error: 'Credential ID is required', - }) - .min(1, 'Credential ID is required'), -}) - -export const oauthTokenPostQuerySchema = z.object({ - userId: z.string().min(1).optional(), -}) - -export const oauthTokenPostHeadersSchema = z.object({ - [MANAGED_OAUTH_DELEGATION_HEADER]: z.string().min(1).optional(), -}) - const oauthTokenResponseSchema = z.object({ accessToken: z.string(), credentialType: z.enum(['oauth', 'managed_oauth', 'service_account']).optional(), @@ -130,31 +97,9 @@ const oauthTokenResponseSchema = z.object({ .optional(), }) -/** Token material a resolved credential yields, on the wire and in-process alike. */ +/** Token material a resolved credential yields to its in-process callers. */ export type OAuthTokenResponse = z.output -export const oauthTokenGetContract = defineRouteContract({ - method: 'GET', - path: '/api/auth/oauth/token', - query: oauthTokenGetQuerySchema, - response: { - mode: 'json', - schema: oauthTokenResponseSchema, - }, -}) - -export const oauthTokenPostContract = defineRouteContract({ - method: 'POST', - path: '/api/auth/oauth/token', - query: oauthTokenPostQuerySchema, - headers: oauthTokenPostHeadersSchema, - body: oauthTokenRequestBodySchema, - response: { - mode: 'json', - schema: oauthTokenResponseSchema, - }, -}) - export const shopifyAuthorizeQuerySchema = z.object({ shop: z.string().optional(), returnUrl: z.string().optional(), diff --git a/apps/sim/lib/atlassian/discovery.ts b/apps/sim/lib/atlassian/discovery.ts index 33c9aa4f96a..911956baa21 100644 --- a/apps/sim/lib/atlassian/discovery.ts +++ b/apps/sim/lib/atlassian/discovery.ts @@ -1,6 +1,7 @@ import { sha256Hex } from '@sim/security/hash' import { parseRetryAfter } from '@sim/utils/retry' import { LRUCache } from 'lru-cache' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type HTTPError, isRetryableError, @@ -158,7 +159,7 @@ export function fetchAtlassianDiscoveryJson( ): Promise { return retryWithExponentialBackoff( async () => { - const response = await fetch(url, { + const response = await outboundFetch(url, { method: 'GET', headers, signal: AbortSignal.timeout(DISCOVERY_REQUEST_TIMEOUT_MS), diff --git a/apps/sim/lib/auth/connectors/managed-oauth.ts b/apps/sim/lib/auth/connectors/managed-oauth.ts index aad3acd77b4..d6cf2e178ad 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.ts @@ -6,6 +6,7 @@ import type { GenericOAuthConfig } from 'better-auth/plugins' import { OAuth2Client, type TokenPayload } from 'google-auth-library' import { createRemoteJWKSet, type JWTPayload, jwtVerify } from 'jose' import { buildConnectorProviders } from '@/lib/auth/connectors/providers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { boundedAccountSites } from '@/lib/credential-groups/account-sites' import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' @@ -207,7 +208,7 @@ function httpsOrigin(value: string): string | null { */ async function listAtlassianSites(accessToken: string): Promise { try { - const response = await fetch(ATLASSIAN_SITES_URL, { + const response = await outboundFetch(ATLASSIAN_SITES_URL, { headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, signal: AbortSignal.timeout(ATLASSIAN_USER_INFO_TIMEOUT_MS), }) @@ -246,7 +247,7 @@ export function createAtlassianManagedOAuthConnector( throw new Error('Atlassian returned an incomplete authorization') } const sites = listAtlassianSites(tokens.accessToken) - const response = await fetch(ATLASSIAN_USER_INFO_URL, { + const response = await outboundFetch(ATLASSIAN_USER_INFO_URL, { headers: { Accept: 'application/json', Authorization: `Bearer ${tokens.accessToken}`, @@ -354,7 +355,7 @@ function requireMicrosoftIdentityClaims(payload: JWTPayload): MicrosoftIdentityC * not consented to Graph. */ async function fetchMicrosoftAccessTokenSubject(accessToken: string): Promise { - const response = await fetch(MICROSOFT_OIDC_USER_INFO_URL, { + const response = await outboundFetch(MICROSOFT_OIDC_USER_INFO_URL, { headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, signal: AbortSignal.timeout(MICROSOFT_OIDC_USER_INFO_TIMEOUT_MS), }) @@ -500,7 +501,7 @@ async function fetchManagedOAuthProfile( ): Promise { const { providerId, userInfo } = options const url = typeof userInfo.url === 'function' ? userInfo.url(tokens) : userInfo.url - const response = await fetch(url, { + const response = await outboundFetch(url, { method: userInfo.method ?? 'GET', headers: { Accept: 'application/json', @@ -706,7 +707,7 @@ function createAttioManagedOAuthConnector(): ManagedOAuthConnectorConfig { Accept: 'application/json', Authorization: `Bearer ${tokens.accessToken}`, } - const selfResponse = await fetch('https://api.attio.com/v2/self', { + const selfResponse = await outboundFetch('https://api.attio.com/v2/self', { headers, signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), }) @@ -722,7 +723,7 @@ function createAttioManagedOAuthConnector(): ManagedOAuthConnectorConfig { identity.authorized_by_workspace_member_id, 'Attio workspace member id' ) - const memberResponse = await fetch( + const memberResponse = await outboundFetch( `https://api.attio.com/v2/workspace_members/${encodeURIComponent(memberId)}`, { headers, signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS) } ) @@ -791,7 +792,7 @@ function createBitbucketManagedOAuthConnector(): ManagedOAuthConnectorConfig { Accept: 'application/json', Authorization: `Bearer ${tokens.accessToken}`, } - const userResponse = await fetch(`${BITBUCKET_API_BASE}/user`, { + const userResponse = await outboundFetch(`${BITBUCKET_API_BASE}/user`, { headers, signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), }) @@ -804,7 +805,7 @@ function createBitbucketManagedOAuthConnector(): ManagedOAuthConnectorConfig { } const user = asProfileRecord(userBody, 'Bitbucket') const accountId = requireIdentityField(user.account_id, 'Bitbucket account id') - const emailsResponse = await fetch(`${BITBUCKET_API_BASE}/user/emails`, { + const emailsResponse = await outboundFetch(`${BITBUCKET_API_BASE}/user/emails`, { headers, signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), }) diff --git a/apps/sim/lib/auth/connectors/providers.ts b/apps/sim/lib/auth/connectors/providers.ts index 29b8290cb8d..359d29ee43d 100644 --- a/apps/sim/lib/auth/connectors/providers.ts +++ b/apps/sim/lib/auth/connectors/providers.ts @@ -9,6 +9,7 @@ import type { GenericOAuthConfig } from 'better-auth/plugins' import { syntheticConnectorEmail } from '@/lib/auth/connector-email' import { env } from '@/lib/core/config/env' import { inspectConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -144,7 +145,7 @@ function salesforceConnector(providerId: string, loginHost: string): GenericOAut redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/${providerId}`, getUserInfo: async (tokens) => { try { - const response = await fetch(userInfoUrl, { + const response = await outboundFetch(userInfoUrl, { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -218,7 +219,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-email`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -254,7 +255,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-calendar`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -290,7 +291,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-drive`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -326,7 +327,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-docs`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -362,7 +363,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-sheets`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -399,7 +400,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-contacts`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -435,7 +436,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-forms`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -471,7 +472,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-ads`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -506,7 +507,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-bigquery`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -543,7 +544,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-vault`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -580,7 +581,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-groups`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -617,7 +618,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-chat`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -654,7 +655,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-meet`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -690,7 +691,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-tasks`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -727,7 +728,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/vertex-ai`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -984,7 +985,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Wealthbox user profile') - const response = await fetch('https://api.crmworkspace.com/v1/me', { + const response = await outboundFetch('https://api.crmworkspace.com/v1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1065,7 +1066,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Pipedrive user profile') - const response = await fetch('https://api.pipedrive.com/v1/users/me', { + const response = await outboundFetch('https://api.pipedrive.com/v1/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1112,7 +1113,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching HubSpot user profile') - const response = await fetch( + const response = await outboundFetch( `https://api.hubapi.com/oauth/v1/access-tokens/${tokens.accessToken}` ) @@ -1207,7 +1208,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { // pkce:false (which connected) then restoring pkce:true + this verifier. if (codeVerifier) tokenParams.set('code_verifier', codeVerifier) - const response = await fetch('https://accounts.zoho.com/oauth/v2/token', { + const response = await outboundFetch('https://accounts.zoho.com/oauth/v2/token', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: tokenParams, @@ -1254,7 +1255,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { // token response api_domain so every API call targets the correct // host instead of assuming desk.zoho.com. Stored inside the scope // string (survives refreshes, which never rewrite scope) and read - // back in /api/auth/oauth/token as `apiDomain`. + // back during credential token resolution as `apiDomain`. const deskBase = deriveZohoDeskBaseFromApiDomain( typeof data.api_domain === 'string' ? data.api_domain : undefined ) @@ -1275,7 +1276,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, getUserInfo: async (tokens) => { try { - const response = await fetch('https://accounts.zoho.com/oauth/user/info', { + const response = await outboundFetch('https://accounts.zoho.com/oauth/user/info', { headers: { Authorization: `Zoho-oauthtoken ${tokens.accessToken}` }, }) @@ -1355,7 +1356,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { // rejects the request shape (invalid_request). if (codeVerifier) tokenParams.set('code_verifier', codeVerifier) - const response = await fetch('https://accounts.zoho.com/oauth/v2/token', { + const response = await outboundFetch('https://accounts.zoho.com/oauth/v2/token', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: tokenParams.toString(), @@ -1406,7 +1407,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, getUserInfo: async (tokens) => { try { - const response = await fetch('https://accounts.zoho.com/oauth/user/info', { + const response = await outboundFetch('https://accounts.zoho.com/oauth/user/info', { headers: { Authorization: `Zoho-oauthtoken ${tokens.accessToken}` }, }) @@ -1471,7 +1472,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/x`, getUserInfo: async (tokens) => { try { - const response = await fetch( + const response = await outboundFetch( 'https://api.x.com/2/users/me?user.fields=profile_image_url,username,name,verified', { headers: { @@ -1528,7 +1529,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { scope: getCanonicalScopesForProvider('tiktok').join(','), }, getToken: async ({ code, redirectURI }) => { - const response = await fetch('https://open.tiktokapis.com/v2/oauth/token/', { + const response = await outboundFetch('https://open.tiktokapis.com/v2/oauth/token/', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ @@ -1559,7 +1560,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, getUserInfo: async (tokens) => { try { - const response = await fetch( + const response = await outboundFetch( 'https://open.tiktokapis.com/v2/user/info/?fields=open_id,display_name,avatar_url', { headers: { @@ -1643,7 +1644,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }), getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.atlassian.com/me', { + const response = await outboundFetch('https://api.atlassian.com/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1704,7 +1705,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }), getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.atlassian.com/me', { + const response = await outboundFetch('https://api.atlassian.com/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1755,7 +1756,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/airtable`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.airtable.com/v0/meta/whoami', { + const response = await outboundFetch('https://api.airtable.com/v0/meta/whoami', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1805,7 +1806,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { const basicAuth = Buffer.from( `${env.BITBUCKET_CLIENT_ID as string}:${env.BITBUCKET_CLIENT_SECRET as string}` ).toString('base64') - const response = await fetch('https://bitbucket.org/site/oauth2/access_token', { + const response = await outboundFetch('https://bitbucket.org/site/oauth2/access_token', { method: 'POST', headers: { Authorization: `Basic ${basicAuth}`, @@ -1846,7 +1847,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { getUserInfo: async (tokens) => { try { const signal = AbortSignal.timeout(15_000) - const response = await fetch('https://api.bitbucket.org/2.0/user', { + const response = await outboundFetch('https://api.bitbucket.org/2.0/user', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1909,7 +1910,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/notion`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.notion.com/v1/users/me', { + const response = await outboundFetch('https://api.notion.com/v1/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, 'Notion-Version': '2022-06-28', @@ -1984,7 +1985,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { getUserInfo: async (tokens) => { try { const signal = AbortSignal.timeout(15_000) - const response = await fetch(MONDAY_API_URL, { + const response = await outboundFetch(MONDAY_API_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -2061,7 +2062,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/reddit`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://oauth.reddit.com/api/v1/me', { + const response = await outboundFetch('https://oauth.reddit.com/api/v1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, 'User-Agent': REDDIT_USER_AGENT, @@ -2108,7 +2109,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/clickup`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.clickup.com/api/v2/user', { + const response = await outboundFetch('https://api.clickup.com/api/v2/user', { headers: { Authorization: `Bearer ${tokens.accessToken}`, 'Content-Type': 'application/json', @@ -2159,7 +2160,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { accessType: 'offline', getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.linear.app/graphql', { + const response = await outboundFetch('https://api.linear.app/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -2239,7 +2240,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { * `/v2/self` requires no scope and reports who authorized the token. * @see https://docs.attio.com/rest-api/endpoint-reference/meta/identify */ - const selfResponse = await fetch('https://api.attio.com/v2/self', { + const selfResponse = await outboundFetch('https://api.attio.com/v2/self', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) @@ -2269,7 +2270,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { * `user_management:read`, which Sim always requests for Attio. * @see https://docs.attio.com/rest-api/endpoint-reference/workspace-members/get-a-workspace-member */ - const memberResponse = await fetch( + const memberResponse = await outboundFetch( `https://api.attio.com/v2/workspace_members/${encodeURIComponent(memberId)}`, { headers: { Authorization: `Bearer ${tokens.accessToken}` } } ) @@ -2327,7 +2328,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/box`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.box.com/2.0/users/me', { + const response = await outboundFetch('https://api.box.com/2.0/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2378,12 +2379,15 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.dropboxapi.com/2/users/get_current_account', { - method: 'POST', - headers: { - Authorization: `Bearer ${tokens.accessToken}`, - }, - }) + const response = await outboundFetch( + 'https://api.dropboxapi.com/2/users/get_current_account', + { + method: 'POST', + headers: { + Authorization: `Bearer ${tokens.accessToken}`, + }, + } + ) if (!response.ok) { const errorText = await response.text() @@ -2429,7 +2433,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/asana`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://app.asana.com/api/1.0/users/me', { + const response = await outboundFetch('https://app.asana.com/api/1.0/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2479,7 +2483,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/slack`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://slack.com/api/auth.test', { + const response = await outboundFetch('https://slack.com/api/auth.test', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2558,7 +2562,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Webflow user info') - const response = await fetch('https://api.webflow.com/v2/token/introspect', { + const response = await outboundFetch('https://api.webflow.com/v2/token/introspect', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2609,7 +2613,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching LinkedIn user profile') - const response = await fetch('https://api.linkedin.com/v2/userinfo', { + const response = await outboundFetch('https://api.linkedin.com/v2/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2659,7 +2663,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Zoom user profile') - const response = await fetch('https://api.zoom.us/v2/users/me', { + const response = await outboundFetch('https://api.zoom.us/v2/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2707,7 +2711,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Spotify user profile') - const response = await fetch('https://api.spotify.com/v1/me', { + const response = await outboundFetch('https://api.spotify.com/v1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2755,7 +2759,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching WordPress.com user profile') - const response = await fetch('https://public-api.wordpress.com/rest/v1.1/me', { + const response = await outboundFetch('https://public-api.wordpress.com/rest/v1.1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2807,7 +2811,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching DocuSign user profile') - const response = await fetch(getDocusignOAuthUrl('/oauth/userinfo'), { + const response = await outboundFetch(getDocusignOAuthUrl('/oauth/userinfo'), { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2864,7 +2868,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { const signal = AbortSignal.timeout(15_000) - const response = await fetch('https://api.ramp.com/developer/v1/business', { + const response = await outboundFetch('https://api.ramp.com/developer/v1/business', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, redirect: 'error', signal, @@ -2922,7 +2926,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { const signal = AbortSignal.timeout(15_000) - const response = await fetch('https://api.helpscout.net/v2/users/me', { + const response = await outboundFetch('https://api.helpscout.net/v2/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: 'application/json' }, redirect: 'error', signal, @@ -2983,7 +2987,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Cal.com user profile') - const response = await fetch('https://api.cal.com/v2/me', { + const response = await outboundFetch('https://api.cal.com/v2/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, 'cal-api-version': '2024-08-13', diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index 1be033d4597..a20683223f0 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -26,8 +26,8 @@ import { } from '@/lib/credential-groups/provider-adapter' import type { CredentialGroupStandardOAuthProvider } from '@/lib/credential-groups/providers' import { getCredentialGroupProviderService } from '@/lib/credential-groups/providers' -import { refreshOAuthToken } from '@/lib/oauth' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' const OAUTH_DISCOVERY_TIMEOUT_MS = 10_000 const OAUTH_DISCOVERY_MAX_BYTES = 256 * 1024 diff --git a/apps/sim/lib/credentials/application/managed-oauth-delegation.ts b/apps/sim/lib/credentials/application/managed-oauth-delegation.ts index d5c68994745..3c1e0e9b039 100644 --- a/apps/sim/lib/credentials/application/managed-oauth-delegation.ts +++ b/apps/sim/lib/credentials/application/managed-oauth-delegation.ts @@ -1,12 +1,5 @@ import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' -import { - InvalidInternalDelegationTokenError, - verifyInternalDelegationToken, -} from '@/lib/auth/internal' -import { - bindInternalExecutorDelegation, - InvalidInternalDelegationBindingError, -} from '@/lib/auth/internal-delegation' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' import { MANAGED_OAUTH_DELEGATION_AUDIENCE } from '@/lib/credentials/application/authorization' import { createExecutorPrincipalFromDelegationOrigin } from '@/lib/internal/principals/executor' import type { ExecutorDelegationOrigin } from '@/executor/types' @@ -18,35 +11,10 @@ export class InvalidManagedOAuthDelegationError extends Error { } } -/** Authenticates and binds an executor delegation to one managed credential ID. */ -export async function authenticateManagedOAuthDelegation( - authorization: string, - credentialId: string -): Promise { - if (!authorization.startsWith('Bearer ')) throw new InvalidManagedOAuthDelegationError() - - try { - const claims = await verifyInternalDelegationToken(authorization.slice('Bearer '.length)) - return await bindInternalExecutorDelegation(claims, { - audience: MANAGED_OAUTH_DELEGATION_AUDIENCE, - resourceScope: { credentialId }, - }) - } catch (error) { - if ( - error instanceof InvalidInternalDelegationTokenError || - error instanceof InvalidInternalDelegationBindingError - ) { - throw new InvalidManagedOAuthDelegationError() - } - throw error - } -} - /** - * In-process sibling of {@link authenticateManagedOAuthDelegation}: binds the - * executor's own delegation origin to one managed credential without minting and - * re-verifying a delegation JWT — see {@link createExecutorPrincipalFromDelegationOrigin} - * for why that loses nothing. + * Binds the executor's own delegation origin to one managed credential without + * minting and re-verifying a delegation JWT — see + * {@link createExecutorPrincipalFromDelegationOrigin} for why that loses nothing. */ export async function bindExecutorManagedOAuthDelegation( origin: ExecutorDelegationOrigin, diff --git a/apps/sim/lib/oauth/apollo.ts b/apps/sim/lib/oauth/apollo.ts index 90b88fe2b6b..a84b522957e 100644 --- a/apps/sim/lib/oauth/apollo.ts +++ b/apps/sim/lib/oauth/apollo.ts @@ -1,6 +1,7 @@ import { generateId } from '@sim/utils/id' import type { GenericOAuthConfig } from 'better-auth/plugins' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' export const APOLLO_TOKEN_URL = 'https://app.apollo.io/api/v1/oauth/token' @@ -39,8 +40,7 @@ export function createApolloProvider( tokenUrl: APOLLO_TOKEN_URL, authentication: 'post', getUserInfo: async (tokens) => { - // boundary-raw-fetch: Apollo's external current-user endpoint verifies the OAuth token's identity. - const response = await fetch('https://app.apollo.io/api/v1/users/api_profile', { + const response = await outboundFetch('https://app.apollo.io/api/v1/users/api_profile', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, redirect: 'error', signal: AbortSignal.timeout(15_000), diff --git a/apps/sim/lib/oauth/atlassian-token.ts b/apps/sim/lib/oauth/atlassian-token.ts index dd2e4403979..bb13b1c5a1c 100644 --- a/apps/sim/lib/oauth/atlassian-token.ts +++ b/apps/sim/lib/oauth/atlassian-token.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { getOAuth2Tokens } from 'better-auth/oauth2' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' const logger = createLogger('AtlassianOAuth') @@ -24,7 +25,7 @@ export async function exchangeAtlassianAuthorizationCode({ redirectUri, codeVerifier, }: ExchangeAtlassianAuthorizationCodeInput) { - const response = await fetch(ATLASSIAN_TOKEN_URL, { + const response = await outboundFetch(ATLASSIAN_TOKEN_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify({ diff --git a/apps/sim/lib/oauth/connect-scope.server.test.ts b/apps/sim/lib/oauth/connect-scope.server.test.ts new file mode 100644 index 00000000000..b55bc89e78a --- /dev/null +++ b/apps/sim/lib/oauth/connect-scope.server.test.ts @@ -0,0 +1,71 @@ +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetActiveConnectDraft, mockLoadWorkspaceApplicationContext } = vi.hoisted(() => ({ + mockGetActiveConnectDraft: vi.fn(), + mockLoadWorkspaceApplicationContext: vi.fn(), +})) + +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +vi.mock('@/lib/credentials/connect-draft', () => ({ + getActiveConnectDraft: mockGetActiveConnectDraft, +})) +vi.mock('@/lib/workspaces/application/context', () => ({ + loadWorkspaceApplicationContext: mockLoadWorkspaceApplicationContext, +})) + +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { resolveConnectDraftOutboundScope } from '@/lib/oauth/connect-scope.server' + +async function organizationSeenByVendorCall( + draftOwner: { workspaceId: string | null; organizationId: string | null }, + requestedProviderId: string +): Promise { + mockGetActiveConnectDraft.mockResolvedValue({ + id: 'draft-1', + providerId: 'instagram', + ...draftOwner, + }) + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId: 'draft-1', + userId: 'user-1', + providerId: requestedProviderId, + }) + let organizationId: string | null | undefined + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (id) => { + organizationId = id + return { kind: 'direct' } + }) + await inDraftScope(() => resolveCurrentOutboundRoute()) + return organizationId +} + +describe('resolveConnectDraftOutboundScope', () => { + beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + mockLoadWorkspaceApplicationContext.mockResolvedValue({ + workspaceId: 'workspace-1', + workspaceOrganizationId: 'org-1', + }) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + + it("routes vendor calls through the draft workspace's organization", async () => { + await expect( + organizationSeenByVendorCall( + { workspaceId: 'workspace-1', organizationId: null }, + 'instagram' + ) + ).resolves.toBe('org-1') + expect(mockGetActiveConnectDraft).toHaveBeenCalledWith('draft-1', 'user-1') + }) + + it('leaves vendor calls unscoped when the draft belongs to another provider', async () => { + await expect( + organizationSeenByVendorCall({ workspaceId: 'workspace-1', organizationId: null }, 'trello') + ).resolves.toBeUndefined() + }) +}) diff --git a/apps/sim/lib/oauth/connect-scope.server.ts b/apps/sim/lib/oauth/connect-scope.server.ts new file mode 100644 index 00000000000..29530cd8318 --- /dev/null +++ b/apps/sim/lib/oauth/connect-scope.server.ts @@ -0,0 +1,29 @@ +import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' +import { captureOutboundScope } from '@/lib/core/network/context.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { getActiveConnectDraft } from '@/lib/credentials/connect-draft' + +type OutboundScopeRunner = (run: () => Promise) => Promise + +/** + * Resolves the outbound scope for a custom OAuth callback's vendor calls (code exchange, + * identity lookup) from the workspace or organization that owns its connect draft, so an + * organization with a dedicated egress gateway connects from its own static IPs. + * + * Ownership comes from the caller's own unexpired draft row, never from the request, and + * is resolved once so every vendor call in the callback reuses it. Without a draft bound + * to this user and provider, the runner leaves calls unscoped. + */ +export async function resolveConnectDraftOutboundScope(params: { + draftId: string | undefined + userId: string + providerId: string +}): Promise { + if (!isOutboundRoutingEnabled() || !params.draftId) return (run) => run() + const draft = await getActiveConnectDraft(params.draftId, params.userId) + if (!draft || draft.providerId !== params.providerId) return (run) => run() + return withResourceOutboundScope( + { workspaceId: draft.workspaceId, organizationId: draft.organizationId }, + async () => captureOutboundScope() + ) +} diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts index 572d3ecf8ec..8284e7e2164 100644 --- a/apps/sim/lib/oauth/credential-service.test.ts +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -60,10 +60,13 @@ vi.mock('@/lib/oauth/microsoft', () => ({ vi.mock('@/lib/oauth/oauth', () => ({ OAUTH_PROVIDERS: {}, - refreshOAuthToken: hoisted.refreshOAuthToken, TOKEN_REFRESH_TIMEOUT_MS: 15_000, })) +vi.mock('@/lib/oauth/refresh.server', () => ({ + refreshOAuthToken: hoisted.refreshOAuthToken, +})) + vi.mock('@/lib/oauth/quickbooks-client-config', () => ({ decryptQuickBooksOAuthClientConfig: hoisted.decryptQuickBooksOAuthClientConfig, })) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 846076c444d..f3131851b34 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -65,8 +65,9 @@ import { isMicrosoftProvider, PROACTIVE_REFRESH_THRESHOLD_DAYS, } from '@/lib/oauth/microsoft' -import { refreshOAuthToken, TOKEN_REFRESH_TIMEOUT_MS } from '@/lib/oauth/oauth' +import { TOKEN_REFRESH_TIMEOUT_MS } from '@/lib/oauth/oauth' import { decryptQuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' import { clearOAuthRefreshDeadFlag, getOAuthRefreshCoordinationIdentity, @@ -1086,8 +1087,8 @@ const SERVICE_ACCOUNT_TOKEN_RESOLVERS: Record encryptionMock) vi.mock('@/lib/oauth/github-installation', () => githubInstallationMock) vi.mock('@/lib/oauth/oauth', () => ({ OAUTH_PROVIDERS: {}, - refreshOAuthToken: vi.fn(), TOKEN_REFRESH_TIMEOUT_MS: 15_000, })) +vi.mock('@/lib/oauth/refresh.server', () => ({ refreshOAuthToken: vi.fn() })) import { resolveServiceAccountToken } from '@/lib/oauth/credential-service' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' diff --git a/apps/sim/lib/oauth/github-installation.ts b/apps/sim/lib/oauth/github-installation.ts index 839ea1d48fe..0806c86f744 100644 --- a/apps/sim/lib/oauth/github-installation.ts +++ b/apps/sim/lib/oauth/github-installation.ts @@ -1,6 +1,7 @@ import { createHash, createPrivateKey, createSign } from 'node:crypto' import { z } from 'zod' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { GitHubInstallationBinding, @@ -166,7 +167,7 @@ async function request( operation: GitHubInstallationOperation = 'installation' ): Promise { const requestSignal = AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]) - const response = await fetch(`${API_URL}${path}`, { + const response = await outboundFetch(`${API_URL}${path}`, { method: body === undefined ? 'GET' : 'POST', headers: { Accept: 'application/vnd.github+json', diff --git a/apps/sim/lib/oauth/github-repositories.test.ts b/apps/sim/lib/oauth/github-repositories.test.ts index 13db90f5d98..85baf77cf67 100644 --- a/apps/sim/lib/oauth/github-repositories.test.ts +++ b/apps/sim/lib/oauth/github-repositories.test.ts @@ -8,7 +8,7 @@ import { verifyGitHubRepositoriesIdentity, } from '@/lib/oauth/github-repositories' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' -import { refreshOAuthToken } from '@/lib/oauth/oauth' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' const tokenResponse = { access_token: 'ghu_access', diff --git a/apps/sim/lib/oauth/github-repositories.ts b/apps/sim/lib/oauth/github-repositories.ts index 320c2658bd0..37398dea7d8 100644 --- a/apps/sim/lib/oauth/github-repositories.ts +++ b/apps/sim/lib/oauth/github-repositories.ts @@ -2,6 +2,7 @@ import { generateId } from '@sim/utils/id' import { getOAuth2Tokens, type OAuth2Tokens } from 'better-auth/oauth2' import type { GenericOAuthConfig } from 'better-auth/plugins' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { OAuthIdentityVerificationError, @@ -60,7 +61,7 @@ export async function verifyGitHubRepositoriesIdentity(accessToken: string) { const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS) let response: Response try { - response = await fetch(`${API_BASE_URL}${path}`, { + response = await outboundFetch(`${API_BASE_URL}${path}`, { headers, signal, redirect: 'error', @@ -170,7 +171,7 @@ export function createGitHubRepositoriesProvider({ getToken: async ({ code, codeVerifier, redirectURI: callbackURI }) => { if (!codeVerifier) throw new Error('GitHub authorization requires a PKCE verifier') const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS) - const response = await fetch(GITHUB_TOKEN_URL, { + const response = await outboundFetch(GITHUB_TOKEN_URL, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/google-service-account-transport.ts b/apps/sim/lib/oauth/google-service-account-transport.ts index 85bd48c1849..1f3001ccf52 100644 --- a/apps/sim/lib/oauth/google-service-account-transport.ts +++ b/apps/sim/lib/oauth/google-service-account-transport.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { parseRetryAfter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isRetryableError, readBoundedHttpErrorPayload, @@ -61,7 +62,7 @@ export async function exchangeGoogleServiceAccountJwt( stage = 'awaiting_response' currentStatus = undefined lastResponse = undefined - const response = await fetch(tokenUri, { + const response = await outboundFetch(tokenUri, { method: 'POST', redirect: 'error', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, diff --git a/apps/sim/lib/oauth/greenhouse.ts b/apps/sim/lib/oauth/greenhouse.ts index 13b4e05e2e8..0c8990d52b2 100644 --- a/apps/sim/lib/oauth/greenhouse.ts +++ b/apps/sim/lib/oauth/greenhouse.ts @@ -3,6 +3,7 @@ import type { OAuth2Tokens } from 'better-auth/oauth2' import type { GenericOAuthConfig } from 'better-auth/plugins' import { z } from 'zod' import { syntheticConnectorEmail } from '@/lib/auth/connector-email' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -90,8 +91,7 @@ export function createGreenhouseProvider({ const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS) let response: Response try { - // boundary-raw-fetch: Greenhouse OAuth token exchange and introspection use external form requests. - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/monday.ts b/apps/sim/lib/oauth/monday.ts index ace87d76433..e563c91023f 100644 --- a/apps/sim/lib/oauth/monday.ts +++ b/apps/sim/lib/oauth/monday.ts @@ -1,6 +1,7 @@ import type { OAuth2Tokens } from 'better-auth/oauth2' import { decodeJwt } from 'jose' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -73,7 +74,7 @@ export async function exchangeMondayAuthorizationCode({ redirectUri, }: ExchangeMondayAuthorizationCodeParams): Promise { const signal = AbortSignal.timeout(MONDAY_OAUTH_TOKEN_TIMEOUT_MS) - const response = await fetch(MONDAY_OAUTH_TOKEN_URL, { + const response = await outboundFetch(MONDAY_OAUTH_TOKEN_URL, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index 1554aab2b6e..397a7e6802b 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -37,7 +37,8 @@ afterAll(resetEnvMock) import { buildConnectorProviders } from '@/lib/auth/connectors/providers' import { DEFAULT_MAX_ERROR_BODY_BYTES } from '@/lib/core/utils/stream-limits' -import { getPerRequestOAuthLinkScopes, OAUTH_PROVIDERS, refreshOAuthToken } from '@/lib/oauth' +import { getPerRequestOAuthLinkScopes, OAUTH_PROVIDERS } from '@/lib/oauth' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' import { getMissingRequiredScopes } from '@/lib/oauth/utils' import { PowerBIBlock } from '@/blocks/blocks/powerbi' @@ -286,7 +287,7 @@ describe('Bitbucket OAuth Connector', () => { describe('OAuth Token Refresh', () => { describe('Basic Auth Providers', () => { - it.concurrent('sends Basic Auth credentials and keeps them out of the body', async () => { + it('sends Basic Auth credentials and keeps them out of the body', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -418,7 +419,7 @@ describe('OAuth Token Refresh', () => { } ) - it.concurrent('should refresh TikTok with client_key instead of client_id', async () => { + it('should refresh TikTok with client_key instead of client_id', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -434,7 +435,7 @@ describe('OAuth Token Refresh', () => { expect(bodyParams.get('client_id')).toBeNull() }) - it.concurrent('should preserve Cal.com bearer refresh authentication', async () => { + it('should preserve Cal.com bearer refresh authentication', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -454,7 +455,7 @@ describe('OAuth Token Refresh', () => { expect(bodyParams.get('refresh_token')).toBeNull() }) - it.concurrent('should send Notion request with Basic Auth header and JSON body', async () => { + it('should send Notion request with Basic Auth header and JSON body', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -494,7 +495,7 @@ describe('OAuth Token Refresh', () => { }) describe('Error Handling', () => { - it.concurrent('should return the canonical error for partial OAuth configuration', async () => { + it('should return the canonical error for partial OAuth configuration', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const result = await withMockFetch(mockFetch, () => @@ -508,7 +509,7 @@ describe('OAuth Token Refresh', () => { }) expect(mockFetch).not.toHaveBeenCalled() }) - it.concurrent('should return failure with errorCode for HTTP error responses', async () => { + it('should return failure with errorCode for HTTP error responses', async () => { const mockFetch = vi.fn().mockResolvedValue({ ok: false, status: 400, @@ -528,7 +529,7 @@ describe('OAuth Token Refresh', () => { } }) - it.concurrent('should return failure for Slack-style body errors', async () => { + it('should return failure for Slack-style body errors', async () => { const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, @@ -545,37 +546,32 @@ describe('OAuth Token Refresh', () => { } }) - it.concurrent( - 'should redact literal and encoded credentials echoed by a provider', - async () => { - const refreshToken = 'refresh/with space' - const formEncodedRefreshToken = new URLSearchParams({ value: refreshToken }) - .toString() - .slice('value='.length) - const mockFetch = vi - .fn() - .mockResolvedValue( - new Response( - `provider echo: ${refreshToken}, ${encodeURIComponent(refreshToken)}, ${formEncodedRefreshToken}, and google_client_secret`, - { status: 400 } - ) + it('should redact literal and encoded credentials echoed by a provider', async () => { + const refreshToken = 'refresh/with space' + const formEncodedRefreshToken = new URLSearchParams({ value: refreshToken }) + .toString() + .slice('value='.length) + const mockFetch = vi + .fn() + .mockResolvedValue( + new Response( + `provider echo: ${refreshToken}, ${encodeURIComponent(refreshToken)}, ${formEncodedRefreshToken}, and google_client_secret`, + { status: 400 } ) - - const result = await withMockFetch(mockFetch, () => - refreshOAuthToken('google', refreshToken) ) - expect(result.ok).toBe(false) - if (!result.ok) { - expect(result.message).not.toContain(refreshToken) - expect(result.message).not.toContain(encodeURIComponent(refreshToken)) - expect(result.message).not.toContain(formEncodedRefreshToken) - expect(result.message).not.toContain('google_client_secret') - } + const result = await withMockFetch(mockFetch, () => refreshOAuthToken('google', refreshToken)) + + expect(result.ok).toBe(false) + if (!result.ok) { + expect(result.message).not.toContain(refreshToken) + expect(result.message).not.toContain(encodeURIComponent(refreshToken)) + expect(result.message).not.toContain(formEncodedRefreshToken) + expect(result.message).not.toContain('google_client_secret') } - ) + }) - it.concurrent('should redact a secret from a successful HTTP body error', async () => { + it('should redact a secret from a successful HTTP body error', async () => { const refreshToken = 'slack-refresh-secret' const mockFetch = vi .fn() @@ -590,7 +586,7 @@ describe('OAuth Token Refresh', () => { } }) - it.concurrent('uses canonical endpoints without following credential redirects', async () => { + it('uses canonical endpoints without following credential redirects', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) await withMockFetch(mockFetch, () => refreshOAuthToken('google', 'test_refresh_token')) @@ -601,23 +597,20 @@ describe('OAuth Token Refresh', () => { ) }) - it.concurrent( - 'should reject oversized OAuth error responses without materializing them', - async () => { - const mockFetch = vi - .fn() - .mockResolvedValue( - new Response('x'.repeat(DEFAULT_MAX_ERROR_BODY_BYTES + 1), { status: 400 }) - ) - - const result = await withMockFetch(mockFetch, () => - refreshOAuthToken('google', 'test_refresh_token') + it('should reject oversized OAuth error responses without materializing them', async () => { + const mockFetch = vi + .fn() + .mockResolvedValue( + new Response('x'.repeat(DEFAULT_MAX_ERROR_BODY_BYTES + 1), { status: 400 }) ) - expect(result.ok).toBe(false) - if (!result.ok) expect(result.message).toContain('exceeds maximum size') - } - ) + const result = await withMockFetch(mockFetch, () => + refreshOAuthToken('google', 'test_refresh_token') + ) + + expect(result.ok).toBe(false) + if (!result.ok) expect(result.message).toContain('exceeds maximum size') + }) }) describe('Token Response Handling', () => { @@ -647,7 +640,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('should bound successful token responses before parsing them', async () => { + it('should bound successful token responses before parsing them', async () => { const mockFetch = vi .fn() .mockResolvedValue(new Response('x'.repeat(DEFAULT_MAX_ERROR_BODY_BYTES + 1))) @@ -660,7 +653,7 @@ describe('OAuth Token Refresh', () => { if (!result.ok) expect(result.message).toContain('exceeds maximum size') }) - it.concurrent('refreshes Monday with JSON body credentials and rotates its token', async () => { + it('refreshes Monday with JSON body credentials and rotates its token', async () => { const expiresAtSeconds = Math.floor(Date.now() / 1000) + 2700 const mockFetch = createMockFetch({ json: { @@ -695,7 +688,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('rejects a Monday refresh response that omits token rotation', async () => { + it('rejects a Monday refresh response that omits token rotation', async () => { const mockFetch = createMockFetch({ json: { access_token: oauthTestJwt({ exp: Math.floor(Date.now() / 1000) + 3600 }), @@ -713,7 +706,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('should use original refresh token when new one is not provided', async () => { + it('should use original refresh token when new one is not provided', async () => { const refreshToken = 'original_refresh_token' const mockFetch = vi.fn().mockResolvedValue({ @@ -734,7 +727,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('should return failure when access token is missing', async () => { + it('should return failure when access token is missing', async () => { const refreshToken = 'test_refresh_token' const mockFetch = vi.fn().mockResolvedValue({ @@ -751,7 +744,7 @@ describe('OAuth Token Refresh', () => { }) describe('Instagram Token Refresh', () => { - it.concurrent('validates and rotates the long-lived token response', async () => { + it('validates and rotates the long-lived token response', async () => { const mockFetch = vi.fn().mockResolvedValue( Response.json({ access_token: 'new_instagram_access_token', @@ -776,7 +769,7 @@ describe('OAuth Token Refresh', () => { ) }) - it.concurrent('rejects malformed long-lived token response fields', async () => { + it('rejects malformed long-lived token response fields', async () => { const mockFetch = vi.fn().mockResolvedValue( Response.json({ access_token: 'new_instagram_access_token', @@ -794,7 +787,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('extracts nested Meta error codes from a bounded response', async () => { + it('extracts nested Meta error codes from a bounded response', async () => { const mockFetch = vi .fn() .mockResolvedValue( diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index ed881ddac3d..a38a7d12554 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -1,10 +1,3 @@ -import { - type OAuthClientCapabilityField, - type OAuthClientCapabilityId, - requireOAuthClientCapability, -} from '@sim/deployment-config/env-capabilities' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { AffinityIcon, AgentMailIcon, @@ -201,40 +194,14 @@ import { ZoomIcon, ZoomInfoIcon, } from '@/components/icons' -import { env } from '@/lib/core/config/env' import { isSlackExtendedScopesEnabled } from '@/lib/core/config/env-flags' -import { redactExactSensitiveValues } from '@/lib/core/security/redaction' -import { - DEFAULT_MAX_ERROR_BODY_BYTES, - readResponseTextWithLimit, -} from '@/lib/core/utils/stream-limits' -import { APOLLO_TOKEN_URL } from '@/lib/oauth/apollo' -import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' -import { - GITHUB_TOKEN_URL, - parseGitHubRepositoriesTokenResponse, -} from '@/lib/oauth/github-repositories' -import { - GREENHOUSE_TOKEN_URL, - getGreenhouseRefreshErrorCode, - parseGreenhouseTokens, -} from '@/lib/oauth/greenhouse' -import { parseInstagramLongLivedToken } from '@/lib/oauth/instagram' -import { MONDAY_OAUTH_TOKEN_URL, resolveMondayAccessTokenExpiresAt } from '@/lib/oauth/monday' -import type { QuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' -import { QUICKBOOKS_TOKEN_URL } from '@/lib/oauth/quickbooks-constants' -import { isCredentialRevocationError } from '@/lib/oauth/refresh-error-codes' import { SALESFORCE_ADDITIONAL_PROVIDER_IDS, - SALESFORCE_LOGIN_HOSTS, SALESFORCE_PROVIDER_ID_LABELS, } from '@/lib/oauth/salesforce' -import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' import type { OAuthProviderConfig } from './types' -const logger = createLogger('OAuth') - /** * Slack scopes requested only where the app is approved for them, gated by * {@link isSlackExtendedScopesEnabled}. Slack rejects the entire authorization @@ -4053,720 +4020,6 @@ export const OAUTH_PROVIDERS: Record = { }, } -interface ProviderAuthConfig { - tokenEndpoint: string - clientId: string - clientSecret: string - useBasicAuth: boolean - additionalHeaders?: Record - supportsRefreshTokenRotation?: boolean - /** - * If true, the refresh token is sent in the Authorization header as Bearer token - * instead of in the request body. Used by Cal.com. - */ - refreshTokenInAuthHeader?: boolean - /** - * If true, the token endpoint expects a JSON body with Content-Type: application/json - * instead of the default application/x-www-form-urlencoded. Used by Notion. - */ - useJsonBody?: boolean - /** - * Token refresh strategy. `instagram_long_lived` uses Meta's GET - * `refresh_access_token?grant_type=ig_refresh_token` flow instead of a - * standard OAuth refresh_token POST. - */ - refreshStrategy?: 'standard' | 'instagram_long_lived' - /** - * Body param name to use for the client identifier instead of the standard `client_id`. - * TikTok requires `client_key` instead. - */ - clientIdParamName?: string - /** Greenhouse requires grant parameters in the query string, with credentials in Basic auth. */ - refreshParamsInQuery?: boolean -} - -function getConfiguredClientCredentials( - providerId: TCapabilityId, - clientIdField: NoInfer>, - clientSecretField?: NoInfer> -): Pick { - const { values } = requireOAuthClientCapability(providerId, env) - return { - clientId: values[clientIdField], - clientSecret: clientSecretField ? values[clientSecretField] : '', - } -} - -/** - * Get OAuth provider configuration for token refresh - */ -function getProviderAuthConfig( - provider: string, - clientOverride?: Pick -): ProviderAuthConfig { - if (clientOverride && provider !== 'quickbooks') { - throw new Error(`OAuth client override is not supported for provider ${provider}`) - } - switch (provider) { - case 'greenhouse': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'greenhouse', - 'GREENHOUSE_CLIENT_ID', - 'GREENHOUSE_CLIENT_SECRET' - ) - return { - tokenEndpoint: GREENHOUSE_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - refreshParamsInQuery: true, - } - } - case 'google': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'google', - 'GOOGLE_CLIENT_ID', - 'GOOGLE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://oauth2.googleapis.com/token', - clientId, - clientSecret, - useBasicAuth: false, - } - } - case 'x': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'x', - 'X_CLIENT_ID', - 'X_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.x.com/2/oauth2/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'tiktok': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'tiktok', - 'TIKTOK_CLIENT_ID', - 'TIKTOK_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://open.tiktokapis.com/v2/oauth/token/', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - // TikTok requires `client_key` in the token request body instead of `client_id`. - clientIdParamName: 'client_key', - } - } - case 'confluence': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'confluence', - 'CONFLUENCE_CLIENT_ID', - 'CONFLUENCE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://auth.atlassian.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - useJsonBody: true, - supportsRefreshTokenRotation: true, - } - } - case 'jira': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'jira', - 'JIRA_CLIENT_ID', - 'JIRA_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://auth.atlassian.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - useJsonBody: true, - supportsRefreshTokenRotation: true, - } - } - case 'calcom': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'calcom', - 'CALCOM_CLIENT_ID' - ) - return { - tokenEndpoint: 'https://app.cal.com/api/auth/oauth/refreshToken', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - // Cal.com requires refresh token in Authorization header, not body - refreshTokenInAuthHeader: true, - } - } - case 'airtable': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'airtable', - 'AIRTABLE_CLIENT_ID', - 'AIRTABLE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://airtable.com/oauth2/v1/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'bitbucket': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'bitbucket', - 'BITBUCKET_CLIENT_ID', - 'BITBUCKET_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://bitbucket.org/site/oauth2/access_token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'github-repositories': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'github-repositories', - 'GITHUB_APP_CLIENT_ID', - 'GITHUB_APP_CLIENT_SECRET' - ) - return { - tokenEndpoint: GITHUB_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: false, - additionalHeaders: { Accept: 'application/json' }, - supportsRefreshTokenRotation: true, - } - } - case 'notion': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'notion', - 'NOTION_CLIENT_ID', - 'NOTION_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.notion.com/v1/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - useJsonBody: true, - } - } - case 'microsoft': - case 'outlook': - case 'onedrive': - case 'sharepoint': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'microsoft', - 'MICROSOFT_CLIENT_ID', - 'MICROSOFT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://login.microsoftonline.com/common/oauth2/v2.0/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'clickup': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'clickup', - 'CLICKUP_CLIENT_ID', - 'CLICKUP_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.clickup.com/api/v2/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'linear': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'linear', - 'LINEAR_CLIENT_ID', - 'LINEAR_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.linear.app/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'attio': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'attio', - 'ATTIO_CLIENT_ID', - 'ATTIO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.attio.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - } - } - case 'box': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'box', - 'BOX_CLIENT_ID', - 'BOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.box.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - // Box refresh tokens are single-use: "the Refresh Token is invalidated and a - // new Refresh Token is returned" and "A Refresh Token is valid for 60 days and - // can be used to obtain a new Access Token and Refresh Token only once." - // (developer.box.com/guides/authentication/tokens/refresh). Without rotation the - // new token is discarded and the credential dies on the second refresh. - supportsRefreshTokenRotation: true, - } - } - case 'docusign': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'docusign', - 'DOCUSIGN_CLIENT_ID', - 'DOCUSIGN_CLIENT_SECRET' - ) - return { - tokenEndpoint: getDocusignOAuthUrl('/oauth/token'), - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'dropbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'dropbox', - 'DROPBOX_CLIENT_ID', - 'DROPBOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.dropboxapi.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'slack': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'slack', - 'SLACK_CLIENT_ID', - 'SLACK_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://slack.com/api/oauth.v2.access', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'reddit': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'reddit', - 'REDDIT_CLIENT_ID', - 'REDDIT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://www.reddit.com/api/v1/access_token', - clientId, - clientSecret, - useBasicAuth: true, - additionalHeaders: { - 'User-Agent': REDDIT_USER_AGENT, - }, - } - } - case 'ramp': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'ramp', - 'RAMP_CLIENT_ID', - 'RAMP_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.ramp.com/developer/v1/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: false, - } - } - case 'helpscout': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'helpscout', - 'HELPSCOUT_CLIENT_ID', - 'HELPSCOUT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.helpscout.net/v2/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'wealthbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'wealthbox', - 'WEALTHBOX_CLIENT_ID', - 'WEALTHBOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.crmworkspace.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'webflow': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'webflow', - 'WEBFLOW_CLIENT_ID', - 'WEBFLOW_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.webflow.com/oauth/access_token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'asana': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'asana', - 'ASANA_CLIENT_ID', - 'ASANA_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.asana.com/-/oauth_token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'apollo': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'apollo', - 'APOLLO_CLIENT_ID', - 'APOLLO_CLIENT_SECRET' - ) - return { - tokenEndpoint: APOLLO_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'pipedrive': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'pipedrive', - 'PIPEDRIVE_CLIENT_ID', - 'PIPEDRIVE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://oauth.pipedrive.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'quickbooks': { - if (!clientOverride) { - throw new Error('QuickBooks OAuth client configuration is missing') - } - return { - tokenEndpoint: QUICKBOOKS_TOKEN_URL, - clientId: clientOverride.clientId, - clientSecret: clientOverride.clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'hubspot': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'hubspot', - 'HUBSPOT_CLIENT_ID', - 'HUBSPOT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.hubapi.com/oauth/v1/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'linkedin': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'linkedin', - 'LINKEDIN_CLIENT_ID', - 'LINKEDIN_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://www.linkedin.com/oauth/v2/accessToken', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'instagram': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'instagram', - 'INSTAGRAM_CLIENT_ID', - 'INSTAGRAM_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://graph.instagram.com/refresh_access_token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - refreshStrategy: 'instagram_long_lived', - } - } - case 'salesforce': - case 'salesforce-sandbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'salesforce', - 'SALESFORCE_CLIENT_ID', - 'SALESFORCE_CLIENT_SECRET' - ) - // A refresh token is only redeemable at the authorization server that - // issued it: a sandbox token posted to login.salesforce.com fails with - // `invalid_grant`. One Connected App's consumer key is valid at both - // hosts, so only the endpoint differs. - return { - tokenEndpoint: `https://${SALESFORCE_LOGIN_HOSTS[provider]}/services/oauth2/token`, - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'shopify': - throw new Error('Shopify refresh requires its persisted installation context') - case 'zoom': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoom', - 'ZOOM_CLIENT_ID', - 'ZOOM_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://zoom.us/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'wordpress': { - // WordPress.com does NOT support refresh tokens - // Users will need to re-authorize when tokens expire (~2 weeks) - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'wordpress', - 'WORDPRESS_CLIENT_ID', - 'WORDPRESS_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://public-api.wordpress.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'spotify': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'spotify', - 'SPOTIFY_CLIENT_ID', - 'SPOTIFY_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.spotify.com/api/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: false, - } - } - case 'monday': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'monday', - 'MONDAY_CLIENT_ID', - 'MONDAY_CLIENT_SECRET' - ) - return { - tokenEndpoint: MONDAY_OAUTH_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: false, - useJsonBody: true, - supportsRefreshTokenRotation: true, - } - } - case 'manageengine-sdp': { - // ServiceDesk Plus Cloud authenticates through Zoho, so the grant is the - // same one Zoho Desk uses and shares its client credentials: scopes are - // chosen per authorization request, not per API-console client, so one - // registered client serves both products. - // - // Rotation stays off for the same reason as zoho-desk below - Zoho's - // refresh_token grant returns a new access token but no new refresh token. - // accounts.zoho.com is correct because the authorize and code-exchange - // legs in lib/auth/connectors/providers.ts are pinned to the US accounts - // server, so every refresh token in the system is US-issued. Data - // residency for API calls is honored separately, via the block's data - // center selector. - // Keyed on the `zoho-desk` capability, which is what - // `resolveOAuthClientCapabilityId('manageengine-sdp')` aliases to — the - // capability names the env pair, not the product. - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoho-desk', - 'ZOHO_CLIENT_ID', - 'ZOHO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'zoho-desk': { - // Zoho's refresh_token grant returns a new access token but no new refresh - // token, so rotation stays off (the existing refresh token is preserved). - // The refresh must target the accounts server of the data center that issued - // the token - "if location=eu, you will need to make access token request to - // https://accounts.zoho.eu" (zoho.com/accounts/protocol/oauth/multi-dc.html). - // accounts.zoho.com is correct here because the authorize and code-exchange - // legs in lib/auth/connectors/providers.ts are also pinned to the US accounts - // server, so every refresh token in the system is US-issued. Making refresh - // DC-aware requires making the grant DC-aware first (read the `accounts-server` - // callback param) and threading the credential's persisted `__zoho_domain__` - // marker into refreshOAuthToken, which today only receives the token string. - // Data residency for API calls is already honored via that persisted Desk base. - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoho-desk', - 'ZOHO_CLIENT_ID', - 'ZOHO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - default: - throw new Error(`Unsupported provider: ${provider}`) - } -} - -/** - * Build the authentication request headers and body for OAuth token refresh - */ -function buildAuthRequest( - config: ProviderAuthConfig, - refreshToken: string -): { headers: Record; bodyParams: Record; useJsonBody?: boolean } { - const headers: Record = { - 'Content-Type': config.useJsonBody ? 'application/json' : 'application/x-www-form-urlencoded', - ...config.additionalHeaders, - } - - const bodyParams: Record = { - grant_type: 'refresh_token', - } - - // Handle refresh token placement - if (config.refreshTokenInAuthHeader) { - // Cal.com style: refresh token in Authorization header as Bearer token - headers.Authorization = `Bearer ${refreshToken}` - } else { - // Standard OAuth: refresh token in request body - bodyParams.refresh_token = refreshToken - } - - if (config.useBasicAuth) { - // Use Basic Authentication - credentials in Authorization header only - const basicAuth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64') - headers.Authorization = `Basic ${basicAuth}` - } else { - // Use body credentials - include client credentials in request body - bodyParams[config.clientIdParamName || 'client_id'] = config.clientId - if (config.clientSecret) { - bodyParams.client_secret = config.clientSecret - } - } - - return { headers, bodyParams, useJsonBody: config.useJsonBody } -} - -/** - * Resolves the key {@link getProviderAuthConfig} is switched on for a stored - * credential's provider id. - * - * Normally that is the base provider, because every service in a family - * refreshes against the same endpoint with the same client. A provider id - * listed in a service's `additionalProviderIds` is the exception: it names a - * *different* authorization server for the same service, so it must reach - * `getProviderAuthConfig` intact — collapsing it to the base would silently - * refresh a sandbox token against the production endpoint. - */ -function getBaseProviderForService(providerId: string): string { - if (providerId in OAUTH_PROVIDERS) { - return providerId - } - - for (const [baseProvider, config] of Object.entries(OAUTH_PROVIDERS)) { - for (const service of Object.values(config.services)) { - if (service.providerId === providerId) { - return baseProvider - } - if (service.additionalProviderIds?.includes(providerId)) { - return providerId - } - } - } - - throw new Error(`Unknown OAuth provider: ${providerId}`) -} - export interface RefreshTokenSuccess { ok: true accessToken: string @@ -4783,322 +4036,11 @@ export interface RefreshTokenFailure { export type RefreshTokenResult = RefreshTokenSuccess | RefreshTokenFailure -function extractErrorCode(value: unknown): string | undefined { - if (value && typeof value === 'object' && 'error' in value) { - const error = (value as { error: unknown }).error - if (typeof error === 'string') return error - if (error && typeof error === 'object' && 'code' in error) { - const code = (error as { code: unknown }).code - if (typeof code === 'string' || typeof code === 'number') return String(code) - } - } - return undefined -} - -function safeOAuthErrorCode(value: unknown, secrets: string[]): string | undefined { - const errorCode = extractErrorCode(value) - if (!errorCode) return undefined - const safeCode = redactExactSensitiveValues(errorCode, secrets).trim().toLowerCase() - return /^[a-z0-9][a-z0-9._:-]{0,127}$/.test(safeCode) ? safeCode : undefined -} - /** - * Hard deadline on the token-endpoint exchange. This function does not coalesce + * Hard deadline on the token-endpoint exchange. `refreshOAuthToken` does not coalesce * on its own; its sole production caller (`performCoalescedRefresh` in the OAuth * utils) shares one in-flight refresh across concurrent callers for a credential. * Without this bound a hung endpoint would wedge every joiner on that key until * the undici socket defaults (~5 min) gave up. */ export const TOKEN_REFRESH_TIMEOUT_MS = 15_000 - -function parseOAuthResponse(responseText: string): unknown { - try { - return JSON.parse(responseText) - } catch { - return responseText - } -} - -function oauthResponseRecord(value: unknown): Record | undefined { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : undefined -} - -const OAUTH_RESPONSE_OMITTED = '[token endpoint response omitted]' - -/** A revoked grant is its owner's to reconnect, not a fault of ours, so it logs at WARN. */ -function logRefreshRejection(errorCode: string | undefined, providerId: string) { - return isCredentialRevocationError(errorCode, providerId) - ? logger.warn.bind(logger) - : logger.error.bind(logger) -} - -async function refreshInstagramLongLivedToken( - config: ProviderAuthConfig, - longLivedToken: string, - providerId: string -): Promise { - const url = new URL(config.tokenEndpoint) - url.searchParams.set('grant_type', 'ig_refresh_token') - url.searchParams.set('access_token', longLivedToken) - - const response = await fetch(url.toString(), { - method: 'GET', - redirect: 'error', - signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), - }) - - const responseText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Instagram token refresh response', - }) - const responseData = parseOAuthResponse(responseText) - - if (!response.ok) { - const exactSecrets = [longLivedToken, config.clientSecret ?? ''] - const errorCode = safeOAuthErrorCode(responseData, exactSecrets) - logRefreshRejection(errorCode, providerId)('Instagram long-lived token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, - } - } - - const payload = parseInstagramLongLivedToken(responseData) - if (!payload) { - logger.warn('Invalid Instagram refresh response', { providerId }) - return { ok: false, message: 'Invalid Instagram token refresh response' } - } - - logger.info('Instagram long-lived token refreshed successfully', { - expiresIn: payload.expires_in, - providerId, - }) - - // Instagram returns a new long-lived token; store it as both access and refresh. - return { - ok: true, - accessToken: payload.access_token, - expiresIn: payload.expires_in, - refreshToken: payload.access_token, - } -} - -export async function refreshOAuthToken( - providerId: string, - refreshToken: string, - clientOverride?: Pick -): Promise { - const exactSecrets = [refreshToken] - try { - const provider = getBaseProviderForService(providerId) - - const config = getProviderAuthConfig(provider, clientOverride) - if (config.clientSecret) exactSecrets.push(config.clientSecret) - - if (config.refreshStrategy === 'instagram_long_lived') { - return await refreshInstagramLongLivedToken(config, refreshToken, providerId) - } - - const { headers, bodyParams, useJsonBody } = buildAuthRequest(config, refreshToken) - - // Microsoft refresh tokens are resource-independent. Keep the Power BI audience explicit. - if (providerId === 'microsoft-powerbi') { - bodyParams.scope = OAUTH_PROVIDERS.microsoft.services['microsoft-powerbi'].scopes.join(' ') - } - - const tokenUrl = new URL(config.tokenEndpoint) - if (config.refreshParamsInQuery) { - tokenUrl.search = new URLSearchParams(bodyParams).toString() - } - const response = await fetch(tokenUrl.toString(), { - method: 'POST', - headers, - body: config.refreshParamsInQuery - ? undefined - : useJsonBody - ? JSON.stringify(bodyParams) - : new URLSearchParams(bodyParams).toString(), - redirect: 'error', - signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), - }) - - const responseText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'OAuth token refresh response', - }) - const responseData = parseOAuthResponse(responseText) - - if (!response.ok) { - const errorCode = - (provider === 'greenhouse' ? getGreenhouseRefreshErrorCode(responseData) : undefined) ?? - safeOAuthErrorCode(responseData, exactSecrets) - - logRefreshRejection(errorCode, providerId)('Token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - hasClientId: !!config.clientId, - hasClientSecret: !!config.clientSecret, - hasRefreshToken: !!refreshToken, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, - } - } - - const data = oauthResponseRecord(responseData) - if (!data) { - logger.warn('Invalid OAuth token refresh response', { providerId }) - return { ok: false, message: 'Invalid OAuth token refresh response' } - } - - if ( - data.ok === false || - (provider === 'github-repositories' && typeof data.error === 'string') - ) { - const errorCode = safeOAuthErrorCode(data, exactSecrets) - logRefreshRejection(errorCode, providerId)('Token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - hasClientId: !!config.clientId, - hasClientSecret: !!config.clientSecret, - hasRefreshToken: !!refreshToken, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${OAUTH_RESPONSE_OMITTED}`, - } - } - - if (provider === 'greenhouse') { - const tokens = parseGreenhouseTokens(data) - return { - ok: true, - accessToken: tokens.accessToken, - refreshToken: tokens.refreshToken, - expiresIn: Math.max( - 1, - Math.ceil((tokens.accessTokenExpiresAt.getTime() - Date.now()) / 1000) - ), - refreshTokenExpiresIn: Math.max( - 1, - Math.ceil((tokens.refreshTokenExpiresAt.getTime() - Date.now()) / 1000) - ), - } - } - - if (provider === 'github-repositories') { - const tokens = parseGitHubRepositoriesTokenResponse(data) - return { - ok: true, - accessToken: tokens.access_token, - refreshToken: tokens.refresh_token, - expiresIn: tokens.expires_in, - refreshTokenExpiresIn: tokens.refresh_token_expires_in, - } - } - - const accessToken = - typeof data.access_token === 'string' && data.access_token.length > 0 - ? data.access_token - : undefined - - let newRefreshToken: string | undefined - if ( - config.supportsRefreshTokenRotation && - typeof data.refresh_token === 'string' && - data.refresh_token.length > 0 - ) { - newRefreshToken = data.refresh_token - logger.info(`Received new refresh token from ${provider}`) - } - if (provider === 'monday' && !newRefreshToken) { - logger.warn('Monday token refresh response omitted its rotating refresh token') - return { ok: false, message: 'Invalid Monday token refresh response' } - } - if (provider === 'quickbooks' && !newRefreshToken) { - logger.warn('QuickBooks token refresh response omitted its rotating refresh token') - return { ok: false, message: 'Invalid QuickBooks token refresh response' } - } - - const rawExpiresIn = data.expires_in ?? data.expiresIn - const parsedExpiresIn = - typeof rawExpiresIn === 'number' || typeof rawExpiresIn === 'string' - ? Number(rawExpiresIn) - : Number.NaN - const responseExpiresIn = - Number.isFinite(parsedExpiresIn) && parsedExpiresIn > 0 ? parsedExpiresIn : undefined - const expiresIn = - provider === 'monday' && accessToken - ? Math.max( - 1, - Math.ceil( - (resolveMondayAccessTokenExpiresAt(accessToken, responseExpiresIn).getTime() - - Date.now()) / - 1000 - ) - ) - : (responseExpiresIn ?? 3600) - - const rawRefreshTokenExpiresIn = data.x_refresh_token_expires_in - const parsedRefreshTokenExpiresIn = - typeof rawRefreshTokenExpiresIn === 'number' || typeof rawRefreshTokenExpiresIn === 'string' - ? Number(rawRefreshTokenExpiresIn) - : Number.NaN - const refreshTokenExpiresIn = - provider === 'quickbooks' && - Number.isSafeInteger(parsedRefreshTokenExpiresIn) && - parsedRefreshTokenExpiresIn > 0 - ? parsedRefreshTokenExpiresIn - : undefined - - if (!accessToken) { - // Log only the shape, never `data` itself - on a partial success it can - // carry live tokens. - logger.warn('No access token found in refresh response', { - providerId, - responseKeys: Object.keys(data ?? {}), - }) - return { ok: false, message: 'No access token in refresh response' } - } - - logger.info('Token refreshed successfully with expiration', { - expiresIn, - hasNewRefreshToken: !!newRefreshToken, - provider, - }) - - return { - ok: true, - accessToken, - expiresIn, - refreshToken: newRefreshToken ?? refreshToken, - ...(refreshTokenExpiresIn ? { refreshTokenExpiresIn } : {}), - } - } catch (error) { - const normalized = toError(error) - const message = - normalized.name === 'PayloadSizeLimitError' || normalized.message.startsWith('OAuth client ') - ? normalized.message - : 'Token refresh failed' - logger.error('Error refreshing token', { errorType: normalized.name }) - return { ok: false, message } - } -} diff --git a/apps/sim/lib/oauth/quickbooks.ts b/apps/sim/lib/oauth/quickbooks.ts index 2f4f9b6d840..9f1990d4480 100644 --- a/apps/sim/lib/oauth/quickbooks.ts +++ b/apps/sim/lib/oauth/quickbooks.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -150,7 +151,7 @@ export async function revokeQuickBooksToken( let response: Response try { - response = await fetch(QUICKBOOKS_REVOCATION_URL, { + response = await outboundFetch(QUICKBOOKS_REVOCATION_URL, { method: 'POST', headers: { Accept: 'application/json', @@ -183,7 +184,7 @@ export async function fetchQuickBooksConnectionProfile( clientConfig: Pick ): Promise { const realmId = normalizeQuickBooksRealmId(callbackRealmId) - const response = await fetch(getQuickBooksUserInfoUrl(clientConfig.environment), { + const response = await outboundFetch(getQuickBooksUserInfoUrl(clientConfig.environment), { headers: buildQuickBooksHeaders(accessToken), signal: AbortSignal.timeout(QUICKBOOKS_OAUTH_REQUEST_TIMEOUT_MS), }) @@ -254,7 +255,7 @@ export async function exchangeQuickBooksAuthorizationCode(params: { signal?: AbortSignal }): Promise { const clientConfig = normalizeQuickBooksOAuthClientConfig(params.clientConfig) - const response = await fetch(QUICKBOOKS_TOKEN_URL, { + const response = await outboundFetch(QUICKBOOKS_TOKEN_URL, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/refresh.server.ts b/apps/sim/lib/oauth/refresh.server.ts new file mode 100644 index 00000000000..4342a9a14fc --- /dev/null +++ b/apps/sim/lib/oauth/refresh.server.ts @@ -0,0 +1,1069 @@ +import { + type OAuthClientCapabilityField, + type OAuthClientCapabilityId, + requireOAuthClientCapability, +} from '@sim/deployment-config/env-capabilities' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { redactExactSensitiveValues } from '@/lib/core/security/redaction' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { APOLLO_TOKEN_URL } from '@/lib/oauth/apollo' +import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' +import { + GITHUB_TOKEN_URL, + parseGitHubRepositoriesTokenResponse, +} from '@/lib/oauth/github-repositories' +import { + GREENHOUSE_TOKEN_URL, + getGreenhouseRefreshErrorCode, + parseGreenhouseTokens, +} from '@/lib/oauth/greenhouse' +import { parseInstagramLongLivedToken } from '@/lib/oauth/instagram' +import { MONDAY_OAUTH_TOKEN_URL, resolveMondayAccessTokenExpiresAt } from '@/lib/oauth/monday' +import { + OAUTH_PROVIDERS, + type RefreshTokenResult, + TOKEN_REFRESH_TIMEOUT_MS, +} from '@/lib/oauth/oauth' +import type { QuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' +import { QUICKBOOKS_TOKEN_URL } from '@/lib/oauth/quickbooks-constants' +import { isCredentialRevocationError } from '@/lib/oauth/refresh-error-codes' +import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' +import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' + +const logger = createLogger('OAuth') + +interface ProviderAuthConfig { + tokenEndpoint: string + clientId: string + clientSecret: string + useBasicAuth: boolean + additionalHeaders?: Record + supportsRefreshTokenRotation?: boolean + /** + * If true, the refresh token is sent in the Authorization header as Bearer token + * instead of in the request body. Used by Cal.com. + */ + refreshTokenInAuthHeader?: boolean + /** + * If true, the token endpoint expects a JSON body with Content-Type: application/json + * instead of the default application/x-www-form-urlencoded. Used by Notion. + */ + useJsonBody?: boolean + /** + * Token refresh strategy. `instagram_long_lived` uses Meta's GET + * `refresh_access_token?grant_type=ig_refresh_token` flow instead of a + * standard OAuth refresh_token POST. + */ + refreshStrategy?: 'standard' | 'instagram_long_lived' + /** + * Body param name to use for the client identifier instead of the standard `client_id`. + * TikTok requires `client_key` instead. + */ + clientIdParamName?: string + /** Greenhouse requires grant parameters in the query string, with credentials in Basic auth. */ + refreshParamsInQuery?: boolean +} + +function getConfiguredClientCredentials( + providerId: TCapabilityId, + clientIdField: NoInfer>, + clientSecretField?: NoInfer> +): Pick { + const { values } = requireOAuthClientCapability(providerId, env) + return { + clientId: values[clientIdField], + clientSecret: clientSecretField ? values[clientSecretField] : '', + } +} + +/** + * Get OAuth provider configuration for token refresh + */ +function getProviderAuthConfig( + provider: string, + clientOverride?: Pick +): ProviderAuthConfig { + if (clientOverride && provider !== 'quickbooks') { + throw new Error(`OAuth client override is not supported for provider ${provider}`) + } + switch (provider) { + case 'greenhouse': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'greenhouse', + 'GREENHOUSE_CLIENT_ID', + 'GREENHOUSE_CLIENT_SECRET' + ) + return { + tokenEndpoint: GREENHOUSE_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + refreshParamsInQuery: true, + } + } + case 'google': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'google', + 'GOOGLE_CLIENT_ID', + 'GOOGLE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://oauth2.googleapis.com/token', + clientId, + clientSecret, + useBasicAuth: false, + } + } + case 'x': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'x', + 'X_CLIENT_ID', + 'X_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.x.com/2/oauth2/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'tiktok': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'tiktok', + 'TIKTOK_CLIENT_ID', + 'TIKTOK_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://open.tiktokapis.com/v2/oauth/token/', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + // TikTok requires `client_key` in the token request body instead of `client_id`. + clientIdParamName: 'client_key', + } + } + case 'confluence': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'confluence', + 'CONFLUENCE_CLIENT_ID', + 'CONFLUENCE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://auth.atlassian.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + useJsonBody: true, + supportsRefreshTokenRotation: true, + } + } + case 'jira': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'jira', + 'JIRA_CLIENT_ID', + 'JIRA_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://auth.atlassian.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + useJsonBody: true, + supportsRefreshTokenRotation: true, + } + } + case 'calcom': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'calcom', + 'CALCOM_CLIENT_ID' + ) + return { + tokenEndpoint: 'https://app.cal.com/api/auth/oauth/refreshToken', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + // Cal.com requires refresh token in Authorization header, not body + refreshTokenInAuthHeader: true, + } + } + case 'airtable': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'airtable', + 'AIRTABLE_CLIENT_ID', + 'AIRTABLE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://airtable.com/oauth2/v1/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'bitbucket': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'bitbucket', + 'BITBUCKET_CLIENT_ID', + 'BITBUCKET_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://bitbucket.org/site/oauth2/access_token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'github-repositories': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'github-repositories', + 'GITHUB_APP_CLIENT_ID', + 'GITHUB_APP_CLIENT_SECRET' + ) + return { + tokenEndpoint: GITHUB_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: false, + additionalHeaders: { Accept: 'application/json' }, + supportsRefreshTokenRotation: true, + } + } + case 'notion': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'notion', + 'NOTION_CLIENT_ID', + 'NOTION_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.notion.com/v1/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + useJsonBody: true, + } + } + case 'microsoft': + case 'outlook': + case 'onedrive': + case 'sharepoint': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'microsoft', + 'MICROSOFT_CLIENT_ID', + 'MICROSOFT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://login.microsoftonline.com/common/oauth2/v2.0/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'clickup': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'clickup', + 'CLICKUP_CLIENT_ID', + 'CLICKUP_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.clickup.com/api/v2/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'linear': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'linear', + 'LINEAR_CLIENT_ID', + 'LINEAR_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.linear.app/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'attio': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'attio', + 'ATTIO_CLIENT_ID', + 'ATTIO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.attio.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + } + } + case 'box': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'box', + 'BOX_CLIENT_ID', + 'BOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.box.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + // Box refresh tokens are single-use: "the Refresh Token is invalidated and a + // new Refresh Token is returned" and "A Refresh Token is valid for 60 days and + // can be used to obtain a new Access Token and Refresh Token only once." + // (developer.box.com/guides/authentication/tokens/refresh). Without rotation the + // new token is discarded and the credential dies on the second refresh. + supportsRefreshTokenRotation: true, + } + } + case 'docusign': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'docusign', + 'DOCUSIGN_CLIENT_ID', + 'DOCUSIGN_CLIENT_SECRET' + ) + return { + tokenEndpoint: getDocusignOAuthUrl('/oauth/token'), + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'dropbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'dropbox', + 'DROPBOX_CLIENT_ID', + 'DROPBOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.dropboxapi.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'slack': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'slack', + 'SLACK_CLIENT_ID', + 'SLACK_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://slack.com/api/oauth.v2.access', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'reddit': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'reddit', + 'REDDIT_CLIENT_ID', + 'REDDIT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://www.reddit.com/api/v1/access_token', + clientId, + clientSecret, + useBasicAuth: true, + additionalHeaders: { + 'User-Agent': REDDIT_USER_AGENT, + }, + } + } + case 'ramp': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'ramp', + 'RAMP_CLIENT_ID', + 'RAMP_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.ramp.com/developer/v1/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: false, + } + } + case 'helpscout': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'helpscout', + 'HELPSCOUT_CLIENT_ID', + 'HELPSCOUT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.helpscout.net/v2/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'wealthbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'wealthbox', + 'WEALTHBOX_CLIENT_ID', + 'WEALTHBOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.crmworkspace.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'webflow': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'webflow', + 'WEBFLOW_CLIENT_ID', + 'WEBFLOW_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.webflow.com/oauth/access_token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'asana': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'asana', + 'ASANA_CLIENT_ID', + 'ASANA_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.asana.com/-/oauth_token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'apollo': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'apollo', + 'APOLLO_CLIENT_ID', + 'APOLLO_CLIENT_SECRET' + ) + return { + tokenEndpoint: APOLLO_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'pipedrive': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'pipedrive', + 'PIPEDRIVE_CLIENT_ID', + 'PIPEDRIVE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://oauth.pipedrive.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'quickbooks': { + if (!clientOverride) { + throw new Error('QuickBooks OAuth client configuration is missing') + } + return { + tokenEndpoint: QUICKBOOKS_TOKEN_URL, + clientId: clientOverride.clientId, + clientSecret: clientOverride.clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'hubspot': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'hubspot', + 'HUBSPOT_CLIENT_ID', + 'HUBSPOT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.hubapi.com/oauth/v1/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'linkedin': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'linkedin', + 'LINKEDIN_CLIENT_ID', + 'LINKEDIN_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://www.linkedin.com/oauth/v2/accessToken', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'instagram': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'instagram', + 'INSTAGRAM_CLIENT_ID', + 'INSTAGRAM_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://graph.instagram.com/refresh_access_token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + refreshStrategy: 'instagram_long_lived', + } + } + case 'salesforce': + case 'salesforce-sandbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'salesforce', + 'SALESFORCE_CLIENT_ID', + 'SALESFORCE_CLIENT_SECRET' + ) + // A refresh token is only redeemable at the authorization server that + // issued it: a sandbox token posted to login.salesforce.com fails with + // `invalid_grant`. One Connected App's consumer key is valid at both + // hosts, so only the endpoint differs. + return { + tokenEndpoint: `https://${SALESFORCE_LOGIN_HOSTS[provider]}/services/oauth2/token`, + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'shopify': + throw new Error('Shopify refresh requires its persisted installation context') + case 'zoom': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoom', + 'ZOOM_CLIENT_ID', + 'ZOOM_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://zoom.us/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'wordpress': { + // WordPress.com does NOT support refresh tokens + // Users will need to re-authorize when tokens expire (~2 weeks) + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'wordpress', + 'WORDPRESS_CLIENT_ID', + 'WORDPRESS_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://public-api.wordpress.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'spotify': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'spotify', + 'SPOTIFY_CLIENT_ID', + 'SPOTIFY_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.spotify.com/api/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: false, + } + } + case 'monday': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'monday', + 'MONDAY_CLIENT_ID', + 'MONDAY_CLIENT_SECRET' + ) + return { + tokenEndpoint: MONDAY_OAUTH_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: false, + useJsonBody: true, + supportsRefreshTokenRotation: true, + } + } + case 'manageengine-sdp': { + // ServiceDesk Plus Cloud authenticates through Zoho, so the grant is the + // same one Zoho Desk uses and shares its client credentials: scopes are + // chosen per authorization request, not per API-console client, so one + // registered client serves both products. + // + // Rotation stays off for the same reason as zoho-desk below - Zoho's + // refresh_token grant returns a new access token but no new refresh token. + // accounts.zoho.com is correct because the authorize and code-exchange + // legs in lib/auth/connectors/providers.ts are pinned to the US accounts + // server, so every refresh token in the system is US-issued. Data + // residency for API calls is honored separately, via the block's data + // center selector. + // Keyed on the `zoho-desk` capability, which is what + // `resolveOAuthClientCapabilityId('manageengine-sdp')` aliases to — the + // capability names the env pair, not the product. + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoho-desk', + 'ZOHO_CLIENT_ID', + 'ZOHO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'zoho-desk': { + // Zoho's refresh_token grant returns a new access token but no new refresh + // token, so rotation stays off (the existing refresh token is preserved). + // The refresh must target the accounts server of the data center that issued + // the token - "if location=eu, you will need to make access token request to + // https://accounts.zoho.eu" (zoho.com/accounts/protocol/oauth/multi-dc.html). + // accounts.zoho.com is correct here because the authorize and code-exchange + // legs in lib/auth/connectors/providers.ts are also pinned to the US accounts + // server, so every refresh token in the system is US-issued. Making refresh + // DC-aware requires making the grant DC-aware first (read the `accounts-server` + // callback param) and threading the credential's persisted `__zoho_domain__` + // marker into refreshOAuthToken, which today only receives the token string. + // Data residency for API calls is already honored via that persisted Desk base. + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoho-desk', + 'ZOHO_CLIENT_ID', + 'ZOHO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + default: + throw new Error(`Unsupported provider: ${provider}`) + } +} + +/** + * Build the authentication request headers and body for OAuth token refresh + */ +function buildAuthRequest( + config: ProviderAuthConfig, + refreshToken: string +): { headers: Record; bodyParams: Record; useJsonBody?: boolean } { + const headers: Record = { + 'Content-Type': config.useJsonBody ? 'application/json' : 'application/x-www-form-urlencoded', + ...config.additionalHeaders, + } + + const bodyParams: Record = { + grant_type: 'refresh_token', + } + + // Handle refresh token placement + if (config.refreshTokenInAuthHeader) { + // Cal.com style: refresh token in Authorization header as Bearer token + headers.Authorization = `Bearer ${refreshToken}` + } else { + // Standard OAuth: refresh token in request body + bodyParams.refresh_token = refreshToken + } + + if (config.useBasicAuth) { + // Use Basic Authentication - credentials in Authorization header only + const basicAuth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64') + headers.Authorization = `Basic ${basicAuth}` + } else { + // Use body credentials - include client credentials in request body + bodyParams[config.clientIdParamName || 'client_id'] = config.clientId + if (config.clientSecret) { + bodyParams.client_secret = config.clientSecret + } + } + + return { headers, bodyParams, useJsonBody: config.useJsonBody } +} + +/** + * Resolves the key {@link getProviderAuthConfig} is switched on for a stored + * credential's provider id. + * + * Normally that is the base provider, because every service in a family + * refreshes against the same endpoint with the same client. A provider id + * listed in a service's `additionalProviderIds` is the exception: it names a + * *different* authorization server for the same service, so it must reach + * `getProviderAuthConfig` intact — collapsing it to the base would silently + * refresh a sandbox token against the production endpoint. + */ +function getBaseProviderForService(providerId: string): string { + if (providerId in OAUTH_PROVIDERS) { + return providerId + } + + for (const [baseProvider, config] of Object.entries(OAUTH_PROVIDERS)) { + for (const service of Object.values(config.services)) { + if (service.providerId === providerId) { + return baseProvider + } + if (service.additionalProviderIds?.includes(providerId)) { + return providerId + } + } + } + + throw new Error(`Unknown OAuth provider: ${providerId}`) +} + +function extractErrorCode(value: unknown): string | undefined { + if (value && typeof value === 'object' && 'error' in value) { + const error = (value as { error: unknown }).error + if (typeof error === 'string') return error + if (error && typeof error === 'object' && 'code' in error) { + const code = (error as { code: unknown }).code + if (typeof code === 'string' || typeof code === 'number') return String(code) + } + } + return undefined +} + +function safeOAuthErrorCode(value: unknown, secrets: string[]): string | undefined { + const errorCode = extractErrorCode(value) + if (!errorCode) return undefined + const safeCode = redactExactSensitiveValues(errorCode, secrets).trim().toLowerCase() + return /^[a-z0-9][a-z0-9._:-]{0,127}$/.test(safeCode) ? safeCode : undefined +} + +function parseOAuthResponse(responseText: string): unknown { + try { + return JSON.parse(responseText) + } catch { + return responseText + } +} + +function oauthResponseRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined +} + +const OAUTH_RESPONSE_OMITTED = '[token endpoint response omitted]' + +/** A revoked grant is its owner's to reconnect, not a fault of ours, so it logs at WARN. */ +function logRefreshRejection(errorCode: string | undefined, providerId: string) { + return isCredentialRevocationError(errorCode, providerId) + ? logger.warn.bind(logger) + : logger.error.bind(logger) +} + +async function refreshInstagramLongLivedToken( + config: ProviderAuthConfig, + longLivedToken: string, + providerId: string +): Promise { + const url = new URL(config.tokenEndpoint) + url.searchParams.set('grant_type', 'ig_refresh_token') + url.searchParams.set('access_token', longLivedToken) + + const response = await outboundFetch(url.toString(), { + method: 'GET', + redirect: 'error', + signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), + }) + + const responseText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Instagram token refresh response', + }) + const responseData = parseOAuthResponse(responseText) + + if (!response.ok) { + const exactSecrets = [longLivedToken, config.clientSecret ?? ''] + const errorCode = safeOAuthErrorCode(responseData, exactSecrets) + logRefreshRejection(errorCode, providerId)('Instagram long-lived token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, + } + } + + const payload = parseInstagramLongLivedToken(responseData) + if (!payload) { + logger.warn('Invalid Instagram refresh response', { providerId }) + return { ok: false, message: 'Invalid Instagram token refresh response' } + } + + logger.info('Instagram long-lived token refreshed successfully', { + expiresIn: payload.expires_in, + providerId, + }) + + // Instagram returns a new long-lived token; store it as both access and refresh. + return { + ok: true, + accessToken: payload.access_token, + expiresIn: payload.expires_in, + refreshToken: payload.access_token, + } +} + +/** + * Exchanges a stored refresh token at its provider's token endpoint. The request + * leaves through the current outbound scope, so an organization with a dedicated + * egress gateway refreshes from its own static IPs. + */ +export async function refreshOAuthToken( + providerId: string, + refreshToken: string, + clientOverride?: Pick +): Promise { + const exactSecrets = [refreshToken] + try { + const provider = getBaseProviderForService(providerId) + + const config = getProviderAuthConfig(provider, clientOverride) + if (config.clientSecret) exactSecrets.push(config.clientSecret) + + if (config.refreshStrategy === 'instagram_long_lived') { + return await refreshInstagramLongLivedToken(config, refreshToken, providerId) + } + + const { headers, bodyParams, useJsonBody } = buildAuthRequest(config, refreshToken) + + // Microsoft refresh tokens are resource-independent. Keep the Power BI audience explicit. + if (providerId === 'microsoft-powerbi') { + bodyParams.scope = OAUTH_PROVIDERS.microsoft.services['microsoft-powerbi'].scopes.join(' ') + } + + const tokenUrl = new URL(config.tokenEndpoint) + if (config.refreshParamsInQuery) { + tokenUrl.search = new URLSearchParams(bodyParams).toString() + } + const response = await outboundFetch(tokenUrl.toString(), { + method: 'POST', + headers, + body: config.refreshParamsInQuery + ? undefined + : useJsonBody + ? JSON.stringify(bodyParams) + : new URLSearchParams(bodyParams).toString(), + redirect: 'error', + signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), + }) + + const responseText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'OAuth token refresh response', + }) + const responseData = parseOAuthResponse(responseText) + + if (!response.ok) { + const errorCode = + (provider === 'greenhouse' ? getGreenhouseRefreshErrorCode(responseData) : undefined) ?? + safeOAuthErrorCode(responseData, exactSecrets) + + logRefreshRejection(errorCode, providerId)('Token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + hasClientId: !!config.clientId, + hasClientSecret: !!config.clientSecret, + hasRefreshToken: !!refreshToken, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, + } + } + + const data = oauthResponseRecord(responseData) + if (!data) { + logger.warn('Invalid OAuth token refresh response', { providerId }) + return { ok: false, message: 'Invalid OAuth token refresh response' } + } + + if ( + data.ok === false || + (provider === 'github-repositories' && typeof data.error === 'string') + ) { + const errorCode = safeOAuthErrorCode(data, exactSecrets) + logRefreshRejection(errorCode, providerId)('Token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + hasClientId: !!config.clientId, + hasClientSecret: !!config.clientSecret, + hasRefreshToken: !!refreshToken, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${OAUTH_RESPONSE_OMITTED}`, + } + } + + if (provider === 'greenhouse') { + const tokens = parseGreenhouseTokens(data) + return { + ok: true, + accessToken: tokens.accessToken, + refreshToken: tokens.refreshToken, + expiresIn: Math.max( + 1, + Math.ceil((tokens.accessTokenExpiresAt.getTime() - Date.now()) / 1000) + ), + refreshTokenExpiresIn: Math.max( + 1, + Math.ceil((tokens.refreshTokenExpiresAt.getTime() - Date.now()) / 1000) + ), + } + } + + if (provider === 'github-repositories') { + const tokens = parseGitHubRepositoriesTokenResponse(data) + return { + ok: true, + accessToken: tokens.access_token, + refreshToken: tokens.refresh_token, + expiresIn: tokens.expires_in, + refreshTokenExpiresIn: tokens.refresh_token_expires_in, + } + } + + const accessToken = + typeof data.access_token === 'string' && data.access_token.length > 0 + ? data.access_token + : undefined + + let newRefreshToken: string | undefined + if ( + config.supportsRefreshTokenRotation && + typeof data.refresh_token === 'string' && + data.refresh_token.length > 0 + ) { + newRefreshToken = data.refresh_token + logger.info(`Received new refresh token from ${provider}`) + } + if (provider === 'monday' && !newRefreshToken) { + logger.warn('Monday token refresh response omitted its rotating refresh token') + return { ok: false, message: 'Invalid Monday token refresh response' } + } + if (provider === 'quickbooks' && !newRefreshToken) { + logger.warn('QuickBooks token refresh response omitted its rotating refresh token') + return { ok: false, message: 'Invalid QuickBooks token refresh response' } + } + + const rawExpiresIn = data.expires_in ?? data.expiresIn + const parsedExpiresIn = + typeof rawExpiresIn === 'number' || typeof rawExpiresIn === 'string' + ? Number(rawExpiresIn) + : Number.NaN + const responseExpiresIn = + Number.isFinite(parsedExpiresIn) && parsedExpiresIn > 0 ? parsedExpiresIn : undefined + const expiresIn = + provider === 'monday' && accessToken + ? Math.max( + 1, + Math.ceil( + (resolveMondayAccessTokenExpiresAt(accessToken, responseExpiresIn).getTime() - + Date.now()) / + 1000 + ) + ) + : (responseExpiresIn ?? 3600) + + const rawRefreshTokenExpiresIn = data.x_refresh_token_expires_in + const parsedRefreshTokenExpiresIn = + typeof rawRefreshTokenExpiresIn === 'number' || typeof rawRefreshTokenExpiresIn === 'string' + ? Number(rawRefreshTokenExpiresIn) + : Number.NaN + const refreshTokenExpiresIn = + provider === 'quickbooks' && + Number.isSafeInteger(parsedRefreshTokenExpiresIn) && + parsedRefreshTokenExpiresIn > 0 + ? parsedRefreshTokenExpiresIn + : undefined + + if (!accessToken) { + // Log only the shape, never `data` itself - on a partial success it can + // carry live tokens. + logger.warn('No access token found in refresh response', { + providerId, + responseKeys: Object.keys(data ?? {}), + }) + return { ok: false, message: 'No access token in refresh response' } + } + + logger.info('Token refreshed successfully with expiration', { + expiresIn, + hasNewRefreshToken: !!newRefreshToken, + provider, + }) + + return { + ok: true, + accessToken, + expiresIn, + refreshToken: newRefreshToken ?? refreshToken, + ...(refreshTokenExpiresIn ? { refreshTokenExpiresIn } : {}), + } + } catch (error) { + const normalized = toError(error) + const message = + normalized.name === 'PayloadSizeLimitError' || normalized.message.startsWith('OAuth client ') + ? normalized.message + : 'Token refresh failed' + logger.error('Error refreshing token', { errorType: normalized.name }) + return { ok: false, message } + } +} diff --git a/apps/sim/lib/oauth/salesforce-instance.server.ts b/apps/sim/lib/oauth/salesforce-instance.server.ts index 86cb61de311..ed11613ad5c 100644 --- a/apps/sim/lib/oauth/salesforce-instance.server.ts +++ b/apps/sim/lib/oauth/salesforce-instance.server.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateSalesforceInstanceUrl } from '@/lib/core/security/input-validation' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { isSalesforceLoginOrigin, SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' @@ -14,7 +15,7 @@ export async function fetchSalesforceInstanceUrl( const loginHost = SALESFORCE_LOGIN_HOSTS[providerId] if (!loginHost) return undefined try { - const response = await fetch(`https://${loginHost}/services/oauth2/userinfo`, { + const response = await outboundFetch(`https://${loginHost}/services/oauth2/userinfo`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, redirect: 'error', signal: AbortSignal.timeout(10_000), diff --git a/apps/sim/lib/oauth/shopify-installation.ts b/apps/sim/lib/oauth/shopify-installation.ts index b3bada458c2..e2908ce9d35 100644 --- a/apps/sim/lib/oauth/shopify-installation.ts +++ b/apps/sim/lib/oauth/shopify-installation.ts @@ -5,6 +5,7 @@ import { generateId } from '@sim/utils/id' import { toRecord } from '@sim/utils/object' import { and, eq, sql } from 'drizzle-orm' import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -95,7 +96,7 @@ async function exchangeToken( const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout let response: Response try { - response = await fetch(`https://${shopDomain}/admin/oauth/access_token`, { + response = await outboundFetch(`https://${shopDomain}/admin/oauth/access_token`, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ @@ -152,7 +153,7 @@ async function fetchShopAccountId( ): Promise { const timeout = AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS) const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout - const response = await fetch( + const response = await outboundFetch( `https://${shopDomain}/admin/api/${SHOPIFY_API_VERSION}/graphql.json`, { method: 'POST', diff --git a/apps/sim/lib/oauth/shopify.ts b/apps/sim/lib/oauth/shopify.ts index fffa57bf0e1..b80e31e6f5d 100644 --- a/apps/sim/lib/oauth/shopify.ts +++ b/apps/sim/lib/oauth/shopify.ts @@ -1,4 +1,5 @@ import { processCredentialDraft } from '@/lib/credentials/draft-processor' +import { resolveConnectDraftOutboundScope } from '@/lib/oauth/connect-scope.server' import { connectShopifyInstallation } from '@/lib/oauth/shopify-installation' interface CompleteShopifyOAuthConnectionParams { @@ -13,7 +14,12 @@ interface CompleteShopifyOAuthConnectionParams { export async function completeShopifyOAuthConnection( params: CompleteShopifyOAuthConnectionParams ): Promise { - const accountId = await connectShopifyInstallation(params) + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId: params.draftId, + userId: params.userId, + providerId: 'shopify', + }) + const accountId = await inDraftScope(() => connectShopifyInstallation(params)) await processCredentialDraft({ draftId: params.draftId, userId: params.userId, diff --git a/apps/sim/lib/oauth/token-resolution.test.ts b/apps/sim/lib/oauth/token-resolution.test.ts index 9c0de2e8eb0..e5a704003d5 100644 --- a/apps/sim/lib/oauth/token-resolution.test.ts +++ b/apps/sim/lib/oauth/token-resolution.test.ts @@ -26,7 +26,6 @@ vi.mock('@/lib/credentials/application/managed-oauth-delegation', () => ({ this.name = 'InvalidManagedOAuthDelegationError' } }, - authenticateManagedOAuthDelegation: vi.fn(), })) vi.mock('@/lib/credentials/application/resolve-managed-oauth-token', () => ({ diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index 7776067db83..9615d075f91 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -59,7 +59,7 @@ export interface ResolveCredentialTokenInput { * The call is an Assistant turn acting with the person's own connected account, so the * tool's policy is the one the Assistant projects for it (GitHub tools bind the person's * GitHub account, never a workspace token). Set only by the in-process executor for an - * authenticated Assistant turn; the token route never forwards it. + * authenticated Assistant turn. */ assistantConnectedAccount?: boolean /** Correlation id used by the credential service's own logging. */ @@ -91,7 +91,7 @@ interface OAuthCredentialContext { accountId?: string | null } -export function validateOAuthCredentialContext( +function validateOAuthCredentialContext( credential: OAuthCredentialContext ): { ok: true } | { ok: false; error: string } { if (credential.providerId !== 'quickbooks') return { ok: true } @@ -583,18 +583,17 @@ export interface ResolveCredentialAccessTokenInput authenticate: () => AuthResult | Promise /** * Proves a delegation for one managed credential: a workflow execution (the - * route verifies the delegation JWT header; the executor binds its delegation - * origin in-process) or a Chat turn acting as the signed-in user. Absent, - * managed credentials are rejected with `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. + * executor binds its delegation origin in-process) or a Chat turn acting as the + * signed-in user. Absent, managed credentials are rejected with + * `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. * Must throw {@link InvalidManagedOAuthDelegationError} on an invalid delegation. */ resolveManagedPrincipal?: (credentialId: string) => Promise } /** - * Authorized application dispatch behind `POST /api/auth/oauth/token`. Every server - * surface that needs a credential token — the route and the in-process tool - * executor — goes through here, so the managed / service-account / plain-OAuth + * Authorized application dispatch for a credential token. Every server surface + * that needs one goes through here, so the managed / service-account / plain-OAuth * dispatch, authorization, refresh, audit, and analytics cannot drift between them. */ export async function resolveCredentialAccessToken( diff --git a/scripts/check-test-types.baseline.json b/scripts/check-test-types.baseline.json index ad34b315e79..e75a4e2540c 100644 --- a/scripts/check-test-types.baseline.json +++ b/scripts/check-test-types.baseline.json @@ -19,12 +19,6 @@ "TS2345": 2, "TS2554": 5 }, - "apps/sim/app/api/auth/oauth/token/route.test.ts": { - "TS2322": 1, - "TS2345": 1, - "TS2554": 11, - "TS2739": 2 - }, "apps/sim/app/api/auth/oauth2/authorize/route.test.ts": { "TS2345": 2 }, diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index 1199b592d45..1d35047b0bf 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -1680,10 +1680,6 @@ "apps/sim/lib/api/contracts/oauth-connections.ts#disconnectOAuthBodySchema", "apps/sim/lib/api/contracts/oauth-connections.ts#oauthAccountSummarySchema", "apps/sim/lib/api/contracts/oauth-connections.ts#oauthConnectionSchema", - "apps/sim/lib/api/contracts/oauth-connections.ts#oauthTokenGetQuerySchema", - "apps/sim/lib/api/contracts/oauth-connections.ts#oauthTokenPostHeadersSchema", - "apps/sim/lib/api/contracts/oauth-connections.ts#oauthTokenPostQuerySchema", - "apps/sim/lib/api/contracts/oauth-connections.ts#oauthTokenRequestBodySchema", "apps/sim/lib/api/contracts/oauth-connections.ts#quickBooksCallbackQuerySchema", "apps/sim/lib/api/contracts/oauth-connections.ts#trelloTokenBodySchema", "apps/sim/lib/api/contracts/oauth-provider.ts#RegisterOAuthClientBody", From c2c57ce956efc576b5e7ad7c5c50a085776549b0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:11:31 -0800 Subject: [PATCH 09/32] docs(egress): document outboundFetch in the integration rules --- CLAUDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CLAUDE.md b/CLAUDE.md index dc98057cab4..1fee41c7650 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,6 +60,7 @@ Build order **Tools** → **Block** → **Icon** → optional **Trigger**, from - **Tool IDs are `snake_case`** (`service_action`), registered in `tools/registry.ts`; blocks register alphabetically in `blocks/registry-maps.ts`. - **Type coercions go in `tools.config.params`** (runs at execution), never `tools.config.tool` (runs at serialization, where `Number()` destroys `` references). +- **Requests made for a workspace use `outboundFetch`** (`@/lib/core/network/outbound-fetch.server`), never the global `fetch`, so an organization with a dedicated egress gateway keeps its static IPs; AWS SDK clients signing with customer credentials take `organizationAwsRequestHandler()`. Sim's own infrastructure stays direct (`check:organization-egress`). - A surface listing integrations to a viewer reads `useIntegrationAccess`, never a raw catalog (`check:integration-access`). - Table column types are registry entries in `apps/sim/lib/table/column-types/`; never add a per-type `case` elsewhere except the one documented import-path switch (`coerceValue` in `lib/table/import.ts`), which every new type extends: a missed arm fails silently. Use `/add-column-type`. From 2d50fc92223f236be78042adbe25798dd2a0cca3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:13:25 -0800 Subject: [PATCH 10/32] test(egress): use the central workspace context mock in the new scope tests --- .../app/api/cron/renew-subscriptions/route.test.ts | 13 +++++++------ apps/sim/lib/oauth/connect-scope.server.test.ts | 12 +++++++----- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts index fd546bc9ca4..4736ddff1ac 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts @@ -13,19 +13,21 @@ import { } from '@sim/testing' import { authInternalMock, authInternalMockFns } from '@sim/testing/mocks/auth-internal.mock' import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { + workspaceContextMock, + workspaceContextMockFns, +} from '@sim/testing/mocks/workspace-context.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetCredentialOwner, mockLoadWorkspaceApplicationContext } = vi.hoisted(() => ({ +const { mockGetCredentialOwner } = vi.hoisted(() => ({ mockGetCredentialOwner: vi.fn(), - mockLoadWorkspaceApplicationContext: vi.fn(), })) +const { mockLoadWorkspaceApplicationContext } = workspaceContextMockFns vi.mock('@/lib/auth/internal', () => authInternalMock) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) vi.mock('@/lib/core/network/config.server', () => networkConfigMock) -vi.mock('@/lib/workspaces/application/context', () => ({ - loadWorkspaceApplicationContext: mockLoadWorkspaceApplicationContext, -})) +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) vi.mock('@/lib/webhooks/provider-subscription-utils', () => ({ getCredentialOwner: mockGetCredentialOwner, getNotificationUrl: vi.fn(() => 'https://sim.test/api/webhooks/trigger/hook-1'), @@ -90,7 +92,6 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { }) afterEach(() => { - vi.unstubAllGlobals() networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) }) diff --git a/apps/sim/lib/oauth/connect-scope.server.test.ts b/apps/sim/lib/oauth/connect-scope.server.test.ts index b55bc89e78a..d6687a59665 100644 --- a/apps/sim/lib/oauth/connect-scope.server.test.ts +++ b/apps/sim/lib/oauth/connect-scope.server.test.ts @@ -1,18 +1,20 @@ import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { + workspaceContextMock, + workspaceContextMockFns, +} from '@sim/testing/mocks/workspace-context.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetActiveConnectDraft, mockLoadWorkspaceApplicationContext } = vi.hoisted(() => ({ +const { mockGetActiveConnectDraft } = vi.hoisted(() => ({ mockGetActiveConnectDraft: vi.fn(), - mockLoadWorkspaceApplicationContext: vi.fn(), })) +const { mockLoadWorkspaceApplicationContext } = workspaceContextMockFns vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@/lib/credentials/connect-draft', () => ({ getActiveConnectDraft: mockGetActiveConnectDraft, })) -vi.mock('@/lib/workspaces/application/context', () => ({ - loadWorkspaceApplicationContext: mockLoadWorkspaceApplicationContext, -})) +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { resolveConnectDraftOutboundScope } from '@/lib/oauth/connect-scope.server' From 2d9052e535211e51415d20e9d5cd5934880d1e29 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:23:49 -0800 Subject: [PATCH 11/32] fix(egress): keep direct AWS requests on the client's own handler and reuse gateway sockets --- .../organization-aws-handler.server.ts | 137 +++++++++++------- .../lib/core/network/proxy-agent.server.ts | 7 +- apps/sim/providers/bedrock/index.ts | 6 +- 3 files changed, 93 insertions(+), 57 deletions(-) diff --git a/apps/sim/lib/core/network/organization-aws-handler.server.ts b/apps/sim/lib/core/network/organization-aws-handler.server.ts index f40bf33fae5..1853e5689c7 100644 --- a/apps/sim/lib/core/network/organization-aws-handler.server.ts +++ b/apps/sim/lib/core/network/organization-aws-handler.server.ts @@ -1,7 +1,7 @@ import type http from 'node:http' import type { Duplex } from 'node:stream' import { createLogger } from '@sim/logger' -import { NodeHttpHandler } from '@smithy/node-http-handler' +import { NodeHttp2Handler, NodeHttpHandler } from '@smithy/node-http-handler' import { Agent, type AgentConnectOpts } from 'agent-base' import { isOutboundRoutingEnabled, type OutboundGateway } from '@/lib/core/network/config.server' import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' @@ -10,12 +10,14 @@ import { AGENT_OPTIONS, type AwsRequestHandlerOptions, awsRequestHandler, - environmentProxyAgent, } from '@/lib/core/network/proxy-agent.server' import { OutboundRoutingError } from '@/lib/core/network/routing' const logger = createLogger('OrganizationAwsRequestHandler') +type AwsHttpHandler = NodeHttpHandler | NodeHttp2Handler +type HandleArguments = Parameters + /** * Pools connections tunnelled through one organization's gateway. Its sockets carry only that * organization's requests, so a keep-alive socket is never reused across organizations or @@ -26,55 +28,54 @@ class GatewayAgent extends Agent { super(AGENT_OPTIONS) } - connect(_req: http.ClientRequest, options: AgentConnectOpts): Promise { + /** + * The gateway only carries HTTPS. agent-base otherwise infers this from the call stack, which + * differs between opening and freeing a socket, so the socket is never found to reuse. + */ + isSecureEndpoint(): boolean { + return true + } + + connect(req: http.ClientRequest, options: AgentConnectOpts): Promise { return connectThroughGateway( this.gateway, { - protocol: options.secureEndpoint ? 'https:' : 'http:', + protocol: req.protocol, hostname: options.host ?? '', port: Number(options.port), }, { profile: 'configuredEndpoint' } ) } - - /** A replaced gateway finishes its in-flight requests and keeps no idle sockets. */ - retire(): void { - this.on('free', (socket: Duplex) => socket.destroy()) - for (const sockets of Object.values(this.freeSockets)) { - for (const socket of sockets ?? []) socket.destroy() - } - } } -const gatewayAgents = new Map() - -function gatewayAgentFor(gateway: OutboundGateway): GatewayAgent { - const current = gatewayAgents.get(gateway.organizationId) - if (current?.gatewayId === gateway.id) return current.agent - current?.agent.retire() - const agent = new GatewayAgent(gateway) - gatewayAgents.set(gateway.organizationId, { gatewayId: gateway.id, agent }) - return agent -} +const gatewayHandlers = new Map< + string, + { gatewayId: string; agent: GatewayAgent; handler: NodeHttpHandler } +>() /** - * Chooses each connection's agent from the current outbound scope: the organization's - * {@link GatewayAgent} on a gateway route, otherwise the environment proxy agent (a direct - * keep-alive connection when no proxy applies). It never pools a socket itself, and its own - * socket limit is unbounded so a request is never queued here behind another organization's; - * the agent it delegates to applies the SDK's limit. + * The organization's shared gateway handler. A reassigned gateway's agent stops keeping + * sockets alive, so Node closes each one as its in-flight request finishes and none is handed + * to a request bound for the new gateway. */ -class OrganizationRoutingAgent extends Agent { - async connect(req: http.ClientRequest, options: AgentConnectOpts): Promise { - const route = await currentRouteOrDirect(options.host) - if (route.kind === 'gateway') return gatewayAgentFor(route.gateway) - return environmentProxyAgent().connect(req, options) +function gatewayHandlerFor(gateway: OutboundGateway): NodeHttpHandler { + const current = gatewayHandlers.get(gateway.organizationId) + if (current?.gatewayId === gateway.id) return current.handler + if (current) { + current.agent.keepAlive = false + for (const sockets of Object.values(current.agent.freeSockets)) { + for (const socket of sockets ?? []) socket.destroy() + } } + const agent = new GatewayAgent(gateway) + const handler = new NodeHttpHandler({ httpAgent: agent, httpsAgent: agent }) + gatewayHandlers.set(gateway.organizationId, { gatewayId: gateway.id, agent, handler }) + return handler } /** A request outside any outbound scope goes direct with a warning, as `outboundFetch` does. */ -async function currentRouteOrDirect(host: string | undefined) { +async function currentRouteOrDirect(host: string) { try { return await resolveCurrentOutboundRoute() } catch (error) { @@ -84,27 +85,65 @@ async function currentRouteOrDirect(host: string | undefined) { } } -let routingAgent: OrganizationRoutingAgent | undefined +/** + * Picks each request's handler from the outbound scope it is sent in: an organization with a + * gateway gets that gateway's handler, and every other request goes to `direct`, which is the + * handler the client would use with no organization routing at all. + */ +class OrganizationRoutedHandler { + readonly metadata: AwsHttpHandler['metadata'] + + constructor(private readonly direct: AwsHttpHandler) { + this.metadata = direct.metadata + } + + async handle(...[request, options]: HandleArguments) { + const route = await currentRouteOrDirect(request.hostname) + const handler = route.kind === 'gateway' ? gatewayHandlerFor(route.gateway) : this.direct + return handler.handle(request, options) + } + + updateHttpClientConfig(...args: Parameters): void { + this.direct.updateHttpClientConfig(...args) + } + + httpHandlerConfigs(): ReturnType { + return this.direct.httpHandlerConfigs() + } + + destroy(): void { + this.direct.destroy() + } +} /** - * `requestHandler` for an AWS SDK v3 client that signs with a customer's credentials. When - * outbound routing is enabled, every request is routed by the outbound scope it is sent in, - * like `outboundFetch`: an organization with a dedicated egress gateway reaches AWS from its - * static IPs (passing `aws:SourceIp` conditions), any other route connects as - * {@link awsRequestHandler} would, and a call outside any scope goes direct with a warning. A - * blocked route or an unavailable gateway fails the request with an `OutboundRoutingError`. - * When routing is disabled this is exactly {@link awsRequestHandler}. + * `requestHandler` for an AWS SDK v3 client that signs with a customer's credentials. Inside the + * outbound scope of an organization with a dedicated egress gateway, each request reaches AWS + * through that gateway, from the organization's static IPs (passing `aws:SourceIp` conditions). + * Every other request, including one outside any scope (which also logs a warning), uses the + * same handler the client gets from {@link awsRequestHandler}: the SDK's own default, or the + * operator's `HTTPS_PROXY`. A blocked route or an unavailable gateway fails the request with an + * `OutboundRoutingError`. With routing disabled this is exactly {@link awsRequestHandler}. + * + * Pass `http2` for a client whose SDK default is HTTP/2 (Bedrock Runtime), so its direct + * requests keep it. Gateway requests always use HTTP/1.1 through the tunnel. * * Never use it for Sim's own AWS usage (its buckets, queues, email, secrets, metrics, or * Bedrock on Sim's keys), which must not leave through a customer's gateway. */ export function organizationAwsRequestHandler( - options?: AwsRequestHandlerOptions -): ReturnType { - if (!isOutboundRoutingEnabled()) return awsRequestHandler(options) - routingAgent ??= new OrganizationRoutingAgent({ - keepAlive: true, - maxSockets: Number.POSITIVE_INFINITY, - }) - return new NodeHttpHandler({ ...options, httpAgent: routingAgent, httpsAgent: routingAgent }) + options?: AwsRequestHandlerOptions & { http2?: boolean } +): ReturnType | OrganizationRoutedHandler { + const { http2, ...rest } = options ?? {} + // No options stays `undefined`, so the client keeps its SDK default handler and settings. + const handlerOptions = Object.keys(rest).length > 0 ? rest : undefined + const configured = awsRequestHandler(handlerOptions) + if (!isOutboundRoutingEnabled()) return configured + const direct = + configured instanceof NodeHttpHandler + ? configured + : http2 + ? new NodeHttp2Handler({ ...handlerOptions, disableConcurrentStreams: true }) + : new NodeHttpHandler(handlerOptions) + return new OrganizationRoutedHandler(direct) } diff --git a/apps/sim/lib/core/network/proxy-agent.server.ts b/apps/sim/lib/core/network/proxy-agent.server.ts index bed3e5fab6d..29736a1d5f9 100644 --- a/apps/sim/lib/core/network/proxy-agent.server.ts +++ b/apps/sim/lib/core/network/proxy-agent.server.ts @@ -46,11 +46,8 @@ class EnvironmentProxyAgent extends Agent { let sharedAgent: EnvironmentProxyAgent | null = null -/** - * The process-wide {@link EnvironmentProxyAgent}. Without a configured proxy every connection - * takes its direct keep-alive agent. - */ -export function environmentProxyAgent(): EnvironmentProxyAgent { +/** The process-wide {@link EnvironmentProxyAgent}; only meaningful while a proxy is configured. */ +function environmentProxyAgent(): EnvironmentProxyAgent { sharedAgent ??= new EnvironmentProxyAgent(AGENT_OPTIONS) return sharedAgent } diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index 282bbdea3a1..ff8cc53f30a 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -147,7 +147,7 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { return { credentials: { accessKeyId, secretAccessKey, sessionToken }, credentialKey: null, - requestHandler: organizationAwsRequestHandler(), + requestHandler: organizationAwsRequestHandler({ http2: true }), } } return { @@ -157,7 +157,7 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { session.credentialId, sha256Hex([accessKeyId, secretAccessKey].join('\0')), ].join(':'), - requestHandler: organizationAwsRequestHandler(), + requestHandler: organizationAwsRequestHandler({ http2: true }), } } @@ -179,7 +179,7 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { secretAccessKey: request.bedrockSecretKey, }, credentialKey: `${request.bedrockAccessKeyId}:${request.bedrockSecretKey}`, - requestHandler: organizationAwsRequestHandler(), + requestHandler: organizationAwsRequestHandler({ http2: true }), } } // The deployment's own credentials stay on Sim's network, never an organization's route. From 0ee5738c2b232e4191da77d1d352d6e40fd6d030 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:26:16 -0800 Subject: [PATCH 12/32] fix(egress): keep Sim-owned endpoints direct, fail closed for connectors, and harden the audit --- .../lib/core/network/outbound-fetch.server.ts | 18 +++- apps/sim/lib/embeddings/client.ts | 5 +- apps/sim/lib/internal/stagehand/operations.ts | 15 ++-- apps/sim/lib/knowledge/documents/processor.ts | 4 +- .../documents/secure-fetch.server.ts | 6 +- apps/sim/lib/tools/falai-pricing.ts | 7 +- .../providers/azure-anthropic/index.test.ts | 5 +- apps/sim/providers/azure-anthropic/index.ts | 3 +- apps/sim/providers/azure-openai/index.test.ts | 5 +- apps/sim/providers/azure-openai/index.ts | 3 +- apps/sim/providers/kie/transport.ts | 4 +- scripts/check-organization-egress.test.ts | 19 +++++ scripts/check-organization-egress.ts | 82 +++++++++++++++---- 13 files changed, 126 insertions(+), 50 deletions(-) diff --git a/apps/sim/lib/core/network/outbound-fetch.server.ts b/apps/sim/lib/core/network/outbound-fetch.server.ts index a87ae320b45..0cdc7ecb851 100644 --- a/apps/sim/lib/core/network/outbound-fetch.server.ts +++ b/apps/sim/lib/core/network/outbound-fetch.server.ts @@ -22,12 +22,26 @@ let gatewayTransport: ReturnType | * Never use it for Sim's own infrastructure (its storage, queues, or internal services), which * must not leave through a customer's gateway. */ -export const outboundFetch: typeof fetch = async (input, init) => { +export const outboundFetch: typeof fetch = (input, init) => routedFetch(input, init, false) + +/** + * {@link outboundFetch} for a caller that always runs inside an outbound scope, such as a + * knowledge connector sync: a missing scope throws `MISSING_SCOPE` instead of going direct, so a + * lost scope fails loudly rather than leaving from a shared IP. + */ +export const scopedOutboundFetch: typeof fetch = (input, init) => routedFetch(input, init, true) + +async function routedFetch( + input: RequestInfo | URL, + init: RequestInit | undefined, + requireScope: boolean +): Promise { let route: Awaited> try { route = await resolveCurrentOutboundRoute() } catch (error) { - if (!(error instanceof OutboundRoutingError) || error.code !== 'MISSING_SCOPE') throw error + const missingScope = error instanceof OutboundRoutingError && error.code === 'MISSING_SCOPE' + if (!missingScope || requireScope) throw error logger.warn('Outbound request made without an organization scope', { host: requestHost(input), }) diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index f05794d96bf..2f51e66ba31 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -582,8 +582,9 @@ async function callEmbeddingAPI( const timeout = setTimeout(() => controller.abort(), EMBEDDING_REQUEST_TIMEOUT_MS) try { - // Ollama is the deployment's own server, never an organization's vendor. - const requestFetch = providerId === 'ollama' ? fetch : outboundFetch + const operatorOwned = providerId === 'ollama' || providerId === 'azure-openai' + // egress-direct: Ollama and the Azure OpenAI override are the operator's own deployments + const requestFetch = operatorOwned ? fetch : outboundFetch const response = await requestFetch(request.apiUrl, { method: 'POST', headers: request.headers, diff --git a/apps/sim/lib/internal/stagehand/operations.ts b/apps/sim/lib/internal/stagehand/operations.ts index 5f04c231a25..f8ccd994121 100644 --- a/apps/sim/lib/internal/stagehand/operations.ts +++ b/apps/sim/lib/internal/stagehand/operations.ts @@ -1,7 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import { isSensitiveKey, REDACTED_MARKER } from '@/lib/core/security/redaction' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' @@ -99,14 +98,12 @@ async function getLiveViewUrl(sessionId: string, signal?: AbortSignal): Promise< const browserbaseApiKey = getBrowserbaseApiKey() if (!browserbaseApiKey) return null try { - const response = await outboundFetch( - `https://api.browserbase.com/v1/sessions/${sessionId}/debug`, - { - method: 'GET', - headers: { 'X-BB-API-Key': browserbaseApiKey }, - signal, - } - ) + // egress-direct: Sim's own Browserbase project, like the Stagehand session itself + const response = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/debug`, { + method: 'GET', + headers: { 'X-BB-API-Key': browserbaseApiKey }, + signal, + }) if (!response.ok) return null const data = await readResponseJsonWithLimit(response, { maxBytes: MAX_BROWSERBASE_DEBUG_RESPONSE_BYTES, diff --git a/apps/sim/lib/knowledge/documents/processor.ts b/apps/sim/lib/knowledge/documents/processor.ts index 8e6db7e35ef..c97522d6905 100644 --- a/apps/sim/lib/knowledge/documents/processor.ts +++ b/apps/sim/lib/knowledge/documents/processor.ts @@ -18,7 +18,6 @@ import { } from '@/lib/chunkers' import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import { env } from '@/lib/core/config/env' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { recordProviderCooldown, waitForProviderAdmission, @@ -670,7 +669,8 @@ async function makeOCRRequest( const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal try { - const response = await outboundFetch(endpoint, { + // egress-direct: Sim's own Azure OCR deployment, configured by the operator + const response = await fetch(endpoint, { method: 'POST', headers, body: typeof body === 'string' ? body : JSON.stringify(body), diff --git a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts index 6a04079cfc6..385e441836d 100644 --- a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts +++ b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts @@ -1,4 +1,4 @@ -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type SecureFetchOptions, type SecureFetchResponse, @@ -84,8 +84,8 @@ export async function fetchWithRetry( ]) const init = { ...options, signal: requestSignal } const response = retryOptions.fetcher - ? await retryOptions.fetcher(url, init, outboundFetch) - : await outboundFetch(url, init) + ? await retryOptions.fetcher(url, init, scopedOutboundFetch) + : await scopedOutboundFetch(url, init) if ( !response.ok && diff --git a/apps/sim/lib/tools/falai-pricing.ts b/apps/sim/lib/tools/falai-pricing.ts index 312fb376718..ef362e6a789 100644 --- a/apps/sim/lib/tools/falai-pricing.ts +++ b/apps/sim/lib/tools/falai-pricing.ts @@ -6,7 +6,6 @@ import { isRecordLike } from '@sim/utils/object' import { parseRetryAfter } from '@sim/utils/retry' import { LRUCache } from 'lru-cache' import { coalesceLocally } from '@/lib/concurrency/singleflight' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DeadlineExceededError, withinDeadline } from '@/lib/core/utils/deadline' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' @@ -152,7 +151,8 @@ async function fetchFalAIBillingEvent( let response: Response try { - response = await outboundFetch(url, { + // egress-direct: Sim's own fal billing account, not a request for the workspace + response = await fetch(url, { headers: { Authorization: `Key ${apiKey}`, }, @@ -200,7 +200,8 @@ async function estimateFalAICallCost( ): Promise<{ costDollars?: number; error?: string }> { let response: Response try { - response = await outboundFetch('https://api.fal.ai/v1/models/pricing/estimate', { + // egress-direct: Sim's own fal billing account, not a request for the workspace + response = await fetch('https://api.fal.ai/v1/models/pricing/estimate', { method: 'POST', headers: { Authorization: `Key ${apiKey}`, diff --git a/apps/sim/providers/azure-anthropic/index.test.ts b/apps/sim/providers/azure-anthropic/index.test.ts index 5c25b45a424..a2bc29ecbc2 100644 --- a/apps/sim/providers/azure-anthropic/index.test.ts +++ b/apps/sim/providers/azure-anthropic/index.test.ts @@ -5,7 +5,6 @@ import { } from '@sim/testing/mocks/input-validation.mock' import { providersModelsMock } from '@sim/testing/mocks/providers-models.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { ProviderRequest } from '@/providers/types' const { mockAnthropic, anthropicArgs, mockExecuteAnthropic, sentinelFetch } = vi.hoisted(() => { @@ -75,14 +74,14 @@ describe('azureAnthropicProvider — SSRF pinning', () => { expect(buildClientOptions()).toMatchObject({ fetch: sentinelFetch }) }) - it('sends a trusted server env endpoint through the unpinned outbound fetch', async () => { + it('does not pin when the endpoint comes from trusted server env', async () => { setEnv({ AZURE_ANTHROPIC_ENDPOINT: 'https://trusted.services.ai.azure.com' }) await azureAnthropicProvider.executeRequest(request({ azureEndpoint: undefined })) expect(mockValidate).not.toHaveBeenCalled() expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(buildClientOptions().fetch).toBe(outboundFetch) + expect(buildClientOptions()).not.toHaveProperty('fetch') }) it('keeps the registry model in core and resolves a separate Azure wire model', async () => { diff --git a/apps/sim/providers/azure-anthropic/index.ts b/apps/sim/providers/azure-anthropic/index.ts index c6d814c8782..7737dcd4abc 100644 --- a/apps/sim/providers/azure-anthropic/index.ts +++ b/apps/sim/providers/azure-anthropic/index.ts @@ -1,7 +1,6 @@ import Anthropic from '@anthropic-ai/sdk' import { createLogger } from '@sim/logger' import { env } from '@/lib/core/config/env' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { executeAnthropicProviderRequest } from '@/providers/anthropic/core' @@ -82,7 +81,7 @@ export const azureAnthropicProvider: ProviderConfig = { baseURL, apiKey, maxRetries: PROVIDER_MAX_RETRIES, - fetch: pinnedFetch ?? outboundFetch, + ...(pinnedFetch ? { fetch: pinnedFetch } : {}), defaultHeaders: { 'anthropic-version': anthropicVersion, }, diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index 1a0c513b632..d68dbc3a97d 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -11,7 +11,6 @@ import { providersTraceEnrichmentMock } from '@sim/testing/mocks/providers-trace import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock' import { toolsMock, toolsMockFns } from '@sim/testing/mocks/tools.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { AgentStreamEvent } from '@/providers/stream-events' import type { ProviderRequest, ProviderToolConfig } from '@/providers/types' @@ -181,7 +180,7 @@ describe('azureOpenAIProvider — SSRF pinning', () => { expect(azureOpenAIArgs()[0]).toMatchObject({ fetch: sentinelFetch }) }) - it('constructs the AzureOpenAI client with the unpinned outbound fetch for a trusted env endpoint', async () => { + it('constructs the AzureOpenAI client without a custom fetch for a trusted env endpoint', async () => { mockIsChatCompletionsEndpoint.mockReturnValue(true) setEnv({ AZURE_OPENAI_ENDPOINT: @@ -195,7 +194,7 @@ describe('azureOpenAIProvider — SSRF pinning', () => { await azureOpenAIProvider.executeRequest(request({ azureEndpoint: undefined })) expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(azureOpenAIArgs()[0].fetch).toBe(outboundFetch) + expect(azureOpenAIArgs()[0]).not.toHaveProperty('fetch') }) it('preserves a custom deployment name through Chat Completions routing', async () => { diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index 2fc9791f4df..37db6e20761 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -14,7 +14,6 @@ import type { } from 'openai/resources/chat/completions' import type { ReasoningEffort } from 'openai/resources/shared' import { env } from '@/lib/core/config/env' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' @@ -105,7 +104,7 @@ async function executeChatCompletionsRequest( apiVersion: azureApiVersion, endpoint: azureEndpoint, ...openAICompatTransport(), - fetch: pinnedFetch ?? outboundFetch, + ...(pinnedFetch ? { fetch: pinnedFetch } : {}), }) const allMessages: ChatCompletionMessageParam[] = [] diff --git a/apps/sim/providers/kie/transport.ts b/apps/sim/providers/kie/transport.ts index 4281b6e96f6..fa5c83e8f9d 100644 --- a/apps/sim/providers/kie/transport.ts +++ b/apps/sim/providers/kie/transport.ts @@ -52,9 +52,9 @@ function withKieThinkingFlag( * are labeled as JSON. Streams are never buffered: they are typed * `text/event-stream`, and a failed streaming request comes back as JSON. */ -export function createKieFetch(baseFetch?: typeof fetch): typeof fetch { +export function createKieFetch(baseFetch: typeof fetch): typeof fetch { return async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const response = await (baseFetch ?? fetch)(input, withKieThinkingFlag(input, init)) + const response = await baseFetch(input, withKieThinkingFlag(input, init)) const contentType = response.headers.get('content-type')?.toLowerCase() if (response.status !== 200 || (contentType && !contentType.includes('application/json'))) { return response diff --git a/scripts/check-organization-egress.test.ts b/scripts/check-organization-egress.test.ts index 37780d7a599..2bc072e59c9 100644 --- a/scripts/check-organization-egress.test.ts +++ b/scripts/check-organization-egress.test.ts @@ -11,6 +11,25 @@ const b = await globalThis.fetch(url)`) ).toEqual([1, 2]) }) + it('reports an alias, a fallback, a shorthand property and another global object', () => { + expect( + lines(`const f = fetch +const g = custom ?? fetch +new OpenAI({ fetch }) +function run(transport = fetch) {} +await window.fetch(url) +await self['fetch'](url)`) + ).toEqual([1, 2, 3, 4, 5, 6]) + }) + + it('ignores fetch in type positions and as a property key', () => { + expect( + lines(`type Fetch = typeof fetch +const transport: typeof globalThis.fetch = outboundFetch +new Anthropic({ fetch: outboundFetch })`) + ).toEqual([]) + }) + it('ignores a client method or a local binding named fetch', () => { expect( lines(`class Client { async fetch(path: string) { return this.fetch(path) } } diff --git a/scripts/check-organization-egress.ts b/scripts/check-organization-egress.ts index 1b7a4cbdd6a..0edef0e48df 100644 --- a/scripts/check-organization-egress.ts +++ b/scripts/check-organization-egress.ts @@ -48,7 +48,15 @@ const SCAN_DIRS = [ 'apps/sim/lib/webhooks', ] -const SKIP_DIRS = new Set(['node_modules', 'dist', '.next', '.turbo', 'coverage', 'client']) +const SKIP_DIRS = new Set([ + 'node_modules', + 'dist', + '.next', + '.turbo', + 'coverage', + 'client', + '__fixtures__', +]) const ANNOTATION = '// egress-direct:' const ANNOTATION_REACH = 2 @@ -116,9 +124,46 @@ function shadowsGlobal(node: ts.Node, name: string): boolean { return false } +const GLOBAL_OBJECTS = new Set(['globalThis', 'window', 'self', 'global']) + /** - * Every call to the global `fetch` (`fetch(…)`, `globalThis.fetch(…)`) without an - * `egress-direct` annotation on one of the two lines above it. + * Whether this `fetch` identifier is a value reference to the global: a call, but also an alias + * (`const f = fetch`), a fallback (`custom ?? fetch`), a shorthand property (`{ fetch }`) or a + * default parameter. Names in declarations, property keys, member names and type positions are + * not references. + */ +function refersToGlobalFetch(node: ts.Identifier): boolean { + const parent = node.parent + if (ts.isPropertyAccessExpression(parent) && parent.name === node) return false + if ( + (ts.isPropertyAssignment(parent) || + ts.isMethodDeclaration(parent) || + ts.isPropertyDeclaration(parent) || + ts.isPropertySignature(parent) || + ts.isMethodSignature(parent) || + ts.isGetAccessorDeclaration(parent) || + ts.isSetAccessorDeclaration(parent) || + ts.isParameter(parent) || + ts.isVariableDeclaration(parent) || + ts.isBindingElement(parent) || + ts.isFunctionDeclaration(parent) || + ts.isImportSpecifier(parent) || + ts.isExportSpecifier(parent) || + ts.isImportClause(parent)) && + parent.name === node + ) { + return false + } + if (ts.isQualifiedName(parent) && parent.right === node) return false + if (ts.isBindingElement(parent) && parent.propertyName === node) return false + if (ts.isImportSpecifier(parent) || ts.isExportSpecifier(parent)) return false + if (ts.isTypeQueryNode(parent) || ts.isTypeReferenceNode(parent)) return false + return !shadowsGlobal(node, 'fetch') +} + +/** + * Every reference to the global `fetch` (a call, an alias, `globalThis.fetch`, `window['fetch']`) + * without an `egress-direct` annotation on one of the two lines above it. */ export function findDirectFetches(file: string, source: string): DirectFetch[] { if (!source.includes('fetch')) return [] @@ -126,21 +171,24 @@ export function findDirectFetches(file: string, source: string): DirectFetch[] { const lines = source.split('\n') const found: DirectFetch[] = [] + const report = (node: ts.Node) => { + const line = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line + const annotated = lines + .slice(Math.max(0, line - ANNOTATION_REACH), line) + .some((text) => text.trim().startsWith(ANNOTATION)) + if (!annotated) found.push({ line: line + 1, text: lines[line].trim() }) + } + const visit = (node: ts.Node): void => { - if (ts.isCallExpression(node)) { - const callee = node.expression - const isGlobalFetch = - (ts.isIdentifier(callee) && callee.text === 'fetch' && !shadowsGlobal(callee, 'fetch')) || - (ts.isPropertyAccessExpression(callee) && - callee.name.text === 'fetch' && - ts.isIdentifier(callee.expression) && - callee.expression.text === 'globalThis') - if (isGlobalFetch) { - const line = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line - const annotated = lines - .slice(Math.max(0, line - ANNOTATION_REACH), line) - .some((text) => text.trim().startsWith(ANNOTATION)) - if (!annotated) found.push({ line: line + 1, text: lines[line].trim() }) + if (ts.isIdentifier(node) && node.text === 'fetch' && refersToGlobalFetch(node)) report(node) + else if ( + (ts.isPropertyAccessExpression(node) && node.name.text === 'fetch') || + (ts.isElementAccessExpression(node) && + ts.isStringLiteralLike(node.argumentExpression) && + node.argumentExpression.text === 'fetch') + ) { + if (ts.isIdentifier(node.expression) && GLOBAL_OBJECTS.has(node.expression.text)) { + report(node) } } ts.forEachChild(node, visit) From 3c894e284a0476a662556f62afb41aa02ab529be Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:26:36 -0800 Subject: [PATCH 13/32] chore(egress): drop references to the removed token route --- apps/sim/lib/credentials/token-service-accounts/descriptors.ts | 2 +- apps/sim/tools/index.test.ts | 3 --- apps/sim/tools/pipedrive/utils.ts | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/credentials/token-service-accounts/descriptors.ts b/apps/sim/lib/credentials/token-service-accounts/descriptors.ts index 40e9e464a84..6e1d2ca1c3a 100644 --- a/apps/sim/lib/credentials/token-service-accounts/descriptors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/descriptors.ts @@ -110,7 +110,7 @@ export interface TokenServiceAccountDescriptor { * HTTP auth scheme the pasted token requires at execution time. Defaults to * `bearer` (`Authorization: Bearer `); `x-api-token` providers (e.g. * Pipedrive) send the token in an `x-api-token` header instead, and the - * token route surfaces this so tool header builders can switch schemes. + * resolved credential token carries this so tool header builders can switch schemes. */ authStyle?: 'bearer' | 'x-api-token' /** diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index 1102eac02aa..d357c0ab5bf 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -6810,9 +6810,6 @@ describe('Managed OAuth Credential Delegation', () => { executorDelegationOrigin, }) ) - expect( - fetchMock.mock.calls.some(([url]) => String(url).includes('/api/auth/oauth/token')) - ).toBe(false) }) it('fails before transport when managed credential delegation lacks current workflow authority', async () => { diff --git a/apps/sim/tools/pipedrive/utils.ts b/apps/sim/tools/pipedrive/utils.ts index fd08dfc4fce..8bbfb2abe1f 100644 --- a/apps/sim/tools/pipedrive/utils.ts +++ b/apps/sim/tools/pipedrive/utils.ts @@ -5,7 +5,7 @@ import type { PipedriveBaseParams } from '@/tools/pipedrive/types' * `Authorization: Bearer`; pasted personal API tokens (token-paste service * accounts) must use the `x-api-token` header instead — Pipedrive documents no * token-format discriminator, so the credential resolver threads an explicit - * `authStyle` signal through the token route into tool params. Works on both + * `authStyle` signal through credential token resolution into tool params. Works on both * `/v1` and `/api/v2` endpoints. */ export function getPipedriveAuthHeaders(params: PipedriveBaseParams): Record { From cd7fb102f6e52dc93bb78b5716df82860e75bd7e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:28:28 -0800 Subject: [PATCH 14/32] improvement(egress): route gemini/vertex sdk, wand, provider model listings, and v1 kb search embeddings through the workspace egress route Pass outboundFetch as the @google/genai httpOptions.fetch for the Google, Vertex, file-attachment, and vision clients. Run wand OpenAI calls, BYOK/hosted model catalog listings, and v1 knowledge search query embeddings inside the canonical workspace's outbound scope after access is verified. --- .../app/api/providers/baseten/models/route.ts | 14 ++++- .../api/providers/fireworks/models/route.ts | 14 ++++- .../providers/ollama-cloud/models/route.ts | 18 +++--- .../api/providers/together/models/route.ts | 14 ++++- .../app/api/v1/knowledge/search/route.test.ts | 37 +++++++++++- apps/sim/app/api/v1/knowledge/search/route.ts | 7 +-- apps/sim/app/api/wand/route.ts | 60 +++++++++---------- apps/sim/lib/internal/vision/client.ts | 2 +- apps/sim/providers/file-attachments.server.ts | 5 +- apps/sim/providers/google/index.ts | 3 +- apps/sim/providers/vertex/index.ts | 2 + 11 files changed, 123 insertions(+), 53 deletions(-) diff --git a/apps/sim/app/api/providers/baseten/models/route.ts b/apps/sim/app/api/providers/baseten/models/route.ts index b73a6711421..ec6eef61c8b 100644 --- a/apps/sim/app/api/providers/baseten/models/route.ts +++ b/apps/sim/app/api/providers/baseten/models/route.ts @@ -10,11 +10,14 @@ import { validationErrorResponse } from '@/lib/api/server' import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('BasetenModelsAPI') +const MODELS_URL = 'https://inference.baseten.co/v1/models' export const GET = withRouteHandler(async (request: NextRequest) => { if (isProviderBlacklisted('baseten')) { @@ -23,6 +26,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } let apiKey: string | undefined + let authorizedWorkspaceId: string | undefined const queryValidation = basetenProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, @@ -34,6 +38,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { if (session?.user?.id) { const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) if (permission) { + authorizedWorkspaceId = workspaceId const byokResult = await getBYOKKey(workspaceId, 'baseten') if (byokResult) { apiKey = byokResult.apiKey @@ -52,13 +57,18 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } try { - const response = await fetch('https://inference.baseten.co/v1/models', { + const init: RequestInit = { headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, cache: 'no-store', - }) + } + const response = authorizedWorkspaceId + ? await withResourceOutboundScope({ workspaceId: authorizedWorkspaceId }, () => + outboundFetch(MODELS_URL, init) + ) + : await fetch(MODELS_URL, init) if (!response.ok) { logger.warn('Failed to fetch Baseten models', { diff --git a/apps/sim/app/api/providers/fireworks/models/route.ts b/apps/sim/app/api/providers/fireworks/models/route.ts index 82754eb53fb..84cef64978c 100644 --- a/apps/sim/app/api/providers/fireworks/models/route.ts +++ b/apps/sim/app/api/providers/fireworks/models/route.ts @@ -11,11 +11,14 @@ import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' import { isHosted } from '@/lib/core/config/env-flags' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('FireworksModelsAPI') +const MODELS_URL = 'https://api.fireworks.ai/inference/v1/models' interface FireworksModel { id: string @@ -36,6 +39,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } let apiKey: string | undefined + let authorizedWorkspaceId: string | undefined const queryValidation = fireworksProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, @@ -47,6 +51,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { if (session?.user?.id) { const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) if (permission) { + authorizedWorkspaceId = workspaceId const byokResult = await getBYOKKey(workspaceId, 'fireworks') if (byokResult) { apiKey = byokResult.apiKey @@ -71,13 +76,18 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } try { - const response = await fetch('https://api.fireworks.ai/inference/v1/models', { + const init: RequestInit = { headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, cache: 'no-store', - }) + } + const response = authorizedWorkspaceId + ? await withResourceOutboundScope({ workspaceId: authorizedWorkspaceId }, () => + outboundFetch(MODELS_URL, init) + ) + : await fetch(MODELS_URL, init) if (!response.ok) { logger.warn('Failed to fetch Fireworks models', { diff --git a/apps/sim/app/api/providers/ollama-cloud/models/route.ts b/apps/sim/app/api/providers/ollama-cloud/models/route.ts index bd5673e0842..1e85fe7134d 100644 --- a/apps/sim/app/api/providers/ollama-cloud/models/route.ts +++ b/apps/sim/app/api/providers/ollama-cloud/models/route.ts @@ -9,6 +9,8 @@ import { import { validationErrorResponse } from '@/lib/api/server' import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' @@ -54,13 +56,15 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } try { - const response = await fetch('https://ollama.com/api/tags', { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - }) + const response = await withResourceOutboundScope({ workspaceId }, () => + outboundFetch('https://ollama.com/api/tags', { + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + cache: 'no-store', + }) + ) if (!response.ok) { logger.warn('Failed to fetch Ollama Cloud models', { diff --git a/apps/sim/app/api/providers/together/models/route.ts b/apps/sim/app/api/providers/together/models/route.ts index dcaa0dc0c5a..5f63f158ef9 100644 --- a/apps/sim/app/api/providers/together/models/route.ts +++ b/apps/sim/app/api/providers/together/models/route.ts @@ -10,11 +10,14 @@ import { validationErrorResponse } from '@/lib/api/server' import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('TogetherModelsAPI') +const MODELS_URL = 'https://api.together.ai/v1/models' /** Together's catalog includes non-text models; only chat models work with chat completions. */ const NON_CHAT_MODEL_TYPES = new Set([ @@ -34,6 +37,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } let apiKey: string | undefined + let authorizedWorkspaceId: string | undefined const queryValidation = togetherProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, @@ -45,6 +49,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { if (session?.user?.id) { const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) if (permission) { + authorizedWorkspaceId = workspaceId const byokResult = await getBYOKKey(workspaceId, 'together') if (byokResult) { apiKey = byokResult.apiKey @@ -63,13 +68,18 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } try { - const response = await fetch('https://api.together.ai/v1/models', { + const init: RequestInit = { headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', }, cache: 'no-store', - }) + } + const response = authorizedWorkspaceId + ? await withResourceOutboundScope({ workspaceId: authorizedWorkspaceId }, () => + outboundFetch(MODELS_URL, init) + ) + : await fetch(MODELS_URL, init) if (!response.ok) { logger.warn('Failed to fetch Together models', { diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index 11eb1b1df66..cf80f10ad50 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -30,9 +30,14 @@ import { knowledgeTagsServiceMock, knowledgeTagsServiceMockFns, } from '@sim/testing/mocks/knowledge-tags-service.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { v1MiddlewareMock, v1MiddlewareMockFns } from '@sim/testing/mocks/v1-middleware.mock' +import { + workspaceContextMock, + workspaceContextMockFns, +} from '@sim/testing/mocks/workspace-context.mock' import { getErrorMessage } from '@sim/utils/errors' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockResolveV1KnowledgeReadAccess, mockExecuteKnowledgeSearch, mockRetrievalStatus } = vi.hoisted(() => ({ @@ -84,7 +89,10 @@ vi.mock('@/app/api/v1/knowledge/utils', () => ({ })) vi.mock('@/lib/knowledge/tags/service', () => knowledgeTagsServiceMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { POST } from '@/app/api/v1/knowledge/search/route' const { mockGetDocumentTagDefinitions } = knowledgeTagsServiceMockFns @@ -143,6 +151,10 @@ describe('v1 knowledge search route — per-KB embedding model', () => { mockRecordSearchEmbeddingUsage.mockResolvedValue(undefined) }) + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + it('fails a search whose retrieval ran out of time instead of returning partial rows', async () => { const access = { kind: 'user' as const, userId: 'user-1', tokens: ['reader-token'] } mockResolveV1KnowledgeReadAccess.mockResolvedValue({ @@ -250,6 +262,29 @@ describe('v1 knowledge search route — per-KB embedding model', () => { ) }) + it("embeds the query through the workspace organization's outbound route", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + workspaceContextMockFns.mockLoadWorkspaceApplicationContext.mockResolvedValueOnce({ + workspaceId: 'ws-1', + workspaceOrganizationId: 'org-1', + }) + mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({ + hasAccess: true, + knowledgeBase: baseKb('kb-1', 'text-embedding-3-small'), + }) + mockGenerateSearchEmbedding.mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + return { embedding: [0.1, 0.2, 0.3], isBYOK: false } + }) + + const res = await POST( + createMockRequest('POST', { workspaceId: 'ws-1', knowledgeBaseIds: 'kb-1', query: 'hello' }) + ) + + expect(res.status).toBe(200) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + }) + it('rejects cross-KB queries with mixed vector widths, which store in different columns', async () => { mockCheckKnowledgeBaseAccess .mockResolvedValueOnce({ diff --git a/apps/sim/app/api/v1/knowledge/search/route.ts b/apps/sim/app/api/v1/knowledge/search/route.ts index fd239a218e0..b1b4b1bc333 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.ts @@ -6,6 +6,7 @@ import { resolveSystemBillingAttribution, } from '@/lib/billing/core/attribution' import { checkSearchUsageLimits } from '@/lib/billing/core/usage-gate-cache' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' import { toKbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' @@ -253,10 +254,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { structuredFilters, }) } else if (hasQuery) { - const queryEmbeddingResult = await generateSearchEmbedding( - query!, - queryEmbeddingTarget!, - workspaceId + const queryEmbeddingResult = await withResourceOutboundScope({ workspaceId }, () => + generateSearchEmbedding(query!, queryEmbeddingTarget!, workspaceId) ) queryEmbeddingIsBYOK = queryEmbeddingResult.isBYOK retrieved = await retrieveKnowledgeSearch({ diff --git a/apps/sim/app/api/wand/route.ts b/apps/sim/app/api/wand/route.ts index fbe1e60d148..651b11638a2 100644 --- a/apps/sim/app/api/wand/route.ts +++ b/apps/sim/app/api/wand/route.ts @@ -18,6 +18,8 @@ import { recordUsage } from '@/lib/billing/core/usage-log' import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' import { env } from '@/lib/core/config/env' import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { enrichSandboxCapabilities } from '@/lib/execution/remote-sandbox/wand-enricher' @@ -48,6 +50,19 @@ if (!useWandAzure && !openaiApiKey) { logger.info(`Using ${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} for wand generation`) } +const wandApiUrl = useWandAzure + ? `${azureEndpoint?.replace(/\/$/, '')}/openai/v1/responses?api-version=${azureApiVersion}` + : 'https://api.openai.com/v1/responses' + +/** + * Sends a wand completion request. Sim's own Azure OpenAI deployment stays direct; OpenAI calls + * (BYOK or Sim's key) are made for the workspace, so they leave through its egress route. + */ +function postWandRequest(workspaceId: string, init: RequestInit): Promise { + if (useWandAzure) return fetch(wandApiUrl, init) + return withResourceOutboundScope({ workspaceId }, () => outboundFetch(wandApiUrl, init)) +} + interface ChatMessage { role: 'user' | 'assistant' | 'system' content: string @@ -342,26 +357,22 @@ export const POST = withRouteHandler(async (req: NextRequest) => { messages.push({ role: 'user', content: prompt }) + const headers: Record = { + 'Content-Type': 'application/json', + 'OpenAI-Beta': 'responses=v1', + } + + if (useWandAzure) { + headers['api-key'] = azureApiKey! + } else { + headers.Authorization = `Bearer ${activeOpenAIKey}` + } + if (stream) { try { logger.info(`About to create stream with model: ${useWandAzure ? wandModelName : 'gpt-4o'}`) - const apiUrl = useWandAzure - ? `${azureEndpoint?.replace(/\/$/, '')}/openai/v1/responses?api-version=${azureApiVersion}` - : 'https://api.openai.com/v1/responses' - - const headers: Record = { - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'responses=v1', - } - - if (useWandAzure) { - headers['api-key'] = azureApiKey! - } else { - headers.Authorization = `Bearer ${activeOpenAIKey}` - } - - const response = await fetch(apiUrl, { + const response = await postWandRequest(workspaceId, { method: 'POST', headers, body: JSON.stringify({ @@ -567,22 +578,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { } } - const apiUrl = useWandAzure - ? `${azureEndpoint?.replace(/\/$/, '')}/openai/v1/responses?api-version=${azureApiVersion}` - : 'https://api.openai.com/v1/responses' - - const headers: Record = { - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'responses=v1', - } - - if (useWandAzure) { - headers['api-key'] = azureApiKey! - } else { - headers.Authorization = `Bearer ${activeOpenAIKey}` - } - - const response = await fetch(apiUrl, { + const response = await postWandRequest(workspaceId, { method: 'POST', headers, body: JSON.stringify({ diff --git a/apps/sim/lib/internal/vision/client.ts b/apps/sim/lib/internal/vision/client.ts index 9eaefc201e8..8d7dcb53330 100644 --- a/apps/sim/lib/internal/vision/client.ts +++ b/apps/sim/lib/internal/vision/client.ts @@ -122,7 +122,7 @@ async function analyzeWithGemini( signal?.throwIfAborted() const base64Payload = await fetchGeminiImage(input, signal) const { mediaType, base64Data } = parseDataImage(base64Payload) - const ai = new GoogleGenAI({ apiKey: input.apiKey }) + const ai = new GoogleGenAI({ apiKey: input.apiKey, httpOptions: { fetch: outboundFetch } }) const response = await ai.models.generateContent({ model: input.model, contents: [ diff --git a/apps/sim/providers/file-attachments.server.ts b/apps/sim/providers/file-attachments.server.ts index 9a91605569d..a08209d2770 100644 --- a/apps/sim/providers/file-attachments.server.ts +++ b/apps/sim/providers/file-attachments.server.ts @@ -149,7 +149,10 @@ export async function uploadLargeFilesToProvider( if (groups.length === 0) return const maxBytes = getProviderAttachmentMaxBytes(providerId) - const ai = providerId === 'google' ? new GoogleGenAI({ apiKey: request.apiKey }) : null + const ai = + providerId === 'google' + ? new GoogleGenAI({ apiKey: request.apiKey, httpOptions: { fetch: outboundFetch } }) + : null for (const group of groups) { const [representative] = group diff --git a/apps/sim/providers/google/index.ts b/apps/sim/providers/google/index.ts index e381ef9da79..ca279cd9766 100644 --- a/apps/sim/providers/google/index.ts +++ b/apps/sim/providers/google/index.ts @@ -1,5 +1,6 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { executeGeminiRequest } from '@/providers/gemini/core' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' @@ -30,7 +31,7 @@ export const googleProvider: ProviderConfig = { logger.info('Creating Google Gemini client', { model: request.model }) - const ai = new GoogleGenAI({ apiKey: request.apiKey }) + const ai = new GoogleGenAI({ apiKey: request.apiKey, httpOptions: { fetch: outboundFetch } }) return executeGeminiRequest({ ai, diff --git a/apps/sim/providers/vertex/index.ts b/apps/sim/providers/vertex/index.ts index ea5edba6328..83c0d255d17 100644 --- a/apps/sim/providers/vertex/index.ts +++ b/apps/sim/providers/vertex/index.ts @@ -2,6 +2,7 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { OAuth2Client } from 'google-auth-library' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateGoogleCloudLocation, validateGoogleCloudProject, @@ -91,6 +92,7 @@ export const vertexProvider: ProviderConfig = { googleAuthOptions: { authClient, }, + httpOptions: { fetch: outboundFetch }, }) return executeGeminiRequest({ From 194c7b725448ebe1cb8a315809e27e3f341c88dc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 20:48:27 -0800 Subject: [PATCH 15/32] improvement(egress): scope generic oauth connect callbacks to their draft and report egress outages as 503 Generic OAuth connectors' custom getToken/getUserInfo now run in the outbound scope of the connect draft named by Better Auth's verified OAuth state, so an organization with a dedicated egress gateway connects from its own IPs. The Salesforce instance lookup in account.create.before uses the same draft scope. An OutboundRoutingError during OAuth refresh is no longer swallowed into a generic failed refresh: it propagates through the coalesced refresh without marking the credential dead or revoked, and token resolution maps it (and the service-account path) to 503 instead of 401. --- apps/sim/lib/auth/auth.ts | 17 +++++-- .../lib/oauth/connect-scope.server.test.ts | 50 ++++++++++++++++++- apps/sim/lib/oauth/connect-scope.server.ts | 47 +++++++++++++++++ apps/sim/lib/oauth/credential-service.test.ts | 12 +++++ apps/sim/lib/oauth/credential-service.ts | 4 ++ apps/sim/lib/oauth/oauth.test.ts | 8 +++ apps/sim/lib/oauth/refresh.server.ts | 6 ++- apps/sim/lib/oauth/token-resolution.test.ts | 23 +++++++++ apps/sim/lib/oauth/token-resolution.ts | 22 ++++++++ 9 files changed, 182 insertions(+), 7 deletions(-) diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index 6fb31d22b92..abf0a012663 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -159,6 +159,10 @@ import { isEmailVerificationEffectivelyEnabled } from '@/lib/messaging/email/ver import { scheduleLifecycleEmail } from '@/lib/messaging/lifecycle' import { APP_ENTRY_PATH } from '@/lib/navigation/paths' import { formatApolloAuthorizationUrl } from '@/lib/oauth/apollo' +import { + resolveConnectDraftOutboundScope, + withConnectDraftOutboundScope, +} from '@/lib/oauth/connect-scope.server' import { clearRecordedRevocation } from '@/lib/oauth/credential-service' import { getMicrosoftRefreshTokenExpiry, @@ -508,9 +512,14 @@ export const auth = betterAuth({ } if (account.accessToken && isSalesforceOAuthProviderId(account.providerId)) { - const instanceUrl = await fetchSalesforceInstanceUrl( - account.providerId, - account.accessToken + const { accessToken } = account + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId: reconnectDraftId, + userId: account.userId, + providerId: account.providerId, + }) + const instanceUrl = await inDraftScope(() => + fetchSalesforceInstanceUrl(account.providerId, accessToken) ) if (instanceUrl) { modifiedAccount.scope = withSalesforceInstanceScope(instanceUrl, account.scope) @@ -1476,7 +1485,7 @@ export const auth = betterAuth({ overrideDefaultEmailVerification: true, }), genericOAuth({ - config: buildConnectorProviders(), + config: buildConnectorProviders().map(withConnectDraftOutboundScope), }), /** * Sim as an OAuth 2.0 authorization server (auth-code + PKCE, refresh diff --git a/apps/sim/lib/oauth/connect-scope.server.test.ts b/apps/sim/lib/oauth/connect-scope.server.test.ts index d6687a59665..eb79eeabb12 100644 --- a/apps/sim/lib/oauth/connect-scope.server.test.ts +++ b/apps/sim/lib/oauth/connect-scope.server.test.ts @@ -5,11 +5,13 @@ import { } from '@sim/testing/mocks/workspace-context.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetActiveConnectDraft } = vi.hoisted(() => ({ +const { mockGetActiveConnectDraft, mockGetOAuthState } = vi.hoisted(() => ({ mockGetActiveConnectDraft: vi.fn(), + mockGetOAuthState: vi.fn(), })) const { mockLoadWorkspaceApplicationContext } = workspaceContextMockFns +vi.mock('better-auth/api', () => ({ getOAuthState: mockGetOAuthState })) vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@/lib/credentials/connect-draft', () => ({ getActiveConnectDraft: mockGetActiveConnectDraft, @@ -17,7 +19,10 @@ vi.mock('@/lib/credentials/connect-draft', () => ({ vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' -import { resolveConnectDraftOutboundScope } from '@/lib/oauth/connect-scope.server' +import { + resolveConnectDraftOutboundScope, + withConnectDraftOutboundScope, +} from '@/lib/oauth/connect-scope.server' async function organizationSeenByVendorCall( draftOwner: { workspaceId: string | null; organizationId: string | null }, @@ -71,3 +76,44 @@ describe('resolveConnectDraftOutboundScope', () => { ).resolves.toBeUndefined() }) }) + +describe('withConnectDraftOutboundScope', () => { + beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + + it("runs a generic connector's code exchange in its connect draft's organization", async () => { + mockGetOAuthState.mockResolvedValue({ + callbackURL: '/workspace/ws-1/integrations?credentialDraftId=draft-1', + link: { userId: 'user-1', email: 'user@example.com' }, + }) + mockGetActiveConnectDraft.mockResolvedValue({ + id: 'draft-1', + providerId: 'jira', + workspaceId: null, + organizationId: 'org-1', + }) + let organizationId: string | null | undefined + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (id) => { + organizationId = id + return { kind: 'direct' } + }) + const connector = withConnectDraftOutboundScope({ + providerId: 'jira', + clientId: 'client', + getToken: async () => { + await resolveCurrentOutboundRoute() + return { accessToken: 'token' } + }, + }) + + await connector.getToken?.({ code: 'code', redirectURI: 'https://sim.test/callback' }) + + expect(organizationId).toBe('org-1') + expect(mockGetActiveConnectDraft).toHaveBeenCalledWith('draft-1', 'user-1') + }) +}) diff --git a/apps/sim/lib/oauth/connect-scope.server.ts b/apps/sim/lib/oauth/connect-scope.server.ts index 29530cd8318..b6d24075384 100644 --- a/apps/sim/lib/oauth/connect-scope.server.ts +++ b/apps/sim/lib/oauth/connect-scope.server.ts @@ -1,7 +1,10 @@ +import { getOAuthState } from 'better-auth/api' +import type { GenericOAuthConfig } from 'better-auth/plugins' import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' import { captureOutboundScope } from '@/lib/core/network/context.server' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { getActiveConnectDraft } from '@/lib/credentials/connect-draft' +import { parseCredentialDraftIdFromCallbackUrl } from '@/lib/credentials/draft-processor' type OutboundScopeRunner = (run: () => Promise) => Promise @@ -27,3 +30,47 @@ export async function resolveConnectDraftOutboundScope(params: { async () => captureOutboundScope() ) } + +/** One draft lookup per callback, shared by its code exchange and identity lookup. */ +const callbackDraftScopes = new WeakMap>() + +/** + * Runs a generic OAuth callback step in the scope of the connect draft named by Better Auth's + * server-verified OAuth state (`link.userId` started the link, `callbackURL` carries the draft + * id). A sign-in callback, which has no `link`, runs unscoped. + */ +async function runInOAuthCallbackDraftScope( + providerId: string, + run: () => Promise +): Promise { + const state = await getOAuthState() + const userId = state?.link?.userId + if (!userId) return run() + let inDraftScope = callbackDraftScopes.get(state) + if (!inDraftScope) { + inDraftScope = resolveConnectDraftOutboundScope({ + draftId: parseCredentialDraftIdFromCallbackUrl(state.callbackURL), + userId, + providerId, + }) + callbackDraftScopes.set(state, inDraftScope) + } + return (await inDraftScope)(run) +} + +/** + * Wraps a generic OAuth connector's custom code exchange and identity lookup so their vendor + * calls leave through the connecting organization's egress gateway. Better Auth's built-in + * exchange (providers without `getToken`) uses its own fetch and is not routed. + */ +export function withConnectDraftOutboundScope(config: GenericOAuthConfig): GenericOAuthConfig { + const { providerId, getToken, getUserInfo } = config + return { + ...config, + getToken: + getToken && ((data) => runInOAuthCallbackDraftScope(providerId, () => getToken(data))), + getUserInfo: + getUserInfo && + ((tokens) => runInOAuthCallbackDraftScope(providerId, () => getUserInfo(tokens))), + } +} diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts index 8284e7e2164..339f6d4f5af 100644 --- a/apps/sim/lib/oauth/credential-service.test.ts +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -91,6 +91,7 @@ vi.mock('@/lib/oauth/refresh-error-codes', () => ({ isTerminalRefreshError: vi.fn(() => false), })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { CredentialRevokedError } from '@/lib/oauth/credential-revoked' import { getCredentialTerminalRefreshError, @@ -906,6 +907,17 @@ describe('OAuth access-token refresh headroom', () => { ) }) + it('surfaces an unavailable egress route without flagging the credential dead or revoked', async () => { + mocks.refreshOAuthToken.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const row = createOAuthAccount(WAITED_GOOGLE_REMAINING_MS) + queueTableRows(account, [row]) + await expect(refreshTokenIfNeeded('test', row, RAW_ACCOUNT_ID)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(markCredentialDead).not.toHaveBeenCalled() + expect(dbChainMockFns.set).not.toHaveBeenCalled() + }) + it('answers a Microsoft caller from its valid token while refresh-token aging refreshes', async () => { vi.mocked(isMicrosoftProvider).mockReturnValue(true) const provider = createDeferred() diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index f3131851b34..5bd06e7c281 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -9,6 +9,7 @@ import { and, desc, eq, isNotNull, sql } from 'drizzle-orm' import { withLeaderLock } from '@/lib/concurrency/leader-lock' import { coalesceLocally } from '@/lib/concurrency/singleflight' import { env } from '@/lib/core/config/env' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { decryptSecret } from '@/lib/core/security/encryption' import { waitWithAbort } from '@/lib/core/utils/concurrency' import { mintAtlassianServiceAccountToken } from '@/lib/credentials/atlassian-client-credentials' @@ -1261,6 +1262,7 @@ function isRevocationReprobeDue(chain: RevocableChain | null | undefined): boole type RefreshOutcome = | { kind: 'token'; accessToken: string } | { kind: 'revoked'; errorCode: string } + | { kind: 'egress_unavailable'; error: OutboundRoutingError } function tokenOutcome(accessToken: string | null): RefreshOutcome | null { return accessToken ? { kind: 'token', accessToken } : null @@ -1851,6 +1853,7 @@ async function performCoalescedRefresh({ logger.info('Successfully refreshed access token', logContext) return { kind: 'token', accessToken: result.accessToken } } catch (error) { + if (error instanceof OutboundRoutingError) return { kind: 'egress_unavailable', error } logger.error('Refresh failed inside leader path', { ...logContext, ...(privacyMode === 'selector' ? {} : { error: toError(error).message }), @@ -2136,6 +2139,7 @@ export async function refreshTokenIfNeeded( refreshed: result.refreshed, } } + if (result.outcome?.kind === 'egress_unavailable') throw result.outcome.error if (result.outcome?.kind === 'revoked') { throw new CredentialRevokedError( `The ${credential.providerId} credential was revoked by the provider (${result.outcome.errorCode}). Reconnect it to continue.`, diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index 397a7e6802b..ee37f34fdc7 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -36,6 +36,7 @@ beforeAll(() => { afterAll(resetEnvMock) import { buildConnectorProviders } from '@/lib/auth/connectors/providers' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES } from '@/lib/core/utils/stream-limits' import { getPerRequestOAuthLinkScopes, OAUTH_PROVIDERS } from '@/lib/oauth' import { refreshOAuthToken } from '@/lib/oauth/refresh.server' @@ -509,6 +510,13 @@ describe('OAuth Token Refresh', () => { }) expect(mockFetch).not.toHaveBeenCalled() }) + it('throws an unavailable egress route instead of reporting a failed refresh', async () => { + const mockFetch = vi.fn().mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + withMockFetch(mockFetch, () => refreshOAuthToken('google', 'test_refresh_token')) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) it('should return failure with errorCode for HTTP error responses', async () => { const mockFetch = vi.fn().mockResolvedValue({ ok: false, diff --git a/apps/sim/lib/oauth/refresh.server.ts b/apps/sim/lib/oauth/refresh.server.ts index 4342a9a14fc..862d1d90d67 100644 --- a/apps/sim/lib/oauth/refresh.server.ts +++ b/apps/sim/lib/oauth/refresh.server.ts @@ -7,6 +7,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { env } from '@/lib/core/config/env' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { redactExactSensitiveValues } from '@/lib/core/security/redaction' import { DEFAULT_MAX_ERROR_BODY_BYTES, @@ -855,7 +856,9 @@ async function refreshInstagramLongLivedToken( /** * Exchanges a stored refresh token at its provider's token endpoint. The request * leaves through the current outbound scope, so an organization with a dedicated - * egress gateway refreshes from its own static IPs. + * egress gateway refreshes from its own static IPs. An unavailable egress route + * throws its `OutboundRoutingError` rather than reporting a failed refresh, so it + * is never read as the provider rejecting the token. */ export async function refreshOAuthToken( providerId: string, @@ -1058,6 +1061,7 @@ export async function refreshOAuthToken( ...(refreshTokenExpiresIn ? { refreshTokenExpiresIn } : {}), } } catch (error) { + if (error instanceof OutboundRoutingError) throw error const normalized = toError(error) const message = normalized.name === 'PayloadSizeLimitError' || normalized.message.startsWith('OAuth client ') diff --git a/apps/sim/lib/oauth/token-resolution.test.ts b/apps/sim/lib/oauth/token-resolution.test.ts index e5a704003d5..d5a4d7be280 100644 --- a/apps/sim/lib/oauth/token-resolution.test.ts +++ b/apps/sim/lib/oauth/token-resolution.test.ts @@ -36,6 +36,7 @@ vi.mock('@/lib/credentials/managed-oauth', () => credentialsManagedOauthMock) vi.unmock('@/lib/oauth/utils') +import { OutboundRoutingError } from '@/lib/core/network/routing' import { OrchestrationError } from '@/lib/core/orchestration/types' import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' @@ -171,6 +172,28 @@ describe('resolveCredentialToken', () => { expect(auditMockFns.mockRecordAuditRollup).not.toHaveBeenCalled() }) + it('reports an unavailable egress route as 503, not a failed credential', async () => { + mockAuthorizeCredentialUseForAuth.mockResolvedValue({ + ok: true, + requesterUserId: 'user-1', + credentialOwnerUserId: 'owner-1', + }) + mockGetCredential.mockResolvedValue({ providerId: 'google' }) + mockRefreshTokenIfNeeded.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + const result = await resolveCredentialToken(INTERNAL_AUTH, { + requestId: 'req-1', + resolvedCredential: null, + credentialId: 'cred-1', + }) + + expect(result).toEqual({ + ok: false, + status: 503, + error: "The organization's egress route is unavailable", + }) + }) + it('authorizes service-account credentials before minting a token', async () => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: false, error: 'Unauthorized' }) diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index 9615d075f91..0c7cedc641f 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -8,6 +8,7 @@ import { } from '@/lib/api/contracts/oauth-connections' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' import { type AuthResult, AuthType } from '@/lib/auth/hybrid' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' import { resolveManagedOAuthCredentialToken } from '@/lib/credentials/application/resolve-managed-oauth-token' @@ -41,6 +42,16 @@ import { extractZohoDeskBaseFromScope } from '@/tools/zoho_desk/host-allowlist' const logger = createLogger('OAuthTokenResolution') +/** + * The organization's egress gateway is down or refuses the route: an outage on Sim's side, + * never a sign the credential is bad, so the caller is told to retry rather than reconnect. + */ +const EGRESS_ROUTE_UNAVAILABLE: Extract = { + ok: false, + status: 503, + error: "The organization's egress route is unavailable", +} + /** * Duck type of the inbound request, used only so audit rows record IP and user agent. * In-process callers omit it. @@ -272,6 +283,10 @@ export async function completeOAuthCredentialToken(params: { }) return { ok: false, status: 401, code: OAUTH_CREDENTIAL_REVOKED, error: error.message } } + if (error instanceof OutboundRoutingError) { + logger.warn('Egress route unavailable for token refresh', { requestId, code: error.code }) + return EGRESS_ROUTE_UNAVAILABLE + } logger.error('Failed to refresh access token:', error, { requestId }) return { ok: false, status: 401, error: 'Failed to refresh access token' } } @@ -444,6 +459,13 @@ export async function resolveCredentialToken( }, } } catch (error) { + if (error instanceof OutboundRoutingError) { + logger.warn('Egress route unavailable for service account token', { + requestId, + code: error.code, + }) + return EGRESS_ROUTE_UNAVAILABLE + } logger.error('Service account token error:', error, { requestId }) if (error instanceof TokenServiceAccountValidationError) { // Classified provider outages are infra failures, not bad credentials. From 817d9257451beb26f81c5dde51210f459e806518 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 21:06:29 -0800 Subject: [PATCH 16/32] improvement(egress): route generic oauth and credential-group code exchanges through the organization gateway Connectors without their own getToken now get a default one that performs Better Auth's generic OAuth exchange (authorizationCodeRequest, discovery-resolved token endpoint, basic/post client auth, PKCE, static tokenUrlParams, redirect refusal, getOAuth2Tokens) but sends it with outboundFetch inside the connect draft's scope. Credential-group managed OAuth exchanges the code the same way and reports an unavailable egress route as 503. A managed OAuth refresh that hits an unavailable egress route reports a retryable 503 refresh failure instead of an unhandled error, without marking the credential for reauthorization. --- .../standard-oauth-provider.test.ts | 36 +++- .../standard-oauth-provider.ts | 17 +- .../sim/lib/credentials/managed-oauth.test.ts | 31 +++ apps/sim/lib/credentials/managed-oauth.ts | 16 +- .../lib/oauth/authorization-code.server.ts | 119 +++++++++++ .../lib/oauth/connect-scope.server.test.ts | 199 ++++++++++++++++++ apps/sim/lib/oauth/connect-scope.server.ts | 12 +- 7 files changed, 417 insertions(+), 13 deletions(-) create mode 100644 apps/sim/lib/oauth/authorization-code.server.ts diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts index 12180693dd9..340f39ad6f9 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts @@ -1,15 +1,19 @@ import { createHash } from 'node:crypto' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { resetUrlsMock, urlsMockFns } from '@sim/testing/mocks/urls.mock' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { CredentialGroupOAuthContext } from '@/lib/credential-groups/enrollments' import type { CredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth-state' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' -const { mockGetToken, mockVerifyIdentity } = vi.hoisted(() => ({ +const { connectorState, mockGetToken, mockVerifyIdentity } = vi.hoisted(() => ({ + connectorState: { builtInExchange: false }, mockGetToken: vi.fn(), mockVerifyIdentity: vi.fn(), })) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + vi.mock('@/lib/auth/connectors/managed-oauth', () => ({ getManagedOAuthConnectorProviderConfig: (providerId: string) => { if (providerId === 'google-calendar') { @@ -22,7 +26,7 @@ vi.mock('@/lib/auth/connectors/managed-oauth', () => ({ redirectURI: 'https://sim.example.com/api/auth/oauth2/callback/google-calendar', accessType: 'offline', scopes: ['calendar.read', 'profile'], - getToken: mockGetToken, + getToken: connectorState.builtInExchange ? undefined : mockGetToken, managedOAuth: { additionalScopes: ['openid'], requiresRefreshToken: true, @@ -71,6 +75,8 @@ vi.mock('@/lib/auth/connectors/managed-oauth', () => ({ }, })) +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { createStandardOAuthCredentialGroupProviderAdapter } from '@/lib/credential-groups/standard-oauth-provider' urlsMockFns.mockGetBaseUrl.mockReturnValue('https://sim.example.com') @@ -138,6 +144,12 @@ async function exchange() { } describe('standard OAuth Credential Group provider', () => { + afterEach(() => { + connectorState.builtInExchange = false + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + beforeEach(() => { mockGetToken.mockResolvedValue({ tokenType: 'Bearer', @@ -312,4 +324,22 @@ describe('standard OAuth Credential Group provider', () => { }) await expect(exchange()).rejects.toMatchObject({ statusCode: 502 }) }) + + it("sends a connector's built-in code exchange through its organization's egress route", async () => { + connectorState.builtInExchange = true + const sharedIpFetch = vi.fn(async () => + Response.json({ token_type: 'Bearer', access_token: 'access-1' }) + ) + vi.stubGlobal('fetch', sharedIpFetch) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect(runWithOutboundOrganization('org-1', exchange)).rejects.toMatchObject({ + statusCode: 503, + }) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + expect(sharedIpFetch).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index a20683223f0..173a1a40d37 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -4,7 +4,6 @@ import { applyDefaultAccessTokenExpiry, createAuthorizationURL, type OAuth2Tokens, - validateAuthorizationCode, } from 'better-auth/oauth2' import { type ConnectorProviderConfig, @@ -13,6 +12,7 @@ import { type ManagedOAuthConnectorConfig, } from '@/lib/auth/connectors/managed-oauth' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { credentialGroupOAuthNonceMatches } from '@/lib/credential-groups/oauth-state' import type { @@ -26,6 +26,7 @@ import { } from '@/lib/credential-groups/provider-adapter' import type { CredentialGroupStandardOAuthProvider } from '@/lib/credential-groups/providers' import { getCredentialGroupProviderService } from '@/lib/credential-groups/providers' +import { exchangeAuthorizationCode } from '@/lib/oauth/authorization-code.server' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' import { refreshOAuthToken } from '@/lib/oauth/refresh.server' @@ -207,7 +208,7 @@ function generatePkceVerifier(): string { return randomBytes(64).toString('base64url') } -async function exchangeAuthorizationCode(params: { +async function exchangeConnectorCode(params: { connector: ConnectorProviderConfig code: string codeVerifier?: string @@ -217,7 +218,7 @@ async function exchangeAuthorizationCode(params: { const { connector, code, codeVerifier, redirectUri, tokenEndpoint } = params const tokens = connector.getToken ? await connector.getToken({ code, redirectURI: redirectUri, codeVerifier }) - : await validateAuthorizationCode({ + : await exchangeAuthorizationCode({ headers: connector.authorizationHeaders, code, codeVerifier, @@ -307,14 +308,20 @@ export function createStandardOAuthCredentialGroupProviderAdapter( const endpoints = await resolveOAuthEndpoints(current.connector, service.name) let tokens: OAuth2Tokens try { - tokens = await exchangeAuthorizationCode({ + tokens = await exchangeConnectorCode({ connector: current.connector, code, ...(attempt.codeVerifier ? { codeVerifier: attempt.codeVerifier } : {}), redirectUri: attempt.redirectUri, tokenEndpoint: endpoints.tokenEndpoint, }) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) { + throw new CredentialGroupOAuthError( + `${service.name} authorization is temporarily unavailable.`, + 503 + ) + } throw new CredentialGroupOAuthError( `${service.name} could not complete authorization. Please try again.`, 502 diff --git a/apps/sim/lib/credentials/managed-oauth.test.ts b/apps/sim/lib/credentials/managed-oauth.test.ts index 74b8427d31d..ccf33d7e2db 100644 --- a/apps/sim/lib/credentials/managed-oauth.test.ts +++ b/apps/sim/lib/credentials/managed-oauth.test.ts @@ -34,6 +34,7 @@ vi.mock('@/lib/credential-groups/slack-managed-users', () => ({ verifySlackUserIdentity: vi.fn(), })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { credentialGroupScopePolicyVersion } from '@/lib/credential-groups/provider-adapter' import { SLACK_MANAGED_USER_SCOPES, @@ -519,4 +520,34 @@ describe('managed OAuth token resolution', () => { ) expect(mocks.encryptSecret).not.toHaveBeenCalled() }) + + it('reports an unavailable egress route during refresh as a retryable outage without reauthorization', async () => { + const row = mondayCredentialRow() + dbChainMockFns.limit.mockResolvedValueOnce([row]).mockResolvedValueOnce([row]) + mocks.decryptSecret.mockResolvedValue({ + decrypted: JSON.stringify({ + type: 'managed-oauth-token-set', + version: 1, + tokenType: 'Bearer', + accessToken: 'expired-access-token', + refreshToken: 'old-refresh-token', + }), + }) + mocks.getAdapter.mockReturnValue({ + getPolicy: vi.fn().mockResolvedValue({ + authorizationAppId: row.authorizationAppId, + scopeVersion: 1, + requiredScopes: row.grantedScopes, + }), + hasRequiredScopes: vi.fn().mockReturnValue(true), + refreshToken: vi.fn().mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')), + isTerminalRefreshError: vi.fn().mockReturnValue(true), + }) + + await expect(resolveManagedOAuthToken(mondayTokenResolutionParams())).rejects.toMatchObject({ + code: 'MANAGED_CREDENTIAL_REFRESH_FAILED', + statusCode: 503, + }) + expect(dbChainMockFns.set).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/credentials/managed-oauth.ts b/apps/sim/lib/credentials/managed-oauth.ts index f674b09aa31..6df2330bc5c 100644 --- a/apps/sim/lib/credentials/managed-oauth.ts +++ b/apps/sim/lib/credentials/managed-oauth.ts @@ -6,6 +6,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { and, eq } from 'drizzle-orm' import { LRUCache } from 'lru-cache' import type { WorkspaceAuthorizationContext } from '@/lib/core/application' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { type ResourceOwner, resourceScopeFields, @@ -25,6 +26,7 @@ import { import { getCredentialGroupProviderAdapterByProviderId } from '@/lib/credential-groups/provider-registry' import { isScopedCredentialGroupsAvailable } from '@/lib/credential-groups/scoped-availability' import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' +import type { RefreshTokenResult } from '@/lib/oauth' import { isSalesforceOAuthProviderId } from '@/lib/oauth/salesforce' import { fetchSalesforceInstanceUrl } from '@/lib/oauth/salesforce-instance.server' import { loadActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/context' @@ -524,7 +526,19 @@ async function resolveManagedOAuthAccessToken( } } - const refreshed = await adapter.refreshToken(currentTokenSet.refreshToken) + let refreshed: RefreshTokenResult + try { + refreshed = await adapter.refreshToken(currentTokenSet.refreshToken) + } catch (error) { + if (!(error instanceof OutboundRoutingError)) throw error + return { + error: new ManagedOAuthCredentialError( + 'MANAGED_CREDENTIAL_REFRESH_FAILED', + 'Managed credential refresh is temporarily unavailable', + 503 + ), + } + } if (!refreshed.ok) { const terminal = adapter.isTerminalRefreshError(refreshed.errorCode) if (terminal) { diff --git a/apps/sim/lib/oauth/authorization-code.server.ts b/apps/sim/lib/oauth/authorization-code.server.ts new file mode 100644 index 00000000000..c93b07a8673 --- /dev/null +++ b/apps/sim/lib/oauth/authorization-code.server.ts @@ -0,0 +1,119 @@ +import { toRecord } from '@sim/utils/object' +import { + authorizationCodeRequest, + getOAuth2Tokens, + type OAuth2Tokens, + type validateAuthorizationCode, +} from 'better-auth/oauth2' +import type { GenericOAuthConfig } from 'better-auth/plugins' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' + +type AuthorizationCodeExchange = Parameters[0] +type GetToken = NonNullable + +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]) +const JSON_MEDIA_TYPE = /^application\/(?:[\w!#$%&*.^`~-]*\+)?json$/i +const TEXT_MEDIA_TYPES = new Set([ + 'image/svg', + 'application/xml', + 'application/xhtml', + 'application/html', +]) + +/** + * Reads a successful response the way Better Auth's `betterFetch` does: JSON and text bodies are + * JSON-parsed when they parse (else kept as text), anything else is a Blob. + */ +async function readResponseData(response: Response): Promise { + const mediaType = response.headers.get('content-type')?.split(';')[0] ?? '' + if ( + !mediaType || + JSON_MEDIA_TYPE.test(mediaType) || + TEXT_MEDIA_TYPES.has(mediaType) || + mediaType.startsWith('text/') + ) { + const text = await response.text() + try { + return JSON.parse(text) + } catch { + return text + } + } + return response.blob() +} + +/** + * Better Auth's `validateAuthorizationCode`, sent through {@link outboundFetch} so the code + * exchange leaves through the current outbound scope's egress route. The request is built by + * Better Auth's own `authorizationCodeRequest` and the response parsed by its `getOAuth2Tokens`; + * like Better Auth, it refuses redirects and throws on a non-2xx answer. + */ +export async function exchangeAuthorizationCode({ + tokenEndpoint, + ...request +}: AuthorizationCodeExchange): Promise { + const { body, headers } = await authorizationCodeRequest(request) + const response = await outboundFetch(tokenEndpoint, { + method: 'POST', + body, + headers, + redirect: 'manual', + }) + if (response.type === 'opaqueredirect' || REDIRECT_STATUSES.has(response.status)) { + await response.body?.cancel().catch(() => {}) + throw new Error(`The OAuth endpoint "${tokenEndpoint}" returned an HTTP redirect`) + } + if (!response.ok) { + await response.body?.cancel().catch(() => {}) + throw new Error(`OAuth token endpoint responded with ${response.status}`) + } + return getOAuth2Tokens(toRecord(await readResponseData(response))) +} + +/** The plugin's token endpoint: the discovery document's when it answers, else `tokenUrl`. */ +async function resolveTokenEndpoint(config: GenericOAuthConfig): Promise { + let tokenEndpoint = config.tokenUrl + if (config.discoveryUrl) { + const response = await outboundFetch(config.discoveryUrl, { + method: 'GET', + headers: config.discoveryHeaders, + }) + if (response.ok) { + const discovery = await readResponseData(response) + if (discovery) { + const endpoint = toRecord(discovery).token_endpoint + tokenEndpoint = typeof endpoint === 'string' ? endpoint : undefined + } + } else { + await response.body?.cancel().catch(() => {}) + } + } + if (!tokenEndpoint) throw new Error(`OAuth token endpoint not found for ${config.providerId}`) + return tokenEndpoint +} + +/** + * The code exchange Better Auth's generic OAuth callback performs for a connector without its + * own `getToken` (discovery, client authentication, PKCE verifier, static `tokenUrlParams`), sent + * through {@link outboundFetch}. Undefined when `tokenUrlParams` reads the callback request, + * which a `getToken` never sees. + */ +export function createOutboundCodeExchange(config: GenericOAuthConfig): GetToken | undefined { + const { tokenUrlParams } = config + if (typeof tokenUrlParams === 'function') return undefined + return async ({ code, redirectURI, codeVerifier }) => + exchangeAuthorizationCode({ + headers: config.authorizationHeaders, + code, + codeVerifier, + redirectURI, + options: { + clientId: config.clientId, + clientSecret: config.clientSecret, + redirectURI: config.redirectURI, + }, + tokenEndpoint: await resolveTokenEndpoint(config), + authentication: config.authentication, + additionalParams: tokenUrlParams, + }) +} diff --git a/apps/sim/lib/oauth/connect-scope.server.test.ts b/apps/sim/lib/oauth/connect-scope.server.test.ts index eb79eeabb12..2319497fd4a 100644 --- a/apps/sim/lib/oauth/connect-scope.server.test.ts +++ b/apps/sim/lib/oauth/connect-scope.server.test.ts @@ -3,6 +3,8 @@ import { workspaceContextMock, workspaceContextMockFns, } from '@sim/testing/mocks/workspace-context.mock' +import { validateAuthorizationCode } from 'better-auth/oauth2' +import { type GenericOAuthConfig, genericOAuth } from 'better-auth/plugins' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockGetActiveConnectDraft, mockGetOAuthState } = vi.hoisted(() => ({ @@ -117,3 +119,200 @@ describe('withConnectDraftOutboundScope', () => { expect(mockGetActiveConnectDraft).toHaveBeenCalledWith('draft-1', 'user-1') }) }) + +const TOKEN_RESPONSE = { + token_type: 'Bearer', + access_token: 'access-1', + refresh_token: 'refresh-1', + expires_in: 3600, + scope: 'openid email', +} +const CODE = { code: 'code-1', redirectURI: 'https://sim.test/api/auth/oauth2/callback/idp' } +type CodeData = typeof CODE & { codeVerifier?: string } + +interface RecordedRequest { + url: string + method: string + redirect: string + headers: Record + body: string +} + +/** Records every request sent through the global `fetch`, the transport of both exchanges. */ +function stubVendor(respond: (url: string) => Response = () => Response.json(TOKEN_RESPONSE)) { + const requests: RecordedRequest[] = [] + vi.stubGlobal('fetch', async (input: RequestInfo | URL, init?: RequestInit) => { + const request = new Request(input, init) + requests.push({ + url: request.url, + method: request.method, + redirect: request.redirect, + headers: Object.fromEntries(request.headers), + body: await request.text(), + }) + return respond(request.url) + }) + return requests +} + +/** The exchange Better Auth's generic OAuth plugin performs for a configured connector. */ +async function pluginExchange(config: GenericOAuthConfig, data: CodeData) { + const initialized = await genericOAuth({ config: [config] }).init?.({ + socialProviders: [], + } as never) + const provider = initialized?.context?.socialProviders?.[0] + if (!provider) throw new Error('generic OAuth provider was not registered') + return provider.validateAuthorizationCode(data) +} + +async function compareExchanges( + config: GenericOAuthConfig, + data: CodeData, + respond?: (url: string) => Response +) { + const betterAuthRequests = stubVendor(respond) + const betterAuthTokens = await pluginExchange(config, data) + const routedRequests = stubVendor(respond) + const routedTokens = await pluginExchange(withConnectDraftOutboundScope(config), data) + return { betterAuthRequests, betterAuthTokens, routedRequests, routedTokens } +} + +describe('default code exchange for connectors without getToken', () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(new Date('2026-11-13T12:00:00Z')) + mockGetOAuthState.mockResolvedValue(null) + }) + + afterEach(() => { + vi.useRealTimers() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + + const base = { + clientId: 'client-1', + clientSecret: 'secret-1', + redirectURI: 'https://sim.test/api/auth/oauth2/callback/idp', + } + + it.each<[string, GenericOAuthConfig, CodeData]>([ + [ + 'basic client authentication with PKCE', + { + ...base, + providerId: 'microsoft-teams', + tokenUrl: 'https://login.idp.test/common/oauth2/v2.0/token', + authentication: 'basic', + pkce: true, + }, + { ...CODE, codeVerifier: 'verifier-1' }, + ], + [ + 'post client authentication with custom headers and a default expiry', + { + ...base, + providerId: 'hubspot', + tokenUrl: 'https://api.idp.test/oauth/v1/token', + authentication: 'post', + authorizationHeaders: { 'x-tenant': 'tenant-1' }, + accessTokenExpiresIn: 7200, + }, + CODE, + ], + [ + 'a token endpoint resolved from discovery', + { + ...base, + providerId: 'google-email', + discoveryUrl: 'https://accounts.idp.test/.well-known/openid-configuration', + discoveryHeaders: { 'x-discovery': '1' }, + }, + CODE, + ], + ])('sends the request Better Auth sends for %s', async (_label, config, data) => { + const respond = (url: string) => + url.endsWith('openid-configuration') + ? Response.json({ token_endpoint: 'https://oauth2.idp.test/token' }) + : Response.json( + config.accessTokenExpiresIn + ? { ...TOKEN_RESPONSE, expires_in: undefined } + : TOKEN_RESPONSE + ) + const result = await compareExchanges(config, data, respond) + + expect(result.routedRequests.length).toBeGreaterThan(0) + expect(result.routedRequests).toEqual(result.betterAuthRequests) + expect(result.routedTokens).toEqual(result.betterAuthTokens) + }) + + it("sends a connector's static token parameters the way Better Auth's callback does", async () => { + const tokenUrl = 'https://idp.test/token' + const tokenUrlParams = { audience: 'api.idp.test', grant_type: 'ignored' } + const config: GenericOAuthConfig = { ...base, providerId: 'idp', tokenUrl, tokenUrlParams } + const betterAuthRequests = stubVendor() + await validateAuthorizationCode({ + headers: config.authorizationHeaders, + ...CODE, + options: { + clientId: config.clientId, + clientSecret: config.clientSecret, + redirectURI: config.redirectURI, + }, + tokenEndpoint: tokenUrl, + authentication: config.authentication, + additionalParams: tokenUrlParams, + }) + const routedRequests = stubVendor() + await withConnectDraftOutboundScope(config).getToken?.(CODE) + + expect(routedRequests).toEqual(betterAuthRequests) + }) + + it.each([ + ['an error status', () => Response.json({ error: 'invalid_grant' }, { status: 400 })], + [ + 'a redirect', + () => new Response(null, { status: 302, headers: { location: 'http://169.254.169.254/' } }), + ], + ])('fails the exchange on %s like Better Auth does', async (_label, respond) => { + const config: GenericOAuthConfig = { + ...base, + providerId: 'idp', + tokenUrl: 'https://idp.test/token', + } + stubVendor(respond) + await expect(pluginExchange(config, CODE)).rejects.toThrow() + const routedRequests = stubVendor(respond) + await expect(pluginExchange(withConnectDraftOutboundScope(config), CODE)).rejects.toThrow() + expect(routedRequests).toHaveLength(1) + }) + + it("runs the default exchange in the connect draft's organization", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + mockGetOAuthState.mockResolvedValue({ + callbackURL: '/workspace/ws-1/integrations?credentialDraftId=draft-1', + link: { userId: 'user-1', email: 'user@example.com' }, + }) + mockGetActiveConnectDraft.mockResolvedValue({ + id: 'draft-1', + providerId: 'microsoft-teams', + workspaceId: null, + organizationId: 'org-1', + }) + const routedOrganizations: Array = [] + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (id) => { + routedOrganizations.push(id) + return { kind: 'direct' } + }) + stubVendor() + const connector = withConnectDraftOutboundScope({ + ...base, + providerId: 'microsoft-teams', + tokenUrl: 'https://login.idp.test/common/oauth2/v2.0/token', + authentication: 'basic', + }) + + await expect(connector.getToken?.(CODE)).resolves.toMatchObject({ accessToken: 'access-1' }) + expect(routedOrganizations).toEqual(['org-1']) + }) +}) diff --git a/apps/sim/lib/oauth/connect-scope.server.ts b/apps/sim/lib/oauth/connect-scope.server.ts index b6d24075384..3f6d3d07747 100644 --- a/apps/sim/lib/oauth/connect-scope.server.ts +++ b/apps/sim/lib/oauth/connect-scope.server.ts @@ -5,6 +5,7 @@ import { captureOutboundScope } from '@/lib/core/network/context.server' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { getActiveConnectDraft } from '@/lib/credentials/connect-draft' import { parseCredentialDraftIdFromCallbackUrl } from '@/lib/credentials/draft-processor' +import { createOutboundCodeExchange } from '@/lib/oauth/authorization-code.server' type OutboundScopeRunner = (run: () => Promise) => Promise @@ -59,12 +60,15 @@ async function runInOAuthCallbackDraftScope( } /** - * Wraps a generic OAuth connector's custom code exchange and identity lookup so their vendor - * calls leave through the connecting organization's egress gateway. Better Auth's built-in - * exchange (providers without `getToken`) uses its own fetch and is not routed. + * Wraps a generic OAuth connector's code exchange and identity lookup so their vendor calls + * leave through the connecting organization's egress gateway. A connector without its own + * `getToken` gets Better Auth's exchange sent through `outboundFetch`, since the plugin's + * built-in exchange uses its own fetch. Better Auth still fetches a `discoveryUrl` document and + * a default `userInfoUrl` itself, unrouted. */ export function withConnectDraftOutboundScope(config: GenericOAuthConfig): GenericOAuthConfig { - const { providerId, getToken, getUserInfo } = config + const { providerId, getUserInfo } = config + const getToken = config.getToken ?? createOutboundCodeExchange(config) return { ...config, getToken: From f785e34b73fdc2328431e27cc4683803e3446637 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 21:10:16 -0800 Subject: [PATCH 17/32] improvement(egress): share the missing-scope route rule, bound gateway handlers, and simplify workspace fetches --- .../app/api/providers/baseten/models/route.ts | 9 +-- .../api/providers/fireworks/models/route.ts | 9 +-- .../providers/ollama-cloud/models/route.ts | 19 +++-- .../api/providers/together/models/route.ts | 9 +-- apps/sim/app/api/wand/route.ts | 5 +- apps/sim/lib/core/network/gateway.server.ts | 2 +- .../organization-aws-handler.server.ts | 72 ++++++++----------- .../lib/core/network/outbound-fetch.server.ts | 54 +++++++------- .../lib/core/network/resource-scope.server.ts | 15 ++++ .../core/security/input-validation.server.ts | 3 +- .../lib/oauth/authorization-code.server.ts | 15 ++-- apps/sim/lib/oauth/connect-scope.server.ts | 4 +- apps/sim/lib/oauth/token-resolution.ts | 5 +- apps/sim/providers/bedrock/index.ts | 19 +++-- 14 files changed, 114 insertions(+), 126 deletions(-) diff --git a/apps/sim/app/api/providers/baseten/models/route.ts b/apps/sim/app/api/providers/baseten/models/route.ts index ec6eef61c8b..222cf091f59 100644 --- a/apps/sim/app/api/providers/baseten/models/route.ts +++ b/apps/sim/app/api/providers/baseten/models/route.ts @@ -10,8 +10,7 @@ import { validationErrorResponse } from '@/lib/api/server' import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' -import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' @@ -64,11 +63,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { }, cache: 'no-store', } - const response = authorizedWorkspaceId - ? await withResourceOutboundScope({ workspaceId: authorizedWorkspaceId }, () => - outboundFetch(MODELS_URL, init) - ) - : await fetch(MODELS_URL, init) + const response = await fetchForWorkspace(authorizedWorkspaceId, MODELS_URL, init) if (!response.ok) { logger.warn('Failed to fetch Baseten models', { diff --git a/apps/sim/app/api/providers/fireworks/models/route.ts b/apps/sim/app/api/providers/fireworks/models/route.ts index 84cef64978c..f4b4b6c8c00 100644 --- a/apps/sim/app/api/providers/fireworks/models/route.ts +++ b/apps/sim/app/api/providers/fireworks/models/route.ts @@ -11,8 +11,7 @@ import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' import { isHosted } from '@/lib/core/config/env-flags' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' -import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' @@ -83,11 +82,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { }, cache: 'no-store', } - const response = authorizedWorkspaceId - ? await withResourceOutboundScope({ workspaceId: authorizedWorkspaceId }, () => - outboundFetch(MODELS_URL, init) - ) - : await fetch(MODELS_URL, init) + const response = await fetchForWorkspace(authorizedWorkspaceId, MODELS_URL, init) if (!response.ok) { logger.warn('Failed to fetch Fireworks models', { diff --git a/apps/sim/app/api/providers/ollama-cloud/models/route.ts b/apps/sim/app/api/providers/ollama-cloud/models/route.ts index 1e85fe7134d..0fa53c58726 100644 --- a/apps/sim/app/api/providers/ollama-cloud/models/route.ts +++ b/apps/sim/app/api/providers/ollama-cloud/models/route.ts @@ -9,8 +9,7 @@ import { import { validationErrorResponse } from '@/lib/api/server' import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' -import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' @@ -56,15 +55,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } try { - const response = await withResourceOutboundScope({ workspaceId }, () => - outboundFetch('https://ollama.com/api/tags', { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - }) - ) + const response = await fetchForWorkspace(workspaceId, 'https://ollama.com/api/tags', { + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + cache: 'no-store', + }) if (!response.ok) { logger.warn('Failed to fetch Ollama Cloud models', { diff --git a/apps/sim/app/api/providers/together/models/route.ts b/apps/sim/app/api/providers/together/models/route.ts index 5f63f158ef9..3feb07434a6 100644 --- a/apps/sim/app/api/providers/together/models/route.ts +++ b/apps/sim/app/api/providers/together/models/route.ts @@ -10,8 +10,7 @@ import { validationErrorResponse } from '@/lib/api/server' import { getBYOKKey } from '@/lib/api-key/byok' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' -import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' @@ -75,11 +74,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { }, cache: 'no-store', } - const response = authorizedWorkspaceId - ? await withResourceOutboundScope({ workspaceId: authorizedWorkspaceId }, () => - outboundFetch(MODELS_URL, init) - ) - : await fetch(MODELS_URL, init) + const response = await fetchForWorkspace(authorizedWorkspaceId, MODELS_URL, init) if (!response.ok) { logger.warn('Failed to fetch Together models', { diff --git a/apps/sim/app/api/wand/route.ts b/apps/sim/app/api/wand/route.ts index 651b11638a2..1c90843eb50 100644 --- a/apps/sim/app/api/wand/route.ts +++ b/apps/sim/app/api/wand/route.ts @@ -18,8 +18,7 @@ import { recordUsage } from '@/lib/billing/core/usage-log' import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' import { env } from '@/lib/core/config/env' import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' -import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { enrichSandboxCapabilities } from '@/lib/execution/remote-sandbox/wand-enricher' @@ -60,7 +59,7 @@ const wandApiUrl = useWandAzure */ function postWandRequest(workspaceId: string, init: RequestInit): Promise { if (useWandAzure) return fetch(wandApiUrl, init) - return withResourceOutboundScope({ workspaceId }, () => outboundFetch(wandApiUrl, init)) + return fetchForWorkspace(workspaceId, wandApiUrl, init) } interface ChatMessage { diff --git a/apps/sim/lib/core/network/gateway.server.ts b/apps/sim/lib/core/network/gateway.server.ts index 6fa1819dbcf..c50ec38031b 100644 --- a/apps/sim/lib/core/network/gateway.server.ts +++ b/apps/sim/lib/core/network/gateway.server.ts @@ -184,7 +184,7 @@ export function createGatewayDispatcher( options ).then( (socket) => callback(null, socket), - (error: OutboundRoutingError) => callback(error, null) + (error: Error) => callback(error, null) ) }, }) diff --git a/apps/sim/lib/core/network/organization-aws-handler.server.ts b/apps/sim/lib/core/network/organization-aws-handler.server.ts index 1853e5689c7..dbeb1403da9 100644 --- a/apps/sim/lib/core/network/organization-aws-handler.server.ts +++ b/apps/sim/lib/core/network/organization-aws-handler.server.ts @@ -1,19 +1,16 @@ import type http from 'node:http' import type { Duplex } from 'node:stream' -import { createLogger } from '@sim/logger' import { NodeHttp2Handler, NodeHttpHandler } from '@smithy/node-http-handler' import { Agent, type AgentConnectOpts } from 'agent-base' +import { LRUCache } from 'lru-cache' import { isOutboundRoutingEnabled, type OutboundGateway } from '@/lib/core/network/config.server' -import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { connectThroughGateway } from '@/lib/core/network/gateway.server' +import { resolveOrganizationRoute } from '@/lib/core/network/outbound-fetch.server' import { AGENT_OPTIONS, type AwsRequestHandlerOptions, awsRequestHandler, } from '@/lib/core/network/proxy-agent.server' -import { OutboundRoutingError } from '@/lib/core/network/routing' - -const logger = createLogger('OrganizationAwsRequestHandler') type AwsHttpHandler = NodeHttpHandler | NodeHttp2Handler type HandleArguments = Parameters @@ -49,42 +46,38 @@ class GatewayAgent extends Agent { } } -const gatewayHandlers = new Map< - string, - { gatewayId: string; agent: GatewayAgent; handler: NodeHttpHandler } ->() +interface GatewayHandler { + gatewayId: string + agent: GatewayAgent + handler: NodeHttpHandler +} /** - * The organization's shared gateway handler. A reassigned gateway's agent stops keeping - * sockets alive, so Node closes each one as its in-flight request finishes and none is handed - * to a request bound for the new gateway. + * Stops keeping a retired gateway's sockets alive, so Node closes each one as its in-flight + * request finishes and none is handed to a later request. */ +function retireGatewayHandler({ agent }: GatewayHandler): void { + agent.keepAlive = false + for (const sockets of Object.values(agent.freeSockets)) { + for (const socket of sockets ?? []) socket.destroy() + } +} + +/** One handler per organization with a gateway; bounded like the routing config itself. */ +const gatewayHandlers = new LRUCache({ + max: 1_000, + dispose: retireGatewayHandler, +}) + function gatewayHandlerFor(gateway: OutboundGateway): NodeHttpHandler { const current = gatewayHandlers.get(gateway.organizationId) if (current?.gatewayId === gateway.id) return current.handler - if (current) { - current.agent.keepAlive = false - for (const sockets of Object.values(current.agent.freeSockets)) { - for (const socket of sockets ?? []) socket.destroy() - } - } const agent = new GatewayAgent(gateway) const handler = new NodeHttpHandler({ httpAgent: agent, httpsAgent: agent }) gatewayHandlers.set(gateway.organizationId, { gatewayId: gateway.id, agent, handler }) return handler } -/** A request outside any outbound scope goes direct with a warning, as `outboundFetch` does. */ -async function currentRouteOrDirect(host: string) { - try { - return await resolveCurrentOutboundRoute() - } catch (error) { - if (!(error instanceof OutboundRoutingError) || error.code !== 'MISSING_SCOPE') throw error - logger.warn('AWS request made without an organization scope', { host }) - return { kind: 'direct' } as const - } -} - /** * Picks each request's handler from the outbound scope it is sent in: an organization with a * gateway gets that gateway's handler, and every other request goes to `direct`, which is the @@ -98,7 +91,7 @@ class OrganizationRoutedHandler { } async handle(...[request, options]: HandleArguments) { - const route = await currentRouteOrDirect(request.hostname) + const route = await resolveOrganizationRoute(request.hostname) const handler = route.kind === 'gateway' ? gatewayHandlerFor(route.gateway) : this.direct return handler.handle(request, options) } @@ -117,19 +110,12 @@ class OrganizationRoutedHandler { } /** - * `requestHandler` for an AWS SDK v3 client that signs with a customer's credentials. Inside the - * outbound scope of an organization with a dedicated egress gateway, each request reaches AWS - * through that gateway, from the organization's static IPs (passing `aws:SourceIp` conditions). - * Every other request, including one outside any scope (which also logs a warning), uses the - * same handler the client gets from {@link awsRequestHandler}: the SDK's own default, or the - * operator's `HTTPS_PROXY`. A blocked route or an unavailable gateway fails the request with an - * `OutboundRoutingError`. With routing disabled this is exactly {@link awsRequestHandler}. - * - * Pass `http2` for a client whose SDK default is HTTP/2 (Bedrock Runtime), so its direct - * requests keep it. Gateway requests always use HTTP/1.1 through the tunnel. - * - * Never use it for Sim's own AWS usage (its buckets, queues, email, secrets, metrics, or - * Bedrock on Sim's keys), which must not leave through a customer's gateway. + * `requestHandler` for an AWS SDK v3 client that signs with a customer's credentials. In the + * outbound scope of an organization with a dedicated egress gateway each request reaches AWS from + * the organization's static IPs (passing `aws:SourceIp` conditions); every other request uses + * the handler {@link awsRequestHandler} gives the client, and routing disabled is exactly that. + * Pass `http2` for a client whose SDK default is HTTP/2 (Bedrock Runtime) so direct requests keep + * it. Never use it for Sim's own AWS usage, which must not leave through a customer's gateway. */ export function organizationAwsRequestHandler( options?: AwsRequestHandlerOptions & { http2?: boolean } diff --git a/apps/sim/lib/core/network/outbound-fetch.server.ts b/apps/sim/lib/core/network/outbound-fetch.server.ts index 0cdc7ecb851..9a22828b0d2 100644 --- a/apps/sim/lib/core/network/outbound-fetch.server.ts +++ b/apps/sim/lib/core/network/outbound-fetch.server.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import type { ResolvedOutboundRoute } from '@/lib/core/network/config.server' import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { OutboundRoutingError } from '@/lib/core/network/routing' import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' @@ -8,26 +9,37 @@ const logger = createLogger('OutboundFetch') let gatewayTransport: ReturnType | undefined /** - * `fetch` for a request Sim makes on an organization's behalf to a destination outside Sim: a - * credential's token endpoint, a customer's tenant host, or a vendor API. When the current - * outbound scope belongs to an organization with a dedicated egress gateway, the request leaves - * through that gateway, so it comes from the organization's static IPs and passes the IP - * allowlists its destinations enforce. Every other route is the global `fetch`, unchanged. - * - * A call made outside any outbound scope goes direct and logs a warning instead of failing with - * `MISSING_SCOPE` the way the SSRF-guarded helpers do: these requests always went direct, and - * failing closed would turn a code path that never established a scope into an outage for every - * organization. A blocked route or an unavailable gateway still fails closed. - * - * Never use it for Sim's own infrastructure (its storage, queues, or internal services), which - * must not leave through a customer's gateway. + * The outbound route for a request made on an organization's behalf. Outside any outbound scope + * it is direct, with a warning, unless `requireScope` is set: these requests always went direct, + * and failing closed with `MISSING_SCOPE` (as the SSRF-guarded helpers do) would turn a code path + * that never established a scope into an outage for every organization. A blocked route or an + * unavailable gateway still throws. + */ +export async function resolveOrganizationRoute( + host: string | undefined, + options: { requireScope?: boolean } = {} +): Promise { + try { + return await resolveCurrentOutboundRoute() + } catch (error) { + const missingScope = error instanceof OutboundRoutingError && error.code === 'MISSING_SCOPE' + if (!missingScope || options.requireScope) throw error + logger.warn('Outbound request made without an organization scope', { host }) + return { kind: 'direct' } + } +} + +/** + * `fetch` for a request Sim makes on an organization's behalf to a destination outside Sim (a + * token endpoint, a customer's tenant host, a vendor API). An organization with a dedicated + * egress gateway sends it from its static IPs; every other route is the global `fetch`, + * unchanged. Never use it for Sim's own infrastructure. */ export const outboundFetch: typeof fetch = (input, init) => routedFetch(input, init, false) /** * {@link outboundFetch} for a caller that always runs inside an outbound scope, such as a - * knowledge connector sync: a missing scope throws `MISSING_SCOPE` instead of going direct, so a - * lost scope fails loudly rather than leaving from a shared IP. + * knowledge connector sync, so a lost scope fails with `MISSING_SCOPE` instead of going direct. */ export const scopedOutboundFetch: typeof fetch = (input, init) => routedFetch(input, init, true) @@ -36,17 +48,7 @@ async function routedFetch( init: RequestInit | undefined, requireScope: boolean ): Promise { - let route: Awaited> - try { - route = await resolveCurrentOutboundRoute() - } catch (error) { - const missingScope = error instanceof OutboundRoutingError && error.code === 'MISSING_SCOPE' - if (!missingScope || requireScope) throw error - logger.warn('Outbound request made without an organization scope', { - host: requestHost(input), - }) - return fetch(input, init) - } + const route = await resolveOrganizationRoute(requestHost(input), { requireScope }) if (route.kind === 'direct') return fetch(input, init) gatewayTransport ??= createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) return gatewayTransport.fetch(input, init) diff --git a/apps/sim/lib/core/network/resource-scope.server.ts b/apps/sim/lib/core/network/resource-scope.server.ts index 85e3b5834b3..1352842b6d4 100644 --- a/apps/sim/lib/core/network/resource-scope.server.ts +++ b/apps/sim/lib/core/network/resource-scope.server.ts @@ -1,5 +1,6 @@ import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { OutboundRoutingError } from '@/lib/core/network/routing' import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { loadWorkspaceApplicationContext } from '@/lib/workspaces/application/context' @@ -17,3 +18,17 @@ export async function withResourceOutboundScope( if (!workspace) throw new OutboundRoutingError('MISSING_SCOPE') return runWithOutboundOrganization(workspace.workspaceOrganizationId, run) } + +/** + * `fetch` for a workspace's request, sent through its organization's outbound route. Without a + * workspace it is a platform request and goes direct. + */ +export function fetchForWorkspace( + workspaceId: string | undefined, + input: string, + init?: RequestInit +): Promise { + // egress-direct: no workspace, so a platform request made with Sim's own key + if (!workspaceId) return fetch(input, init) + return withResourceOutboundScope({ workspaceId }, () => outboundFetch(input, init)) +} diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 28274c14bc7..2c45aef509f 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -464,7 +464,8 @@ export const MAX_JSON_API_RESPONSE_BYTES = 10 * 1024 * 1024 */ const FOLLOWED_REDIRECT_STATUSES: ReadonlySet = new Set([301, 302, 303, 307, 308]) -function isRedirectStatus(status: number): boolean { +/** Whether `status` names a redirect target (301, 302, 303, 307 or 308). */ +export function isRedirectStatus(status: number): boolean { return FOLLOWED_REDIRECT_STATUSES.has(status) } diff --git a/apps/sim/lib/oauth/authorization-code.server.ts b/apps/sim/lib/oauth/authorization-code.server.ts index c93b07a8673..7d1fe569d46 100644 --- a/apps/sim/lib/oauth/authorization-code.server.ts +++ b/apps/sim/lib/oauth/authorization-code.server.ts @@ -7,11 +7,11 @@ import { } from 'better-auth/oauth2' import type { GenericOAuthConfig } from 'better-auth/plugins' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { isRedirectStatus } from '@/lib/core/security/input-validation.server' type AuthorizationCodeExchange = Parameters[0] type GetToken = NonNullable -const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]) const JSON_MEDIA_TYPE = /^application\/(?:[\w!#$%&*.^`~-]*\+)?json$/i const TEXT_MEDIA_TYPES = new Set([ 'image/svg', @@ -59,13 +59,14 @@ export async function exchangeAuthorizationCode({ headers, redirect: 'manual', }) - if (response.type === 'opaqueredirect' || REDIRECT_STATUSES.has(response.status)) { + const redirected = response.type === 'opaqueredirect' || isRedirectStatus(response.status) + if (redirected || !response.ok) { await response.body?.cancel().catch(() => {}) - throw new Error(`The OAuth endpoint "${tokenEndpoint}" returned an HTTP redirect`) - } - if (!response.ok) { - await response.body?.cancel().catch(() => {}) - throw new Error(`OAuth token endpoint responded with ${response.status}`) + throw new Error( + redirected + ? `The OAuth endpoint "${tokenEndpoint}" returned an HTTP redirect` + : `OAuth token endpoint responded with ${response.status}` + ) } return getOAuth2Tokens(toRecord(await readResponseData(response))) } diff --git a/apps/sim/lib/oauth/connect-scope.server.ts b/apps/sim/lib/oauth/connect-scope.server.ts index 3f6d3d07747..b756db8bbe2 100644 --- a/apps/sim/lib/oauth/connect-scope.server.ts +++ b/apps/sim/lib/oauth/connect-scope.server.ts @@ -63,8 +63,8 @@ async function runInOAuthCallbackDraftScope( * Wraps a generic OAuth connector's code exchange and identity lookup so their vendor calls * leave through the connecting organization's egress gateway. A connector without its own * `getToken` gets Better Auth's exchange sent through `outboundFetch`, since the plugin's - * built-in exchange uses its own fetch. Better Auth still fetches a `discoveryUrl` document and - * a default `userInfoUrl` itself, unrouted. + * built-in exchange uses its own fetch. Better Auth's own discovery reads for the authorize URL + * and callback stay unrouted; the exchange resolves its token endpoint through `outboundFetch`. */ export function withConnectDraftOutboundScope(config: GenericOAuthConfig): GenericOAuthConfig { const { providerId, getUserInfo } = config diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index 0c7cedc641f..90675352425 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -83,10 +83,7 @@ export interface ResolveCredentialTokenInput { scopes?: string[] /** Google domain-wide-delegation subject for service-account credentials. */ impersonateEmail?: string - /** - * Asserted acting user. When the caller authenticated with an internal JWT it - * must equal the token subject, so a forged assertion cannot widen access. - */ + /** Acting user whose credential access is enforced; absent when access is not enforced. */ callerUserId?: string auditRequest?: CredentialAuditRequest /** Credential lookup already performed by {@link resolveCredentialAccessToken}'s dispatch. */ diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index ff8cc53f30a..ec67c109c64 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -71,6 +71,9 @@ import { const logger = createLogger('BedrockProvider') +/** Bedrock Runtime defaults to HTTP/2, which a customer's direct requests keep. */ +const customerRequestHandler = () => organizationAwsRequestHandler({ http2: true }) + function enrichLastModelSegmentFromBedrockResponse( timeSegments: TimeSegment[], response: ConverseResponse, @@ -138,7 +141,7 @@ function enrichLastModelSegmentFromBedrockResponse( function resolveBedrockClientCredentials(request: ProviderRequest): { credentials: BedrockRuntimeClientConfig['credentials'] credentialKey: string | null - requestHandler: BedrockRuntimeClientConfig['requestHandler'] + createRequestHandler: () => BedrockRuntimeClientConfig['requestHandler'] } { const session = request.bedrockSession if (session) { @@ -147,7 +150,7 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { return { credentials: { accessKeyId, secretAccessKey, sessionToken }, credentialKey: null, - requestHandler: organizationAwsRequestHandler({ http2: true }), + createRequestHandler: customerRequestHandler, } } return { @@ -157,7 +160,7 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { session.credentialId, sha256Hex([accessKeyId, secretAccessKey].join('\0')), ].join(':'), - requestHandler: organizationAwsRequestHandler({ http2: true }), + createRequestHandler: customerRequestHandler, } } @@ -179,14 +182,14 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { secretAccessKey: request.bedrockSecretKey, }, credentialKey: `${request.bedrockAccessKeyId}:${request.bedrockSecretKey}`, - requestHandler: organizationAwsRequestHandler({ http2: true }), + createRequestHandler: customerRequestHandler, } } // The deployment's own credentials stay on Sim's network, never an organization's route. return { credentials: getAwsCredentials(), credentialKey: 'default-chain', - requestHandler: awsRequestHandler(), + createRequestHandler: awsRequestHandler, } } @@ -219,8 +222,10 @@ export const bedrockProvider: ProviderConfig = { region, }) - const { credentials, credentialKey, requestHandler } = resolveBedrockClientCredentials(request) - const buildClient = () => new BedrockRuntimeClient({ region, credentials, requestHandler }) + const { credentials, credentialKey, createRequestHandler } = + resolveBedrockClientCredentials(request) + const buildClient = () => + new BedrockRuntimeClient({ region, credentials, requestHandler: createRequestHandler() }) const client = credentialKey === null ? buildClient() From 05db88c64196160901bf81a20a8110c68a6b8b8a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 21:14:31 -0800 Subject: [PATCH 18/32] test(egress): tighten the new egress tests and shrink the non-null baseline --- .../cron/renew-subscriptions/route.test.ts | 1 + .../lib/core/network/gateway.server.test.ts | 19 +++++++------------ .../standard-oauth-provider.test.ts | 2 +- .../sim/lib/internal/sailpoint/client.test.ts | 1 + .../application/slack-search/home.test.ts | 9 +++------ .../lib/oauth/connect-scope.server.test.ts | 3 +-- apps/sim/lib/oauth/credential-service.test.ts | 4 +--- scripts/check-explicit-any.baseline.json | 2 +- 8 files changed, 16 insertions(+), 25 deletions(-) diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts index 4736ddff1ac..a43938ecc37 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts @@ -71,6 +71,7 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { beforeEach(() => { graphCalls.length = 0 + routeOrganizationId = undefined networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (organizationId) => { routeOrganizationId = organizationId diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts index f2543557b3e..aabbf035890 100644 --- a/apps/sim/lib/core/network/gateway.server.test.ts +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -304,7 +304,7 @@ describe('organization gateways over real TLS CONNECT sockets', () => { await transport.dispatcher.destroy() } }) - it("sends a credential request through the organization's gateway", async () => { + it("sends outboundFetch through the scoped organization's gateway", async () => { const start = admissions.length await runWithOutboundOrganization('org_b', async () => { const response = await outboundFetch(url(), { @@ -313,7 +313,7 @@ describe('organization gateways over real TLS CONNECT sockets', () => { redirect: 'error', }) expect(response.headers.get('x-via-proxy')).toBe('yes') - expect(await response.text()).toBe('grant_type=client_credentials') + await response.body?.cancel() }) expect(admissions.slice(start)).toEqual([ { token: 'Bearer bravo', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, @@ -340,21 +340,16 @@ describe('organization gateways over real TLS CONNECT sockets', () => { return error?.$response?.headers['x-via-proxy'] } - it("sends a request through the organization's gateway", async () => { + it('tunnels each organization with its own credential and never shares its pooled socket', async () => { const start = admissions.length expect(await viaProxy('org_a')).toBe('yes') - expect(admissions.slice(start)).toEqual([ - { token: 'Bearer alpha', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, - ]) - }) - - it("never reuses one organization's pooled socket for another", async () => { - expect(await viaProxy('org_a')).toBe('yes') - const start = admissions.length expect(await viaProxy('org_b')).toBe('yes') expect(await viaProxy('org_a')).toBe('yes') expect(await viaProxy('org_b')).toBe('yes') - expect(admissions.slice(start).map((entry) => entry.token)).toEqual(['Bearer bravo']) + expect(admissions.slice(start)).toEqual([ + { token: 'Bearer alpha', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, + { token: 'Bearer bravo', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, + ]) }) }) diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts index 340f39ad6f9..7c88bb31b56 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts @@ -325,7 +325,7 @@ describe('standard OAuth Credential Group provider', () => { await expect(exchange()).rejects.toMatchObject({ statusCode: 502 }) }) - it("sends a connector's built-in code exchange through its organization's egress route", async () => { + it("fails a connector's built-in code exchange closed with 503 when its organization's gateway is unavailable", async () => { connectorState.builtInExchange = true const sharedIpFetch = vi.fn(async () => Response.json({ token_type: 'Bearer', access_token: 'access-1' }) diff --git a/apps/sim/lib/internal/sailpoint/client.test.ts b/apps/sim/lib/internal/sailpoint/client.test.ts index c955e7fcdaa..68fdf122f0e 100644 --- a/apps/sim/lib/internal/sailpoint/client.test.ts +++ b/apps/sim/lib/internal/sailpoint/client.test.ts @@ -61,6 +61,7 @@ describe('SailPoint client', () => { await vi.waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(1)) release?.(tokenResponse('shared')) await expect(Promise.all([first, second])).resolves.toEqual(['shared', 'shared']) + expect(mockFetch).toHaveBeenCalledTimes(1) }) it('lets one token waiter abort without cancelling the shared exchange', async () => { diff --git a/apps/sim/lib/knowledge/application/slack-search/home.test.ts b/apps/sim/lib/knowledge/application/slack-search/home.test.ts index 69dd10e838c..6fb36abd101 100644 --- a/apps/sim/lib/knowledge/application/slack-search/home.test.ts +++ b/apps/sim/lib/knowledge/application/slack-search/home.test.ts @@ -69,6 +69,7 @@ function publish(signal = new AbortController().signal) { }) } beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) mocks.installation.mockResolvedValue(installation) mocks.publish.mockResolvedValue({ status: 200, data: { ok: true } }) }) @@ -136,16 +137,12 @@ describe('static Home delivery', () => { }) it("calls Slack from the installation's organization outbound scope", async () => { networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) - networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (organizationId) => ({ - organizationId, - })) - const routes: unknown[] = [] mocks.publish.mockImplementationOnce(async () => { - routes.push(await resolveCurrentOutboundRoute()) + await resolveCurrentOutboundRoute() return { status: 200, data: { ok: true } } }) await publish() - expect(routes).toEqual([{ organizationId: 'org1' }]) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org1') }) it.each([ { enabled: false }, diff --git a/apps/sim/lib/oauth/connect-scope.server.test.ts b/apps/sim/lib/oauth/connect-scope.server.test.ts index 2319497fd4a..0849da2d9c3 100644 --- a/apps/sim/lib/oauth/connect-scope.server.test.ts +++ b/apps/sim/lib/oauth/connect-scope.server.test.ts @@ -282,9 +282,8 @@ describe('default code exchange for connectors without getToken', () => { } stubVendor(respond) await expect(pluginExchange(config, CODE)).rejects.toThrow() - const routedRequests = stubVendor(respond) + stubVendor(respond) await expect(pluginExchange(withConnectDraftOutboundScope(config), CODE)).rejects.toThrow() - expect(routedRequests).toHaveLength(1) }) it("runs the default exchange in the connect draft's organization", async () => { diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts index 339f6d4f5af..a70ea094f58 100644 --- a/apps/sim/lib/oauth/credential-service.test.ts +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -907,15 +907,13 @@ describe('OAuth access-token refresh headroom', () => { ) }) - it('surfaces an unavailable egress route without flagging the credential dead or revoked', async () => { + it('rethrows an unavailable egress route instead of a revoked or failed refresh', async () => { mocks.refreshOAuthToken.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) const row = createOAuthAccount(WAITED_GOOGLE_REMAINING_MS) queueTableRows(account, [row]) await expect(refreshTokenIfNeeded('test', row, RAW_ACCOUNT_ID)).rejects.toBeInstanceOf( OutboundRoutingError ) - expect(markCredentialDead).not.toHaveBeenCalled() - expect(dbChainMockFns.set).not.toHaveBeenCalled() }) it('answers a Microsoft caller from its valid token while refresh-token aging refreshes', async () => { diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 438f2e1f027..87291fb3530 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1132,7 +1132,7 @@ "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/pull/route.ts": 1, "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/push/route.ts": 1, "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/route.ts": 1, - "apps/sim/app/api/wand/route.ts": 2, + "apps/sim/app/api/wand/route.ts": 1, "apps/sim/app/api/workflows/[id]/execute/route.test.ts": 5, "apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.test.ts": 1, "apps/sim/app/credential-groups/enroll/[token]/page.test.tsx": 1, From 0730c2482b270b5da43b1e2a3ed92328d7ba319e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 21:16:24 -0800 Subject: [PATCH 19/32] fix(egress): type the routed AWS handler's config passthrough --- apps/sim/lib/core/network/organization-aws-handler.server.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/core/network/organization-aws-handler.server.ts b/apps/sim/lib/core/network/organization-aws-handler.server.ts index dbeb1403da9..2141e3a197f 100644 --- a/apps/sim/lib/core/network/organization-aws-handler.server.ts +++ b/apps/sim/lib/core/network/organization-aws-handler.server.ts @@ -96,8 +96,9 @@ class OrganizationRoutedHandler { return handler.handle(request, options) } - updateHttpClientConfig(...args: Parameters): void { - this.direct.updateHttpClientConfig(...args) + /** The SDK only tunes the direct handler's own options; the gateway handler keeps its own. */ + updateHttpClientConfig(key: never, value: never): void { + this.direct.updateHttpClientConfig(key, value) } httpHandlerConfigs(): ReturnType { From 388d28219846b6c7c1ed905677742b9a9611587f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 21:31:30 -0800 Subject: [PATCH 20/32] fix(egress): keep the Azure DevOps block's constants off the server fetch module and type the new route tests --- .../cron/renew-subscriptions/route.test.ts | 13 ++-- .../app/api/v1/knowledge/search/route.test.ts | 14 +++-- .../lib/oauth/connect-scope.server.test.ts | 11 ++-- .../azure_devops/get_work_items_batch.ts | 3 +- .../tools/azure_devops/query_work_items.ts | 3 +- apps/sim/tools/azure_devops/utils.ts | 53 ----------------- .../tools/azure_devops/work-items.server.ts | 59 +++++++++++++++++++ 7 files changed, 85 insertions(+), 71 deletions(-) create mode 100644 apps/sim/tools/azure_devops/work-items.server.ts diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts index a43938ecc37..c3c1a7c1ca6 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts @@ -14,6 +14,7 @@ import { import { authInternalMock, authInternalMockFns } from '@sim/testing/mocks/auth-internal.mock' import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { + createMockWorkspaceApplicationContext, workspaceContextMock, workspaceContextMockFns, } from '@sim/testing/mocks/workspace-context.mock' @@ -77,10 +78,12 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { routeOrganizationId = organizationId return { kind: 'direct' } }) - mockLoadWorkspaceApplicationContext.mockResolvedValue({ - workspaceId: 'workspace-1', - workspaceOrganizationId: 'org-1', - }) + mockLoadWorkspaceApplicationContext.mockResolvedValue( + createMockWorkspaceApplicationContext({ + workspaceId: 'workspace-1', + workspaceOrganizationId: 'org-1', + }) + ) mockGetCredentialOwner.mockResolvedValue({ accountId: 'account-1', userId: 'user-1' }) authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockResolvedValue('graph-token') vi.stubGlobal( @@ -112,7 +115,7 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { }, ]) - const response = await GET(createRequest()) + const response = await GET(createRequest(), { params: Promise.resolve({}) }) expect(response.status).toBe(202) await vi.waitFor(() => expect(graphCalls).toHaveLength(1)) diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index cf80f10ad50..375e553473b 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -33,6 +33,7 @@ import { import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { v1MiddlewareMock, v1MiddlewareMockFns } from '@sim/testing/mocks/v1-middleware.mock' import { + createMockWorkspaceApplicationContext, workspaceContextMock, workspaceContextMockFns, } from '@sim/testing/mocks/workspace-context.mock' @@ -264,10 +265,12 @@ describe('v1 knowledge search route — per-KB embedding model', () => { it("embeds the query through the workspace organization's outbound route", async () => { networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) - workspaceContextMockFns.mockLoadWorkspaceApplicationContext.mockResolvedValueOnce({ - workspaceId: 'ws-1', - workspaceOrganizationId: 'org-1', - }) + workspaceContextMockFns.mockLoadWorkspaceApplicationContext.mockResolvedValueOnce( + createMockWorkspaceApplicationContext({ + workspaceId: 'ws-1', + workspaceOrganizationId: 'org-1', + }) + ) mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({ hasAccess: true, knowledgeBase: baseKb('kb-1', 'text-embedding-3-small'), @@ -278,7 +281,8 @@ describe('v1 knowledge search route — per-KB embedding model', () => { }) const res = await POST( - createMockRequest('POST', { workspaceId: 'ws-1', knowledgeBaseIds: 'kb-1', query: 'hello' }) + createMockRequest('POST', { workspaceId: 'ws-1', knowledgeBaseIds: 'kb-1', query: 'hello' }), + { params: Promise.resolve({}) } ) expect(res.status).toBe(200) diff --git a/apps/sim/lib/oauth/connect-scope.server.test.ts b/apps/sim/lib/oauth/connect-scope.server.test.ts index 0849da2d9c3..61c8568eef6 100644 --- a/apps/sim/lib/oauth/connect-scope.server.test.ts +++ b/apps/sim/lib/oauth/connect-scope.server.test.ts @@ -1,5 +1,6 @@ import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { + createMockWorkspaceApplicationContext, workspaceContextMock, workspaceContextMockFns, } from '@sim/testing/mocks/workspace-context.mock' @@ -52,10 +53,12 @@ async function organizationSeenByVendorCall( describe('resolveConnectDraftOutboundScope', () => { beforeEach(() => { networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) - mockLoadWorkspaceApplicationContext.mockResolvedValue({ - workspaceId: 'workspace-1', - workspaceOrganizationId: 'org-1', - }) + mockLoadWorkspaceApplicationContext.mockResolvedValue( + createMockWorkspaceApplicationContext({ + workspaceId: 'workspace-1', + workspaceOrganizationId: 'org-1', + }) + ) }) afterEach(() => { diff --git a/apps/sim/tools/azure_devops/get_work_items_batch.ts b/apps/sim/tools/azure_devops/get_work_items_batch.ts index 03c2c4229b8..07e8ff1f74b 100644 --- a/apps/sim/tools/azure_devops/get_work_items_batch.ts +++ b/apps/sim/tools/azure_devops/get_work_items_batch.ts @@ -6,12 +6,11 @@ import type { import type { AzureDevOpsRawWorkItem } from '@/tools/azure_devops/utils' import { azureDevOpsProjectUrl, - fetchWorkItemsChunk, formatWorkItem, mapWorkItem, WORK_ITEM_LIST_DETAIL_OUTPUT_PROPERTIES, - WORK_ITEMS_BATCH_SIZE, } from '@/tools/azure_devops/utils' +import { fetchWorkItemsChunk, WORK_ITEMS_BATCH_SIZE } from '@/tools/azure_devops/work-items.server' import { AZURE_DEVOPS_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' diff --git a/apps/sim/tools/azure_devops/query_work_items.ts b/apps/sim/tools/azure_devops/query_work_items.ts index 6c0d796e22a..4a10a005c19 100644 --- a/apps/sim/tools/azure_devops/query_work_items.ts +++ b/apps/sim/tools/azure_devops/query_work_items.ts @@ -6,11 +6,10 @@ import type { } from '@/tools/azure_devops/types' import { azureDevOpsProjectUrl, - fetchWorkItemsChunk, formatWorkItem, WORK_ITEM_LIST_DETAIL_OUTPUT_PROPERTIES, - WORK_ITEMS_BATCH_SIZE, } from '@/tools/azure_devops/utils' +import { fetchWorkItemsChunk, WORK_ITEMS_BATCH_SIZE } from '@/tools/azure_devops/work-items.server' import { AZURE_DEVOPS_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' diff --git a/apps/sim/tools/azure_devops/utils.ts b/apps/sim/tools/azure_devops/utils.ts index 496215af603..aba8ada8713 100644 --- a/apps/sim/tools/azure_devops/utils.ts +++ b/apps/sim/tools/azure_devops/utils.ts @@ -1,12 +1,5 @@ import { toStringOrNull } from '@sim/utils/coerce' import { toRecordOrNull } from '@sim/utils/object' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' -import { - DEFAULT_MAX_ERROR_BODY_BYTES, - readResponseJsonWithLimit, - readResponseTextWithLimit, -} from '@/lib/core/utils/stream-limits' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { AzureDevOpsComment, AzureDevOpsWorkItem } from '@/tools/azure_devops/types' import type { OutputProperty } from '@/tools/types' import { safeUrlPathSegment } from '@/tools/url-path' @@ -30,52 +23,6 @@ function pathSegment(value: string, paramName: string): string { return safeUrlPathSegment(decoded, paramName) } -/** Work Items - List accepts at most 200 ids per request. */ -export const WORK_ITEMS_BATCH_SIZE = 200 - -/** - * Fetches one Work Items - List chunk with `$expand=all`, bounded to the buffered - * transfer cap (100 MB) and cancelled with the tool call. - */ -export async function fetchWorkItemsChunk( - params: { organization: string; project: string; accessToken: string }, - ids: Array, - errorPrefix: string, - signal?: AbortSignal -): Promise { - const url = new URL(`${azureDevOpsProjectUrl(params)}/_apis/wit/workitems`) - url.searchParams.set('ids', ids.join(',')) - url.searchParams.set('$expand', 'all') - url.searchParams.set('api-version', '7.2-preview.3') - - const response = await outboundFetch(url.toString(), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: `Basic ${btoa(`:${params.accessToken}`)}`, - }, - signal, - }) - - if (!response.ok) { - const errorBody = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Azure DevOps work items error body', - signal, - }).catch(() => '') - throw new Error(`${errorPrefix} (${response.status}): ${errorBody || response.statusText}`) - } - - const data = await readResponseJsonWithLimit<{ value?: AzureDevOpsRawWorkItem[] }>(response, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - label: 'Azure DevOps work items response', - signal, - }) - return (data.value ?? []).map((raw: AzureDevOpsRawWorkItem) => - mapWorkItem(raw, { includeRawDetails: false }) - ) -} - /** States for Azure DevOps Basic process work items (Issue, Task, Epic). */ export const AZURE_DEVOPS_BASIC_WORK_ITEM_STATES = ['To Do', 'Doing', 'Done'] as const diff --git a/apps/sim/tools/azure_devops/work-items.server.ts b/apps/sim/tools/azure_devops/work-items.server.ts new file mode 100644 index 00000000000..47ba15c00a0 --- /dev/null +++ b/apps/sim/tools/azure_devops/work-items.server.ts @@ -0,0 +1,59 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { AzureDevOpsWorkItem } from '@/tools/azure_devops/types' +import { + type AzureDevOpsRawWorkItem, + azureDevOpsProjectUrl, + mapWorkItem, +} from '@/tools/azure_devops/utils' + +/** Work Items - List accepts at most 200 ids per request. */ +export const WORK_ITEMS_BATCH_SIZE = 200 + +/** + * Fetches one Work Items - List chunk with `$expand=all`, bounded to the buffered + * transfer cap (100 MB) and cancelled with the tool call. + */ +export async function fetchWorkItemsChunk( + params: { organization: string; project: string; accessToken: string }, + ids: Array, + errorPrefix: string, + signal?: AbortSignal +): Promise { + const url = new URL(`${azureDevOpsProjectUrl(params)}/_apis/wit/workitems`) + url.searchParams.set('ids', ids.join(',')) + url.searchParams.set('$expand', 'all') + url.searchParams.set('api-version', '7.2-preview.3') + + const response = await outboundFetch(url.toString(), { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${btoa(`:${params.accessToken}`)}`, + }, + signal, + }) + + if (!response.ok) { + const errorBody = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Azure DevOps work items error body', + signal, + }).catch(() => '') + throw new Error(`${errorPrefix} (${response.status}): ${errorBody || response.statusText}`) + } + + const data = await readResponseJsonWithLimit<{ value?: AzureDevOpsRawWorkItem[] }>(response, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Azure DevOps work items response', + signal, + }) + return (data.value ?? []).map((raw: AzureDevOpsRawWorkItem) => + mapWorkItem(raw, { includeRawDetails: false }) + ) +} From 6b08ac001f2fdfef688dfb95494a9d7ac2a8b97a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 21:52:17 -0800 Subject: [PATCH 21/32] improvement(egress): drop token-resolution inputs left dead by the token route removal resolveCredentialAccessToken now takes the execution's userId and builds the internal-JWT auth itself instead of an authenticate callback; resolveCredentialToken is private. Removes the session/API-key saved-secret refusal no caller can reach, the auditRequest plumbing the executor never sets (recordCredentialAccess keeps it for selectors), and the unparsed oauthTokenResponseSchema in favor of a plain CredentialTokenPayload type. --- apps/sim/executor/utils/credential-token.ts | 7 +- .../lib/api/contracts/oauth-connections.ts | 22 --- apps/sim/lib/oauth/token-resolution.test.ts | 140 +++++++----------- apps/sim/lib/oauth/token-resolution.ts | 95 +++++------- 4 files changed, 98 insertions(+), 166 deletions(-) diff --git a/apps/sim/executor/utils/credential-token.ts b/apps/sim/executor/utils/credential-token.ts index 0deff02814b..b8351efe2ad 100644 --- a/apps/sim/executor/utils/credential-token.ts +++ b/apps/sim/executor/utils/credential-token.ts @@ -1,5 +1,4 @@ import { createLogger } from '@sim/logger' -import { AuthType } from '@/lib/auth/hybrid' import { createCopilotManagedOAuthPrincipal } from '@/lib/credentials/application/copilot-managed-oauth-delegation' import { bindExecutorManagedOAuthDelegation } from '@/lib/credentials/application/managed-oauth-delegation' import { authorizePersonalCredential } from '@/lib/credentials/application/personal-credentials' @@ -118,11 +117,7 @@ export async function resolveExecutorCredentialToken( scopes: params.scopes, impersonateEmail: params.impersonateEmail, callerUserId: userId && params.enforceCredentialAccess ? userId : undefined, - authenticate: () => ({ - success: true, - userId, - authType: AuthType.INTERNAL_JWT, - }), + userId, resolveManagedPrincipal, }) diff --git a/apps/sim/lib/api/contracts/oauth-connections.ts b/apps/sim/lib/api/contracts/oauth-connections.ts index 79630e0c89a..b0869fe6fa1 100644 --- a/apps/sim/lib/api/contracts/oauth-connections.ts +++ b/apps/sim/lib/api/contracts/oauth-connections.ts @@ -78,28 +78,6 @@ const trelloCallbackQuerySchema = z /** Google domain-wide-delegation subject, validated for in-process credential callers. */ export const impersonateEmailSchema = z.string().email() -const oauthTokenResponseSchema = z.object({ - accessToken: z.string(), - credentialType: z.enum(['oauth', 'managed_oauth', 'service_account']).optional(), - idToken: z.string().optional(), - instanceUrl: z.string().optional(), - /** Zoho Desk — the data-center-scoped Desk REST base for this credential. */ - apiDomain: z.string().optional(), - cloudId: z.string().optional(), - domain: z.string().optional(), - providerWorkspaceId: z.string().optional(), - /** Datadog — the application key sent beside the API key (`accessToken`). */ - applicationKey: z.string().optional(), - realmId: z.string().optional(), - quickBooksEnvironment: z.enum(['sandbox', 'production']).optional(), - authStyle: z - .enum(['x-api-token', 'snowflake-workload-identity', 'snowflake-key-pair', 'basic', 'bearer']) - .optional(), -}) - -/** Token material a resolved credential yields to its in-process callers. */ -export type OAuthTokenResponse = z.output - export const shopifyAuthorizeQuerySchema = z.object({ shop: z.string().optional(), returnUrl: z.string().optional(), diff --git a/apps/sim/lib/oauth/token-resolution.test.ts b/apps/sim/lib/oauth/token-resolution.test.ts index d5a4d7be280..2398e9ab7c5 100644 --- a/apps/sim/lib/oauth/token-resolution.test.ts +++ b/apps/sim/lib/oauth/token-resolution.test.ts @@ -40,7 +40,7 @@ import { OutboundRoutingError } from '@/lib/core/network/routing' import { OrchestrationError } from '@/lib/core/orchestration/types' import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' -import { resolveCredentialAccessToken, resolveCredentialToken } from '@/lib/oauth/token-resolution' +import { resolveCredentialAccessToken } from '@/lib/oauth/token-resolution' import type { OAuthService } from '@/lib/oauth/types' import { getToolMetadata } from '@/tools/metadata' @@ -54,18 +54,20 @@ const { mockResolveServiceAccountToken, } = authOAuthUtilsMockFns -const INTERNAL_AUTH = { success: true, userId: 'user-1', authType: 'internal_jwt' } as const +describe('resolveCredentialAccessToken for plain credentials', () => { + beforeEach(() => { + mockResolveOAuthAccountId.mockResolvedValue(null) + }) -describe('resolveCredentialToken', () => { it('fails closed when the credential is not authorized', async () => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: false, error: 'You do not have access to this credential.', }) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', }) @@ -95,9 +97,9 @@ describe('resolveCredentialToken', () => { }) mockRefreshTokenIfNeeded.mockResolvedValue({ accessToken: 'fresh', refreshed: true }) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', workflowId: 'wf-1', }) @@ -106,6 +108,10 @@ describe('resolveCredentialToken', () => { ok: true, token: { accessToken: 'fresh', credentialType: 'oauth', idToken: 'id-token' }, }) + expect(mockAuthorizeCredentialUseForAuth).toHaveBeenCalledWith( + { success: true, userId: 'user-1', authType: 'internal_jwt' }, + { credentialId: 'cred-1', workflowId: 'wf-1', callerUserId: undefined } + ) expect(mockGetCredential).toHaveBeenCalledWith('req-1', 'account-1', 'owner-1') expect(auditMockFns.mockRecordAuditRollup).toHaveBeenCalledWith( expect.objectContaining({ @@ -135,9 +141,9 @@ describe('resolveCredentialToken', () => { mockRefreshTokenIfNeeded.mockResolvedValue({ accessToken: 'fresh', refreshed: false }) await expect( - resolveCredentialToken(INTERNAL_AUTH, { + resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', }) ).resolves.toEqual({ @@ -161,9 +167,9 @@ describe('resolveCredentialToken', () => { mockGetCredential.mockResolvedValue({ providerId: 'google' }) mockRefreshTokenIfNeeded.mockRejectedValue(new Error('refresh token revoked')) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', }) @@ -181,9 +187,9 @@ describe('resolveCredentialToken', () => { mockGetCredential.mockResolvedValue({ providerId: 'google' }) mockRefreshTokenIfNeeded.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', }) @@ -197,17 +203,19 @@ describe('resolveCredentialToken', () => { it('authorizes service-account credentials before minting a token', async () => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: false, error: 'Unauthorized' }) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + mockResolveOAuthAccountId.mockResolvedValue({ + credentialType: 'service_account', + credentialId: 'sa-1', + providerId: 'google', + workspaceId: 'ws-1', + accountId: '', + usedCredentialTable: true, + }) + + const result = await resolveCredentialAccessToken({ requestId: 'req-1', + userId: 'user-1', credentialId: 'cred-1', - resolvedCredential: { - credentialType: 'service_account', - credentialId: 'sa-1', - providerId: 'google', - workspaceId: 'ws-1', - accountId: '', - usedCredentialTable: true, - }, }) expect(result).toEqual({ ok: false, status: 403, error: 'Unauthorized' }) @@ -215,9 +223,9 @@ describe('resolveCredentialToken', () => { }) it('rejects a malformed impersonation subject before touching the credential', async () => { - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', impersonateEmail: 'not-an-email', }) @@ -244,13 +252,11 @@ const EXECUTOR_PRINCIPAL = { } as never describe('resolveCredentialAccessToken', () => { - const authenticate = vi.fn() const resolveManagedPrincipal = vi.fn() beforeEach(() => { mockExecuteManagedToken.mockReset() mockResolveOAuthAccountId.mockResolvedValue(null) - authenticate.mockResolvedValue(INTERNAL_AUTH) resolveManagedPrincipal.mockResolvedValue(EXECUTOR_PRINCIPAL) mockGetToolMetadata.mockReturnValue({ id: 'gmail_send', @@ -266,7 +272,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - authenticate, + userId: 'user-1', }) expect(result).toEqual({ @@ -275,7 +281,7 @@ describe('resolveCredentialAccessToken', () => { code: 'MANAGED_CREDENTIAL_DELEGATION_REQUIRED', error: 'Managed credentials can only be used by an authenticated workflow execution', }) - expect(authenticate).not.toHaveBeenCalled() + expect(mockAuthorizeCredentialUseForAuth).not.toHaveBeenCalled() }) it('maps an invalid delegation to 401 with its message', async () => { @@ -286,7 +292,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -305,7 +311,7 @@ describe('resolveCredentialAccessToken', () => { const result = await resolveCredentialAccessToken({ requestId: 'req-1', credentialId: 'cred-1', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -325,7 +331,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'http_request', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -359,7 +365,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'slack_add_bookmark', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -384,7 +390,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -399,14 +405,11 @@ describe('resolveCredentialAccessToken', () => { it("scopes a managed credential to the tool's required scopes", async () => { mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) mockExecuteManagedToken.mockResolvedValue({ accessToken: 'managed-token', idToken: 'id-1' }) - const auditRequest = { headers: { get: () => null } } - const result = await resolveCredentialAccessToken({ requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - auditRequest, - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -426,9 +429,8 @@ describe('resolveCredentialAccessToken', () => { requiredScopes: ['scope-a'], toolId: 'gmail_send', }, - request: auditRequest, }) - expect(authenticate).not.toHaveBeenCalled() + expect(mockAuthorizeCredentialUseForAuth).not.toHaveBeenCalled() }) it('projects orchestration failures as managed unauthorized', async () => { @@ -441,7 +443,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -455,7 +457,6 @@ describe('resolveCredentialAccessToken', () => { }) describe('ordinary tool credential binding', () => { - const authenticate = async () => INTERNAL_AUTH const serviceAccount = { credentialType: 'service_account', credentialId: 'service-account-1', @@ -481,33 +482,11 @@ describe('ordinary tool credential binding', () => { }) }) - it.each([ - ['a session', { success: true, userId: 'user-1', authType: 'session' }], - ['an API key', { success: true, userId: 'user-1', authType: 'api_key' }], - ] as const)( - 'releases a saved secret to server-side callers only, never %s', - async (_label, auth) => { - const result = await resolveCredentialAccessToken({ - requestId: 'binding-test', - credentialId: 'credential-1', - toolId: 'oracle_fusion_financials_list_payables_invoices', - authenticate: async () => auth, - }) - - expect(result).toMatchObject({ - ok: false, - status: 403, - code: 'SERVICE_ACCOUNT_SESSION_FORBIDDEN', - }) - expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() - } - ) - it('refuses to release a service-account secret without a tool to bind it to', async () => { const result = await resolveCredentialAccessToken({ requestId: 'binding-test', credentialId: 'credential-1', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'SERVICE_ACCOUNT_TOOL_REQUIRED' }) @@ -522,7 +501,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'oracle_fusion_financials_list_payables_invoices', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'CREDENTIAL_PROVIDER_MISMATCH' }) expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() @@ -550,7 +529,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'dynamodb_get', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: true }) expect(mockResolveServiceAccountToken).toHaveBeenCalled() @@ -564,7 +543,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'dynamodb_get', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, @@ -584,7 +563,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'oracle_fusion_financials_list_payables_invoices', - authenticate, + userId: 'user-1', }) ).toEqual({ ok: false, status: 403, error: 'Unauthorized' }) expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() @@ -598,7 +577,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'oracle_fusion_financials_list_payables_invoices', - authenticate, + userId: 'user-1', }) ).toMatchObject({ ok: false, status: 400, code: 'CREDENTIAL_TYPE_MISMATCH' }) expect(mockRefreshTokenIfNeeded).not.toHaveBeenCalled() @@ -625,7 +604,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'fixture-tool', - authenticate, + userId: 'user-1', }) ).toMatchObject({ ok: true, @@ -663,7 +642,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'fixture-tool', - authenticate, + userId: 'user-1', }) ).toMatchObject({ ok: true, @@ -690,7 +669,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'fixture-tool', - authenticate, + userId: 'user-1', }) expect(result.ok).toBe(allowed) if (!allowed) { @@ -702,7 +681,6 @@ describe('ordinary tool credential binding', () => { }) describe('Microsoft app credential binding', () => { - const authenticate = async () => INTERNAL_AUTH const appCredential = { credentialType: 'service_account', credentialId: 'microsoft-app-1', @@ -746,7 +724,7 @@ describe('Microsoft app credential binding', () => { requestId: 'microsoft-app', credentialId: 'microsoft-app-1', toolId: `${provider}-tool`, - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'CREDENTIAL_PROVIDER_MISMATCH' }) expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() @@ -761,7 +739,7 @@ describe('Microsoft app credential binding', () => { requestId: 'microsoft-app', credentialId: 'microsoft-app-1', toolId: `${provider}-tool`, - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: true, token: { credentialType: 'service_account' } }) expect(mockResolveServiceAccountToken).toHaveBeenCalledWith( @@ -781,7 +759,7 @@ describe('Microsoft app credential binding', () => { requestId: 'microsoft-app', credentialId: 'microsoft-app-1', toolId: resetPasswordTool.id, - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'CREDENTIAL_TYPE_MISMATCH' }) expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() @@ -789,8 +767,6 @@ describe('Microsoft app credential binding', () => { }) describe('keyless Google Cloud credential refusals', () => { - const authenticate = async () => INTERNAL_AUTH - beforeEach(() => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: true, @@ -821,7 +797,7 @@ describe('keyless Google Cloud credential refusals', () => { requestId: 'google-keyless', credentialId: 'google-keyless-1', toolId: 'google-drive-tool', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, @@ -839,15 +815,13 @@ describe('keyless Google Cloud credential refusals', () => { requestId: 'servicenow-host', credentialId: 'google-keyless-1', toolId: 'google-drive-tool', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'host_not_allowed' }) }) }) describe('Microsoft app credential cloud refusals', () => { - const authenticate = async () => INTERNAL_AUTH - beforeEach(() => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: true, @@ -878,7 +852,7 @@ describe('Microsoft app credential cloud refusals', () => { requestId: 'microsoft-cloud', credentialId: 'microsoft-app-gov', toolId: 'powerbi-tool', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'cloud_mismatch' }) expect(result.ok ? '' : result.error).toMatch(/Microsoft cloud/) diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index 90675352425..64da57f9b6b 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -2,12 +2,9 @@ import { AuditAction, AuditResourceType, recordAudit, recordAuditRollup } from ' import { type DelegatedPrincipal, resolvePrincipalSubject } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { - impersonateEmailSchema, - type OAuthTokenResponse, -} from '@/lib/api/contracts/oauth-connections' +import { impersonateEmailSchema } from '@/lib/api/contracts/oauth-connections' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' -import { type AuthResult, AuthType } from '@/lib/auth/hybrid' +import { AuthType } from '@/lib/auth/hybrid' import { OutboundRoutingError } from '@/lib/core/network/routing' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' @@ -22,6 +19,7 @@ import { refreshTokenIfNeeded, resolveOAuthAccountId, resolveServiceAccountToken, + type ServiceAccountTokenResult, } from '@/lib/oauth/credential-service' import { extractMicrosoftDataverseEnvironmentUrl, @@ -37,6 +35,7 @@ import { } from '@/lib/oauth/utils' import { captureServerEvent } from '@/lib/posthog/server' import { getToolMetadata } from '@/tools/metadata' +import type { QuickBooksEnvironment } from '@/tools/quickbooks/client' import type { OAuthConfig } from '@/tools/types' import { extractZohoDeskBaseFromScope } from '@/tools/zoho_desk/host-allowlist' @@ -60,10 +59,18 @@ export interface CredentialAuditRequest { headers: { get(name: string): string | null } } -/** Token material a resolved credential yields; taken from the contract so it cannot drift. */ -export type CredentialTokenPayload = OAuthTokenResponse +/** + * Token material a resolved credential yields to the tool that consumes it: everything a + * service-account mint returns, plus the OAuth-only identity fields. + */ +export interface CredentialTokenPayload extends ServiceAccountTokenResult { + credentialType?: 'oauth' | 'managed_oauth' | 'service_account' + idToken?: string + realmId?: string + quickBooksEnvironment?: QuickBooksEnvironment +} -export interface ResolveCredentialTokenInput { +interface ResolveCredentialTokenInput { /** Tool whose canonical provider and credential-kind policy applies before token minting. */ toolId?: string /** @@ -85,7 +92,11 @@ export interface ResolveCredentialTokenInput { impersonateEmail?: string /** Acting user whose credential access is enforced; absent when access is not enforced. */ callerUserId?: string - auditRequest?: CredentialAuditRequest + /** + * User the server-side execution runs as, authorizing non-managed credentials. + * Managed OAuth credentials authenticate through delegation instead. + */ + userId?: string /** Credential lookup already performed by {@link resolveCredentialAccessToken}'s dispatch. */ resolvedCredential: ResolvedCredential | null } @@ -236,7 +247,6 @@ export async function completeOAuthCredentialToken(params: { workspaceId: string | null workflowId?: string executionId?: string - auditRequest?: CredentialAuditRequest }): Promise { const { requestId, @@ -246,7 +256,6 @@ export async function completeOAuthCredentialToken(params: { workspaceId, workflowId, executionId, - auditRequest, } = params const contextValidation = validateOAuthCredentialContext(credential) if (!contextValidation.ok) { @@ -265,7 +274,6 @@ export async function completeOAuthCredentialToken(params: { credentialType: 'oauth', workflowId, executionId, - auditRequest, }) } @@ -343,15 +351,11 @@ function validateToolCredentialBinding( } /** - * Resolves a plain OAuth or service-account credential to a token for an - * authenticated caller. Managed OAuth credentials are dispatched one level up by - * {@link resolveCredentialAccessToken}, which every server surface goes through, - * so authorization, refresh, and audit cannot drift between surfaces. - * - * @param auth Result of authenticating the caller (session or internal JWT). + * Resolves a plain OAuth or service-account credential to a token for server-side + * execution. Managed OAuth credentials are dispatched one level up by + * {@link resolveCredentialAccessToken}. */ -export async function resolveCredentialToken( - auth: AuthResult, +async function resolveCredentialToken( input: ResolveCredentialTokenInput ): Promise { const { @@ -362,7 +366,7 @@ export async function resolveCredentialToken( scopes, impersonateEmail, callerUserId, - auditRequest, + userId, } = input try { @@ -377,25 +381,19 @@ export async function resolveCredentialToken( } const resolved = input.resolvedCredential - const authz = await authorizeCredentialUseForAuth(auth, { - credentialId, - workflowId, - callerUserId, - }) + const authz = await authorizeCredentialUseForAuth( + { success: true, userId, authType: AuthType.INTERNAL_JWT }, + { + credentialId, + workflowId, + callerUserId, + } + ) if (resolved?.credentialType === 'service_account' && resolved.credentialId) { if (!authz.ok) { return { ok: false, status: 403, error: authz.error || 'Unauthorized' } } - // A saved secret never leaves the server: only server-side (internal JWT) callers may resolve it. - if (auth.authType !== AuthType.INTERNAL_JWT) { - return { - ok: false, - status: 403, - code: 'SERVICE_ACCOUNT_SESSION_FORBIDDEN', - error: 'Saved credentials are resolved only during server-side execution', - } - } // A saved secret is released only to a tool of its own provider, never on a bare ID. if (!input.toolId) { @@ -437,7 +435,6 @@ export async function resolveCredentialToken( credentialType: 'service_account', workflowId, executionId, - auditRequest, }) } @@ -585,7 +582,6 @@ export async function resolveCredentialToken( workspaceId: authz.workspaceId ?? null, workflowId, executionId, - auditRequest, }) } catch (error) { logger.error('Error getting access token', error, { requestId }) @@ -595,11 +591,6 @@ export async function resolveCredentialToken( export interface ResolveCredentialAccessTokenInput extends Omit { - /** - * Authenticates the caller for non-managed credentials. Invoked only when the - * credential is not managed OAuth, which authenticates through delegation instead. - */ - authenticate: () => AuthResult | Promise /** * Proves a delegation for one managed credential: a workflow execution (the * executor binds its delegation origin in-process) or a Chat turn acting as the @@ -611,33 +602,28 @@ export interface ResolveCredentialAccessTokenInput } /** - * Authorized application dispatch for a credential token. Every server surface - * that needs one goes through here, so the managed / service-account / plain-OAuth - * dispatch, authorization, refresh, audit, and analytics cannot drift between them. + * Authorized application dispatch for a credential token during server-side + * execution, so the managed / service-account / plain-OAuth dispatch, + * authorization, refresh, audit, and analytics run in one place. */ export async function resolveCredentialAccessToken( input: ResolveCredentialAccessTokenInput ): Promise { - const { requestId, credentialId, toolId, auditRequest } = input + const { requestId, credentialId, toolId } = input const resolved = credentialId ? await resolveOAuthAccountId(credentialId) : null if (resolved?.credentialType !== 'managed_oauth' || !resolved.credentialId) { - const auth = await input.authenticate() - return resolveCredentialToken(auth, { + return resolveCredentialToken({ requestId, credentialId, workflowId: input.workflowId, executionId: input.executionId, scopes: input.scopes, - /** - * In-process callers forward raw subblock state, where an untouched - * field is '' — treated as absent, matching what the wire contract - * (which rejects '') and the old truthy guards always produced. - */ + /** The executor forwards raw subblock state, where an untouched field is '': treated as absent. */ impersonateEmail: input.impersonateEmail || undefined, callerUserId: input.callerUserId, - auditRequest, + userId: input.userId, resolvedCredential: resolved, toolId, assistantConnectedAccount: input.assistantConnectedAccount, @@ -713,7 +699,6 @@ export async function resolveCredentialAccessToken( requiredScopes, toolId, }, - request: auditRequest, }) const subject = resolvePrincipalSubject(principal) From 2c9970ebd4e4dfcefba249ab6a59813ed62dbd0a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 21:54:05 -0800 Subject: [PATCH 22/32] improvement(egress): classify an organization egress outage once as an unavailable orchestration error Add an 'unavailable' OrchestrationErrorCode (503, v2 SERVICE_UNAVAILABLE) and let asOrchestrationError classify an OutboundRoutingError anywhere in the cause chain with a fixed message, so no routing code reaches callers. The standard OAuth credential-group adapter now rethrows routing failures from code exchange and identity verification instead of wrapping them locally. --- apps/sim/app/api/v2/lib/response.ts | 1 + apps/sim/ee/scim/lib/protocol/errors.ts | 1 + apps/sim/lib/core/orchestration/types.test.ts | 20 +++++++++++++++++++ apps/sim/lib/core/orchestration/types.ts | 16 +++++++++++++++ .../standard-oauth-provider.test.ts | 14 ++++++++++--- .../standard-oauth-provider.ts | 9 +++------ apps/sim/lib/folders/status.ts | 1 + 7 files changed, 53 insertions(+), 9 deletions(-) diff --git a/apps/sim/app/api/v2/lib/response.ts b/apps/sim/app/api/v2/lib/response.ts index 1e95d321f71..1e7117e4c72 100644 --- a/apps/sim/app/api/v2/lib/response.ts +++ b/apps/sim/app/api/v2/lib/response.ts @@ -545,6 +545,7 @@ const V2_CODE_BY_ORCHESTRATION_ERROR: Record { } }) }) + +describe('asOrchestrationError', () => { + it('classifies an egress routing failure deep in a cause chain as a 503 without its routing code', () => { + const wrapped = new Error('Failed query: select ...', { + cause: new Error('token exchange failed', { + cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + }), + }) + + const classified = asOrchestrationError(wrapped) + + expect(classified?.code).toBe('unavailable') + expect(statusForOrchestrationError(classified?.code)).toBe(503) + expect(classified?.message).toBe("The organization's egress route is unavailable") + expect(classified?.message).not.toContain('GATEWAY_UNAVAILABLE') + }) +}) diff --git a/apps/sim/lib/core/orchestration/types.ts b/apps/sim/lib/core/orchestration/types.ts index 47374d0c4bc..307f12c84c1 100644 --- a/apps/sim/lib/core/orchestration/types.ts +++ b/apps/sim/lib/core/orchestration/types.ts @@ -1,3 +1,4 @@ +import { OutboundRoutingError } from '@/lib/core/network/routing' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' export type OrchestrationErrorCode = @@ -13,6 +14,12 @@ export type OrchestrationErrorCode = | 'conflict' | 'locked' | 'payload_too_large' + /** + * A dependency Sim reaches on the caller's behalf is temporarily unreachable — + * the organization's egress gateway, say. Retryable; nothing about the caller + * or its credentials is wrong. + */ + | 'unavailable' | 'internal' /** @@ -28,6 +35,7 @@ export function statusForOrchestrationError(code: OrchestrationErrorCode | undef if (code === 'conflict') return 409 if (code === 'locked') return 423 if (code === 'payload_too_large') return 413 + if (code === 'unavailable') return 503 return 500 } @@ -93,6 +101,8 @@ export function throwOrchestrationFailure( ) } +const EGRESS_UNAVAILABLE_MESSAGE = "The organization's egress route is unavailable" + /** * The {@link OrchestrationError} in `error`'s cause chain, or `null` when the * failure is not a classified one. @@ -101,6 +111,10 @@ export function throwOrchestrationFailure( * raised inside a transaction callback in a `DrizzleQueryError` whose own * message is the failed SQL — the same reason the message-matching this * replaced had to dig for a root cause before it could classify anything. + * + * An {@link OutboundRoutingError} anywhere in the chain is the organization's + * egress route failing, so it classifies as `unavailable` with a fixed message: + * its routing code stays in server logs. */ export function asOrchestrationError(error: unknown): OrchestrationError | null { let current: unknown = error @@ -108,6 +122,8 @@ export function asOrchestrationError(error: unknown): OrchestrationError | null if (current instanceof OrchestrationError) return current if (isPayloadSizeLimitError(current)) return new OrchestrationError('payload_too_large', current.message) + if (current instanceof OutboundRoutingError) + return new OrchestrationError('unavailable', EGRESS_UNAVAILABLE_MESSAGE) current = current.cause } return null diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts index 7c88bb31b56..b703d413fb6 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts @@ -77,6 +77,7 @@ vi.mock('@/lib/auth/connectors/managed-oauth', () => ({ import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OutboundRoutingError } from '@/lib/core/network/routing' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { createStandardOAuthCredentialGroupProviderAdapter } from '@/lib/credential-groups/standard-oauth-provider' urlsMockFns.mockGetBaseUrl.mockReturnValue('https://sim.example.com') @@ -244,6 +245,12 @@ describe('standard OAuth Credential Group provider', () => { }) }) + it('classifies an egress outage during identity verification as unavailable, not an invalid identity', async () => { + mockVerifyIdentity.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const failure = await exchange().catch((error: unknown) => error) + expect(statusForOrchestrationError(asOrchestrationError(failure)?.code)).toBe(503) + }) + it('uses the existing Atlassian callback and state-bound identity verification', async () => { const requiredScopes = ['read:me', 'read:jira-work', 'offline_access'] const context: CredentialGroupOAuthContext = { @@ -336,9 +343,10 @@ describe('standard OAuth Credential Group provider', () => { new OutboundRoutingError('GATEWAY_UNAVAILABLE') ) - await expect(runWithOutboundOrganization('org-1', exchange)).rejects.toMatchObject({ - statusCode: 503, - }) + const failure = await runWithOutboundOrganization('org-1', exchange).catch( + (error: unknown) => error + ) + expect(statusForOrchestrationError(asOrchestrationError(failure)?.code)).toBe(503) expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') expect(sharedIpFetch).not.toHaveBeenCalled() }) diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index 173a1a40d37..e18587e4ada 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -316,12 +316,8 @@ export function createStandardOAuthCredentialGroupProviderAdapter( tokenEndpoint: endpoints.tokenEndpoint, }) } catch (error) { - if (error instanceof OutboundRoutingError) { - throw new CredentialGroupOAuthError( - `${service.name} authorization is temporarily unavailable.`, - 503 - ) - } + // An egress outage is classified once, by asOrchestrationError, not as a provider failure. + if (error instanceof OutboundRoutingError) throw error throw new CredentialGroupOAuthError( `${service.name} could not complete authorization. Please try again.`, 502 @@ -340,6 +336,7 @@ export function createStandardOAuthCredentialGroupProviderAdapter( clientId: current.connector.clientId, }) } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new CredentialGroupOAuthError( `${service.name} returned an invalid identity token.`, 502, diff --git a/apps/sim/lib/folders/status.ts b/apps/sim/lib/folders/status.ts index 9c5762c3a1d..17a1617febb 100644 --- a/apps/sim/lib/folders/status.ts +++ b/apps/sim/lib/folders/status.ts @@ -24,5 +24,6 @@ export function folderMutationStatus(errorCode: FolderMutationErrorCode | undefi // A folder collection too large to materialize reaches these routes as a classified // failure too; without this arm it rendered as an unexplained 500. if (errorCode === 'payload_too_large') return 413 + if (errorCode === 'unavailable') return 503 return 500 } From 96aa2b301bab9ed1e626491996e65a80621d8de1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 21:57:33 -0800 Subject: [PATCH 23/32] improvement(egress): read egress outages through the shared unavailable code in token resolution, managed OAuth and Slack --- .../slack-managed-users.test.ts | 23 +++++++++++ .../credential-groups/slack-managed-users.ts | 2 + .../credential-groups/slack-provider.test.ts | 33 ++++++++++++++++ .../lib/credential-groups/slack-provider.ts | 8 +++- .../sim/lib/credentials/managed-oauth.test.ts | 9 ++--- apps/sim/lib/credentials/managed-oauth.ts | 15 +------ apps/sim/lib/oauth/token-resolution.test.ts | 19 +++++++++ apps/sim/lib/oauth/token-resolution.ts | 39 +++++++++++-------- 8 files changed, 111 insertions(+), 37 deletions(-) diff --git a/apps/sim/lib/credential-groups/slack-managed-users.test.ts b/apps/sim/lib/credential-groups/slack-managed-users.test.ts index 73eb082630d..6bad711445c 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.test.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.test.ts @@ -3,6 +3,7 @@ import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { resetUrlsMock, urlsMockFns } from '@sim/testing/mocks/urls.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -28,9 +29,12 @@ const { attempts, redis } = vi.hoisted(() => { }) vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@/lib/core/security/encryption', () => encryptionMock) +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { credentialGroupScopePolicyVersion } from '@/lib/credential-groups/provider-adapter' import { SLACK_MANAGED_USER_SCOPES, @@ -84,6 +88,7 @@ function slackResponse(value: Record): Response { describe('Slack managed-user authorization', () => { beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) resetDbChainMock() attempts.clear() setEnv({ SLACK_SEARCH_APP_ID: '' }) @@ -577,6 +582,24 @@ describe('Slack managed-user authorization', () => { expect(dbChainMockFns.transaction).not.toHaveBeenCalled() }) + it('surfaces an unavailable egress route instead of blaming Slack', async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + runWithOutboundOrganization('organization-1', () => + exchangeSlackUserAuthorization({ + clientId: 'client-id', + clientSecret: 'client-secret', + code: 'single-use-code', + redirectUri: 'https://sim.ai/callback', + }) + ) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('fails closed when Slack omits the user token type', async () => { vi.stubGlobal( 'fetch', diff --git a/apps/sim/lib/credential-groups/slack-managed-users.ts b/apps/sim/lib/credential-groups/slack-managed-users.ts index d46d8d1abd8..1d73135eb21 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.ts @@ -17,6 +17,7 @@ import { compareStrings } from '@sim/utils/string' import { and, eq, inArray, isNull, or } from 'drizzle-orm' import { getRedisClient } from '@/lib/core/config/redis' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' @@ -486,6 +487,7 @@ export async function exchangeSlackUserAuthorization(params: { cache: 'no-store', }) } catch (error) { + if (error instanceof OutboundRoutingError) throw error logger.error('Slack OAuth exchange failed', { error: getErrorMessage(error) }) throw new SlackManagedUsersError('Slack could not complete authorization.', 'provider_error') } diff --git a/apps/sim/lib/credential-groups/slack-provider.test.ts b/apps/sim/lib/credential-groups/slack-provider.test.ts index 5acbe0394cf..11e9277eef7 100644 --- a/apps/sim/lib/credential-groups/slack-provider.test.ts +++ b/apps/sim/lib/credential-groups/slack-provider.test.ts @@ -21,6 +21,7 @@ vi.mock('@/lib/credential-groups/slack-managed-users', () => ({ }), })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import type { CredentialGroupOAuthContext } from '@/lib/credential-groups/enrollments' import { SLACK_MANAGED_USER_SCOPES, @@ -95,6 +96,38 @@ describe('Slack member scope policy', () => { } ) + it('passes an unavailable egress route through instead of blaming Slack', async () => { + const current = context(SLACK_SEARCH_USER_SCOPES) + mocks.exchange.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const policy = await adapter.getPolicy(current.option, { + workspaceId: current.workspaceId, + credentialGroupId: current.credentialGroupId, + }) + await expect( + adapter.exchangeAndVerify({ + context: current, + policy, + code: 'code', + attempt: { + state: 'state', + provider: 'slack', + workspaceId: 'workspace-1', + email: current.email, + nonceHash: 'nonce-hash', + enrollmentId: current.enrollmentId, + credentialGroupId: current.credentialGroupId, + optionId: current.option.id, + authorizationAppId: policy.authorizationAppId, + scopeVersion: policy.scopeVersion, + requiredScopes: policy.requiredScopes, + redirectUri: 'https://sim.fixture.test/api/credential-groups/oauth/slack/callback', + invitationToken: 'invitation', + createdAt: Date.now(), + }, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('accepts a different provider email', async () => { const scopes = SLACK_SEARCH_USER_SCOPES const current = context(scopes) diff --git a/apps/sim/lib/credential-groups/slack-provider.ts b/apps/sim/lib/credential-groups/slack-provider.ts index f94d443e428..9a8b5630db8 100644 --- a/apps/sim/lib/credential-groups/slack-provider.ts +++ b/apps/sim/lib/credential-groups/slack-provider.ts @@ -2,6 +2,7 @@ import { db } from '@sim/db' import { credentialGroup } from '@sim/db/schema' import { normalizeEmail } from '@sim/utils/string' import { and, eq } from 'drizzle-orm' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { resourceScopeColumns, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { getBaseUrl } from '@/lib/core/utils/urls' @@ -210,7 +211,8 @@ export const slackCredentialGroupProviderAdapter: CredentialGroupProviderAdapter code, redirectUri: attempt.redirectUri, }) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new CredentialGroupOAuthError( 'Slack could not complete authorization. Please try again.', 502 @@ -265,7 +267,9 @@ export const slackCredentialGroupProviderAdapter: CredentialGroupProviderAdapter } } catch (error) { await revokeSlackToken(grant.accessToken) - if (error instanceof CredentialGroupOAuthError) throw error + if (error instanceof CredentialGroupOAuthError || error instanceof OutboundRoutingError) { + throw error + } throw new CredentialGroupOAuthError('Slack could not verify the granted access.', 502) } }, diff --git a/apps/sim/lib/credentials/managed-oauth.test.ts b/apps/sim/lib/credentials/managed-oauth.test.ts index ccf33d7e2db..f3407e18e8f 100644 --- a/apps/sim/lib/credentials/managed-oauth.test.ts +++ b/apps/sim/lib/credentials/managed-oauth.test.ts @@ -521,7 +521,7 @@ describe('managed OAuth token resolution', () => { expect(mocks.encryptSecret).not.toHaveBeenCalled() }) - it('reports an unavailable egress route during refresh as a retryable outage without reauthorization', async () => { + it('surfaces an unavailable egress route during refresh without reauthorizing or writing', async () => { const row = mondayCredentialRow() dbChainMockFns.limit.mockResolvedValueOnce([row]).mockResolvedValueOnce([row]) mocks.decryptSecret.mockResolvedValue({ @@ -544,10 +544,9 @@ describe('managed OAuth token resolution', () => { isTerminalRefreshError: vi.fn().mockReturnValue(true), }) - await expect(resolveManagedOAuthToken(mondayTokenResolutionParams())).rejects.toMatchObject({ - code: 'MANAGED_CREDENTIAL_REFRESH_FAILED', - statusCode: 503, - }) + await expect(resolveManagedOAuthToken(mondayTokenResolutionParams())).rejects.toBeInstanceOf( + OutboundRoutingError + ) expect(dbChainMockFns.set).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/credentials/managed-oauth.ts b/apps/sim/lib/credentials/managed-oauth.ts index 6df2330bc5c..401fae30d42 100644 --- a/apps/sim/lib/credentials/managed-oauth.ts +++ b/apps/sim/lib/credentials/managed-oauth.ts @@ -6,7 +6,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { and, eq } from 'drizzle-orm' import { LRUCache } from 'lru-cache' import type { WorkspaceAuthorizationContext } from '@/lib/core/application' -import { OutboundRoutingError } from '@/lib/core/network/routing' import { type ResourceOwner, resourceScopeFields, @@ -526,19 +525,7 @@ async function resolveManagedOAuthAccessToken( } } - let refreshed: RefreshTokenResult - try { - refreshed = await adapter.refreshToken(currentTokenSet.refreshToken) - } catch (error) { - if (!(error instanceof OutboundRoutingError)) throw error - return { - error: new ManagedOAuthCredentialError( - 'MANAGED_CREDENTIAL_REFRESH_FAILED', - 'Managed credential refresh is temporarily unavailable', - 503 - ), - } - } + const refreshed: RefreshTokenResult = await adapter.refreshToken(currentTokenSet.refreshToken) if (!refreshed.ok) { const terminal = adapter.isTerminalRefreshError(refreshed.errorCode) if (terminal) { diff --git a/apps/sim/lib/oauth/token-resolution.test.ts b/apps/sim/lib/oauth/token-resolution.test.ts index 2398e9ab7c5..175a14aedff 100644 --- a/apps/sim/lib/oauth/token-resolution.test.ts +++ b/apps/sim/lib/oauth/token-resolution.test.ts @@ -454,6 +454,25 @@ describe('resolveCredentialAccessToken', () => { error: 'Managed credential not found', }) }) + + it('reports a managed refresh blocked by the egress route as a retryable 503, not unauthorized', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + mockExecuteManagedToken.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + userId: 'user-1', + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: false, + status: 503, + error: "The organization's egress route is unavailable", + }) + }) }) describe('ordinary tool credential binding', () => { diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index 64da57f9b6b..d75844cae69 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -5,7 +5,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { impersonateEmailSchema } from '@/lib/api/contracts/oauth-connections' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' import { AuthType } from '@/lib/auth/hybrid' -import { OutboundRoutingError } from '@/lib/core/network/routing' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' import { resolveManagedOAuthCredentialToken } from '@/lib/credentials/application/resolve-managed-oauth-token' @@ -42,13 +41,20 @@ import { extractZohoDeskBaseFromScope } from '@/tools/zoho_desk/host-allowlist' const logger = createLogger('OAuthTokenResolution') /** - * The organization's egress gateway is down or refuses the route: an outage on Sim's side, - * never a sign the credential is bad, so the caller is told to retry rather than reconnect. + * A failure Sim's side can't currently get past (the organization's egress gateway is down or + * refuses the route), or null. Never a sign the credential is bad, so the caller is told to + * retry rather than reconnect. */ -const EGRESS_ROUTE_UNAVAILABLE: Extract = { - ok: false, - status: 503, - error: "The organization's egress route is unavailable", +function unavailableFailure( + error: unknown +): Extract | null { + const classified = asOrchestrationError(error) + if (classified?.code !== 'unavailable') return null + return { + ok: false, + status: statusForOrchestrationError('unavailable'), + error: classified.message, + } } /** @@ -288,9 +294,10 @@ export async function completeOAuthCredentialToken(params: { }) return { ok: false, status: 401, code: OAUTH_CREDENTIAL_REVOKED, error: error.message } } - if (error instanceof OutboundRoutingError) { - logger.warn('Egress route unavailable for token refresh', { requestId, code: error.code }) - return EGRESS_ROUTE_UNAVAILABLE + const unavailable = unavailableFailure(error) + if (unavailable) { + logger.warn('Egress route unavailable for token refresh', { requestId }) + return unavailable } logger.error('Failed to refresh access token:', error, { requestId }) return { ok: false, status: 401, error: 'Failed to refresh access token' } @@ -453,12 +460,10 @@ async function resolveCredentialToken( }, } } catch (error) { - if (error instanceof OutboundRoutingError) { - logger.warn('Egress route unavailable for service account token', { - requestId, - code: error.code, - }) - return EGRESS_ROUTE_UNAVAILABLE + const unavailable = unavailableFailure(error) + if (unavailable) { + logger.warn('Egress route unavailable for service account token', { requestId }) + return unavailable } logger.error('Service account token error:', error, { requestId }) if (error instanceof TokenServiceAccountValidationError) { @@ -734,6 +739,8 @@ export async function resolveCredentialAccessToken( return { ok: false, status: error.statusCode, code: error.code, error: error.message } } + const unavailable = unavailableFailure(error) + if (unavailable) return unavailable const orchestrationError = asOrchestrationError(error) if (orchestrationError) { return { From 72f97c755d3576dcce225d111ccad81036295771 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:00:52 -0800 Subject: [PATCH 24/32] improvement(egress): move wand and provider model listing behind authorized use cases The wand route and the baseten, fireworks, together and ollama-cloud model routes authorized inline and set the outbound scope per call through fetchForWorkspace. Each now authenticates, parses and presents only; the use case loads the workspace, authorizes, picks the BYOK or Sim key and calls the vendor with scopedOutboundFetch inside the workspace's outbound scope. Wand splits into a workflow operation (write) and a workspace operation (read) so each declares its real minimum role. A model listing for a caller outside the workspace still falls back to Sim's key, direct. fetchForWorkspace has no callers left and is removed. --- .../providers/baseten/models/route.test.ts | 19 +- .../app/api/providers/baseten/models/route.ts | 85 +- .../api/providers/fireworks/models/route.ts | 106 +-- .../ollama-cloud/models/route.test.ts | 19 +- .../providers/ollama-cloud/models/route.ts | 72 +- .../providers/together/models/route.test.ts | 19 +- .../api/providers/together/models/route.ts | 97 +-- apps/sim/app/api/wand/route.ts | 729 +++--------------- apps/sim/lib/api/server/routes/index.ts | 1 + .../api/server/routes/internal-json-route.ts | 10 + .../lib/core/network/resource-scope.server.ts | 15 - .../application/list-models.ts | 175 +++++ .../provider-models/application/operations.ts | 12 + apps/sim/lib/wand/application/generate.ts | 661 ++++++++++++++++ apps/sim/lib/wand/application/operations.ts | 22 + .../check-source-conventions.baseline.json | 1 - 16 files changed, 1049 insertions(+), 994 deletions(-) create mode 100644 apps/sim/lib/provider-models/application/list-models.ts create mode 100644 apps/sim/lib/provider-models/application/operations.ts create mode 100644 apps/sim/lib/wand/application/generate.ts create mode 100644 apps/sim/lib/wand/application/operations.ts diff --git a/apps/sim/app/api/providers/baseten/models/route.test.ts b/apps/sim/app/api/providers/baseten/models/route.test.ts index 2de4e413eb9..f0281e2f13b 100644 --- a/apps/sim/app/api/providers/baseten/models/route.test.ts +++ b/apps/sim/app/api/providers/baseten/models/route.test.ts @@ -1,22 +1,25 @@ import { authMockFns, createMockRequest, resetEnvMock, setEnv } from '@sim/testing' import { jsonResponse } from '@sim/testing/helpers/http' import { apiKeyByokMock, apiKeyByokMockFns } from '@sim/testing/mocks/api-key-byok.mock' -import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock' +import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' +import { workspaceContextMock } from '@sim/testing/mocks/workspace-context.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/providers/utils', () => providersUtilsMock) vi.mock('@/lib/api-key/byok', () => apiKeyByokMock) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) + +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) import { GET } from '@/app/api/providers/baseten/models/route' const mockGetSession = authMockFns.mockGetSession const mockGetBYOKKey = apiKeyByokMockFns.mockGetBYOKKey const { mockFilterBlacklistedModels, mockIsProviderBlacklisted } = providersUtilsMockFns -const { mockGetUserEntityPermissions } = permissionsMockFns +const { mockResolveEffectiveWorkspacePermission } = workspaceAuthzMockFns function setEnvKey(value: string | undefined): void { setEnv({ BASETEN_API_KEY: value }) @@ -38,7 +41,7 @@ describe('GET /api/providers/baseten/models', () => { mockFilterBlacklistedModels.mockImplementation((models: string[]) => models) mockGetBYOKKey.mockResolvedValue(null) mockGetSession.mockResolvedValue(null) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) setEnvKey(undefined) }) @@ -48,8 +51,8 @@ describe('GET /api/providers/baseten/models', () => { it('uses the BYOK key when workspaceId, session, and permission are present', async () => { setEnvKey('env-key') - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue('admin') + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') mockGetBYOKKey.mockResolvedValue({ apiKey: 'byok-key', isBYOK: true }) mockFetch.mockResolvedValueOnce(jsonResponse({ data: [{ id: 'model-a' }] })) @@ -66,8 +69,8 @@ describe('GET /api/providers/baseten/models', () => { it('falls back to the env key when the user lacks workspace permission', async () => { setEnvKey('env-key') - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) mockFetch.mockResolvedValueOnce(jsonResponse({ data: [{ id: 'model-a' }] })) const res = await GET( diff --git a/apps/sim/app/api/providers/baseten/models/route.ts b/apps/sim/app/api/providers/baseten/models/route.ts index 222cf091f59..90890c3e47f 100644 --- a/apps/sim/app/api/providers/baseten/models/route.ts +++ b/apps/sim/app/api/providers/baseten/models/route.ts @@ -1,22 +1,13 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { - basetenProviderModelsQuerySchema, - basetenUpstreamResponseSchema, - providerModelsResponseSchema, -} from '@/lib/api/contracts/providers' +import { basetenProviderModelsQuerySchema } from '@/lib/api/contracts/providers' import { validationErrorResponse } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' -import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' +import { listProviderModels } from '@/lib/provider-models/application/list-models' +import { isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('BasetenModelsAPI') -const MODELS_URL = 'https://inference.baseten.co/v1/models' export const GET = withRouteHandler(async (request: NextRequest) => { if (isProviderBlacklisted('baseten')) { @@ -24,75 +15,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ models: [] }) } - let apiKey: string | undefined - let authorizedWorkspaceId: string | undefined - const queryValidation = basetenProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, }) if (!queryValidation.success) return validationErrorResponse(queryValidation.error) const { workspaceId } = queryValidation.data - if (workspaceId) { - const session = await getSession() - if (session?.user?.id) { - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission) { - authorizedWorkspaceId = workspaceId - const byokResult = await getBYOKKey(workspaceId, 'baseten') - if (byokResult) { - apiKey = byokResult.apiKey - } - } - } - } - - if (!apiKey) { - apiKey = env.BASETEN_API_KEY - } - - if (!apiKey) { - logger.info('No Baseten API key available, returning empty models') - return NextResponse.json({ models: [] }) - } - - try { - const init: RequestInit = { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - } - const response = await fetchForWorkspace(authorizedWorkspaceId, MODELS_URL, init) - if (!response.ok) { - logger.warn('Failed to fetch Baseten models', { - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json({ models: [] }) - } - - const data = basetenUpstreamResponseSchema.parse(await response.json()) - - const allModels: string[] = [] - for (const model of data.data ?? []) { - allModels.push(`baseten/${model.id}`) - } - - const uniqueModels = Array.from(new Set(allModels)) - const models = filterBlacklistedModels(uniqueModels) - - logger.info('Successfully fetched Baseten models', { - count: models.length, - filtered: uniqueModels.length - models.length, - }) - - return NextResponse.json(providerModelsResponseSchema.parse({ models })) - } catch (error) { - logger.error('Error fetching Baseten models', { - error: getErrorMessage(error, 'Unknown error'), - }) - return NextResponse.json({ models: [] }) - } + const principal = workspaceId ? await authenticateOptionalInternalSession() : null + const models = await listProviderModels('baseten', { principal, workspaceId }) + return NextResponse.json({ models }) }) diff --git a/apps/sim/app/api/providers/fireworks/models/route.ts b/apps/sim/app/api/providers/fireworks/models/route.ts index f4b4b6c8c00..7c04ec1dc97 100644 --- a/apps/sim/app/api/providers/fireworks/models/route.ts +++ b/apps/sim/app/api/providers/fireworks/models/route.ts @@ -1,35 +1,13 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { - fireworksProviderModelsQuerySchema, - fireworksUpstreamResponseSchema, - providerModelsResponseSchema, -} from '@/lib/api/contracts/providers' +import { fireworksProviderModelsQuerySchema } from '@/lib/api/contracts/providers' import { validationErrorResponse } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' -import { isHosted } from '@/lib/core/config/env-flags' -import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' +import { listProviderModels } from '@/lib/provider-models/application/list-models' +import { isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('FireworksModelsAPI') -const MODELS_URL = 'https://api.fireworks.ai/inference/v1/models' - -interface FireworksModel { - id: string - object?: string - created?: number - owned_by?: string -} - -interface FireworksModelsResponse { - data: FireworksModel[] - object?: string -} export const GET = withRouteHandler(async (request: NextRequest) => { if (isProviderBlacklisted('fireworks')) { @@ -37,83 +15,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ models: [] }) } - let apiKey: string | undefined - let authorizedWorkspaceId: string | undefined - const queryValidation = fireworksProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, }) if (!queryValidation.success) return validationErrorResponse(queryValidation.error) const { workspaceId } = queryValidation.data - if (workspaceId) { - const session = await getSession() - if (session?.user?.id) { - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission) { - authorizedWorkspaceId = workspaceId - const byokResult = await getBYOKKey(workspaceId, 'fireworks') - if (byokResult) { - apiKey = byokResult.apiKey - } - } - } - } - - /** - * On hosted Sim the platform key is the sim-auto pool's inference key, not a - * workspace credential: enumerating the whole Fireworks catalog from it would - * offer every serverless model as selectable when no key can actually run it - * (`getApiKeyWithBYOK` serves the platform key to catalog models only). - */ - if (!apiKey && !isHosted) { - apiKey = env.FIREWORKS_API_KEY - } - - if (!apiKey) { - logger.info('No Fireworks API key available, returning empty models') - return NextResponse.json({ models: [] }) - } - try { - const init: RequestInit = { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - } - const response = await fetchForWorkspace(authorizedWorkspaceId, MODELS_URL, init) - - if (!response.ok) { - logger.warn('Failed to fetch Fireworks models', { - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json({ models: [] }) - } - - const data: FireworksModelsResponse = fireworksUpstreamResponseSchema.parse( - await response.json() - ) - - const allModels: string[] = [] - for (const model of data.data ?? []) { - allModels.push(`fireworks/${model.id}`) - } - - const uniqueModels = Array.from(new Set(allModels)) - const models = filterBlacklistedModels(uniqueModels) - - logger.info('Successfully fetched Fireworks models', { - count: models.length, - filtered: uniqueModels.length - models.length, - }) - - return NextResponse.json(providerModelsResponseSchema.parse({ models })) - } catch (error) { - logger.error('Error fetching Fireworks models', { - error: getErrorMessage(error, 'Unknown error'), - }) - return NextResponse.json({ models: [] }) - } + const principal = workspaceId ? await authenticateOptionalInternalSession() : null + const models = await listProviderModels('fireworks', { principal, workspaceId }) + return NextResponse.json({ models }) }) diff --git a/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts b/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts index f30499b2a4a..889f652b7e5 100644 --- a/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts +++ b/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts @@ -1,7 +1,8 @@ import { authMockFns, createMockRequest } from '@sim/testing' import { apiKeyByokMock, apiKeyByokMockFns } from '@sim/testing/mocks/api-key-byok.mock' -import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock' +import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' +import { workspaceContextMock } from '@sim/testing/mocks/workspace-context.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch } = vi.hoisted(() => ({ @@ -12,7 +13,9 @@ vi.mock('@/providers/utils', () => providersUtilsMock) vi.mock('@/lib/api-key/byok', () => apiKeyByokMock) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) + +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) import { GET } from '@/app/api/providers/ollama-cloud/models/route' @@ -20,7 +23,7 @@ const { mockGetBYOKKey } = apiKeyByokMockFns const mockGetSession = authMockFns.mockGetSession const { mockFilterBlacklistedModels, mockIsProviderBlacklisted } = providersUtilsMockFns -const { mockGetUserEntityPermissions } = permissionsMockFns +const { mockResolveEffectiveWorkspacePermission } = workspaceAuthzMockFns /** * Builds a request whose query string carries the given workspaceId. Passing @@ -36,8 +39,8 @@ const requestWithWorkspace = (workspaceId?: string) => { /** Grants a session + workspace permission so the BYOK lookup is reached. */ const grantWorkspaceAccess = () => { - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue('admin') + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') } describe('GET /api/providers/ollama-cloud/models', () => { @@ -48,7 +51,7 @@ describe('GET /api/providers/ollama-cloud/models', () => { mockFilterBlacklistedModels.mockImplementation((models: string[]) => models) mockGetBYOKKey.mockResolvedValue(null) mockGetSession.mockResolvedValue(null) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) }) it('returns empty models when the workspace has no stored BYOK key (never falls back to a hosted key)', async () => { @@ -64,8 +67,8 @@ describe('GET /api/providers/ollama-cloud/models', () => { }) it('does not call getBYOKKey when the session user lacks workspace permission', async () => { - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) const res = await GET(requestWithWorkspace('ws-1')) diff --git a/apps/sim/app/api/providers/ollama-cloud/models/route.ts b/apps/sim/app/api/providers/ollama-cloud/models/route.ts index 0fa53c58726..bb8d6006265 100644 --- a/apps/sim/app/api/providers/ollama-cloud/models/route.ts +++ b/apps/sim/app/api/providers/ollama-cloud/models/route.ts @@ -1,18 +1,11 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { - ollamaCloudProviderModelsQuerySchema, - ollamaUpstreamResponseSchema, - providerModelsResponseSchema, -} from '@/lib/api/contracts/providers' +import { ollamaCloudProviderModelsQuerySchema } from '@/lib/api/contracts/providers' import { validationErrorResponse } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' +import { listProviderModels } from '@/lib/provider-models/application/list-models' +import { isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('OllamaCloudModelsAPI') @@ -35,58 +28,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { if (!queryValidation.success) return validationErrorResponse(queryValidation.error) const { workspaceId } = queryValidation.data - let apiKey: string | undefined - if (workspaceId) { - const session = await getSession() - if (session?.user?.id) { - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission) { - const byokResult = await getBYOKKey(workspaceId, 'ollama-cloud') - if (byokResult) { - apiKey = byokResult.apiKey - } - } - } - } - - if (!apiKey) { - logger.info('No Ollama Cloud API key available, returning empty models') - return NextResponse.json({ models: [] }) - } - - try { - const response = await fetchForWorkspace(workspaceId, 'https://ollama.com/api/tags', { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - }) - - if (!response.ok) { - logger.warn('Failed to fetch Ollama Cloud models', { - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json({ models: [] }) - } - - const data = ollamaUpstreamResponseSchema.parse(await response.json()) - - const allModels = data.models.map((model) => `ollama-cloud/${model.name}`) - const uniqueModels = Array.from(new Set(allModels)) - const models = filterBlacklistedModels(uniqueModels) - - logger.info('Successfully fetched Ollama Cloud models', { - count: models.length, - filtered: uniqueModels.length - models.length, - }) - - return NextResponse.json(providerModelsResponseSchema.parse({ models })) - } catch (error) { - logger.error('Error fetching Ollama Cloud models', { - error: getErrorMessage(error, 'Unknown error'), - }) - return NextResponse.json({ models: [] }) - } + const principal = workspaceId ? await authenticateOptionalInternalSession() : null + const models = await listProviderModels('ollama-cloud', { principal, workspaceId }) + return NextResponse.json({ models }) }) diff --git a/apps/sim/app/api/providers/together/models/route.test.ts b/apps/sim/app/api/providers/together/models/route.test.ts index e92adce54f6..6d3b75dbed4 100644 --- a/apps/sim/app/api/providers/together/models/route.test.ts +++ b/apps/sim/app/api/providers/together/models/route.test.ts @@ -1,8 +1,9 @@ import { authMockFns, createMockRequest, resetEnvMock, setEnv } from '@sim/testing' import { jsonResponse } from '@sim/testing/helpers/http' import { apiKeyByokMock, apiKeyByokMockFns } from '@sim/testing/mocks/api-key-byok.mock' -import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock' +import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' +import { workspaceContextMock } from '@sim/testing/mocks/workspace-context.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch } = vi.hoisted(() => ({ @@ -13,14 +14,16 @@ vi.mock('@/providers/utils', () => providersUtilsMock) vi.mock('@/lib/api-key/byok', () => apiKeyByokMock) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) + +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) import { GET } from '@/app/api/providers/together/models/route' const mockGetSession = authMockFns.mockGetSession const mockGetBYOKKey = apiKeyByokMockFns.mockGetBYOKKey const { mockFilterBlacklistedModels, mockIsProviderBlacklisted } = providersUtilsMockFns -const { mockGetUserEntityPermissions } = permissionsMockFns +const { mockResolveEffectiveWorkspacePermission } = workspaceAuthzMockFns /** * Builds a request whose query string carries the given workspaceId. Passing @@ -49,7 +52,7 @@ describe('GET /api/providers/together/models', () => { mockFilterBlacklistedModels.mockImplementation((models: string[]) => models) mockGetBYOKKey.mockResolvedValue(null) mockGetSession.mockResolvedValue(null) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) }) afterAll(() => { @@ -58,8 +61,8 @@ describe('GET /api/providers/together/models', () => { it('uses the BYOK key when a workspace, session, and permission are present', async () => { setEnv({ TOGETHER_API_KEY: 'env-together-key' }) - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue('admin') + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') mockGetBYOKKey.mockResolvedValue({ apiKey: 'byok-together-key' }) mockFetch.mockResolvedValue(jsonResponse([{ id: 'moonshotai/Kimi-K2-Instruct' }])) @@ -74,8 +77,8 @@ describe('GET /api/providers/together/models', () => { it('falls back to the env key when the session user lacks workspace permission', async () => { setEnv({ TOGETHER_API_KEY: 'env-together-key' }) - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) mockFetch.mockResolvedValue(jsonResponse([{ id: 'moonshotai/Kimi-K2-Instruct' }])) const res = await GET(requestWithWorkspace('ws-1')) diff --git a/apps/sim/app/api/providers/together/models/route.ts b/apps/sim/app/api/providers/together/models/route.ts index 3feb07434a6..9e4edde5d74 100644 --- a/apps/sim/app/api/providers/together/models/route.ts +++ b/apps/sim/app/api/providers/together/models/route.ts @@ -1,33 +1,13 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { - providerModelsResponseSchema, - togetherProviderModelsQuerySchema, - togetherUpstreamResponseSchema, -} from '@/lib/api/contracts/providers' +import { togetherProviderModelsQuerySchema } from '@/lib/api/contracts/providers' import { validationErrorResponse } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' -import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' +import { listProviderModels } from '@/lib/provider-models/application/list-models' +import { isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('TogetherModelsAPI') -const MODELS_URL = 'https://api.together.ai/v1/models' - -/** Together's catalog includes non-text models; only chat models work with chat completions. */ -const NON_CHAT_MODEL_TYPES = new Set([ - 'image', - 'video', - 'audio', - 'transcribe', - 'embedding', - 'moderation', - 'rerank', -]) export const GET = withRouteHandler(async (request: NextRequest) => { if (isProviderBlacklisted('together')) { @@ -35,76 +15,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ models: [] }) } - let apiKey: string | undefined - let authorizedWorkspaceId: string | undefined - const queryValidation = togetherProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, }) if (!queryValidation.success) return validationErrorResponse(queryValidation.error) const { workspaceId } = queryValidation.data - if (workspaceId) { - const session = await getSession() - if (session?.user?.id) { - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission) { - authorizedWorkspaceId = workspaceId - const byokResult = await getBYOKKey(workspaceId, 'together') - if (byokResult) { - apiKey = byokResult.apiKey - } - } - } - } - - if (!apiKey) { - apiKey = env.TOGETHER_API_KEY - } - - if (!apiKey) { - logger.info('No Together API key available, returning empty models') - return NextResponse.json({ models: [] }) - } - try { - const init: RequestInit = { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - } - const response = await fetchForWorkspace(authorizedWorkspaceId, MODELS_URL, init) - - if (!response.ok) { - logger.warn('Failed to fetch Together models', { - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json({ models: [] }) - } - - const data = togetherUpstreamResponseSchema.parse(await response.json()) - - const allModels: string[] = [] - for (const model of data) { - if (model.type && NON_CHAT_MODEL_TYPES.has(model.type)) continue - allModels.push(`together/${model.id}`) - } - - const uniqueModels = Array.from(new Set(allModels)) - const models = filterBlacklistedModels(uniqueModels) - - logger.info('Successfully fetched Together models', { - count: models.length, - filtered: uniqueModels.length - models.length, - }) - - return NextResponse.json(providerModelsResponseSchema.parse({ models })) - } catch (error) { - logger.error('Error fetching Together models', { - error: getErrorMessage(error, 'Unknown error'), - }) - return NextResponse.json({ models: [] }) - } + const principal = workspaceId ? await authenticateOptionalInternalSession() : null + const models = await listProviderModels('together', { principal, workspaceId }) + return NextResponse.json({ models }) }) diff --git a/apps/sim/app/api/wand/route.ts b/apps/sim/app/api/wand/route.ts index 1c90843eb50..31eaa8525e0 100644 --- a/apps/sim/app/api/wand/route.ts +++ b/apps/sim/app/api/wand/route.ts @@ -1,31 +1,22 @@ -import { db } from '@sim/db' -import { workflow } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { wandGenerateContract } from '@/lib/api/contracts' import { parseRequest } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { - type BillingAttributionSnapshot, - checkAttributedBillingBlocks, - checkAttributedUsageLimits, - resolveBillingAttribution, - toBillingContext, -} from '@/lib/billing/core/attribution' -import { recordUsage } from '@/lib/billing/core/usage-log' -import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' -import { env } from '@/lib/core/config/env' -import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' -import { fetchForWorkspace } from '@/lib/core/network/resource-scope.server' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { enrichSandboxCapabilities } from '@/lib/execution/remote-sandbox/wand-enricher' -import { enrichTableSchema } from '@/lib/table/llm/wand' -import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' -import { extractResponseText, parseResponsesUsage } from '@/providers/openai/utils' -import { getModelPricing } from '@/providers/utils' +import { + generateWandForWorkflow, + generateWandInWorkspace, + WAND_DEPLOYMENT, + WandBillingRefusedError, + WandEmptyResponseError, + WandNotConfiguredError, + WandStreamStartError, + WandWorkflowDetachedError, + WandWorkflowNotFoundError, +} from '@/lib/wand/application/generate' export const dynamic = 'force-dynamic' export const runtime = 'nodejs' @@ -33,633 +24,127 @@ export const maxDuration = 60 const logger = createLogger('WandGenerateAPI') -const azureApiKey = env.AZURE_OPENAI_API_KEY -const azureEndpoint = env.AZURE_OPENAI_ENDPOINT -const azureApiVersion = env.AZURE_OPENAI_API_VERSION -const wandModelName = env.WAND_OPENAI_MODEL_NAME || 'gpt-4o' -const openaiApiKey = env.OPENAI_API_KEY - -const useWandAzure = azureApiKey && azureEndpoint && azureApiVersion - -if (!useWandAzure && !openaiApiKey) { - logger.warn( - 'Neither Azure OpenAI nor OpenAI API key found. Wand generation API will not function.' - ) -} else { - logger.info(`Using ${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} for wand generation`) -} - -const wandApiUrl = useWandAzure - ? `${azureEndpoint?.replace(/\/$/, '')}/openai/v1/responses?api-version=${azureApiVersion}` - : 'https://api.openai.com/v1/responses' - -/** - * Sends a wand completion request. Sim's own Azure OpenAI deployment stays direct; OpenAI calls - * (BYOK or Sim's key) are made for the workspace, so they leave through its egress route. - */ -function postWandRequest(workspaceId: string, init: RequestInit): Promise { - if (useWandAzure) return fetch(wandApiUrl, init) - return fetchForWorkspace(workspaceId, wandApiUrl, init) -} +const WORKSPACE_REQUIRED = 'Workspace context is required.' -interface ChatMessage { - role: 'user' | 'assistant' | 'system' - content: string +function wandError(error: string, status: number, extra: Record = {}) { + return NextResponse.json({ success: false, error, ...extra }, { status }) } -/** - * Wand enricher function type. - * Enrichers add context to the system prompt based on generationType. - */ -type WandEnricher = ( - workspaceId: string | null, - context: Record -) => Promise - -/** - * Registry of wand enrichers by generationType. - * Each enricher returns additional context to append to the system prompt. - */ -const wandEnrichers: Partial> = { - timestamp: async () => { - const now = new Date() - return `Current date and time context for reference: -- Current UTC timestamp: ${now.toISOString()} -- Current Unix timestamp (seconds): ${Math.floor(now.getTime() / 1000)} -- Current Unix timestamp (milliseconds): ${now.getTime()} -- Current date (UTC): ${now.toISOString().split('T')[0]} -- Current year: ${now.getUTCFullYear()} -- Current month: ${now.getUTCMonth() + 1} -- Current day of month: ${now.getUTCDate()} -- Current day of week: ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][now.getUTCDay()]} - -Use this context to calculate relative dates like "yesterday", "last week", "beginning of this month", etc.` - }, - - 'table-schema': enrichTableSchema, - /** JavaScript, Python, and Shell swap prompt text while retaining sandbox enrichment. */ - 'javascript-function-body': enrichSandboxCapabilities, +function errorDetails(error: unknown) { + if (!(error instanceof Error)) return { message: 'Unknown error' } + return { + name: error.name, + message: error.message || 'Unknown error', + code: 'code' in error ? error.code : undefined, + status: 'status' in error ? error.status : undefined, + stack: error.stack, + } } -async function updateUserStatsForWand( - billingAttribution: BillingAttributionSnapshot, - usage: { - prompt_tokens?: number - completion_tokens?: number - total_tokens?: number - }, - requestId: string, - isBYOK = false -): Promise { - if (!isBillingEnabled) { - return +/** Projects a refusal the use case raised, or `null` for a generation failure. */ +function refusalResponse(error: unknown, target: 'workflow' | 'workspace'): NextResponse | null { + if (error instanceof WandWorkflowNotFoundError) return wandError(error.message, 404) + if (error instanceof WandWorkflowDetachedError) return wandError(error.message, 403) + if ( + error instanceof OrchestrationError && + (error.code === 'forbidden' || error.code === 'not_found') + ) { + if (target === 'workspace') return wandError(WORKSPACE_REQUIRED, 400) + logger.warn('Wand caller lacks write access to the workflow workspace') + return wandError('Unauthorized', 403) } - - if (!usage.total_tokens || usage.total_tokens <= 0) { - return + if (error instanceof WandNotConfiguredError) return wandError(error.message, 503) + if (error instanceof WandBillingRefusedError) { + return wandError(error.message, 402, { scope: error.scope }) } + if (error instanceof WandEmptyResponseError) { + return wandError('Failed to generate content. AI response was empty.', 500) + } + if (error instanceof WandStreamStartError) { + logger.error('Failed to create stream', { ...errorDetails(error.cause), ...WAND_DEPLOYMENT }) + return wandError('An error occurred during wand generation streaming.', 500) + } + return null +} - try { - const promptTokens = usage.prompt_tokens || 0 - const completionTokens = usage.completion_tokens || 0 - - const modelName = useWandAzure ? wandModelName : 'gpt-4o' - let costToStore = 0 - - if (!isBYOK) { - const pricing = getModelPricing(modelName) - const costMultiplier = getCostMultiplier() - let modelCost = 0 - - if (pricing) { - const inputCost = (promptTokens / 1000000) * pricing.input - const outputCost = (completionTokens / 1000000) * pricing.output - modelCost = inputCost + outputCost - } else { - modelCost = (promptTokens / 1000000) * 0.005 + (completionTokens / 1000000) * 0.015 - } - - costToStore = modelCost * costMultiplier - } - - await recordUsage({ - userId: billingAttribution.actorUserId, - workspaceId: billingAttribution.workspaceId ?? undefined, - ...toBillingContext(billingAttribution), - entries: [ - { - category: 'model', - source: 'wand', - description: modelName, - cost: costToStore, - sourceReference: `wand:${requestId}`, - metadata: { inputTokens: promptTokens, outputTokens: completionTokens }, - }, - ], - }) - - await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) - } catch (error) { - logger.error('Failed to update user stats for wand usage', error, { requestId }) +function generationFailedResponse(error: unknown): NextResponse { + logger.error('Wand generation failed', { ...errorDetails(error), ...WAND_DEPLOYMENT }) + + let clientErrorMessage = 'Wand generation failed. Please try again later.' + let status = + error instanceof Error && 'status' in error && typeof error.status === 'number' + ? error.status + : 500 + + if ( + WAND_DEPLOYMENT.useWandAzure && + error instanceof Error && + error.message.includes('DeploymentNotFound') + ) { + clientErrorMessage = + 'Azure OpenAI deployment not found. Please check your model deployment configuration.' + status = 404 + } else if (status === 401) { + clientErrorMessage = 'Authentication failed. Please check your API key configuration.' + } else if (status === 429) { + clientErrorMessage = 'Rate limit exceeded. Please try again later.' + } else if (status >= 500) { + clientErrorMessage = + 'The wand generation service is currently unavailable. Please try again later.' } + + return wandError(clientErrorMessage, status) } export const POST = withRouteHandler(async (req: NextRequest) => { const requestId = generateRequestId() logger.info('Received wand generation request') - const session = await getSession() - if (!session?.user?.id) { + const principal = await authenticateOptionalInternalSession() + if (!principal) { logger.warn('Unauthorized wand generation attempt') - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + return wandError('Unauthorized', 401) } + let target: 'workflow' | 'workspace' | undefined try { const parsed = await parseRequest(wandGenerateContract, req, {}) if (!parsed.success) return parsed.response - const { body } = parsed.data - - const { - prompt, - systemPrompt, - stream = false, - history = [], - workflowId, - workspaceId: requestedWorkspaceId, - generationType, - wandContext = {}, - } = body + const { workflowId, workspaceId, ...generation } = parsed.data.body + const request = { ...generation, requestId } - if (!prompt) { - logger.warn('Invalid request: Missing prompt.') - return NextResponse.json( - { success: false, error: 'Missing required field: prompt.' }, - { status: 400 } - ) - } - - let workspaceId: string | null = null + let result if (workflowId) { - const [workflowRecord] = await db - .select({ workspaceId: workflow.workspaceId }) - .from(workflow) - .where(eq(workflow.id, workflowId)) - .limit(1) - - if (!workflowRecord) { - logger.warn(`Workflow not found: ${workflowId}`) - return NextResponse.json({ success: false, error: 'Workflow not found' }, { status: 404 }) - } - - workspaceId = workflowRecord.workspaceId - - if (workflowRecord.workspaceId) { - const permission = await verifyWorkspaceMembership( - session.user.id, - workflowRecord.workspaceId - ) - if (!permission || (permission !== 'admin' && permission !== 'write')) { - logger.warn( - `User ${session.user.id} does not have write access to workspace for workflow ${workflowId}` - ) - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 403 }) - } - } else { - logger.warn(`Workflow ${workflowId} has no workspaceId; wand request blocked`) - return NextResponse.json( - { - success: false, - error: - 'This workflow is not attached to a workspace. Personal workflows are deprecated and cannot be accessed.', - }, - { status: 403 } - ) - } - } else if (requestedWorkspaceId) { - // No workflow entity to resolve from (e.g. table-schema wand or an - // unhydrated editor); attribute to the workspace the wand is running in, - // but only when the caller is a member so usage can't be misattributed. - const permission = await verifyWorkspaceMembership(session.user.id, requestedWorkspaceId) - if (permission) { - workspaceId = requestedWorkspaceId - } - } - - // Per-member usage must be attributable to an org workspace. The editor always - // supplies a workflow or a workspace the caller belongs to; refuse to run rather - // than stamp usage workspace-less (which would silently skip the per-member cap). - if (!workspaceId) { - return NextResponse.json( - { success: false, error: 'Workspace context is required.' }, - { status: 400 } - ) - } - - /** Wand is an interactive action attributed to the authenticated human. */ - const actorUserId = session.user.id - const billingAttribution = await resolveBillingAttribution({ - actorUserId, - workspaceId, - }) - - let isBYOK = false - let activeOpenAIKey = openaiApiKey - - if (workspaceId && !useWandAzure) { - const byokResult = await getBYOKKey(workspaceId, 'openai') - if (byokResult) { - isBYOK = true - activeOpenAIKey = byokResult.apiKey - logger.info('Using BYOK OpenAI key for wand generation') - } - } - - if (!useWandAzure && !activeOpenAIKey) { - logger.error('AI client not initialized. Missing API key.') - return NextResponse.json( - { success: false, error: 'Wand generation service is not configured.' }, - { status: 503 } - ) - } - - /** - * BYOK skips metered caps but still checks the actor and exact workspace - * payer. Non-BYOK applies the complete attributed gate. - */ - if (isBYOK) { - const blocked = await checkAttributedBillingBlocks(billingAttribution) - if (blocked.blocked) { - return NextResponse.json( - { - success: false, - error: blocked.message || 'Account is not in good standing. Please contact support.', - }, - { status: 402 } - ) - } + target = 'workflow' + result = await generateWandForWorkflow.execute({ + principal, + input: { ...request, workflowId }, + }) + } else if (workspaceId) { + target = 'workspace' + result = await generateWandInWorkspace.execute({ + principal, + input: { ...request, workspaceId }, + }) } else { - const usage = await checkAttributedUsageLimits(billingAttribution) - if (usage.isExceeded) { - return NextResponse.json( - { - success: false, - error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.', - scope: usage.scope, - }, - { status: 402 } - ) - } - } - - let finalSystemPrompt = - systemPrompt || - 'You are a helpful AI assistant. Generate content exactly as requested by the user.' - - // Apply enricher if one exists for this generationType - if (generationType) { - const enricher = wandEnrichers[generationType] - if (enricher) { - const enrichment = await enricher(workspaceId, wandContext) - if (enrichment) { - finalSystemPrompt += `\n\n${enrichment}` - } - } - } - - if (generationType === 'cron-expression') { - finalSystemPrompt += - '\n\nIMPORTANT: Return ONLY the raw cron expression (e.g., "0 9 * * 1-5"). Do NOT wrap it in markdown code blocks, backticks, or quotes. Do NOT include any explanation or text before or after the expression.' + // Per-member usage must be attributable to an org workspace. The editor always + // supplies a workflow or a workspace the caller belongs to; refuse to run rather + // than stamp usage workspace-less (which would silently skip the per-member cap). + return wandError(WORKSPACE_REQUIRED, 400) } - // Both the JavaScript and Python function-body prompts share this type, so - // the reinforcement stays language-neutral. - if (generationType === 'javascript-function-body') { - finalSystemPrompt += - '\n\nIMPORTANT: Return ONLY the raw function body. Do NOT wrap it in markdown code blocks (no ```javascript, no ```python, no ```). Do NOT include any explanation before or after the code.' + if (result.kind === 'content') { + return NextResponse.json({ success: true, content: result.content }) } - - if (generationType === 'json-object') { - finalSystemPrompt += - '\n\nIMPORTANT: Return ONLY the raw JSON object. Do NOT wrap it in markdown code blocks (no ```json or ```). Do NOT include any explanation or text before or after the JSON. The response must start with { and end with }.' - } - - // Separate from json-object: that reinforcement demands braces, which would - // fight a field whose contract is an array. - if (generationType === 'json-array') { - finalSystemPrompt += - '\n\nIMPORTANT: Return ONLY the raw JSON array. Do NOT wrap it in markdown code blocks (no ```json or ```). Do NOT include any explanation or text before or after the JSON. The response must start with [ and end with ].' - } - - const messages: ChatMessage[] = [{ role: 'system', content: finalSystemPrompt }] - - messages.push(...history.filter((msg) => msg.role !== 'system')) - - messages.push({ role: 'user', content: prompt }) - - const headers: Record = { - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'responses=v1', - } - - if (useWandAzure) { - headers['api-key'] = azureApiKey! - } else { - headers.Authorization = `Bearer ${activeOpenAIKey}` - } - - if (stream) { - try { - logger.info(`About to create stream with model: ${useWandAzure ? wandModelName : 'gpt-4o'}`) - - const response = await postWandRequest(workspaceId, { - method: 'POST', - headers, - body: JSON.stringify({ - model: useWandAzure ? wandModelName : 'gpt-4o', - input: messages, - temperature: 0.2, - max_output_tokens: 10000, - stream: true, - }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('API request failed', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - throw new Error(`API request failed: ${response.status} ${response.statusText}`) - } - - logger.info('Stream response received, starting processing') - - const encoder = new TextEncoder() - const decoder = new TextDecoder() - - const readable = new ReadableStream({ - async start(controller) { - const reader = response.body?.getReader() - if (!reader) { - controller.close() - return - } - - let finalUsage: any = null - let usageRecorded = false - - const flushUsage = async () => { - if (usageRecorded || !finalUsage) { - return - } - - usageRecorded = true - await updateUserStatsForWand(billingAttribution, finalUsage, requestId, isBYOK) - } - - try { - let buffer = '' - let chunkCount = 0 - let activeEventType: string | undefined - - while (true) { - const { done, value } = await reader.read() - - if (done) { - logger.info(`Stream completed. Total chunks: ${chunkCount}`) - await flushUsage() - controller.enqueue(encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`)) - controller.close() - break - } - - buffer += decoder.decode(value, { stream: true }) - - const lines = buffer.split('\n') - buffer = lines.pop() || '' - - for (const line of lines) { - const trimmed = line.trim() - if (!trimmed) { - continue - } - - if (trimmed.startsWith('event:')) { - activeEventType = trimmed.slice(6).trim() - continue - } - - if (!trimmed.startsWith('data:')) { - continue - } - - const data = trimmed.slice(5).trim() - if (data === '[DONE]') { - logger.info('Received [DONE] signal') - - await flushUsage() - - controller.enqueue( - encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`) - ) - controller.close() - return - } - - let parsed: any - try { - parsed = JSON.parse(data) - } catch { - continue - } - - const eventType = parsed?.type ?? activeEventType - - if ( - eventType === 'response.error' || - eventType === 'error' || - eventType === 'response.failed' - ) { - throw new Error(parsed?.error?.message || 'Responses stream error') - } - - if ( - eventType === 'response.output_text.delta' || - eventType === 'response.output_json.delta' - ) { - let content = '' - if (typeof parsed.delta === 'string') { - content = parsed.delta - } else if (parsed.delta && typeof parsed.delta.text === 'string') { - content = parsed.delta.text - } else if (parsed.delta && parsed.delta.json !== undefined) { - content = JSON.stringify(parsed.delta.json) - } else if (parsed.json !== undefined) { - content = JSON.stringify(parsed.json) - } else if (typeof parsed.text === 'string') { - content = parsed.text - } - - if (content) { - chunkCount++ - if (chunkCount === 1) { - logger.info('Received first content chunk') - } - - controller.enqueue( - encoder.encode(`data: ${JSON.stringify({ chunk: content })}\n\n`) - ) - } - } - - if (eventType === 'response.completed') { - const usage = parseResponsesUsage(parsed?.response?.usage ?? parsed?.usage) - if (usage) { - finalUsage = { - prompt_tokens: usage.promptTokens, - completion_tokens: usage.completionTokens, - total_tokens: usage.totalTokens, - } - logger.info(`Received usage data: ${JSON.stringify(finalUsage)}`) - } - } - } - } - } catch (streamError: any) { - logger.error('Streaming error', { - name: streamError?.name, - message: streamError?.message || 'Unknown error', - stack: streamError?.stack, - }) - - try { - await flushUsage() - } catch (usageError) { - logger.warn('Failed to record usage after stream error', usageError) - } - - const errorData = `data: ${JSON.stringify({ error: 'Streaming failed', done: true })}\n\n` - controller.enqueue(encoder.encode(errorData)) - controller.close() - } finally { - reader.releaseLock() - } - }, - cancel() {}, - }) - - return new Response(readable, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache, no-transform', - Connection: 'keep-alive', - 'X-Accel-Buffering': 'no', - }, - }) - } catch (error: any) { - logger.error('Failed to create stream', { - name: error?.name, - message: error?.message || 'Unknown error', - code: error?.code, - status: error?.status, - stack: error?.stack, - useWandAzure, - model: useWandAzure ? wandModelName : 'gpt-4o', - endpoint: useWandAzure ? azureEndpoint : 'api.openai.com', - apiVersion: useWandAzure ? azureApiVersion : 'N/A', - }) - - return NextResponse.json( - { success: false, error: 'An error occurred during wand generation streaming.' }, - { status: 500 } - ) - } - } - - const response = await postWandRequest(workspaceId, { - method: 'POST', - headers, - body: JSON.stringify({ - model: useWandAzure ? wandModelName : 'gpt-4o', - input: messages, - temperature: 0.2, - max_output_tokens: 10000, - }), - }) - - if (!response.ok) { - const errorText = await response.text() - const apiError = new Error( - `API request failed: ${response.status} ${response.statusText} - ${errorText}` - ) - ;(apiError as any).status = response.status - throw apiError - } - - const completion = await response.json() - const generatedContent = extractResponseText(completion.output)?.trim() - - if (!generatedContent) { - logger.error(`${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} response was empty or invalid.`) - return NextResponse.json( - { success: false, error: 'Failed to generate content. AI response was empty.' }, - { status: 500 } - ) - } - - logger.info('Wand generation successful') - - const usage = parseResponsesUsage(completion.usage) - if (usage) { - await updateUserStatsForWand( - billingAttribution, - { - prompt_tokens: usage.promptTokens, - completion_tokens: usage.completionTokens, - total_tokens: usage.totalTokens, - }, - requestId, - isBYOK - ) - } - - return NextResponse.json({ success: true, content: generatedContent }) - } catch (error: any) { - logger.error('Wand generation failed', { - name: error?.name, - message: error?.message || 'Unknown error', - code: error?.code, - status: error?.status, - stack: error?.stack, - useWandAzure, - model: useWandAzure ? wandModelName : 'gpt-4o', - endpoint: useWandAzure ? azureEndpoint : 'api.openai.com', - apiVersion: useWandAzure ? azureApiVersion : 'N/A', - }) - - let clientErrorMessage = 'Wand generation failed. Please try again later.' - let status = typeof (error as any)?.status === 'number' ? (error as any).status : 500 - - if (useWandAzure && error?.message?.includes('DeploymentNotFound')) { - clientErrorMessage = - 'Azure OpenAI deployment not found. Please check your model deployment configuration.' - status = 404 - } else if (status === 401) { - clientErrorMessage = 'Authentication failed. Please check your API key configuration.' - } else if (status === 429) { - clientErrorMessage = 'Rate limit exceeded. Please try again later.' - } else if (status >= 500) { - clientErrorMessage = - 'The wand generation service is currently unavailable. Please try again later.' - } - - return NextResponse.json( - { - success: false, - error: clientErrorMessage, + return new Response(result.body, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache, no-transform', + Connection: 'keep-alive', + 'X-Accel-Buffering': 'no', }, - { status } - ) + }) + } catch (error) { + return (target && refusalResponse(error, target)) || generationFailedResponse(error) } }) diff --git a/apps/sim/lib/api/server/routes/index.ts b/apps/sim/lib/api/server/routes/index.ts index e42507d6cc8..23aefbfdbec 100644 --- a/apps/sim/lib/api/server/routes/index.ts +++ b/apps/sim/lib/api/server/routes/index.ts @@ -1,5 +1,6 @@ export { defineInternalBinaryRoute } from '@/lib/api/server/routes/internal-binary-route' export { + authenticateOptionalInternalSession, createInternalSessionOrExecutorAuth, defineInternalJsonRoute, extendInternalErrorPolicy, diff --git a/apps/sim/lib/api/server/routes/internal-json-route.ts b/apps/sim/lib/api/server/routes/internal-json-route.ts index 4e57e7d7df2..87cb9b4008e 100644 --- a/apps/sim/lib/api/server/routes/internal-json-route.ts +++ b/apps/sim/lib/api/server/routes/internal-json-route.ts @@ -62,6 +62,16 @@ export const internalSessionAuth = { }, } as const +/** The session principal when the request is signed in, otherwise `null`, for routes that also serve anonymous callers. */ +export async function authenticateOptionalInternalSession(): Promise { + try { + return await internalSessionAuth.authenticate() + } catch (error) { + if (error instanceof InternalUnauthenticatedError) return null + throw error + } +} + interface InternalSessionOrExecutorAuthOptions { audience: string resourceScope?( diff --git a/apps/sim/lib/core/network/resource-scope.server.ts b/apps/sim/lib/core/network/resource-scope.server.ts index 1352842b6d4..85e3b5834b3 100644 --- a/apps/sim/lib/core/network/resource-scope.server.ts +++ b/apps/sim/lib/core/network/resource-scope.server.ts @@ -1,6 +1,5 @@ import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' import { runWithOutboundOrganization } from '@/lib/core/network/context.server' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { OutboundRoutingError } from '@/lib/core/network/routing' import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { loadWorkspaceApplicationContext } from '@/lib/workspaces/application/context' @@ -18,17 +17,3 @@ export async function withResourceOutboundScope( if (!workspace) throw new OutboundRoutingError('MISSING_SCOPE') return runWithOutboundOrganization(workspace.workspaceOrganizationId, run) } - -/** - * `fetch` for a workspace's request, sent through its organization's outbound route. Without a - * workspace it is a platform request and goes direct. - */ -export function fetchForWorkspace( - workspaceId: string | undefined, - input: string, - init?: RequestInit -): Promise { - // egress-direct: no workspace, so a platform request made with Sim's own key - if (!workspaceId) return fetch(input, init) - return withResourceOutboundScope({ workspaceId }, () => outboundFetch(input, init)) -} diff --git a/apps/sim/lib/provider-models/application/list-models.ts b/apps/sim/lib/provider-models/application/list-models.ts new file mode 100644 index 00000000000..10f6e6d22f2 --- /dev/null +++ b/apps/sim/lib/provider-models/application/list-models.ts @@ -0,0 +1,175 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + basetenUpstreamResponseSchema, + fireworksUpstreamResponseSchema, + ollamaUpstreamResponseSchema, + providerModelsResponseSchema, + togetherUpstreamResponseSchema, +} from '@/lib/api/contracts/providers' +import { getBYOKKey } from '@/lib/api-key/byok' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { env } from '@/lib/core/config/env' +import { isHosted } from '@/lib/core/config/env-flags' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { providerModelOperations } from '@/lib/provider-models/application/operations' +import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/context' +import { filterBlacklistedModels } from '@/providers/utils' + +const logger = createLogger('ProviderModelsList') + +/** Together's catalog includes non-text models; only chat models work with chat completions. */ +const TOGETHER_NON_CHAT_MODEL_TYPES = new Set([ + 'image', + 'video', + 'audio', + 'transcribe', + 'embedding', + 'moderation', + 'rerank', +]) + +interface ModelCatalogSource { + url: string + /** Sim's own key, used when the workspace has none; absent for a BYOK-only provider. */ + platformKey(): string | undefined + /** Upstream body to provider-prefixed model ids. */ + parse(body: unknown): string[] +} + +const MODEL_CATALOGS = { + baseten: { + url: 'https://inference.baseten.co/v1/models', + platformKey: () => env.BASETEN_API_KEY, + parse: (body) => + basetenUpstreamResponseSchema.parse(body).data.map((model) => `baseten/${model.id}`), + }, + fireworks: { + url: 'https://api.fireworks.ai/inference/v1/models', + /** + * On hosted Sim the platform key is the sim-auto pool's inference key, not a workspace + * credential: enumerating the whole Fireworks catalog from it would offer every serverless + * model as selectable when no key can actually run it (`getApiKeyWithBYOK` serves the platform + * key to catalog models only). + */ + platformKey: () => (isHosted ? undefined : env.FIREWORKS_API_KEY), + parse: (body) => + fireworksUpstreamResponseSchema.parse(body).data.map((model) => `fireworks/${model.id}`), + }, + together: { + url: 'https://api.together.ai/v1/models', + platformKey: () => env.TOGETHER_API_KEY, + parse: (body) => + togetherUpstreamResponseSchema + .parse(body) + .filter((model) => !model.type || !TOGETHER_NON_CHAT_MODEL_TYPES.has(model.type)) + .map((model) => `together/${model.id}`), + }, + /** BYOK-only: Sim never supplies a hosted key and never bills usage. */ + 'ollama-cloud': { + url: 'https://ollama.com/api/tags', + platformKey: () => undefined, + parse: (body) => + ollamaUpstreamResponseSchema.parse(body).models.map((model) => `ollama-cloud/${model.name}`), + }, +} as const satisfies Record + +export type ModelCatalogProvider = keyof typeof MODEL_CATALOGS + +/** + * Reads a provider's model catalog with `apiKey`. Every failure is logged and answers an empty + * list, so the model picker degrades instead of erroring. + */ +async function fetchModelCatalog( + provider: ModelCatalogProvider, + apiKey: string | undefined, + fetchImpl: typeof fetch +): Promise { + if (!apiKey) { + logger.info('No provider API key available, returning empty models', { provider }) + return [] + } + const source = MODEL_CATALOGS[provider] + try { + const response = await fetchImpl(source.url, { + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + cache: 'no-store', + }) + if (!response.ok) { + logger.warn('Failed to fetch provider models', { + provider, + status: response.status, + statusText: response.statusText, + }) + return [] + } + const uniqueModels = Array.from(new Set(source.parse(await response.json()))) + const { models } = providerModelsResponseSchema.parse({ + models: filterBlacklistedModels(uniqueModels), + }) + logger.info('Successfully fetched provider models', { + provider, + count: models.length, + filtered: uniqueModels.length - models.length, + }) + return models + } catch (error) { + logger.error('Error fetching provider models', { + provider, + error: getErrorMessage(error, 'Unknown error'), + }) + return [] + } +} + +/** + * Lists a provider's models for a workspace member: the workspace's BYOK key when it has one, + * otherwise Sim's key, sent through the workspace organization's outbound route. + */ +const listWorkspaceProviderModels = defineAuthorizedWorkspaceUseCase({ + operation: providerModelOperations.list, + resolveContext: ({ input }: { input: { provider: ModelCatalogProvider; workspaceId: string } }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: {}, + execute: async ({ input, context }) => { + const byok = await getBYOKKey(context.workspaceId, input.provider) + const apiKey = byok?.apiKey ?? MODEL_CATALOGS[input.provider].platformKey() + return fetchModelCatalog(input.provider, apiKey, scopedOutboundFetch) + }, +}) + +/** A missing workspace or a caller outside it gets the platform catalog, never an error. */ +function isWorkspaceUnavailable(error: unknown): boolean { + return ( + error instanceof OrchestrationError && + (error.code === 'not_found' || error.code === 'forbidden') + ) +} + +/** + * The models a provider offers to the caller. A signed-in workspace member reads them with the + * workspace's key through its egress route; anyone else gets Sim's own catalog, read directly. + */ +export async function listProviderModels( + provider: ModelCatalogProvider, + { principal, workspaceId }: { principal: SessionPrincipal | null; workspaceId?: string } +): Promise { + if (principal && workspaceId) { + try { + return await listWorkspaceProviderModels.execute({ + principal, + input: { provider, workspaceId }, + }) + } catch (error) { + if (!isWorkspaceUnavailable(error)) throw error + } + } + return fetchModelCatalog( + provider, + MODEL_CATALOGS[provider].platformKey(), + // egress-direct: no authorized workspace, so a platform request made with Sim's own key + fetch + ) +} diff --git a/apps/sim/lib/provider-models/application/operations.ts b/apps/sim/lib/provider-models/application/operations.ts new file mode 100644 index 00000000000..0598cf0ad49 --- /dev/null +++ b/apps/sim/lib/provider-models/application/operations.ts @@ -0,0 +1,12 @@ +import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' + +export const providerModelOperations = { + list: defineWorkspaceOperation({ + id: 'provider_models.list', + minimumRole: 'read', + // permission-group-exempt: lists the models a workspace's own provider key can reach; running a model is gated where it executes. + capability: 'none', + workspaceApiKey: 'deny', + principalKinds: ['session'], + }), +} as const diff --git a/apps/sim/lib/wand/application/generate.ts b/apps/sim/lib/wand/application/generate.ts new file mode 100644 index 00000000000..ed060a085a3 --- /dev/null +++ b/apps/sim/lib/wand/application/generate.ts @@ -0,0 +1,661 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { workflow } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' +import type OpenAI from 'openai' +import { getBYOKKey } from '@/lib/api-key/byok' +import { + type BillingAttributionSnapshot, + checkAttributedBillingBlocks, + checkAttributedUsageLimits, + resolveBillingAttribution, + toBillingContext, +} from '@/lib/billing/core/attribution' +import { recordUsage } from '@/lib/billing/core/usage-log' +import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { env } from '@/lib/core/config/env' +import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { enrichSandboxCapabilities } from '@/lib/execution/remote-sandbox/wand-enricher' +import { enrichTableSchema } from '@/lib/table/llm/wand' +import { wandOperations } from '@/lib/wand/application/operations' +import { + type ActiveWorkspaceApplicationContext, + resolveActiveWorkspaceApplicationContext, +} from '@/lib/workspaces/application/context' +import { extractResponseText, parseResponsesUsage } from '@/providers/openai/utils' +import { getModelPricing } from '@/providers/utils' + +const logger = createLogger('WandGenerate') + +const azureApiKey = env.AZURE_OPENAI_API_KEY +const azureEndpoint = env.AZURE_OPENAI_ENDPOINT +const azureApiVersion = env.AZURE_OPENAI_API_VERSION +const wandModelName = env.WAND_OPENAI_MODEL_NAME || 'gpt-4o' +const openaiApiKey = env.OPENAI_API_KEY + +const useWandAzure = Boolean(azureApiKey && azureEndpoint && azureApiVersion) + +if (!useWandAzure && !openaiApiKey) { + logger.warn( + 'Neither Azure OpenAI nor OpenAI API key found. Wand generation API will not function.' + ) +} else { + logger.info(`Using ${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} for wand generation`) +} + +const wandApiUrl = useWandAzure + ? `${azureEndpoint?.replace(/\/$/, '')}/openai/v1/responses?api-version=${azureApiVersion}` + : 'https://api.openai.com/v1/responses' + +const requestModel = useWandAzure ? wandModelName : 'gpt-4o' + +/** Which deployment serves wand requests, for the surface's error logging and messages. */ +export const WAND_DEPLOYMENT = { + useWandAzure, + model: requestModel, + endpoint: useWandAzure ? azureEndpoint : 'api.openai.com', + apiVersion: useWandAzure ? azureApiVersion : 'N/A', +} as const + +/** + * Sends a wand completion request. Sim's own Azure OpenAI deployment stays direct; OpenAI calls + * (BYOK or Sim's key) are made for the workspace, so they leave through its egress route. + */ +function postWandRequest(init: RequestInit): Promise { + // egress-direct: Sim's env-configured Azure OpenAI deployment, not a workspace credential + if (useWandAzure) return fetch(wandApiUrl, init) + return scopedOutboundFetch(wandApiUrl, init) +} + +interface WandChatMessage { + role: 'user' | 'assistant' | 'system' + content: string +} + +/** + * Wand enricher function type. + * Enrichers add context to the system prompt based on generationType. + */ +type WandEnricher = ( + workspaceId: string | null, + context: Record +) => Promise + +/** + * Registry of wand enrichers by generationType. + * Each enricher returns additional context to append to the system prompt. + */ +const wandEnrichers: Partial> = { + timestamp: async () => { + const now = new Date() + return `Current date and time context for reference: +- Current UTC timestamp: ${now.toISOString()} +- Current Unix timestamp (seconds): ${Math.floor(now.getTime() / 1000)} +- Current Unix timestamp (milliseconds): ${now.getTime()} +- Current date (UTC): ${now.toISOString().split('T')[0]} +- Current year: ${now.getUTCFullYear()} +- Current month: ${now.getUTCMonth() + 1} +- Current day of month: ${now.getUTCDate()} +- Current day of week: ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][now.getUTCDay()]} + +Use this context to calculate relative dates like "yesterday", "last week", "beginning of this month", etc.` + }, + + 'table-schema': enrichTableSchema, + /** JavaScript, Python, and Shell swap prompt text while retaining sandbox enrichment. */ + 'javascript-function-body': enrichSandboxCapabilities, +} + +interface WandUsage { + prompt_tokens?: number + completion_tokens?: number + total_tokens?: number +} + +async function updateUserStatsForWand( + billingAttribution: BillingAttributionSnapshot, + usage: WandUsage, + requestId: string, + isBYOK = false +): Promise { + if (!isBillingEnabled) { + return + } + + if (!usage.total_tokens || usage.total_tokens <= 0) { + return + } + + try { + const promptTokens = usage.prompt_tokens || 0 + const completionTokens = usage.completion_tokens || 0 + + const modelName = requestModel + let costToStore = 0 + + if (!isBYOK) { + const pricing = getModelPricing(modelName) + const costMultiplier = getCostMultiplier() + let modelCost = 0 + + if (pricing) { + const inputCost = (promptTokens / 1000000) * pricing.input + const outputCost = (completionTokens / 1000000) * pricing.output + modelCost = inputCost + outputCost + } else { + modelCost = (promptTokens / 1000000) * 0.005 + (completionTokens / 1000000) * 0.015 + } + + costToStore = modelCost * costMultiplier + } + + await recordUsage({ + userId: billingAttribution.actorUserId, + workspaceId: billingAttribution.workspaceId ?? undefined, + ...toBillingContext(billingAttribution), + entries: [ + { + category: 'model', + source: 'wand', + description: modelName, + cost: costToStore, + sourceReference: `wand:${requestId}`, + metadata: { inputTokens: promptTokens, outputTokens: completionTokens }, + }, + ], + }) + + await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) + } catch (error) { + logger.error('Failed to update user stats for wand usage', error, { requestId }) + } +} + +/** The workflow a wand request names does not exist. */ +export class WandWorkflowNotFoundError extends OrchestrationError { + constructor() { + super('not_found', 'Workflow not found') + this.name = 'WandWorkflowNotFoundError' + } +} + +/** The workflow a wand request names belongs to no workspace (a deprecated personal workflow). */ +export class WandWorkflowDetachedError extends OrchestrationError { + constructor() { + super( + 'forbidden', + 'This workflow is not attached to a workspace. Personal workflows are deprecated and cannot be accessed.' + ) + this.name = 'WandWorkflowDetachedError' + } +} + +/** Neither Sim's Azure deployment nor an OpenAI key (Sim's or the workspace's) is available. */ +export class WandNotConfiguredError extends Error { + constructor() { + super('Wand generation service is not configured.') + this.name = 'WandNotConfiguredError' + } +} + +/** The billing gate refused the request; `message` is the client-facing explanation. */ +export class WandBillingRefusedError extends Error { + constructor( + message: string, + readonly scope?: 'actor' | 'payer' | 'member' + ) { + super(message) + this.name = 'WandBillingRefusedError' + } +} + +/** The model answered without any text. */ +export class WandEmptyResponseError extends Error { + constructor() { + super('Wand response was empty or invalid') + this.name = 'WandEmptyResponseError' + } +} + +/** The model's endpoint answered a non-streaming request with an error status. */ +class WandUpstreamError extends Error { + constructor( + readonly status: number, + message: string + ) { + super(message) + this.name = 'WandUpstreamError' + } +} + +/** The streaming request could not be opened. */ +export class WandStreamStartError extends Error { + constructor(cause: unknown) { + super('Failed to create wand stream', { cause }) + this.name = 'WandStreamStartError' + } +} + +interface GenerateWandInput { + prompt: string + systemPrompt?: string + stream: boolean + history: WandChatMessage[] + generationType?: string + wandContext: Record + requestId: string +} + +type GenerateWandResult = + | { kind: 'stream'; body: ReadableStream } + | { kind: 'content'; content: string } + +interface ResponsesStreamEvent { + type?: string + delta?: unknown + json?: unknown + text?: unknown + error?: { message?: string } + response?: { usage?: OpenAI.Responses.ResponseUsage } + usage?: OpenAI.Responses.ResponseUsage +} + +interface ResponsesCompletion { + output: OpenAI.Responses.ResponseOutputItem[] + usage?: OpenAI.Responses.ResponseUsage +} + +/** + * Kept apart from `resolveActiveWorkflowApplicationContext`: wand has always accepted archived + * workflows and answers a workflow without a workspace with its own refusal, not a not-found. + */ +async function resolveWorkflowWorkspaceId(workflowId: string): Promise { + const [workflowRecord] = await db + .select({ workspaceId: workflow.workspaceId }) + .from(workflow) + .where(eq(workflow.id, workflowId)) + .limit(1) + if (!workflowRecord) { + logger.warn('Wand workflow not found', { workflowId }) + throw new WandWorkflowNotFoundError() + } + if (!workflowRecord.workspaceId) { + logger.warn('Wand workflow has no workspace; request blocked', { workflowId }) + throw new WandWorkflowDetachedError() + } + return workflowRecord.workspaceId +} + +/** + * Streams the Responses API SSE body as `data: {chunk}` events, closing with `data: {done}` and + * recording usage once the completion reports it. + */ +function relayResponsesStream( + response: Response, + recordUsageOnce: (usage: WandUsage) => Promise +): ReadableStream { + const encoder = new TextEncoder() + const decoder = new TextDecoder() + + return new ReadableStream({ + async start(controller) { + const reader = response.body?.getReader() + if (!reader) { + controller.close() + return + } + + let finalUsage: WandUsage | null = null + let usageRecorded = false + + const flushUsage = async () => { + if (usageRecorded || !finalUsage) { + return + } + + usageRecorded = true + await recordUsageOnce(finalUsage) + } + + try { + let buffer = '' + let chunkCount = 0 + let activeEventType: string | undefined + + while (true) { + const { done, value } = await reader.read() + + if (done) { + logger.info(`Stream completed. Total chunks: ${chunkCount}`) + await flushUsage() + controller.enqueue(encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`)) + controller.close() + break + } + + buffer += decoder.decode(value, { stream: true }) + + const lines = buffer.split('\n') + buffer = lines.pop() || '' + + for (const line of lines) { + const trimmed = line.trim() + if (!trimmed) { + continue + } + + if (trimmed.startsWith('event:')) { + activeEventType = trimmed.slice(6).trim() + continue + } + + if (!trimmed.startsWith('data:')) { + continue + } + + const data = trimmed.slice(5).trim() + if (data === '[DONE]') { + logger.info('Received [DONE] signal') + + await flushUsage() + + controller.enqueue(encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`)) + controller.close() + return + } + + let parsed: ResponsesStreamEvent | null + try { + parsed = JSON.parse(data) + } catch { + continue + } + + const eventType = parsed?.type ?? activeEventType + + if ( + eventType === 'response.error' || + eventType === 'error' || + eventType === 'response.failed' + ) { + throw new Error(parsed?.error?.message || 'Responses stream error') + } + + if ( + parsed && + (eventType === 'response.output_text.delta' || + eventType === 'response.output_json.delta') + ) { + const content = deltaContent(parsed) + + if (content) { + chunkCount++ + if (chunkCount === 1) { + logger.info('Received first content chunk') + } + + controller.enqueue( + encoder.encode(`data: ${JSON.stringify({ chunk: content })}\n\n`) + ) + } + } + + if (eventType === 'response.completed') { + const usage = parseResponsesUsage(parsed?.response?.usage ?? parsed?.usage) + if (usage) { + finalUsage = { + prompt_tokens: usage.promptTokens, + completion_tokens: usage.completionTokens, + total_tokens: usage.totalTokens, + } + logger.info(`Received usage data: ${JSON.stringify(finalUsage)}`) + } + } + } + } + } catch (streamError) { + logger.error('Streaming error', { + name: streamError instanceof Error ? streamError.name : undefined, + message: (streamError instanceof Error && streamError.message) || 'Unknown error', + stack: streamError instanceof Error ? streamError.stack : undefined, + }) + + try { + await flushUsage() + } catch (usageError) { + logger.warn('Failed to record usage after stream error', usageError) + } + + const errorData = `data: ${JSON.stringify({ error: 'Streaming failed', done: true })}\n\n` + controller.enqueue(encoder.encode(errorData)) + controller.close() + } finally { + reader.releaseLock() + } + }, + cancel() {}, + }) +} + +function deltaContent(event: ResponsesStreamEvent): string { + const { delta } = event + if (typeof delta === 'string') return delta + if (delta && typeof delta === 'object') { + if ('text' in delta && typeof delta.text === 'string') return delta.text + if ('json' in delta && delta.json !== undefined) return JSON.stringify(delta.json) + } + if (event.json !== undefined) return JSON.stringify(event.json) + if (typeof event.text === 'string') return event.text + return '' +} + +/** + * Generates wand content for a workspace member, billed to that workspace. OpenAI requests leave + * through the workspace organization's outbound route. + */ +async function executeWandGeneration({ + principal, + input, + context, +}: { + principal: SessionPrincipal + input: I + context: ActiveWorkspaceApplicationContext +}): Promise { + const { workspaceId } = context + const { prompt, history, generationType, wandContext, requestId } = input + + /** Wand is an interactive action attributed to the authenticated human. */ + const billingAttribution = await resolveBillingAttribution({ + actorUserId: principal.userId, + workspaceId, + }) + + let isBYOK = false + let activeOpenAIKey = openaiApiKey + + if (!useWandAzure) { + const byokResult = await getBYOKKey(workspaceId, 'openai') + if (byokResult) { + isBYOK = true + activeOpenAIKey = byokResult.apiKey + logger.info('Using BYOK OpenAI key for wand generation') + } + } + + if (!useWandAzure && !activeOpenAIKey) { + logger.error('AI client not initialized. Missing API key.') + throw new WandNotConfiguredError() + } + + /** + * BYOK skips metered caps but still checks the actor and exact workspace + * payer. Non-BYOK applies the complete attributed gate. + */ + if (isBYOK) { + const blocked = await checkAttributedBillingBlocks(billingAttribution) + if (blocked.blocked) { + throw new WandBillingRefusedError( + blocked.message || 'Account is not in good standing. Please contact support.' + ) + } + } else { + const usage = await checkAttributedUsageLimits(billingAttribution) + if (usage.isExceeded) { + throw new WandBillingRefusedError( + usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.', + usage.scope + ) + } + } + + let finalSystemPrompt = + input.systemPrompt || + 'You are a helpful AI assistant. Generate content exactly as requested by the user.' + + if (generationType) { + const enricher = wandEnrichers[generationType] + if (enricher) { + const enrichment = await enricher(workspaceId, wandContext) + if (enrichment) { + finalSystemPrompt += `\n\n${enrichment}` + } + } + } + + if (generationType === 'cron-expression') { + finalSystemPrompt += + '\n\nIMPORTANT: Return ONLY the raw cron expression (e.g., "0 9 * * 1-5"). Do NOT wrap it in markdown code blocks, backticks, or quotes. Do NOT include any explanation or text before or after the expression.' + } + + // Both the JavaScript and Python function-body prompts share this type, so + // the reinforcement stays language-neutral. + if (generationType === 'javascript-function-body') { + finalSystemPrompt += + '\n\nIMPORTANT: Return ONLY the raw function body. Do NOT wrap it in markdown code blocks (no ```javascript, no ```python, no ```). Do NOT include any explanation before or after the code.' + } + + if (generationType === 'json-object') { + finalSystemPrompt += + '\n\nIMPORTANT: Return ONLY the raw JSON object. Do NOT wrap it in markdown code blocks (no ```json or ```). Do NOT include any explanation or text before or after the JSON. The response must start with { and end with }.' + } + + // Separate from json-object: that reinforcement demands braces, which would + // fight a field whose contract is an array. + if (generationType === 'json-array') { + finalSystemPrompt += + '\n\nIMPORTANT: Return ONLY the raw JSON array. Do NOT wrap it in markdown code blocks (no ```json or ```). Do NOT include any explanation or text before or after the JSON. The response must start with [ and end with ].' + } + + const messages: WandChatMessage[] = [ + { role: 'system', content: finalSystemPrompt }, + ...history.filter((msg) => msg.role !== 'system'), + { role: 'user', content: prompt }, + ] + + const headers: Record = { + 'Content-Type': 'application/json', + 'OpenAI-Beta': 'responses=v1', + } + + if (useWandAzure && azureApiKey) { + headers['api-key'] = azureApiKey + } else { + headers.Authorization = `Bearer ${activeOpenAIKey}` + } + + const requestBody = { + model: requestModel, + input: messages, + temperature: 0.2, + max_output_tokens: 10000, + } + + const recordWandUsage = (usage: WandUsage) => + updateUserStatsForWand(billingAttribution, usage, requestId, isBYOK) + + if (input.stream) { + let response: Response + try { + logger.info(`About to create stream with model: ${requestModel}`) + + response = await postWandRequest({ + method: 'POST', + headers, + body: JSON.stringify({ ...requestBody, stream: true }), + }) + + if (!response.ok) { + const errorText = await response.text() + logger.error('API request failed', { + status: response.status, + statusText: response.statusText, + error: errorText, + }) + throw new Error(`API request failed: ${response.status} ${response.statusText}`) + } + } catch (error) { + throw new WandStreamStartError(error) + } + + logger.info('Stream response received, starting processing') + return { kind: 'stream', body: relayResponsesStream(response, recordWandUsage) } + } + + const response = await postWandRequest({ + method: 'POST', + headers, + body: JSON.stringify(requestBody), + }) + + if (!response.ok) { + const errorText = await response.text() + throw new WandUpstreamError( + response.status, + `API request failed: ${response.status} ${response.statusText} - ${errorText}` + ) + } + + const completion: ResponsesCompletion = await response.json() + const generatedContent = extractResponseText(completion.output)?.trim() + + if (!generatedContent) { + logger.error(`${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} response was empty or invalid.`) + throw new WandEmptyResponseError() + } + + logger.info('Wand generation successful') + + const usage = parseResponsesUsage(completion.usage) + if (usage) { + await recordWandUsage({ + prompt_tokens: usage.promptTokens, + completion_tokens: usage.completionTokens, + total_tokens: usage.totalTokens, + }) + } + + return { kind: 'content', content: generatedContent } +} + +/** Generates for a workflow's editor, which requires write access to the workflow's workspace. */ +export const generateWandForWorkflow = defineAuthorizedWorkspaceUseCase({ + operation: wandOperations.generateForWorkflow, + resolveContext: async ({ input }: { input: GenerateWandInput & { workflowId: string } }) => + resolveActiveWorkspaceApplicationContext(await resolveWorkflowWorkspaceId(input.workflowId)), + authorizationOptions: {}, + execute: executeWandGeneration, +}) + +/** Generates with no workflow to resolve (a table schema, an unhydrated editor) for a workspace member. */ +export const generateWandInWorkspace = defineAuthorizedWorkspaceUseCase({ + operation: wandOperations.generateInWorkspace, + resolveContext: ({ input }: { input: GenerateWandInput & { workspaceId: string } }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: {}, + execute: executeWandGeneration, +}) diff --git a/apps/sim/lib/wand/application/operations.ts b/apps/sim/lib/wand/application/operations.ts new file mode 100644 index 00000000000..7e7fb9299c4 --- /dev/null +++ b/apps/sim/lib/wand/application/operations.ts @@ -0,0 +1,22 @@ +import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' + +export const wandOperations = { + /** Generating for a workflow's editor requires write access to the workflow's workspace. */ + generateForWorkflow: defineWorkspaceOperation({ + id: 'wand.generate_for_workflow', + minimumRole: 'write', + // permission-group-exempt: an editor assist billed to the workspace; it reads no protected resource the caller cannot already edit. + capability: 'none', + workspaceApiKey: 'deny', + principalKinds: ['session'], + }), + /** Generating with no workflow to resolve requires workspace membership. */ + generateInWorkspace: defineWorkspaceOperation({ + id: 'wand.generate_in_workspace', + minimumRole: 'read', + // permission-group-exempt: an editor assist billed to the workspace; it reads no protected resource the caller cannot already see. + capability: 'none', + workspaceApiKey: 'deny', + principalKinds: ['session'], + }), +} as const diff --git a/scripts/check-source-conventions.baseline.json b/scripts/check-source-conventions.baseline.json index 138033a0538..0129b4138ed 100644 --- a/scripts/check-source-conventions.baseline.json +++ b/scripts/check-source-conventions.baseline.json @@ -646,7 +646,6 @@ "apps/sim/app/api/v1/workflows/[id]/route.ts": 2, "apps/sim/app/api/v1/workflows/route.ts": 2, "apps/sim/app/api/v2/workflows/[workflowId]/execute/route.ts": 2, - "apps/sim/app/api/wand/route.ts": 2, "apps/sim/app/api/webhooks/[id]/route.ts": 2, "apps/sim/app/api/webhooks/agentmail/route.ts": 1, "apps/sim/app/api/webhooks/route.ts": 2, From 29e3fcaa8b9c49f5a240448bb4f4e175b8fdca9b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:01:37 -0800 Subject: [PATCH 25/32] chore(egress): scan the wand and provider-model use cases in check:organization-egress --- scripts/check-organization-egress.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/check-organization-egress.ts b/scripts/check-organization-egress.ts index 0edef0e48df..6bba384138a 100644 --- a/scripts/check-organization-egress.ts +++ b/scripts/check-organization-egress.ts @@ -44,7 +44,9 @@ const SCAN_DIRS = [ 'apps/sim/lib/managed-agents', 'apps/sim/lib/media', 'apps/sim/lib/oauth', + 'apps/sim/lib/provider-models', 'apps/sim/lib/selectors', + 'apps/sim/lib/wand', 'apps/sim/lib/webhooks', ] From adf7aeed87bd51a1037f28d24b825bf82a2daff1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:16:30 -0800 Subject: [PATCH 26/32] fix(egress): declare why the new operations are permission-group exempt and type the Slack provider test fixtures --- apps/sim/lib/credential-groups/slack-provider.test.ts | 3 +++ apps/sim/lib/provider-models/application/operations.ts | 2 +- apps/sim/lib/wand/application/operations.ts | 4 ++-- scripts/check-test-types.baseline.json | 3 --- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/sim/lib/credential-groups/slack-provider.test.ts b/apps/sim/lib/credential-groups/slack-provider.test.ts index 11e9277eef7..b52aac07a4b 100644 --- a/apps/sim/lib/credential-groups/slack-provider.test.ts +++ b/apps/sim/lib/credential-groups/slack-provider.test.ts @@ -122,6 +122,7 @@ describe('Slack member scope policy', () => { requiredScopes: policy.requiredScopes, redirectUri: 'https://sim.fixture.test/api/credential-groups/oauth/slack/callback', invitationToken: 'invitation', + userId: 'user-1', createdAt: Date.now(), }, }) @@ -161,6 +162,7 @@ describe('Slack member scope policy', () => { requiredScopes: policy.requiredScopes, redirectUri: 'https://sim.fixture.test/api/credential-groups/oauth/slack/callback', invitationToken: 'invitation', + userId: 'user-1', createdAt: Date.now(), }, }) @@ -212,6 +214,7 @@ describe('Slack member scope policy', () => { requiredScopes: policy.requiredScopes, redirectUri: 'https://sim.fixture.test/api/credential-groups/oauth/slack/callback', invitationToken: 'invitation', + userId: 'user-1', createdAt: Date.now(), }, }) diff --git a/apps/sim/lib/provider-models/application/operations.ts b/apps/sim/lib/provider-models/application/operations.ts index 0598cf0ad49..5c1170fcb4d 100644 --- a/apps/sim/lib/provider-models/application/operations.ts +++ b/apps/sim/lib/provider-models/application/operations.ts @@ -1,10 +1,10 @@ import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' export const providerModelOperations = { + // permission-group-exempt: lists the models a workspace's own provider key can reach; running a model is gated where it executes. list: defineWorkspaceOperation({ id: 'provider_models.list', minimumRole: 'read', - // permission-group-exempt: lists the models a workspace's own provider key can reach; running a model is gated where it executes. capability: 'none', workspaceApiKey: 'deny', principalKinds: ['session'], diff --git a/apps/sim/lib/wand/application/operations.ts b/apps/sim/lib/wand/application/operations.ts index 7e7fb9299c4..96d11b7daf5 100644 --- a/apps/sim/lib/wand/application/operations.ts +++ b/apps/sim/lib/wand/application/operations.ts @@ -2,19 +2,19 @@ import { defineWorkspaceOperation } from '@/lib/core/application/workspace-opera export const wandOperations = { /** Generating for a workflow's editor requires write access to the workflow's workspace. */ + // permission-group-exempt: an editor assist billed to the workspace; it reads no protected resource the caller cannot already edit. generateForWorkflow: defineWorkspaceOperation({ id: 'wand.generate_for_workflow', minimumRole: 'write', - // permission-group-exempt: an editor assist billed to the workspace; it reads no protected resource the caller cannot already edit. capability: 'none', workspaceApiKey: 'deny', principalKinds: ['session'], }), /** Generating with no workflow to resolve requires workspace membership. */ + // permission-group-exempt: an editor assist billed to the workspace; it reads no protected resource the caller cannot already see. generateInWorkspace: defineWorkspaceOperation({ id: 'wand.generate_in_workspace', minimumRole: 'read', - // permission-group-exempt: an editor assist billed to the workspace; it reads no protected resource the caller cannot already see. capability: 'none', workspaceApiKey: 'deny', principalKinds: ['session'], diff --git a/scripts/check-test-types.baseline.json b/scripts/check-test-types.baseline.json index e75a4e2540c..36e295e7ee2 100644 --- a/scripts/check-test-types.baseline.json +++ b/scripts/check-test-types.baseline.json @@ -1166,9 +1166,6 @@ "apps/sim/lib/credential-groups/slack-managed-users.test.ts": { "TS2339": 1 }, - "apps/sim/lib/credential-groups/slack-provider.test.ts": { - "TS2741": 2 - }, "apps/sim/lib/credential-groups/standard-oauth-provider.test.ts": { "TS2739": 1, "TS2741": 1 From 7f63ada4e1ede828e3eec88dc67e87684bd0aabf Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:26:55 -0800 Subject: [PATCH 27/32] improvement(egress): surface gateway failures as 503 and keep Sim-owned calls direct - wand route and v1 knowledge handleError answer 503 with the egress-unavailable message when an OutboundRoutingError is anywhere in the cause chain - buffer, firecrawl and fireflies internal operations rethrow OutboundRoutingError instead of wrapping it as a provider failure - Cerebras goes direct: a custom fetch replaces the SDK's node-fetch keep-alive transport and its 600s header budget - the Responses core takes a required fetch; OpenAI passes outboundFetch and Sim's env Azure deployment picks the global fetch explicitly - check:organization-egress reports a fetch imported from any module other than the outbound router - narrow organizationAwsRequestHandler to { http2 }, hoist the embeddings fetch choice, reword transport/fal/CLAUDE.md egress docs, shrink explicit-any baseline --- CLAUDE.md | 2 +- apps/sim/app/api/v1/knowledge/utils.test.ts | 16 ++++++- apps/sim/app/api/v1/knowledge/utils.ts | 5 ++ apps/sim/app/api/wand/route.test.ts | 48 +++++++++++++++++++ apps/sim/app/api/wand/route.ts | 7 ++- .../api/server/routes/internal-json-route.ts | 2 +- .../organization-aws-handler.server.ts | 24 ++++------ apps/sim/lib/embeddings/client.ts | 6 +-- apps/sim/lib/internal/buffer/execute-tool.ts | 2 + .../lib/internal/buffer/operations.test.ts | 20 ++++++++ apps/sim/lib/internal/buffer/operations.ts | 2 + .../lib/internal/firecrawl/operations.test.ts | 12 +++++ apps/sim/lib/internal/firecrawl/operations.ts | 2 + .../lib/internal/fireflies/operations.test.ts | 12 +++++ apps/sim/lib/internal/fireflies/operations.ts | 3 +- apps/sim/lib/tools/falai-pricing.ts | 6 ++- apps/sim/providers/azure-openai/index.test.ts | 4 +- apps/sim/providers/azure-openai/index.ts | 6 ++- apps/sim/providers/cerebras/index.ts | 5 +- apps/sim/providers/openai/core.ts | 11 ++--- apps/sim/providers/openai/index.ts | 2 + apps/sim/providers/transport.ts | 8 ++-- scripts/check-explicit-any.baseline.json | 2 - scripts/check-organization-egress.test.ts | 18 ++++++- scripts/check-organization-egress.ts | 41 ++++++++++++++-- 25 files changed, 219 insertions(+), 47 deletions(-) create mode 100644 apps/sim/app/api/wand/route.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 1fee41c7650..3a5b7acdf9a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,7 +60,7 @@ Build order **Tools** → **Block** → **Icon** → optional **Trigger**, from - **Tool IDs are `snake_case`** (`service_action`), registered in `tools/registry.ts`; blocks register alphabetically in `blocks/registry-maps.ts`. - **Type coercions go in `tools.config.params`** (runs at execution), never `tools.config.tool` (runs at serialization, where `Number()` destroys `` references). -- **Requests made for a workspace use `outboundFetch`** (`@/lib/core/network/outbound-fetch.server`), never the global `fetch`, so an organization with a dedicated egress gateway keeps its static IPs; AWS SDK clients signing with customer credentials take `organizationAwsRequestHandler()`. Sim's own infrastructure stays direct (`check:organization-egress`). +- **Requests made for a workspace use `outboundFetch`** (`@/lib/core/network/outbound-fetch.server`), never the global `fetch` or another imported `fetch`, so an organization with a dedicated egress gateway keeps its static IPs; `check:organization-egress` enforces this, and a request that rightly goes direct (Sim's own infrastructure, an operator-local server) takes `// egress-direct: `. AWS SDK clients signing with customer credentials take `organizationAwsRequestHandler()`: no audit catches a missing one, so reviewers check it. - A surface listing integrations to a viewer reads `useIntegrationAccess`, never a raw catalog (`check:integration-access`). - Table column types are registry entries in `apps/sim/lib/table/column-types/`; never add a per-type `case` elsewhere except the one documented import-path switch (`coerceValue` in `lib/table/import.ts`), which every new type extends: a missed arm fails silently. Use `/add-column-type`. diff --git a/apps/sim/app/api/v1/knowledge/utils.test.ts b/apps/sim/app/api/v1/knowledge/utils.test.ts index b864363be78..6a46dbe25a3 100644 --- a/apps/sim/app/api/v1/knowledge/utils.test.ts +++ b/apps/sim/app/api/v1/knowledge/utils.test.ts @@ -10,7 +10,8 @@ vi.mock('@/lib/knowledge/access/scope', () => knowledgeAccessScopeMock) vi.mock('@/lib/knowledge/service', () => knowledgeServiceMock) vi.mock('@/app/api/v1/middleware', () => v1MiddlewareMock) -import { resolveV1KnowledgeReadAccess } from '@/app/api/v1/knowledge/utils' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { handleError, resolveV1KnowledgeReadAccess } from '@/app/api/v1/knowledge/utils' const createUserProvider = knowledgeAccessScopeMockFns.mockCreateUserKnowledgeAccessProvider @@ -34,3 +35,16 @@ describe('v1 knowledge reader identity', () => { expect(createUserProvider).not.toHaveBeenCalled() }) }) + +describe('v1 knowledge error projection', () => { + it('answers 503 when the organization egress route fails under a wrapping error', async () => { + const error = new Error('Search failed', { + cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + }) + const response = handleError('req-1', error, 'Failed to search') + expect(response.status).toBe(503) + await expect(response.json()).resolves.toEqual({ + error: "The organization's egress route is unavailable", + }) + }) +}) diff --git a/apps/sim/app/api/v1/knowledge/utils.ts b/apps/sim/app/api/v1/knowledge/utils.ts index abeb501818c..83378c16e7e 100644 --- a/apps/sim/app/api/v1/knowledge/utils.ts +++ b/apps/sim/app/api/v1/knowledge/utils.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' import { validationErrorResponseFromError } from '@/lib/api/server' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { createUserKnowledgeAccessProvider, WORKSPACE_ACCESS_SCOPE, @@ -123,5 +124,9 @@ export function handleError( } logger.error(`${defaultMessage}:`, error, { requestId }) + const orchestrationError = asOrchestrationError(error) + if (orchestrationError?.code === 'unavailable') { + return NextResponse.json({ error: orchestrationError.message }, { status: 503 }) + } return NextResponse.json({ error: defaultMessage }, { status: 500 }) } diff --git a/apps/sim/app/api/wand/route.test.ts b/apps/sim/app/api/wand/route.test.ts new file mode 100644 index 00000000000..29dd5ccb819 --- /dev/null +++ b/apps/sim/app/api/wand/route.test.ts @@ -0,0 +1,48 @@ +/** + * @vitest-environment node + */ +import { createMockRequest, createRouteContext } from '@sim/testing' +import { authMockFns } from '@sim/testing/mocks/auth.mock' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGenerateInWorkspace } = vi.hoisted(() => ({ mockGenerateInWorkspace: vi.fn() })) + +vi.mock('@/lib/wand/application/generate', async (importOriginal) => ({ + ...(await importOriginal()), + generateWandInWorkspace: { execute: mockGenerateInWorkspace }, +})) + +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { WandStreamStartError } from '@/lib/wand/application/generate' +import { POST } from '@/app/api/wand/route' + +const EGRESS_UNAVAILABLE = "The organization's egress route is unavailable" + +function wandRequest(stream: boolean) { + return createMockRequest('POST', { prompt: 'Write a cron', workspaceId: 'ws-1', stream }) +} + +describe('wand route egress failures', () => { + beforeEach(() => { + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: 'user-1' }, + session: { id: 'session-1' }, + }) + }) + + it('answers 503 when the gateway fails a non-streaming generation', async () => { + mockGenerateInWorkspace.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const response = await POST(wandRequest(false), createRouteContext({})) + expect(response.status).toBe(503) + await expect(response.json()).resolves.toEqual({ success: false, error: EGRESS_UNAVAILABLE }) + }) + + it('answers 503 when the gateway fails to open a stream', async () => { + mockGenerateInWorkspace.mockRejectedValueOnce( + new WandStreamStartError(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + ) + const response = await POST(wandRequest(true), createRouteContext({})) + expect(response.status).toBe(503) + await expect(response.json()).resolves.toEqual({ success: false, error: EGRESS_UNAVAILABLE }) + }) +}) diff --git a/apps/sim/app/api/wand/route.ts b/apps/sim/app/api/wand/route.ts index 31eaa8525e0..a4b11e4ee0f 100644 --- a/apps/sim/app/api/wand/route.ts +++ b/apps/sim/app/api/wand/route.ts @@ -3,7 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { wandGenerateContract } from '@/lib/api/contracts' import { parseRequest } from '@/lib/api/server' import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' -import { OrchestrationError } from '@/lib/core/orchestration/types' +import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { @@ -145,6 +145,11 @@ export const POST = withRouteHandler(async (req: NextRequest) => { }, }) } catch (error) { + const orchestrationError = asOrchestrationError(error) + if (orchestrationError?.code === 'unavailable') { + logger.error('Wand generation unavailable', { ...errorDetails(error), ...WAND_DEPLOYMENT }) + return wandError(orchestrationError.message, 503) + } return (target && refusalResponse(error, target)) || generationFailedResponse(error) } }) diff --git a/apps/sim/lib/api/server/routes/internal-json-route.ts b/apps/sim/lib/api/server/routes/internal-json-route.ts index 87cb9b4008e..ac9f996b932 100644 --- a/apps/sim/lib/api/server/routes/internal-json-route.ts +++ b/apps/sim/lib/api/server/routes/internal-json-route.ts @@ -62,7 +62,7 @@ export const internalSessionAuth = { }, } as const -/** The session principal when the request is signed in, otherwise `null`, for routes that also serve anonymous callers. */ +/** The session principal when the request is signed in; returns `null` instead of throwing when unauthenticated, so the route answers with its own response. */ export async function authenticateOptionalInternalSession(): Promise { try { return await internalSessionAuth.authenticate() diff --git a/apps/sim/lib/core/network/organization-aws-handler.server.ts b/apps/sim/lib/core/network/organization-aws-handler.server.ts index 2141e3a197f..d34232863c3 100644 --- a/apps/sim/lib/core/network/organization-aws-handler.server.ts +++ b/apps/sim/lib/core/network/organization-aws-handler.server.ts @@ -6,11 +6,7 @@ import { LRUCache } from 'lru-cache' import { isOutboundRoutingEnabled, type OutboundGateway } from '@/lib/core/network/config.server' import { connectThroughGateway } from '@/lib/core/network/gateway.server' import { resolveOrganizationRoute } from '@/lib/core/network/outbound-fetch.server' -import { - AGENT_OPTIONS, - type AwsRequestHandlerOptions, - awsRequestHandler, -} from '@/lib/core/network/proxy-agent.server' +import { AGENT_OPTIONS, awsRequestHandler } from '@/lib/core/network/proxy-agent.server' type AwsHttpHandler = NodeHttpHandler | NodeHttp2Handler type HandleArguments = Parameters @@ -118,19 +114,17 @@ class OrganizationRoutedHandler { * Pass `http2` for a client whose SDK default is HTTP/2 (Bedrock Runtime) so direct requests keep * it. Never use it for Sim's own AWS usage, which must not leave through a customer's gateway. */ -export function organizationAwsRequestHandler( - options?: AwsRequestHandlerOptions & { http2?: boolean } -): ReturnType | OrganizationRoutedHandler { - const { http2, ...rest } = options ?? {} - // No options stays `undefined`, so the client keeps its SDK default handler and settings. - const handlerOptions = Object.keys(rest).length > 0 ? rest : undefined - const configured = awsRequestHandler(handlerOptions) +export function organizationAwsRequestHandler(options?: { + http2?: boolean +}): ReturnType | OrganizationRoutedHandler { + // `undefined` without an environment proxy, so the client keeps its SDK default handler. + const configured = awsRequestHandler() if (!isOutboundRoutingEnabled()) return configured const direct = configured instanceof NodeHttpHandler ? configured - : http2 - ? new NodeHttp2Handler({ ...handlerOptions, disableConcurrentStreams: true }) - : new NodeHttpHandler(handlerOptions) + : options?.http2 + ? new NodeHttp2Handler({ disableConcurrentStreams: true }) + : new NodeHttpHandler() return new OrganizationRoutedHandler(direct) } diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index 2f51e66ba31..9b255e95b1c 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -544,6 +544,9 @@ async function callEmbeddingAPI( ): Promise<{ embeddings: number[][]; totalTokens: number; dimensions: number }> { const admissionWaitMs = bulk ? KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS : EMBEDDING_RETRY_BUDGET_MS const admissionIdentity = embeddingAdmissionIdentity({ providerId, quotaCircuitIdentity, isBYOK }) + const operatorOwned = providerId === 'ollama' || providerId === 'azure-openai' + // egress-direct: Ollama and the Azure OpenAI override are the operator's own deployments + const requestFetch = operatorOwned ? fetch : outboundFetch return retryWithExponentialBackoff( async (operationSignal, deadlineAt) => { if (await isEmbeddingQuotaCircuitOpen(admissionIdentity)) { @@ -582,9 +585,6 @@ async function callEmbeddingAPI( const timeout = setTimeout(() => controller.abort(), EMBEDDING_REQUEST_TIMEOUT_MS) try { - const operatorOwned = providerId === 'ollama' || providerId === 'azure-openai' - // egress-direct: Ollama and the Azure OpenAI override are the operator's own deployments - const requestFetch = operatorOwned ? fetch : outboundFetch const response = await requestFetch(request.apiUrl, { method: 'POST', headers: request.headers, diff --git a/apps/sim/lib/internal/buffer/execute-tool.ts b/apps/sim/lib/internal/buffer/execute-tool.ts index 7018e177aef..d2baf8a3983 100644 --- a/apps/sim/lib/internal/buffer/execute-tool.ts +++ b/apps/sim/lib/internal/buffer/execute-tool.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '@sim/utils/errors' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { BufferOperationError } from '@/lib/internal/buffer/errors' import { bufferCreatePostInputSchema, bufferEditPostInputSchema } from '@/lib/internal/buffer/input' import { createBufferPost, editBufferPost } from '@/lib/internal/buffer/operations' @@ -33,6 +34,7 @@ export const executeBufferTool: InternalToolOperationHandler = async (request) = ) } catch (error) { request.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error if (error instanceof BufferOperationError) { return Response.json({ success: false, error: error.message }, { status: error.status }) } diff --git a/apps/sim/lib/internal/buffer/operations.test.ts b/apps/sim/lib/internal/buffer/operations.test.ts index 22cf5ca9d1f..cd632b503c9 100644 --- a/apps/sim/lib/internal/buffer/operations.test.ts +++ b/apps/sim/lib/internal/buffer/operations.test.ts @@ -8,11 +8,31 @@ vi.mock('@/lib/uploads/utils/file.server', () => fileUtilsServerMock) const { mockResolveFileInputToUrl } = fileUtilsServerMockFns +import { OutboundRoutingError } from '@/lib/core/network/routing' import { executeBufferTool } from '@/lib/internal/buffer/execute-tool' import { createBufferPost } from '@/lib/internal/buffer/operations' import bufferPostFixture from '@/tools/buffer/__fixtures__/post.json' describe('Buffer operations', () => { + it('surfaces an egress routing failure instead of a provider error', async () => { + vi.spyOn(globalThis, 'fetch').mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + executeBufferTool({ + toolId: 'buffer_create_post', + input: { + apiKey: 'buffer-key', + channelId: 'channel-1', + mode: 'addToQueue', + text: 'Caption', + }, + headers: new Headers(), + context: { workflowId: 'workflow-1', userId: 'user-1' }, + requestId: 'request-1', + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('creates a text post when optional assets are null', async () => { const providerInputs: Record[] = [] vi.spyOn(globalThis, 'fetch').mockImplementation(async (_input, init) => { diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts index aaff6075ea6..f98af4f57fd 100644 --- a/apps/sim/lib/internal/buffer/operations.ts +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, @@ -172,6 +173,7 @@ async function executePostMutation(args: { result = toRecord(candidate) } catch (error) { context.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error const message = getErrorMessage(error, 'Buffer API request failed') logger.error('Buffer post mutation failed', { requestId: context.requestId, error: message }) throw new BufferOperationError(message, 502) diff --git a/apps/sim/lib/internal/firecrawl/operations.test.ts b/apps/sim/lib/internal/firecrawl/operations.test.ts index aca3d12c46f..4749977f319 100644 --- a/apps/sim/lib/internal/firecrawl/operations.test.ts +++ b/apps/sim/lib/internal/firecrawl/operations.test.ts @@ -11,6 +11,7 @@ import { workspaceFileSecretProvenanceMockFns, } from '@sim/testing/mocks/workspace-file-secret-provenance.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' import { RESOLVED_SECRET_PROVENANCE_FIELD, @@ -91,4 +92,15 @@ describe('executeFirecrawlParse', () => { expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() expect(fetch).not.toHaveBeenCalled() }) + + it('surfaces an egress routing failure instead of a parse error', async () => { + vi.mocked(fetch).mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + executeFirecrawlParse( + { apiKey: 'firecrawl-key', file: FILE, options: { formats: ['markdown'] } }, + createContext() + ) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) }) diff --git a/apps/sim/lib/internal/firecrawl/operations.ts b/apps/sim/lib/internal/firecrawl/operations.ts index efde8ac77f9..d016e90a5b9 100644 --- a/apps/sim/lib/internal/firecrawl/operations.ts +++ b/apps/sim/lib/internal/firecrawl/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES, isPayloadSizeLimitError, @@ -120,6 +121,7 @@ export async function executeFirecrawlParse( }) } catch (error) { context.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error const notReady = docNotReadyResponse(error) if (notReady) return notReady logger.error('Firecrawl parse failed', { diff --git a/apps/sim/lib/internal/fireflies/operations.test.ts b/apps/sim/lib/internal/fireflies/operations.test.ts index b4d305fe1d1..49b66bc1a00 100644 --- a/apps/sim/lib/internal/fireflies/operations.test.ts +++ b/apps/sim/lib/internal/fireflies/operations.test.ts @@ -3,6 +3,7 @@ import { fileUtilsServerMockFns, } from '@sim/testing/mocks/file-utils-server.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' import { RESOLVED_SECRET_PROVENANCE_FIELD, @@ -71,4 +72,15 @@ describe('executeFirefliesUploadAudio', () => { }) ) }) + + it('surfaces an egress routing failure instead of an upload error', async () => { + vi.mocked(fetch).mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + executeFirefliesUploadAudio( + { apiKey: 'fireflies-key', audioUrl: 'https://media.example.com/audio.mp3' }, + createContext() + ) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) }) diff --git a/apps/sim/lib/internal/fireflies/operations.ts b/apps/sim/lib/internal/fireflies/operations.ts index 40464b27222..f4ca58d64f9 100644 --- a/apps/sim/lib/internal/fireflies/operations.ts +++ b/apps/sim/lib/internal/fireflies/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' import type { FirefliesUploadAudioInput } from '@/lib/internal/fireflies/schema' @@ -118,7 +119,7 @@ export async function executeFirefliesUploadAudio( }) return Response.json(data, { status: response.status }) } catch (error) { - if (context.signal?.aborted) throw error + if (context.signal?.aborted || error instanceof OutboundRoutingError) throw error logger.error('Fireflies upload failed', { requestId: context.requestId, error: getErrorMessage(error, 'Unknown error'), diff --git a/apps/sim/lib/tools/falai-pricing.ts b/apps/sim/lib/tools/falai-pricing.ts index ef362e6a789..a4bb1dc2611 100644 --- a/apps/sim/lib/tools/falai-pricing.ts +++ b/apps/sim/lib/tools/falai-pricing.ts @@ -151,7 +151,8 @@ async function fetchFalAIBillingEvent( let response: Response try { - // egress-direct: Sim's own fal billing account, not a request for the workspace + // egress-direct: meters Sim's own fal billing account; the generation call itself is the + // workspace's request and is routed whichever key it carries response = await fetch(url, { headers: { Authorization: `Key ${apiKey}`, @@ -200,7 +201,8 @@ async function estimateFalAICallCost( ): Promise<{ costDollars?: number; error?: string }> { let response: Response try { - // egress-direct: Sim's own fal billing account, not a request for the workspace + // egress-direct: meters Sim's own fal billing account; the generation call itself is the + // workspace's request and is routed whichever key it carries response = await fetch('https://api.fal.ai/v1/models/pricing/estimate', { method: 'POST', headers: { diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index d68dbc3a97d..252adef6e3e 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -121,14 +121,14 @@ describe('azureOpenAIProvider — SSRF pinning', () => { expect(responsesConfig().fetch).toBe(sentinelFetch) }) - it('passes no custom fetch when the endpoint comes from trusted server env', async () => { + it("sends Sim's trusted env endpoint direct, not through the organization router", async () => { setEnv({ AZURE_OPENAI_ENDPOINT: 'https://trusted.openai.azure.com' }) await azureOpenAIProvider.executeRequest(request({ azureEndpoint: undefined })) expect(mockValidate).not.toHaveBeenCalled() expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(responsesConfig().fetch).toBeUndefined() + expect(responsesConfig().fetch).toBe(globalThis.fetch) }) it.each([false, true])( diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index 37db6e20761..307accdd16c 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -767,6 +767,8 @@ export const azureOpenAIProvider: ProviderConfig = { } pinnedFetch = createPinnedFetch(validation.resolvedIP, { profile: 'configuredEndpoint' }) } + // egress-direct: without a user endpoint this is Sim's own Azure deployment + const responsesFetch = pinnedFetch ?? fetch const apiKey = request.apiKey if (!apiKey) { @@ -826,7 +828,7 @@ export const azureOpenAIProvider: ProviderConfig = { 'api-key': apiKey, }, logger, - fetch: pinnedFetch, + fetch: responsesFetch, }) } @@ -848,7 +850,7 @@ export const azureOpenAIProvider: ProviderConfig = { 'api-key': apiKey, }, logger, - fetch: pinnedFetch, + fetch: responsesFetch, }) }, } diff --git a/apps/sim/providers/cerebras/index.ts b/apps/sim/providers/cerebras/index.ts index f0b42f99819..efb04cb21c1 100644 --- a/apps/sim/providers/cerebras/index.ts +++ b/apps/sim/providers/cerebras/index.ts @@ -2,7 +2,6 @@ import { Cerebras } from '@cerebras/cerebras_cloud_sdk' import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -68,10 +67,12 @@ export const cerebrasProvider: ProviderConfig = { const providerStartTimeISO = new Date(providerStartTime).toISOString() try { + // Not routed through the organization gateway: this SDK's default transport is node-fetch + // on a keep-alive agent it stretches to the 600s header budget, and any custom `fetch` + // replaces both with the global fetch, cutting that budget for every organization. const client = new Cerebras({ apiKey: request.apiKey, ...openAICompatTransport(), - fetch: outboundFetch, }) const allMessages = [] diff --git a/apps/sim/providers/openai/core.ts b/apps/sim/providers/openai/core.ts index 9691445bace..2e3df6a39ea 100644 --- a/apps/sim/providers/openai/core.ts +++ b/apps/sim/providers/openai/core.ts @@ -4,7 +4,6 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import type OpenAI from 'openai' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { @@ -142,11 +141,11 @@ export interface ResponsesProviderConfig { headers: Record logger: Logger /** - * Optional fetch implementation. Used to pin the connection to a pre-validated - * IP (DNS-rebinding/SSRF protection) when the endpoint is user-supplied. - * Defaults to {@link outboundFetch}. + * Fetch implementation for the request, chosen by the caller: `outboundFetch` for a vendor + * called on the workspace's behalf, a pinned fetch for a user-supplied endpoint + * (DNS-rebinding/SSRF protection), or the global `fetch` for Sim's own deployment. */ - fetch?: typeof fetch + fetch: typeof fetch } /** @@ -157,7 +156,7 @@ export async function executeResponsesProviderRequest( config: ResponsesProviderConfig ): Promise { const { logger } = config - const fetchImpl = config.fetch ?? outboundFetch + const fetchImpl = config.fetch logger.info(`Preparing ${config.providerLabel} request`, { model: request.model, diff --git a/apps/sim/providers/openai/index.ts b/apps/sim/providers/openai/index.ts index db95598e8c7..b85560709db 100644 --- a/apps/sim/providers/openai/index.ts +++ b/apps/sim/providers/openai/index.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types' @@ -33,6 +34,7 @@ export const openaiProvider: ProviderConfig = { 'OpenAI-Beta': 'responses=v1', }, logger, + fetch: outboundFetch, }) }, } diff --git a/apps/sim/providers/transport.ts b/apps/sim/providers/transport.ts index d882cfe9936..fec962eaee4 100644 --- a/apps/sim/providers/transport.ts +++ b/apps/sim/providers/transport.ts @@ -54,9 +54,11 @@ export interface OpenAICompatTransport { * what makes them safe to set in a constructor that {@link getCachedProviderClient} * memoises — anything varying per request must be passed per call instead. * - * Deliberately stamps no `fetch`: a wrapper would turn the SDK's header-only timer - * into a total deadline that truncates a live stream, and would drop the caller's - * abort signal. + * Stamps no `fetch`. A wrapper that adds its own deadline would turn the SDK's + * header-only timer into a total one that truncates a live stream, and could drop the + * caller's abort signal. A pass-through router such as `outboundFetch` does neither and + * is fine, but providers add it per constructor rather than here: operator-local + * providers (Ollama, vLLM) share this transport and must stay direct. */ export const openAICompatTransport = (): OpenAICompatTransport => ({ timeout: PROVIDER_HEADERS_TIMEOUT_MS, diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 87291fb3530..6f49965e3a1 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -21,7 +21,6 @@ "apps/sim/app/api/schedules/route.test.ts": 1, "apps/sim/app/api/v1/logs/[id]/route.ts": 2, "apps/sim/app/api/v1/logs/route.ts": 3, - "apps/sim/app/api/wand/route.ts": 8, "apps/sim/app/api/webhooks/[id]/route.ts": 1, "apps/sim/app/api/webhooks/route.ts": 7, "apps/sim/app/api/workflows/[id]/execute/route.ts": 11, @@ -1132,7 +1131,6 @@ "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/pull/route.ts": 1, "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/push/route.ts": 1, "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/route.ts": 1, - "apps/sim/app/api/wand/route.ts": 1, "apps/sim/app/api/workflows/[id]/execute/route.test.ts": 5, "apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.test.ts": 1, "apps/sim/app/credential-groups/enroll/[token]/page.test.tsx": 1, diff --git a/scripts/check-organization-egress.test.ts b/scripts/check-organization-egress.test.ts index 2bc072e59c9..2a54a952936 100644 --- a/scripts/check-organization-egress.test.ts +++ b/scripts/check-organization-egress.test.ts @@ -34,8 +34,22 @@ new Anthropic({ fetch: outboundFetch })`) expect( lines(`class Client { async fetch(path: string) { return this.fetch(path) } } await client.fetch(url) -async function run(fetch: typeof globalThis.fetch) { return fetch(url) } -import { fetch } from 'undici' +async function run(fetch: typeof globalThis.fetch) { return fetch(url) }`) + ).toEqual([]) + }) + + it('reports a fetch imported from anywhere but the outbound router, at its import', () => { + expect( + lines(`import { fetch } from 'undici' +await fetch(url) +import nodeFetch, { fetch as undiciFetch } from 'undici' +import fetch from 'node-fetch'`) + ).toEqual([1, 3, 4]) + }) + + it('accepts outboundFetch imported under the name fetch', () => { + expect( + lines(`import { outboundFetch as fetch } from '@/lib/core/network/outbound-fetch.server' await fetch(url)`) ).toEqual([]) }) diff --git a/scripts/check-organization-egress.ts b/scripts/check-organization-egress.ts index 6bba384138a..8b543076bc4 100644 --- a/scripts/check-organization-egress.ts +++ b/scripts/check-organization-egress.ts @@ -14,7 +14,9 @@ * clients, tools, webhooks, providers and connectors), a call to the global `fetch` fails this * check. A call that rightly goes direct (an operator-local server such as Ollama, or a public * catalog read with no workspace behind it) takes `// egress-direct: ` on one of the two - * lines above it. A method named `fetch` on a client class is not the global and is ignored. + * lines above it. A method named `fetch` on a client class is not the global and is ignored. A + * `fetch` imported from another module (undici, node-fetch) bypasses the gateway just the same and + * is reported at its import; only `outboundFetch` may be imported under that name. * * Parsed with the TypeScript AST, so `fetch` inside strings, comments and tool descriptions is * never reported. The tree was swept when this landed, so there is no baseline. @@ -61,6 +63,8 @@ const SKIP_DIRS = new Set([ ]) const ANNOTATION = '// egress-direct:' const ANNOTATION_REACH = 2 +/** The only module a binding of `fetch` may be imported from: it is the router itself. */ +const ROUTED_FETCH_MODULE = '@/lib/core/network/outbound-fetch.server' export interface DirectFetch { line: number @@ -163,6 +167,34 @@ function refersToGlobalFetch(node: ts.Identifier): boolean { return !shadowsGlobal(node, 'fetch') } +/** + * Import bindings of a `fetch` from anywhere but {@link ROUTED_FETCH_MODULE}, such as undici's + * or node-fetch's: each is the global's bypass under another name, whatever it is called locally. + * Uses of the binding are then local and are not reported again. + */ +function importedFetchBindings(declaration: ts.ImportDeclaration): ts.Node[] { + const clause = declaration.importClause + if (!clause || clause.isTypeOnly) return [] + if ( + ts.isStringLiteral(declaration.moduleSpecifier) && + declaration.moduleSpecifier.text === ROUTED_FETCH_MODULE + ) { + return [] + } + const bindings: ts.Node[] = [] + if (clause.name?.text === 'fetch') bindings.push(clause.name) + const named = clause.namedBindings + if (named && ts.isNamespaceImport(named) && named.name.text === 'fetch') bindings.push(named) + if (named && ts.isNamedImports(named)) { + for (const element of named.elements) { + if (element.isTypeOnly) continue + const imported = element.propertyName ?? element.name + if (imported.text === 'fetch' || element.name.text === 'fetch') bindings.push(element) + } + } + return bindings +} + /** * Every reference to the global `fetch` (a call, an alias, `globalThis.fetch`, `window['fetch']`) * without an `egress-direct` annotation on one of the two lines above it. @@ -182,8 +214,11 @@ export function findDirectFetches(file: string, source: string): DirectFetch[] { } const visit = (node: ts.Node): void => { - if (ts.isIdentifier(node) && node.text === 'fetch' && refersToGlobalFetch(node)) report(node) - else if ( + if (ts.isImportDeclaration(node)) { + for (const binding of importedFetchBindings(node)) report(binding) + } else if (ts.isIdentifier(node) && node.text === 'fetch' && refersToGlobalFetch(node)) { + report(node) + } else if ( (ts.isPropertyAccessExpression(node) && node.name.text === 'fetch') || (ts.isElementAccessExpression(node) && ts.isStringLiteralLike(node.argumentExpression) && From 0e8b47ddf5e717f4fc5ffe013b441b7310aa0230 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:28:19 -0800 Subject: [PATCH 28/32] improvement(egress): propagate egress outages and route remaining workspace vendor calls - Salesforce identity lookup rethrows OutboundRoutingError instead of reporting an unverifiable account; the account.create.after lookup runs in the connect draft's outbound scope and is skipped when the before hook stored the instance. - GitHub installation read admission runs in the knowledge base owner's outbound scope with scopedOutboundFetch and fails on an unavailable route instead of denying the source. - Google Drive/Admin SDK connector fetches use scopedOutboundFetch. - GCS and BigQuery drains mint service-account tokens through outboundFetch via gaxios fetchImplementation (shared createServiceAccountJwt). --- .../google-drive/google-drive-errors.ts | 7 +-- .../google-drive/workspace-drives.test.ts | 18 +++++++ apps/sim/lib/auth/auth.ts | 40 ++++++++++---- .../lib/data-drains/destinations/bigquery.ts | 12 ++--- apps/sim/lib/data-drains/destinations/gcs.ts | 12 ++--- .../destinations/google-jwt.test.ts | 53 +++++++++++++++++++ .../data-drains/destinations/google-jwt.ts | 17 ++++++ .../access/github-installation.test.ts | 33 +++++++++++- .../knowledge/access/github-installation.ts | 28 +++++++--- .../oauth/salesforce-instance.server.test.ts | 31 +++++++++++ .../lib/oauth/salesforce-instance.server.ts | 10 +++- 11 files changed, 220 insertions(+), 41 deletions(-) create mode 100644 apps/sim/lib/data-drains/destinations/google-jwt.test.ts create mode 100644 apps/sim/lib/data-drains/destinations/google-jwt.ts create mode 100644 apps/sim/lib/oauth/salesforce-instance.server.test.ts diff --git a/apps/sim/connectors/google-drive/google-drive-errors.ts b/apps/sim/connectors/google-drive/google-drive-errors.ts index 377a949cc2f..41ae12d4fbb 100644 --- a/apps/sim/connectors/google-drive/google-drive-errors.ts +++ b/apps/sim/connectors/google-drive/google-drive-errors.ts @@ -1,4 +1,4 @@ -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' import { attachRetryHeaders, isRetryableError, @@ -160,7 +160,8 @@ export async function readGoogleDriveApiError( * Fetches a Google API, retrying errors whose structured body identifies a * transient rejection. Shared by every Google call a connector makes — Drive * and the Admin SDK use the same error envelope and the same rate-limit - * reasons. + * reasons. Requires an outbound scope: a lost scope fails closed instead of + * going direct. */ export async function fetchGoogleDriveWithRetry( url: string, @@ -170,7 +171,7 @@ export async function fetchGoogleDriveWithRetry( ): Promise { return retryWithExponentialBackoff( async () => { - const response = await outboundFetch(url, options) + const response = await scopedOutboundFetch(url, options) if (response.ok) return response const error = await readGoogleDriveApiError(response, operation) diff --git a/apps/sim/connectors/google-drive/workspace-drives.test.ts b/apps/sim/connectors/google-drive/workspace-drives.test.ts index 8a49c131304..d26b1181fa6 100644 --- a/apps/sim/connectors/google-drive/workspace-drives.test.ts +++ b/apps/sim/connectors/google-drive/workspace-drives.test.ts @@ -1,7 +1,11 @@ +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import type { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' import { listGoogleWorkspaceDrives } from '@/connectors/google-drive/workspace-drives' +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + const mockFetch = vi.fn() const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { @@ -16,9 +20,23 @@ beforeEach(() => { afterEach(() => { vi.useRealTimers() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) }) describe('Google Workspace shared-drive enumeration', () => { + it('refuses to reach Google outside an outbound scope when egress routing is on', async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (organizationId) => { + if (organizationId === undefined) throw new OutboundRoutingError('MISSING_SCOPE') + return { kind: 'direct' } + }) + await expect(listGoogleWorkspaceDrives('token')).rejects.toMatchObject({ + code: 'MISSING_SCOPE', + }) + expect(mockFetch).not.toHaveBeenCalled() + }) + it('returns one bounded page using only the delegated user token', async () => { const controller = new AbortController() mockFetch.mockResolvedValueOnce( diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index abf0a012663..2e959523a29 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -174,7 +174,11 @@ import { MICROSOFT_DATAVERSE_PROVIDER_ID, } from '@/lib/oauth/microsoft-dataverse' import { clearOAuthRefreshDeadFlag } from '@/lib/oauth/refresh-coordination' -import { isSalesforceOAuthProviderId, withSalesforceInstanceScope } from '@/lib/oauth/salesforce' +import { + extractSalesforceInstanceUrl, + isSalesforceOAuthProviderId, + withSalesforceInstanceScope, +} from '@/lib/oauth/salesforce' import { fetchSalesforceInstanceUrl } from '@/lib/oauth/salesforce-instance.server' import { extractSlackTeamId, fanOutSlackTokenChain } from '@/lib/oauth/slack' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' @@ -265,6 +269,20 @@ function resolveStripeBilling() { const stripeBilling = resolveStripeBilling() +/** + * Resolves a connecting Salesforce token's instance from the egress route of the workspace or + * organization whose connect draft started the callback. + */ +async function fetchConnectingSalesforceInstanceUrl(params: { + draftId: string | undefined + userId: string + providerId: string + accessToken: string +}): Promise { + const inDraftScope = await resolveConnectDraftOutboundScope(params) + return inDraftScope(() => fetchSalesforceInstanceUrl(params.providerId, params.accessToken)) +} + /** * A fresh Slack connect re-issues the installation's rotating token chain, * invalidating the copies held by sibling account rows for the same team (Slack @@ -512,15 +530,12 @@ export const auth = betterAuth({ } if (account.accessToken && isSalesforceOAuthProviderId(account.providerId)) { - const { accessToken } = account - const inDraftScope = await resolveConnectDraftOutboundScope({ + const instanceUrl = await fetchConnectingSalesforceInstanceUrl({ draftId: reconnectDraftId, userId: account.userId, providerId: account.providerId, + accessToken: account.accessToken, }) - const instanceUrl = await inDraftScope(() => - fetchSalesforceInstanceUrl(account.providerId, accessToken) - ) if (instanceUrl) { modifiedAccount.scope = withSalesforceInstanceScope(instanceUrl, account.scope) } @@ -727,11 +742,14 @@ export const auth = betterAuth({ updates.accessTokenExpiresAt = new Date(Date.now() + 2 * 60 * 60 * 1000) } - if (account.accessToken) { - const instanceUrl = await fetchSalesforceInstanceUrl( - account.providerId, - account.accessToken - ) + // The before hook already stored the instance when its lookup succeeded. + if (account.accessToken && !extractSalesforceInstanceUrl(account.scope)) { + const instanceUrl = await fetchConnectingSalesforceInstanceUrl({ + draftId: credentialDraftBinding?.draftId, + userId: account.userId, + providerId: account.providerId, + accessToken: account.accessToken, + }) if (instanceUrl) { updates.scope = withSalesforceInstanceScope(instanceUrl, account.scope) } diff --git a/apps/sim/lib/data-drains/destinations/bigquery.ts b/apps/sim/lib/data-drains/destinations/bigquery.ts index 6e1cf38fe8a..80d4a174516 100644 --- a/apps/sim/lib/data-drains/destinations/bigquery.ts +++ b/apps/sim/lib/data-drains/destinations/bigquery.ts @@ -2,11 +2,11 @@ import { createHash } from 'node:crypto' import { createLogger } from '@sim/logger' import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' -import { JWT } from 'google-auth-library' +import type { JWT } from 'google-auth-library' import { z } from 'zod' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { createServiceAccountJwt } from '@/lib/data-drains/destinations/google-jwt' import { - type ParsedServiceAccount, parseNdjsonObjects, parseServiceAccount, refineServiceAccountJson, @@ -75,10 +75,6 @@ const bigqueryCredentialsSchema = z export type BigQueryDestinationConfig = z.infer export type BigQueryDestinationCredentials = z.infer -function buildJwt(account: ParsedServiceAccount): JWT { - return new JWT({ email: account.clientEmail, key: account.privateKey, scopes: SCOPES }) -} - async function getAccessToken(jwt: JWT, forceRefresh = false): Promise { if (forceRefresh) { /** Clearing `credentials` forces `getAccessToken` to mint a new token instead of returning the cached one. */ @@ -289,7 +285,7 @@ export const bigqueryDestination: DrainDestination< */ async test({ config, credentials, signal }) { const account = parseServiceAccount(credentials.serviceAccountJson) - const jwt = buildJwt(account) + const jwt = createServiceAccountJwt(account, SCOPES) const token = await getAccessToken(jwt) const url = `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(config.projectId)}/datasets/${encodeURIComponent(config.datasetId)}/tables/${encodeURIComponent(config.tableId)}?fields=id` const perAttempt = AbortSignal.any([signal, AbortSignal.timeout(PER_ATTEMPT_TIMEOUT_MS)]) @@ -307,7 +303,7 @@ export const bigqueryDestination: DrainDestination< openSession({ config, credentials }) { const account = parseServiceAccount(credentials.serviceAccountJson) - const jwt = buildJwt(account) + const jwt = createServiceAccountJwt(account, SCOPES) return { async deliver({ body, metadata, signal }) { const rows = parseNdjsonObjects(body, { requireObject: true }) as Record[] diff --git a/apps/sim/lib/data-drains/destinations/gcs.ts b/apps/sim/lib/data-drains/destinations/gcs.ts index 0be0d2e08c1..62056470309 100644 --- a/apps/sim/lib/data-drains/destinations/gcs.ts +++ b/apps/sim/lib/data-drains/destinations/gcs.ts @@ -2,13 +2,13 @@ import { createLogger } from '@sim/logger' import { interruptibleSleep } from '@sim/utils/helpers' import { generateShortId } from '@sim/utils/id' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' -import { JWT } from 'google-auth-library' +import type { JWT } from 'google-auth-library' import { z } from 'zod' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { createServiceAccountJwt } from '@/lib/data-drains/destinations/google-jwt' import { buildObjectKey, normalizePrefix, - type ParsedServiceAccount, parseServiceAccount, refineServiceAccountJson, } from '@/lib/data-drains/destinations/utils' @@ -85,10 +85,6 @@ const gcsCredentialsSchema = z export type GCSDestinationConfig = z.infer export type GCSDestinationCredentials = z.infer -function buildJwt(account: ParsedServiceAccount): JWT { - return new JWT({ email: account.clientEmail, key: account.privateKey, scopes: [SCOPE] }) -} - async function getAccessToken(jwt: JWT): Promise { const { token } = await jwt.getAccessToken() if (!token) throw new Error('Failed to obtain GCS access token') @@ -263,7 +259,7 @@ export const gcsDestination: DrainDestination networkConfigMock) + +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { asOrchestrationError } from '@/lib/core/orchestration/types' +import { createServiceAccountJwt } from '@/lib/data-drains/destinations/google-jwt' + +const mockFetch = vi.fn() +const { privateKey } = generateKeyPairSync('rsa', { + modulusLength: 2048, + privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, + publicKeyEncoding: { type: 'spki', format: 'pem' }, +}) +const account = { clientEmail: 'drain@project.iam.gserviceaccount.com', privateKey } + +describe('createServiceAccountJwt', () => { + beforeEach(() => { + vi.stubGlobal('fetch', mockFetch) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + + it("mints the access token on the drain organization's egress route", async () => { + mockFetch.mockResolvedValue( + Response.json({ access_token: 'minted', token_type: 'Bearer', expires_in: 3600 }) + ) + const jwt = createServiceAccountJwt(account, ['https://www.googleapis.com/auth/scope']) + const { token } = await runWithOutboundOrganization('org-1', () => jwt.getAccessToken()) + expect(token).toBe('minted') + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + expect(String(mockFetch.mock.calls[0][0])).toBe('https://oauth2.googleapis.com/token') + }) + + it('surfaces an unavailable egress route as Sim unavailability', async () => { + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + const jwt = createServiceAccountJwt(account, ['https://www.googleapis.com/auth/scope']) + const error = await runWithOutboundOrganization('org-1', () => jwt.getAccessToken()).catch( + (caught: unknown) => caught + ) + expect(asOrchestrationError(error)?.code).toBe('unavailable') + expect(mockFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/data-drains/destinations/google-jwt.ts b/apps/sim/lib/data-drains/destinations/google-jwt.ts new file mode 100644 index 00000000000..3c5c2dd3500 --- /dev/null +++ b/apps/sim/lib/data-drains/destinations/google-jwt.ts @@ -0,0 +1,17 @@ +import { JWT } from 'google-auth-library' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import type { ParsedServiceAccount } from '@/lib/data-drains/destinations/utils' + +/** + * A service-account JWT client whose token mint leaves through `outboundFetch`, so an + * organization with a dedicated egress gateway reaches Google's token endpoint from its own + * static IPs. gaxios otherwise mints with its own fetch, outside the organization's route. + */ +export function createServiceAccountJwt(account: ParsedServiceAccount, scopes: string[]): JWT { + return new JWT({ + email: account.clientEmail, + key: account.privateKey, + scopes, + transporterOptions: { fetchImplementation: outboundFetch }, + }) +} diff --git a/apps/sim/lib/knowledge/access/github-installation.test.ts b/apps/sim/lib/knowledge/access/github-installation.test.ts index dd8ef7fb4df..289deb1f2ae 100644 --- a/apps/sim/lib/knowledge/access/github-installation.test.ts +++ b/apps/sim/lib/knowledge/access/github-installation.test.ts @@ -8,13 +8,16 @@ import { githubInstallationMock, githubInstallationMockFns, } from '@sim/testing/mocks/github-installation.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { resolveGitHubInstallationReadGrants } from '@/lib/knowledge/access/github-installation' import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' vi.mock('@/lib/credentials/managed-oauth', () => credentialsManagedOauthMock) vi.mock('@/lib/core/security/encryption', () => encryptionMock) vi.mock('@/lib/oauth/github-installation', () => githubInstallationMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) const mocks = { token: credentialsManagedOauthMockFns.mockResolveManagedOAuthToken, @@ -192,6 +195,34 @@ describe('live GitHub installation reader access', () => { expect(mocks.token).toHaveBeenCalledTimes(1) }) + describe('with organization egress routing', () => { + beforeEach(() => networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true)) + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + + it("reaches GitHub from the knowledge base owner's route when the caller set none", async () => { + queueSources() + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([grant]) + expect(networkConfigMockFns.mockResolveOutboundRoute.mock.calls).toEqual([ + ['org-1'], + ['org-1'], + ]) + }) + + it('fails the admission on an unavailable route instead of denying the source', async () => { + queueSources() + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + await expect(resolveGitHubInstallationReadGrants(input)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + }) + it('bounds one candidate batch without enumerating all organization sources', async () => { await expect( resolveGitHubInstallationReadGrants({ diff --git a/apps/sim/lib/knowledge/access/github-installation.ts b/apps/sim/lib/knowledge/access/github-installation.ts index dea8ffec58a..4488c1b73a4 100644 --- a/apps/sim/lib/knowledge/access/github-installation.ts +++ b/apps/sim/lib/knowledge/access/github-installation.ts @@ -9,7 +9,9 @@ import { import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' @@ -72,7 +74,7 @@ function positiveId(value: unknown): string | null { async function readGitHubJson(path: string, accessToken: string, signal: AbortSignal) { signal.throwIfAborted() - const response = await outboundFetch(`https://api.github.com${path}`, { + const response = await scopedOutboundFetch(`https://api.github.com${path}`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/vnd.github+json', @@ -139,7 +141,9 @@ async function verifyRepository( /** * Proves current reader access before any installation-backed indexed content is selected. * The caller supplies credentials already bound to the verified current member. All positive - * evidence and token reuse are local to this admission; failures grant nothing for that source. + * evidence and token reuse are local to this admission; failures grant nothing for that source, + * except an unavailable egress route, which is Sim's outage and fails the admission. GitHub is + * reached from the knowledge base owner's egress route whether or not the caller set one. */ export async function resolveGitHubInstallationReadGrants(input: { scope: ResourceScope @@ -224,8 +228,12 @@ export async function resolveGitHubInstallationReadGrants(input: { ) .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) const contentById = new Map(credentials.map((entry) => [entry.id, entry])) - const timeout = AbortSignal.timeout(GITHUB_READ_TIMEOUT_MS) - const admissionSignal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout + const outage = new AbortController() + const admissionSignal = AbortSignal.any([ + outage.signal, + AbortSignal.timeout(GITHUB_READ_TIMEOUT_MS), + ...(input.signal ? [input.signal] : []), + ]) const installations = new Map>() const tokens = new Map>() const proofs = new Map>() @@ -301,15 +309,19 @@ export async function resolveGitHubInstallationReadGrants(input: { readerSubjectToken: source.subjectToken, repositoryId: source.repositoryId, }) - } catch { + } catch (error) { + if (asOrchestrationError(error)?.code === 'unavailable') { + outage.abort(error) + throw error + } logger.warn('GitHub did not confirm current Search access', { connectorId: source.connectorId, }) } } } - await Promise.all( - Array.from({ length: Math.min(GITHUB_READ_CONCURRENCY, sources.length) }, worker) + await withResourceOutboundScope(resourceScopeFields(input.scope), () => + Promise.all(Array.from({ length: Math.min(GITHUB_READ_CONCURRENCY, sources.length) }, worker)) ) input.signal?.throwIfAborted() return [...grants.values()] diff --git a/apps/sim/lib/oauth/salesforce-instance.server.test.ts b/apps/sim/lib/oauth/salesforce-instance.server.test.ts new file mode 100644 index 00000000000..460119b1740 --- /dev/null +++ b/apps/sim/lib/oauth/salesforce-instance.server.test.ts @@ -0,0 +1,31 @@ +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { fetchSalesforceInstanceUrl } from '@/lib/oauth/salesforce-instance.server' + +const mockFetch = vi.fn() + +describe('fetchSalesforceInstanceUrl', () => { + beforeEach(() => { + vi.stubGlobal('fetch', mockFetch) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + + it('propagates a blocked egress route instead of reporting an unverifiable account', async () => { + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('ROUTE_BLOCKED') + ) + await expect(fetchSalesforceInstanceUrl('salesforce', 'token')).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(mockFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/oauth/salesforce-instance.server.ts b/apps/sim/lib/oauth/salesforce-instance.server.ts index ed11613ad5c..c2ace3a006b 100644 --- a/apps/sim/lib/oauth/salesforce-instance.server.ts +++ b/apps/sim/lib/oauth/salesforce-instance.server.ts @@ -1,13 +1,18 @@ import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { validateSalesforceInstanceUrl } from '@/lib/core/security/input-validation' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { isSalesforceLoginOrigin, SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' const logger = createLogger('SalesforceAccountDestination') -/** Resolves a token's API origin from Salesforce identity, never from model arguments. */ +/** + * Resolves a token's API origin from Salesforce identity, never from model arguments. An + * unavailable or blocked egress route is Sim's outage, not an unverifiable account, so it + * propagates instead of resolving to `undefined`. + */ export async function fetchSalesforceInstanceUrl( providerId: string, accessToken: string @@ -36,7 +41,8 @@ export async function fetchSalesforceInstanceUrl( if (origin.isValid && origin.sanitized && !isSalesforceLoginOrigin(origin.sanitized)) return origin.sanitized } - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error logger.warn('Could not resolve Salesforce account destination', { providerId }) } return undefined From fe4a771fdc82b3ff2192f39cdea3b4c50e4dddb4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:30:03 -0800 Subject: [PATCH 29/32] fix(egress): reach Confluence from the knowledge base owner's route and fail the admission on an egress outage --- .../knowledge/access/confluence-site.test.ts | 30 +++++++++++- .../lib/knowledge/access/confluence-site.ts | 49 +++++++++++++------ 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/apps/sim/lib/knowledge/access/confluence-site.test.ts b/apps/sim/lib/knowledge/access/confluence-site.test.ts index fc51b7ab720..fe5f1cc6059 100644 --- a/apps/sim/lib/knowledge/access/confluence-site.test.ts +++ b/apps/sim/lib/knowledge/access/confluence-site.test.ts @@ -4,7 +4,9 @@ import { credentialsManagedOauthMockFns, } from '@sim/testing/mocks/credentials-managed-oauth.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { CONFLUENCE_READ_CREDENTIAL_ALTERNATIVES, CONFLUENCE_READ_RESPONSE_MAX_BYTES, @@ -14,6 +16,7 @@ import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' vi.mock('@/lib/credentials/managed-oauth', () => credentialsManagedOauthMock) vi.mock('@/lib/core/security/encryption', () => encryptionMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) const mocks = { token: credentialsManagedOauthMockFns.mockResolveManagedOAuthToken, fetch: vi.fn() } @@ -90,6 +93,31 @@ describe('current Confluence site access', () => { expect(JSON.stringify(mocks.fetch.mock.calls)).not.toContain(binding.apiToken) }) + describe('with organization egress routing', () => { + beforeEach(() => networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true)) + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + + it("reaches Confluence from the knowledge base owner's route when the caller set none", async () => { + queueSources() + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([grant]) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + }) + + it('fails the admission on an unavailable route instead of denying the site', async () => { + queueSources() + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + await expect(resolveConfluenceSiteReadGrants(input)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + }) + it('denies newly revoked site access even if a direct space grant remains indexed', async () => { queueSources() await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([grant]) diff --git a/apps/sim/lib/knowledge/access/confluence-site.ts b/apps/sim/lib/knowledge/access/confluence-site.ts index 0fe5aec71c9..4bfc224422e 100644 --- a/apps/sim/lib/knowledge/access/confluence-site.ts +++ b/apps/sim/lib/knowledge/access/confluence-site.ts @@ -4,7 +4,9 @@ import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' @@ -92,6 +94,11 @@ async function readSiteBinding(key: string): Promise { return { cloudId: value.cloudId, domain: value.domain } } +/** An unavailable egress route is Sim's outage, so it fails the admission instead of denying a source. */ +function isUnavailable(error: unknown): boolean { + return asOrchestrationError(error)?.code === 'unavailable' +} + /** Space grants do not prove that the reader still has Confluence's site-level Can use permission. */ async function verifySite( binding: SiteBinding, @@ -100,7 +107,7 @@ async function verifySite( signal: AbortSignal ): Promise { signal.throwIfAborted() - const response = await outboundFetch( + const response = await scopedOutboundFetch( `https://api.atlassian.com/ex/confluence/${binding.cloudId}/wiki/rest/api/user/current`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, @@ -201,8 +208,12 @@ export async function resolveConfluenceSiteReadGrants(input: { ) ) .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) - const timeout = AbortSignal.timeout(CONFLUENCE_READ_TIMEOUT_MS) - const admissionSignal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout + const outage = new AbortController() + const admissionSignal = AbortSignal.any([ + outage.signal, + AbortSignal.timeout(CONFLUENCE_READ_TIMEOUT_MS), + ...(input.signal ? [input.signal] : []), + ]) const bindings = new Map() for (const content of credentials) { if (!content.key || admissionSignal.aborted) continue @@ -310,7 +321,15 @@ export async function resolveConfluenceSiteReadGrants(input: { .slice(offset, offset + CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY) .map((credentialId) => attempt(binding, identity, credentialId, cancellation.signal)) ) - } catch { + } catch (error) { + const unavailable = + error instanceof AggregateError + ? error.errors.find((attemptError) => isUnavailable(attemptError)) + : undefined + if (unavailable) { + outage.abort(unavailable) + throw unavailable + } if (requests >= MAX_KNOWLEDGE_ACCESS_CANDIDATES) return undefined } finally { cancellation.abort() @@ -333,15 +352,17 @@ export async function resolveConfluenceSiteReadGrants(input: { } } } - await Promise.all( - Array.from( - { - length: Math.min( - proofCount, - CONFLUENCE_READ_CONCURRENCY / CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY - ), - }, - worker + await withResourceOutboundScope(resourceScopeFields(input.scope), () => + Promise.all( + Array.from( + { + length: Math.min( + proofCount, + CONFLUENCE_READ_CONCURRENCY / CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY + ), + }, + worker + ) ) ) input.signal?.throwIfAborted() From 891e31c1c3c0ec1926c290132c68bda0fc3578b4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:39:46 -0800 Subject: [PATCH 30/32] improvement(egress): surface org egress outages as Sim's outage, never a credential failure An OutboundRoutingError (egress gateway down or route blocked) now propagates, or classifies as `unavailable` (503 with the fixed message), instead of reading as a missing, revoked, or invalid credential, a provider outage, or a poll failure: - token refresh: getOAuthToken / resolveCredentialTokenBundle rethrow the egress outcome (one throw in resolveStoredAccessToken); the leader logs it - polling: an egress outage skips the poll without counting toward auto-disable, at the outer catch, mid-batch, and in RSS source backoff (isUncountedPollFailure / skipUncountedPollFailure) - Slack managed users: callSlackApi rethrows; start configuration runs in the owner's scope - Shopify installation exchange and refresh rethrow instead of returning null - credential minters and verifiers (fetchProvider, NetSuite, CrowdStrike, Databricks, Google workload identity, Microsoft Entra, AWS trust probe, Slack bot) rethrow; service-account secret builders pass it through; credential create/update rethrow it so routes answer 503; the client-credential mint cache records and evicts nothing on an outage - AWS credential failure message uses the shared EGRESS_UNAVAILABLE_MESSAGE - withConnectDraftOutboundScope leaves connectors untouched when routing is disabled - Slack agent-session stop restores status in the credential's workspace scope - Fireflies uses scopedOutboundFetch; selector http/harmonic rethrow instead of options unavailable - drop a redundant RefreshTokenResult annotation in managed-oauth --- .../connectors/fireflies/fireflies.test.ts | 26 ++++++++++ apps/sim/connectors/fireflies/fireflies.ts | 4 +- apps/sim/lib/core/orchestration/types.ts | 12 ++++- .../application/slack-managed-users.test.ts | 11 ++++ .../application/slack-managed-users.ts | 25 +++++---- .../slack-managed-users.test.ts | 14 +++++ .../credential-groups/slack-managed-users.ts | 1 + .../aws-service-account.server.test.ts | 18 +++++++ .../credentials/aws-service-account.server.ts | 1 + .../minters/crowdstrike.test.ts | 9 ++++ .../minters/crowdstrike.ts | 2 + .../minters/netsuite.test.ts | 9 ++++ .../minters/netsuite.ts | 4 +- .../coordinated-client-token.test.ts | 24 +++++++++ .../credentials/coordinated-client-token.ts | 2 + .../databricks-service-account.server.test.ts | 33 ++++++++++++ .../databricks-service-account.server.ts | 6 ++- .../google-workload-identity.server.test.ts | 42 +++++++++++++++ .../google-workload-identity.server.ts | 20 ++++++-- apps/sim/lib/credentials/managed-oauth.ts | 3 +- ...crosoft-app-service-account.server.test.ts | 38 ++++++++++++++ .../microsoft-app-service-account.server.ts | 12 ++++- .../orchestration/credential-create.ts | 7 ++- .../credentials/orchestration/index.test.ts | 35 +++++++++++++ .../lib/credentials/orchestration/index.ts | 4 +- .../service-account-secret.test.ts | 30 +++++++++++ .../lib/credentials/service-account-secret.ts | 51 +++++++++---------- .../token-service-accounts/errors.test.ts | 9 ++++ .../token-service-accounts/errors.ts | 9 ++-- apps/sim/lib/internal/aws/credentials.test.ts | 22 ++++++++ apps/sim/lib/internal/aws/failure.ts | 7 ++- .../lib/oauth/connect-scope.server.test.ts | 12 +++++ apps/sim/lib/oauth/connect-scope.server.ts | 2 + .../credential-service-cancellation.test.ts | 17 +++++++ apps/sim/lib/oauth/credential-service.test.ts | 20 +++++--- apps/sim/lib/oauth/credential-service.ts | 13 ++++- .../lib/oauth/shopify-installation.test.ts | 46 +++++++++++++++++ apps/sim/lib/oauth/shopify-installation.ts | 8 ++- .../server/providers/harmonic.test.ts | 9 ++++ .../selectors/server/providers/harmonic.ts | 3 +- .../selectors/server/providers/http.test.ts | 11 ++++ .../lib/selectors/server/providers/http.ts | 3 +- apps/sim/lib/webhooks/polling/gmail.test.ts | 32 +++++++++++- apps/sim/lib/webhooks/polling/gmail.ts | 11 ++-- .../lib/webhooks/polling/google-calendar.ts | 11 ++-- apps/sim/lib/webhooks/polling/google-drive.ts | 11 ++-- .../sim/lib/webhooks/polling/google-sheets.ts | 11 ++-- apps/sim/lib/webhooks/polling/hubspot.ts | 13 +++-- apps/sim/lib/webhooks/polling/imap.ts | 11 ++-- apps/sim/lib/webhooks/polling/outlook.ts | 11 ++-- apps/sim/lib/webhooks/polling/utils.test.ts | 35 +++++++++++++ apps/sim/lib/webhooks/polling/utils.ts | 43 +++++++++++++--- .../lib/webhooks/slack-custom-ingress.test.ts | 40 +++++++++++++++ apps/sim/lib/webhooks/slack-custom-ingress.ts | 5 +- .../src/mocks/webhooks-polling-utils.mock.ts | 31 +++++++++-- 55 files changed, 768 insertions(+), 131 deletions(-) create mode 100644 apps/sim/lib/credentials/coordinated-client-token.test.ts create mode 100644 apps/sim/lib/oauth/shopify-installation.test.ts diff --git a/apps/sim/connectors/fireflies/fireflies.test.ts b/apps/sim/connectors/fireflies/fireflies.test.ts index 88c7f015300..283c7a62044 100644 --- a/apps/sim/connectors/fireflies/fireflies.test.ts +++ b/apps/sim/connectors/fireflies/fireflies.test.ts @@ -1,8 +1,14 @@ +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + +import { OutboundRoutingError } from '@/lib/core/network/routing' import { firefliesConnector } from '@/connectors/fireflies/fireflies' beforeEach(() => { vi.useRealTimers() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) }) interface GraphQLCall { @@ -157,6 +163,26 @@ describe('fireflies listDocuments', () => { }) }) +describe('fireflies outbound scope', () => { + it('refuses to call Fireflies outside a sync scope instead of leaving through the shared egress', async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockImplementationOnce(async (organizationId) => { + if (organizationId === undefined) throw new OutboundRoutingError('MISSING_SCOPE') + return { kind: 'direct' } + }) + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + const error = await firefliesConnector + .listDocuments('key', {}, undefined, {}) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(OutboundRoutingError) + expect(error).toMatchObject({ code: 'MISSING_SCOPE' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) + describe('fireflies getDocument', () => { it.each([null, {}, transcript('different')])( 'rejects malformed or mismatched successful transcript metadata', diff --git a/apps/sim/connectors/fireflies/fireflies.ts b/apps/sim/connectors/fireflies/fireflies.ts index bd077bb53a6..5fb91073388 100644 --- a/apps/sim/connectors/fireflies/fireflies.ts +++ b/apps/sim/connectors/fireflies/fireflies.ts @@ -1,7 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' -import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { isRetryableError, @@ -201,7 +201,7 @@ async function firefliesGraphQL( return retryWithExponentialBackoff( async () => { /** One retry layer owns transport, HTTP, and GraphQL semantic failures. */ - const response = await outboundFetch(FIREFLIES_GRAPHQL_URL, { + const response = await scopedOutboundFetch(FIREFLIES_GRAPHQL_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/core/orchestration/types.ts b/apps/sim/lib/core/orchestration/types.ts index 307f12c84c1..6dd21c156d1 100644 --- a/apps/sim/lib/core/orchestration/types.ts +++ b/apps/sim/lib/core/orchestration/types.ts @@ -101,7 +101,8 @@ export function throwOrchestrationFailure( ) } -const EGRESS_UNAVAILABLE_MESSAGE = "The organization's egress route is unavailable" +/** The fixed caller-facing text for an organization egress outage; the routing code stays in logs. */ +export const EGRESS_UNAVAILABLE_MESSAGE = "The organization's egress route is unavailable" /** * The {@link OrchestrationError} in `error`'s cause chain, or `null` when the @@ -129,6 +130,15 @@ export function asOrchestrationError(error: unknown): OrchestrationError | null return null } +/** + * Whether `error` is a failure Sim's side cannot currently get past (an `unavailable` failure, + * such as the organization's egress route being down): never a verdict on the caller, its + * credentials, or the resource, so nothing may be recorded against them. + */ +export function isUnavailableFailure(error: unknown): boolean { + return asOrchestrationError(error)?.code === 'unavailable' +} + /** Transport metadata available to an application operation for audit capture. */ export interface OrchestrationRequestContext { headers: { get(name: string): string | null } diff --git a/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts b/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts index 9c2fc51f4a9..2a9c5ac637b 100644 --- a/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts +++ b/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts @@ -199,6 +199,17 @@ describe('scoped Slack setup', () => { ) }) + it("verifies the custom bot through the owning organization's egress route", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + queueTableRows(credentialGroup, [group]) + mocks.create.mockImplementation(async () => { + await resolveCurrentOutboundRoute() + return { state: 'state-1', authorizationUrl: 'https://slack.com/oauth' } + }) + await expect(start()).resolves.toMatchObject({ state: 'state-1' }) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('organization-1') + }) + it("exchanges the Slack grant through the owning organization's egress route", async () => { networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) queueTableRows(credentialGroup, [group]) diff --git a/apps/sim/lib/credential-groups/application/slack-managed-users.ts b/apps/sim/lib/credential-groups/application/slack-managed-users.ts index 9d8a5bc8597..be9431bc820 100644 --- a/apps/sim/lib/credential-groups/application/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/application/slack-managed-users.ts @@ -102,17 +102,20 @@ export const startSlackCredentialGroupConfiguration: OperationUseCase< ) if (scope.kind === 'organization') await requireOrganizationAccountsSetup(scope.organizationId, input.credentialGroupId) - return createSlackManagedUsersAttempt({ - ...resourceScopeFields(scope), - userId: principal.userId, - credentialGroupId: input.credentialGroupId, - slackBotCredentialId: input.slackBotCredentialId, - appId: input.appId, - teamId: input.teamId, - clientId: input.clientId, - clientSecret: input.clientSecret, - requiredScopes: input.requiredScopes, - }) + const owner = resourceScopeFields(scope) + return withResourceOutboundScope(owner, () => + createSlackManagedUsersAttempt({ + ...owner, + userId: principal.userId, + credentialGroupId: input.credentialGroupId, + slackBotCredentialId: input.slackBotCredentialId, + appId: input.appId, + teamId: input.teamId, + clientId: input.clientId, + clientSecret: input.clientSecret, + requiredScopes: input.requiredScopes, + }) + ) }, } diff --git a/apps/sim/lib/credential-groups/slack-managed-users.test.ts b/apps/sim/lib/credential-groups/slack-managed-users.test.ts index 6bad711445c..4574ac8920a 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.test.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.test.ts @@ -46,6 +46,7 @@ import { exchangeAndConfigureSlackManagedUsers, exchangeSlackUserAuthorization, loadSlackManagedUsersAttempt, + verifySlackCustomBotAppIdentity, } from '@/lib/credential-groups/slack-managed-users' const shared = { flag: featureFlagsMockFns.mockIsFeatureEnabled } @@ -600,6 +601,19 @@ describe('Slack managed-user authorization', () => { ).rejects.toBeInstanceOf(OutboundRoutingError) }) + it('surfaces an unavailable egress route while verifying a custom bot instead of blaming Slack', async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + runWithOutboundOrganization('organization-1', () => + verifySlackCustomBotAppIdentity('xoxb-token') + ) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('fails closed when Slack omits the user token type', async () => { vi.stubGlobal( 'fetch', diff --git a/apps/sim/lib/credential-groups/slack-managed-users.ts b/apps/sim/lib/credential-groups/slack-managed-users.ts index 1d73135eb21..b8986c2a165 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.ts @@ -371,6 +371,7 @@ async function callSlackApi(method: string, accessToken: string, body?: URLSearc cache: 'no-store', }) } catch (error) { + if (error instanceof OutboundRoutingError) throw error logger.error('Slack API verification failed', { method, error: getErrorMessage(error) }) throw new SlackManagedUsersError('Slack could not verify the authorization.', 'provider_error') } diff --git a/apps/sim/lib/credentials/aws-service-account.server.test.ts b/apps/sim/lib/credentials/aws-service-account.server.test.ts index f99f068f03d..0a81e34d879 100644 --- a/apps/sim/lib/credentials/aws-service-account.server.test.ts +++ b/apps/sim/lib/credentials/aws-service-account.server.test.ts @@ -24,6 +24,7 @@ vi.mock('@aws-sdk/client-sts', async (importActual) => ({ import { AssumeRoleWithWebIdentityCommand, GetCallerIdentityCommand } from '@aws-sdk/client-sts' import { credential } from '@sim/db/schema' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { getAwsCredentialConfig, resolveAwsSessionCredentials, @@ -134,6 +135,23 @@ describe('saving a keyless AWS role', () => { ).rejects.toThrow() expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() }) + + it('surfaces an unavailable egress route during the any-subject check instead of asking to retry', async () => { + roleTrusting(true) + const pinned = hoisted.send.getMockImplementation() + hoisted.send.mockImplementation(async (command: unknown) => { + const probesUnboundSubject = + command instanceof AssumeRoleWithWebIdentityCommand && + command.input.WebIdentityToken !== `token-for:${SUBJECT}` + if (probesUnboundSubject) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + return pinned?.(command) + }) + + await expect( + verifyAndEncryptAwsCredential(keylessReconnect.serialized, keylessReconnect.owner) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) }) describe('choosing the audience of a keyless AWS role', () => { diff --git a/apps/sim/lib/credentials/aws-service-account.server.ts b/apps/sim/lib/credentials/aws-service-account.server.ts index dd8c0c29795..1b10fc7defa 100644 --- a/apps/sim/lib/credentials/aws-service-account.server.ts +++ b/apps/sim/lib/credentials/aws-service-account.server.ts @@ -385,6 +385,7 @@ async function assertRolePinsSubject( } catch (error) { if (error instanceof Error && error.name === 'AccessDenied') return if (error instanceof KeylessCredentialRefusedError) throw error + if (error instanceof OutboundRoutingError) throw error throw new AwsCredentialError( `Sim could not confirm the role's trust policy limits it to this credential: ${failureClause(error)}. Try again.` ) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts index 795bdc4ab3f..21e47062c73 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts @@ -1,5 +1,6 @@ import { jsonResponse } from '@sim/testing/helpers/http' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { mintCrowdStrikeServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/crowdstrike' const FIELDS = { @@ -162,6 +163,14 @@ describe('CrowdStrike API client token exchange', () => { }) }) + it('surfaces an unavailable egress route instead of a provider failure', async () => { + mockFetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + }) + it('refuses a token response without an access token', async () => { mockFetch.mockResolvedValue(jsonResponse({ expires_in: 1799 })) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts index 85474bf2121..e026c2629d1 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts @@ -1,3 +1,4 @@ +import { OutboundRoutingError } from '@/lib/core/network/routing' import type { ClientCredentialAccountFields, ClientCredentialAccountMintOptions, @@ -68,6 +69,7 @@ export async function mintCrowdStrikeServiceAccountToken( ) } catch (error) { options?.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error if (error instanceof CrowdStrikeTokenRedirectError) { throw new TokenServiceAccountValidationError('invalid_credentials', 400, { step: STEP, diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts index 57ffd1443f7..aabe7b394ec 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts @@ -2,6 +2,7 @@ import { generateKeyPairSync } from 'node:crypto' import { jsonResponse } from '@sim/testing/helpers/http' import { jwtVerify } from 'jose' import { describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { mintNetSuiteServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/netsuite' const ORIGIN = 'https://1234567-sb1.suitetalk.api.netsuite.com' @@ -257,4 +258,12 @@ describe('mintNetSuiteServiceAccountToken', () => { logDetail: { step: 'netsuite_token_mint', reason: 'network error reaching provider' }, }) }) + + it('surfaces an unavailable egress route instead of a provider outage', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new OutboundRoutingError('ROUTE_BLOCKED'))) + + await expect(mintNetSuiteServiceAccountToken(FIELDS)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts index 5f122915936..ec3b5995b45 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts @@ -3,6 +3,7 @@ import { generateId } from '@sim/utils/id' import { truncate } from '@sim/utils/string' import { SignJWT } from 'jose' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -121,7 +122,8 @@ async function exchangeNetSuiteToken( redirect: 'error', signal, }) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new TokenServiceAccountValidationError('provider_unavailable', 502, { step: TOKEN_MINT_STEP, reason: 'network error reaching provider', diff --git a/apps/sim/lib/credentials/coordinated-client-token.test.ts b/apps/sim/lib/credentials/coordinated-client-token.test.ts new file mode 100644 index 00000000000..12ad2dc109d --- /dev/null +++ b/apps/sim/lib/credentials/coordinated-client-token.test.ts @@ -0,0 +1,24 @@ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { resolveCoordinatedClientToken } from '@/lib/credentials/coordinated-client-token' + +beforeEach(resetDbChainMock) + +describe('resolveCoordinatedClientToken', () => { + it("surfaces an unavailable egress route as Sim's outage and remembers no failure", async () => { + const exchange = vi.fn().mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + resolveCoordinatedClientToken({ + namespace: 'fixture', + application: ['fixture-client'], + clientSecret: 'fixture-secret', + step: 'fixture_token', + exchange, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(exchange).toHaveBeenCalledOnce() + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/credentials/coordinated-client-token.ts b/apps/sim/lib/credentials/coordinated-client-token.ts index c61d1eabe3e..3d8dc25dfbc 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.ts @@ -4,6 +4,7 @@ import { clientCredentialToken } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { and, eq, inArray, lt, sql } from 'drizzle-orm' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { assessTokenRequestFailure, @@ -521,6 +522,7 @@ export async function resolveCoordinatedClientToken( } catch (error) { signal?.throwIfAborted() if (error instanceof TokenServiceAccountValidationError) throw error + if (error instanceof OutboundRoutingError) throw error throw unavailable(step, 'token coordination or exchange failed') } } diff --git a/apps/sim/lib/credentials/databricks-service-account.server.test.ts b/apps/sim/lib/credentials/databricks-service-account.server.test.ts index 5659f3cfb28..ee1d40d5c92 100644 --- a/apps/sim/lib/credentials/databricks-service-account.server.test.ts +++ b/apps/sim/lib/credentials/databricks-service-account.server.test.ts @@ -25,6 +25,7 @@ vi.mock('@/lib/credentials/workload-identity/issuer', () => ({ })) import { credential } from '@sim/db/schema' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { EgressRefusedError } from '@/lib/core/network/transport.server' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { @@ -108,6 +109,14 @@ describe('saving a Databricks connection', () => { } ) + it('surfaces an unavailable egress route instead of failing verification', async () => { + secureFetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + verifyAndEncryptDatabricksCredential(patConfig(HOST), owner) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('reports a host the egress policy refuses as not allowed', async () => { secureFetch.mockRejectedValue(new EgressRefusedError('url resolves to a private address')) @@ -223,6 +232,22 @@ describe('saving a keyless Databricks connection', () => { verifyAndEncryptDatabricksCredential(JSON.stringify(config), reconnectOwner) ).rejects.toThrow(/this credential: Could not reach the Databricks workspace\. Try again\.$/) }) + + it('surfaces an unavailable egress route during the any-subject check instead of asking to retry', async () => { + principalTrusting(true) + const trusted = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit) => { + const body = new URLSearchParams(String(init?.body)) + if (url.endsWith('/oidc/v1/token') && body.get('subject_token') !== `token-for:${SUBJECT}`) { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + return trusted?.(url, init) + }) + + await expect( + verifyAndEncryptDatabricksCredential(JSON.stringify(config), reconnectOwner) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) }) describe('a keyless Databricks connection with its own audience', () => { @@ -436,6 +461,14 @@ describe('getting a Databricks token for a run', () => { }) }) + it('surfaces an unavailable egress route instead of an unreachable workspace', async () => { + fetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(resolveDatabricksCredentialToken('cred-1')).rejects.toBeInstanceOf( + OutboundRoutingError + ) + }) + it('reports a token reply over the size cap as oversized, not as a refused request', async () => { fetch.mockResolvedValue(jsonResponse({ access_token: 't'.repeat(1024 * 1024) }, 200)) diff --git a/apps/sim/lib/credentials/databricks-service-account.server.ts b/apps/sim/lib/credentials/databricks-service-account.server.ts index 70d25eae18d..ef2f6419496 100644 --- a/apps/sim/lib/credentials/databricks-service-account.server.ts +++ b/apps/sim/lib/credentials/databricks-service-account.server.ts @@ -4,6 +4,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { eq } from 'drizzle-orm' import { z } from 'zod' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { EgressRefusedError } from '@/lib/core/network/transport.server' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation' @@ -171,8 +172,9 @@ async function requestWorkspaceToken( redirect: 'error', signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }) - } catch { + } catch (error) { signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error // A network failure, timeout, or refused redirect says nothing about the credential. throw new DatabricksCredentialError('Could not reach the Databricks workspace', 502) } @@ -353,6 +355,7 @@ async function verifyWorkspaceIdentity( } catch (error) { // A keyless refusal already names its cause and code; wrapping it would lose both. if (error instanceof KeylessCredentialRefusedError) throw error + if (error instanceof OutboundRoutingError) throw error if (error instanceof EgressRefusedError) { throw new DatabricksCredentialError( 'Could not verify the Databricks credential: the workspace host is not allowed' @@ -389,6 +392,7 @@ async function assertPolicyPinsSubject( providerFailureCode(error.status) === 'invalid_credentials' if (refused) return if (error instanceof KeylessCredentialRefusedError) throw error + if (error instanceof OutboundRoutingError) throw error throw new DatabricksCredentialError( `Sim could not confirm the federation policy limits this service principal to this credential: ${failureClause(error)}. Try again.` ) diff --git a/apps/sim/lib/credentials/google-workload-identity.server.test.ts b/apps/sim/lib/credentials/google-workload-identity.server.test.ts index 4fcfbc78b84..e7c70b2f24f 100644 --- a/apps/sim/lib/credentials/google-workload-identity.server.test.ts +++ b/apps/sim/lib/credentials/google-workload-identity.server.test.ts @@ -17,6 +17,7 @@ vi.mock('@/lib/credentials/workload-identity/issuer', () => ({ `sim-oidc|${audience}|${subject}`, })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { resolveGoogleWorkloadIdentityToken, verifyAndEncryptGoogleWorkloadIdentityCredential, @@ -305,6 +306,32 @@ describe('saving a keyless Google Cloud credential', () => { expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() }) + it.each([ + { + label: 'exchanging this credential’s token', + fails: (url: string, body: string) => + url === 'https://sts.googleapis.com/v1/token' && body.includes(SUBJECT), + }, + { + label: 'confirming the subject is pinned', + fails: (url: string, _body: string, auth: string) => + url.startsWith('https://iamcredentials.') && auth !== `Bearer fed:${SUBJECT}`, + }, + ])('surfaces an unavailable egress route while $label', async ({ fails }) => { + fakeGoogle(fetch, { canImpersonate: (subject) => subject === SUBJECT }) + const google = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit = {}) => { + const auth = new Headers(init.headers).get('authorization') ?? '' + if (fails(url, String(init.body ?? ''), auth)) { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + return google?.(url, init) + }) + + await expect(save({})).rejects.toBeInstanceOf(OutboundRoutingError) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) + it('saves when the provider’s attribute condition refuses every other subject', async () => { const google = fakeGoogle(fetch, { acceptSubject: (subject) => subject === SUBJECT }) @@ -671,6 +698,21 @@ describe('saving a keyless credential with domain-wide delegation', () => { expect(error.message).not.toContain('Manage Domain Wide Delegation') }) + it.each([ + { label: 'exchanging the delegated assertion', url: 'https://oauth2.googleapis.com/token' }, + { label: 'reading as the delegated user', url: GMAIL_PROFILE }, + ])('surfaces an unavailable egress route while $label', async ({ url: failing }) => { + fakeGoogle(fetch, { canImpersonate: (subject) => subject === SUBJECT }) + const google = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit = {}) => { + if (url === failing) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + return google?.(url, init) + }) + + await expect(save({ delegation })).rejects.toBeInstanceOf(OutboundRoutingError) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) + it('names the API to enable when a delegated read is refused', async () => { fakeGoogle(fetch, { canImpersonate: (subject) => subject === SUBJECT, diff --git a/apps/sim/lib/credentials/google-workload-identity.server.ts b/apps/sim/lib/credentials/google-workload-identity.server.ts index 41deb3dd685..9b9b6183ffe 100644 --- a/apps/sim/lib/credentials/google-workload-identity.server.ts +++ b/apps/sim/lib/credentials/google-workload-identity.server.ts @@ -4,6 +4,7 @@ import { LRUCache } from 'lru-cache' import { z } from 'zod' import { coalesceLocally } from '@/lib/concurrency/singleflight' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { encryptSecret } from '@/lib/core/security/encryption' import { GOOGLE_DELEGATION_SERVICE_NAMES, @@ -282,7 +283,8 @@ async function postJson( redirect: 'error', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), }) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new GoogleWorkloadIdentityError('Could not reach Google Cloud. Try again.') } const payload = (await response.json().catch(() => ({}))) as Record @@ -292,7 +294,7 @@ async function postJson( /** * The HTTP status of a bearer-authorized GET, with the same redirect and * timeout guards as {@link postJson}. The body is discarded unread. - * Undefined when Google could not be reached. + * Undefined when Google could not be reached; an unavailable egress route throws. */ async function getStatus(url: string, bearer: string): Promise { try { @@ -303,7 +305,8 @@ async function getStatus(url: string, bearer: string): Promise undefined) return response.status - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error return undefined } } @@ -546,7 +549,8 @@ async function exchangeDelegationAssertion( let response: Awaited> try { response = await exchangeGoogleServiceAccountJwt(OAUTH_TOKEN_URL, assertion) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new GoogleWorkloadIdentityError('Could not reach Google. Try again.') } const payload = parsePayload(response.body) @@ -646,6 +650,7 @@ async function assertSubjectPinned( await step() } catch (error) { if (isRefusal(error)) return true + if (error instanceof OutboundRoutingError) throw error throw unconfirmed() } return false @@ -818,7 +823,12 @@ export async function verifyAndEncryptGoogleWorkloadIdentityCredential( assertSubjectPinned(config, owner), ]) if (verified.status === 'rejected') { - if (verified.reason instanceof GoogleWorkloadIdentityError) throw verified.reason + if ( + verified.reason instanceof GoogleWorkloadIdentityError || + verified.reason instanceof OutboundRoutingError + ) { + throw verified.reason + } throw new GoogleWorkloadIdentityError( `Could not verify the Google Cloud keyless credential: ${getErrorMessage(verified.reason, 'Unknown error')}` ) diff --git a/apps/sim/lib/credentials/managed-oauth.ts b/apps/sim/lib/credentials/managed-oauth.ts index 401fae30d42..f674b09aa31 100644 --- a/apps/sim/lib/credentials/managed-oauth.ts +++ b/apps/sim/lib/credentials/managed-oauth.ts @@ -25,7 +25,6 @@ import { import { getCredentialGroupProviderAdapterByProviderId } from '@/lib/credential-groups/provider-registry' import { isScopedCredentialGroupsAvailable } from '@/lib/credential-groups/scoped-availability' import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' -import type { RefreshTokenResult } from '@/lib/oauth' import { isSalesforceOAuthProviderId } from '@/lib/oauth/salesforce' import { fetchSalesforceInstanceUrl } from '@/lib/oauth/salesforce-instance.server' import { loadActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/context' @@ -525,7 +524,7 @@ async function resolveManagedOAuthAccessToken( } } - const refreshed: RefreshTokenResult = await adapter.refreshToken(currentTokenSet.refreshToken) + const refreshed = await adapter.refreshToken(currentTokenSet.refreshToken) if (!refreshed.ok) { const terminal = adapter.isTerminalRefreshError(refreshed.errorCode) if (terminal) { diff --git a/apps/sim/lib/credentials/microsoft-app-service-account.server.test.ts b/apps/sim/lib/credentials/microsoft-app-service-account.server.test.ts index 7b2c5b34075..fd86f262baf 100644 --- a/apps/sim/lib/credentials/microsoft-app-service-account.server.test.ts +++ b/apps/sim/lib/credentials/microsoft-app-service-account.server.test.ts @@ -24,6 +24,7 @@ vi.mock('@/lib/credentials/workload-identity/issuer', () => ({ })) import { credential } from '@sim/db/schema' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { MicrosoftAppCredentialError, resolveMicrosoftAppCredentialToken, @@ -205,6 +206,26 @@ describe('saving a Microsoft app credential', () => { ) }) + it.each([ + { label: 'Entra', failing: (url: string) => url === TOKEN_URL }, + { + label: 'Microsoft Graph', + failing: (url: string) => url.includes('/v1.0/servicePrincipals('), + }, + ])('surfaces an unavailable egress route while reaching $label', async ({ failing }) => { + fakeMicrosoft(fetch) + const microsoft = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit) => { + if (failing(url)) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + return microsoft?.(url, init) + }) + + await expect(save({ authType: 'client_secret', clientSecret: SECRET })).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) + it('signs a PS256 certificate assertion Entra can verify with the uploaded certificate', async () => { const tokens = fakeMicrosoft(fetch) @@ -306,6 +327,23 @@ describe('saving a Microsoft app credential', () => { expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() }) + it('surfaces an unavailable egress route while confirming the subject is pinned', async () => { + fakeMicrosoft(fetch, { acceptAssertion: (assertion) => assertion.endsWith(`|${SUBJECT}`) }) + const microsoft = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit) => { + const assertion = new URLSearchParams(String(init?.body)).get('client_assertion') ?? '' + if (url === TOKEN_URL && !assertion.endsWith(`|${SUBJECT}`)) { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + return microsoft?.(url, init) + }) + + await expect(save({ authType: 'federated' }, reconnectOwner)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) + it('refuses a subject issued to another member, without contacting Entra', async () => { fakeMicrosoft(fetch) const grant = issueWorkloadIdentitySubject('microsoft', { workspaceId: 'ws-1' }, 'user-2') diff --git a/apps/sim/lib/credentials/microsoft-app-service-account.server.ts b/apps/sim/lib/credentials/microsoft-app-service-account.server.ts index f5f478ad836..fdb616aca10 100644 --- a/apps/sim/lib/credentials/microsoft-app-service-account.server.ts +++ b/apps/sim/lib/credentials/microsoft-app-service-account.server.ts @@ -12,6 +12,7 @@ import { LRUCache } from 'lru-cache' import { z } from 'zod' import { coalesceLocally } from '@/lib/concurrency/singleflight' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { MICROSOFT_CLOUD_IDS, @@ -423,8 +424,9 @@ async function requestEntraToken( redirect: 'error', signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }) - } catch { + } catch (error) { signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error throw new MicrosoftAppCredentialError('Could not reach Microsoft Entra. Try again.', 502) } const payload = (await response.json().catch(() => ({}))) as { @@ -570,6 +572,7 @@ async function assertFederationPinsSubject( error.status !== undefined && providerFailureCode(error.status) === 'invalid_credentials' if (refused) return + if (error instanceof OutboundRoutingError) throw error throw new MicrosoftAppCredentialError( 'Sim could not confirm this app trusts only this credential. Try again.' ) @@ -661,7 +664,12 @@ export async function verifyAndEncryptMicrosoftAppCredential( config.authType === 'federated' ? assertFederationPinsSubject(config, owner) : undefined, ]) if (verified.status === 'rejected') { - if (verified.reason instanceof MicrosoftAppCredentialError) throw verified.reason + if ( + verified.reason instanceof MicrosoftAppCredentialError || + verified.reason instanceof OutboundRoutingError + ) { + throw verified.reason + } throw new MicrosoftAppCredentialError( `Could not verify the Microsoft app credential: ${getErrorMessage(verified.reason, 'Unknown error')}` ) diff --git a/apps/sim/lib/credentials/orchestration/credential-create.ts b/apps/sim/lib/credentials/orchestration/credential-create.ts index 222b505c8ee..6e7c6064c43 100644 --- a/apps/sim/lib/credentials/orchestration/credential-create.ts +++ b/apps/sim/lib/credentials/orchestration/credential-create.ts @@ -8,7 +8,10 @@ import { generateId } from '@sim/utils/id' import { and, eq } from 'drizzle-orm' import { normalizeCredentialEnvKey } from '@/lib/api/contracts/credentials' import { acquireOrganizationUserMutationLocks } from '@/lib/billing/organizations/membership' -import type { OrchestrationRequestContext } from '@/lib/core/orchestration/types' +import { + isUnavailableFailure, + type OrchestrationRequestContext, +} from '@/lib/core/orchestration/types' import { type ResourceOwner, resourceScopeColumns, @@ -664,6 +667,8 @@ export async function createCredentialRecord( return failure('Credential source data failed validation checks', 'validation') } + // Sim's egress route is down: the surface answers 503, never a verdict on the credential. + if (isUnavailableFailure(error)) throw error const errAsRecord = typeof error === 'object' && error !== null ? (error as Record) : {} logger.error('Credential create failure details', { diff --git a/apps/sim/lib/credentials/orchestration/index.test.ts b/apps/sim/lib/credentials/orchestration/index.test.ts index 0178be1a078..fc4da856e9c 100644 --- a/apps/sim/lib/credentials/orchestration/index.test.ts +++ b/apps/sim/lib/credentials/orchestration/index.test.ts @@ -84,6 +84,7 @@ vi.mock('@/lib/credentials/token-service-accounts/errors', () => ({ })) vi.mock('@/lib/posthog/server', () => posthogServerMock) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { createServiceAccountCredential, deleteCredentialRecord, @@ -179,6 +180,23 @@ describe('performUpdateCredential — service-account secret rotation', () => { expect(result.previousDisplayName).toBe(OLD_EMAIL) }) + it('surfaces an unavailable egress route instead of an internal failure', async () => { + mockCredential() + mockStoredBlob({ type: 'service_account', client_email: OLD_EMAIL }) + mockVerifyAndBuildServiceAccountSecret.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + performUpdateCredential({ + credentialId: 'cred-1', + userId: 'user-1', + serviceAccountJson: NEW_GOOGLE_KEY, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(dbChainMockFns.set).not.toHaveBeenCalled() + }) + it('requires the complete OCI tuple and treats an omitted passphrase as clearing it', async () => { mockCredential({ providerId: 'oci-api-key-service-account', @@ -696,6 +714,23 @@ describe('createServiceAccountCredential', () => { } ) + it('surfaces an unavailable egress route instead of an internal failure', async () => { + mockVerifyAndBuildServiceAccountSecret.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + createServiceAccountCredential({ + workspaceId: 'workspace-1', + userId: 'user-1', + providerId: 'google-service-account', + displayName: 'Google signer', + serviceAccountJson: NEW_GOOGLE_KEY, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) + it('rejects an existing service-account source instead of discarding the submitted secret', async () => { mockVerifyAndBuildServiceAccountSecret.mockResolvedValue({ providerId: 'zoom-service-account', diff --git a/apps/sim/lib/credentials/orchestration/index.ts b/apps/sim/lib/credentials/orchestration/index.ts index 70ab4e4d3d6..e09985c55f8 100644 --- a/apps/sim/lib/credentials/orchestration/index.ts +++ b/apps/sim/lib/credentials/orchestration/index.ts @@ -11,7 +11,7 @@ import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { and, eq, sql } from 'drizzle-orm' import type { NextRequest } from 'next/server' -import { OrchestrationError } from '@/lib/core/orchestration/types' +import { isUnavailableFailure, OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeColumns, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' @@ -618,6 +618,8 @@ export async function updateCredentialRecord( errorCode: 'conflict', } } + // Sim's egress route is down: the surface answers 503, never a verdict on the credential. + if (isUnavailableFailure(error)) throw error logger.error('Failed to update credential', { error }) return { success: false, error: 'Internal server error', errorCode: 'internal' } } diff --git a/apps/sim/lib/credentials/service-account-secret.test.ts b/apps/sim/lib/credentials/service-account-secret.test.ts index 3bf33e761d1..b85f8a1323a 100644 --- a/apps/sim/lib/credentials/service-account-secret.test.ts +++ b/apps/sim/lib/credentials/service-account-secret.test.ts @@ -91,6 +91,7 @@ vi.mock('@/lib/credentials/client-credential-accounts/server', () => ({ : undefined, })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { AtlassianValidationError } from '@/lib/credentials/atlassian-service-account' import { ServiceAccountSecretError, @@ -98,6 +99,7 @@ import { } from '@/lib/credentials/service-account-secret' import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, + DATABRICKS_SERVICE_ACCOUNT_PROVIDER_ID, POSTGRESQL_SERVICE_ACCOUNT_PROVIDER_ID, SLACK_CUSTOM_BOT_PROVIDER_ID, } from '@/lib/oauth/types' @@ -112,6 +114,34 @@ describe('verifyAndBuildServiceAccountSecret', () => { mockNormalizeDomain.mockImplementation((raw: string) => raw.trim().toLowerCase()) }) + it('surfaces an unavailable egress route while verifying a Slack bot token', async () => { + mockFetchSlackTeamId.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + verifyAndBuildServiceAccountSecret(SLACK_CUSTOM_BOT_PROVIDER_ID, { + signingSecret: 'sec', + botToken: 'xoxb-1', + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + + it('surfaces an unavailable egress route while verifying a Databricks connection', async () => { + inputValidationMockFns.mockSecureFetchWithValidation.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + verifyAndBuildServiceAccountSecret(DATABRICKS_SERVICE_ACCOUNT_PROVIDER_ID, { + serviceAccountJson: JSON.stringify({ + authType: 'personal_access_token', + host: 'dbc-1234abcd-5678.cloud.databricks.com', + token: 'dapi-secret', + }), + owner: { workspaceId: 'ws-1', actorUserId: 'user-1' }, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('verifies a Slack bot token and encrypts the derived blob', async () => { mockFetchSlackTeamId.mockResolvedValue({ teamId: 'T1', userId: 'U_BOT', teamName: 'Acme' }) const result = await verifyAndBuildServiceAccountSecret(SLACK_CUSTOM_BOT_PROVIDER_ID, { diff --git a/apps/sim/lib/credentials/service-account-secret.ts b/apps/sim/lib/credentials/service-account-secret.ts index cb2bbea5176..e8c127f1e2e 100644 --- a/apps/sim/lib/credentials/service-account-secret.ts +++ b/apps/sim/lib/credentials/service-account-secret.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { serviceAccountJsonSchema } from '@/lib/api/contracts/credentials' import { getValidationErrorMessage } from '@/lib/api/server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { encryptSecret } from '@/lib/core/security/encryption' import { mintAtlassianServiceAccountToken } from '@/lib/credentials/atlassian-client-credentials' import { @@ -320,6 +321,7 @@ async function buildSlackCustomBotSecret( botUserId = auth.userId teamName = auth.teamName } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new ServiceAccountSecretError( `Could not verify the Slack bot token: ${getErrorMessage(error)}` ) @@ -392,6 +394,25 @@ async function buildGoogleServiceAccountSecret( } } +/** + * Rethrows a keyless-capable provider's verification failure as the save reports it: a keyless + * refusal and an unavailable egress route keep their own class (the surface maps both), the + * provider's own error becomes its message, and anything else is named for `label`. + */ +function throwVerificationFailure( + error: unknown, + ProviderError: abstract new (...args: never[]) => Error, + label: string +): never { + if (error instanceof KeylessCredentialRefusedError || error instanceof OutboundRoutingError) { + throw error + } + if (error instanceof ProviderError) throw new ServiceAccountSecretError(error.message) + throw new ServiceAccountSecretError( + `Could not verify the ${label}: ${getErrorMessage(error, 'Unknown error')}` + ) +} + /** * A keyless Google credential: Workload Identity Federation, no stored key, * optionally with domain-wide delegation to one Workspace user. @@ -405,13 +426,7 @@ async function buildGoogleWorkloadIdentitySecret( try { result = await verifyAndEncryptGoogleWorkloadIdentityCredential(serviceAccountJson, owner) } catch (error) { - if (error instanceof KeylessCredentialRefusedError) throw error - if (error instanceof GoogleWorkloadIdentityError) { - throw new ServiceAccountSecretError(error.message) - } - throw new ServiceAccountSecretError( - `Could not verify the Google Cloud keyless credential: ${getErrorMessage(error, 'Unknown error')}` - ) + throwVerificationFailure(error, GoogleWorkloadIdentityError, 'Google Cloud keyless credential') } const principal: ServiceAccountPrincipal = { kind: 'user', id: result.serviceAccountEmail } return { @@ -444,11 +459,7 @@ async function buildAwsServiceAccountSecret( try { result = await verifyAndEncryptAwsCredential(fields.serviceAccountJson, fields.owner) } catch (error) { - if (error instanceof KeylessCredentialRefusedError) throw error - if (error instanceof AwsCredentialError) throw new ServiceAccountSecretError(error.message) - throw new ServiceAccountSecretError( - `Could not verify the AWS credential: ${getErrorMessage(error, 'Unknown error')}` - ) + throwVerificationFailure(error, AwsCredentialError, 'AWS credential') } const principal: ServiceAccountPrincipal = { kind: 'user', id: result.callerArn } return { @@ -471,13 +482,7 @@ async function buildDatabricksServiceAccountSecret( try { result = await verifyAndEncryptDatabricksCredential(fields.serviceAccountJson, fields.owner) } catch (error) { - if (error instanceof KeylessCredentialRefusedError) throw error - if (error instanceof DatabricksCredentialError) { - throw new ServiceAccountSecretError(error.message) - } - throw new ServiceAccountSecretError( - `Could not verify the Databricks credential: ${getErrorMessage(error, 'Unknown error')}` - ) + throwVerificationFailure(error, DatabricksCredentialError, 'Databricks credential') } const principal: ServiceAccountPrincipal = { kind: 'user', id: result.identity } return { @@ -501,13 +506,7 @@ async function buildMicrosoftAppServiceAccountSecret( try { result = await verifyAndEncryptMicrosoftAppCredential(fields.serviceAccountJson, fields.owner) } catch (error) { - if (error instanceof KeylessCredentialRefusedError) throw error - if (error instanceof MicrosoftAppCredentialError) { - throw new ServiceAccountSecretError(error.message) - } - throw new ServiceAccountSecretError( - `Could not verify the Microsoft app credential: ${getErrorMessage(error, 'Unknown error')}` - ) + throwVerificationFailure(error, MicrosoftAppCredentialError, 'Microsoft app credential') } // The app's service principal is the identity every app-only token acts as. const principal: ServiceAccountPrincipal = { diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts index 5c5f9dc205d..cc6178c68f3 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { fetchProvider, isHostNotFound, @@ -55,6 +56,14 @@ describe('token service-account error helpers', () => { expect(error.logDetail).toEqual({ step: 'self', reason: 'network error reaching provider' }) }) + it("surfaces an unavailable egress route as Sim's outage, not the provider's", async () => { + mockFetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(fetchProvider(PROVIDER_URL, {}, 'self')).rejects.toBeInstanceOf( + OutboundRoutingError + ) + }) + it('never includes the URL in logDetail', async () => { mockFetch.mockRejectedValue(new Error(`ECONNREFUSED ${PROVIDER_URL}`)) diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.ts b/apps/sim/lib/credentials/token-service-accounts/errors.ts index 47cbc8faef8..6d483e87b56 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.ts @@ -209,7 +209,8 @@ export function isHostNotFound(error: unknown): boolean { * as raw undici errors — whose `cause` can carry connection details — and are * never blamed on the pasted token. DNS-resolution failures can optionally be * mapped to a different code via {@link FetchProviderOptions}. A blocked or unavailable - * organization egress route is also `provider_unavailable`. + * organization egress route is Sim's outage, not the provider's, so its + * {@link OutboundRoutingError} propagates unchanged. * * Redirects are refused (`redirect: 'error'`) unless `init` names another mode: * every caller sends a secret, and fetch replays custom headers and 307/308 @@ -228,6 +229,7 @@ export async function fetchProvider( return await outboundFetch(url, { ...init, redirect: init.redirect ?? 'error', signal }) } catch (error) { init.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error // Only ENOTFOUND proves the host doesn't exist; EAI_AGAIN is a transient // resolver failure and stays provider_unavailable. if (options?.dnsFailureCode && isHostNotFound(error)) { @@ -238,10 +240,7 @@ export async function fetchProvider( } throw new TokenServiceAccountValidationError('provider_unavailable', 502, { step, - reason: - error instanceof OutboundRoutingError - ? 'organization egress route unavailable' - : 'network error reaching provider', + reason: 'network error reaching provider', }) } } diff --git a/apps/sim/lib/internal/aws/credentials.test.ts b/apps/sim/lib/internal/aws/credentials.test.ts index 4c35cb52b77..64307c1d26d 100644 --- a/apps/sim/lib/internal/aws/credentials.test.ts +++ b/apps/sim/lib/internal/aws/credentials.test.ts @@ -24,6 +24,8 @@ import { IDPRejectedClaimException, InvalidIdentityTokenException, } from '@aws-sdk/client-sts' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' import { awsClientCredentials, withAwsCredential } from '@/lib/internal/aws/credentials' function call(overrides: Partial = {}): InternalToolOperationCall { @@ -212,4 +214,24 @@ describe('withAwsCredential', () => { expect(response.status).toBe(status) expect(handler).not.toHaveBeenCalled() }) + + it("reports an egress outage as Sim's, without naming its routing code", async () => { + mockGetAwsCredentialConfig.mockResolvedValue({ config: {}, owner: { workspaceId: 'ws-1' } }) + mockResolveAwsSessionCredentials.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + const response = (await withAwsCredential(vi.fn())( + call({ + context: { + workflowId: 'wf-1', + workspaceId: 'ws-1', + authorizedServiceAccountCredential: { credentialId: 'cred-1', service: 'aws' }, + }, + }) + )) as Response + + expect(response.status).toBe(503) + await expect(response.json()).resolves.toEqual({ error: EGRESS_UNAVAILABLE_MESSAGE }) + }) }) diff --git a/apps/sim/lib/internal/aws/failure.ts b/apps/sim/lib/internal/aws/failure.ts index a4f4e70f480..241a712a19b 100644 --- a/apps/sim/lib/internal/aws/failure.ts +++ b/apps/sim/lib/internal/aws/failure.ts @@ -2,6 +2,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { isRetryableNetworkError } from '@/lib/core/errors/retryable-infrastructure' import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' const THROTTLING_ERROR_NAMES = new Set(['Throttling', 'ThrottlingException']) @@ -26,7 +27,11 @@ export function isTransientAwsFailure(error: unknown): boolean { ) } -/** The caller-facing reason a saved AWS credential could not be used: the AWS message, never a key. */ +/** + * The caller-facing reason a saved AWS credential could not be used: the AWS message, never a key. + * An egress outage is Sim's, not the credential's, so it reads as that and never names its routing code. + */ export function awsCredentialFailureMessage(error: unknown): string { + if (error instanceof OutboundRoutingError) return EGRESS_UNAVAILABLE_MESSAGE return `Could not use the AWS credential: ${getErrorMessage(error, 'Unknown error')}` } diff --git a/apps/sim/lib/oauth/connect-scope.server.test.ts b/apps/sim/lib/oauth/connect-scope.server.test.ts index 61c8568eef6..200acc2600b 100644 --- a/apps/sim/lib/oauth/connect-scope.server.test.ts +++ b/apps/sim/lib/oauth/connect-scope.server.test.ts @@ -91,6 +91,17 @@ describe('withConnectDraftOutboundScope', () => { networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) }) + it('keeps a connector exactly as configured when the deployment has no outbound routing', () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + const config: GenericOAuthConfig = { + providerId: 'idp', + clientId: 'client-1', + tokenUrl: 'https://idp.test/token', + } + + expect(withConnectDraftOutboundScope(config)).toBe(config) + }) + it("runs a generic connector's code exchange in its connect draft's organization", async () => { mockGetOAuthState.mockResolvedValue({ callbackURL: '/workspace/ws-1/integrations?credentialDraftId=draft-1', @@ -185,6 +196,7 @@ describe('default code exchange for connectors without getToken', () => { vi.useFakeTimers({ toFake: ['Date'] }) vi.setSystemTime(new Date('2026-11-13T12:00:00Z')) mockGetOAuthState.mockResolvedValue(null) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) }) afterEach(() => { diff --git a/apps/sim/lib/oauth/connect-scope.server.ts b/apps/sim/lib/oauth/connect-scope.server.ts index b756db8bbe2..98a80f3678e 100644 --- a/apps/sim/lib/oauth/connect-scope.server.ts +++ b/apps/sim/lib/oauth/connect-scope.server.ts @@ -65,8 +65,10 @@ async function runInOAuthCallbackDraftScope( * `getToken` gets Better Auth's exchange sent through `outboundFetch`, since the plugin's * built-in exchange uses its own fetch. Better Auth's own discovery reads for the authorize URL * and callback stay unrouted; the exchange resolves its token endpoint through `outboundFetch`. + * A deployment without outbound routing keeps the connector exactly as configured. */ export function withConnectDraftOutboundScope(config: GenericOAuthConfig): GenericOAuthConfig { + if (!isOutboundRoutingEnabled()) return config const { providerId, getUserInfo } = config const getToken = config.getToken ?? createOutboundCodeExchange(config) return { diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index 3fe7db1a82f..18c4cb1ee1d 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -3,6 +3,7 @@ import { createDeferred } from '@sim/testing/helpers/deferred' import { queueTableRows, resetDbChainMock } from '@sim/testing/mocks/database.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, RAMP_SERVICE_ACCOUNT_PROVIDER_ID, @@ -286,6 +287,22 @@ describe('client credential token refresh after a failed mint', () => { }) }) + it("records nothing about the credential when Sim's egress route is down, so it asks again at once", async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-egress-outage') + mint.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + vi.advanceTimersByTime(6 * 60_000) + + await expect(resolveFor('ramp-egress-outage')).resolves.toEqual({ + error: expect.any(OutboundRoutingError), + }) + mint.mockResolvedValueOnce({ accessToken: 'next-token', expiresInSeconds: 600 }) + await expect(resolveFor('ramp-egress-outage')).resolves.toEqual({ + value: { accessToken: 'next-token' }, + }) + }) + it('retries a quick transient failure once inside the mint', async () => { const mint = minter() mint.mockRejectedValueOnce(unavailable(503)).mockResolvedValueOnce(token) diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts index a70ea094f58..4fe8913dde0 100644 --- a/apps/sim/lib/oauth/credential-service.test.ts +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -907,14 +907,18 @@ describe('OAuth access-token refresh headroom', () => { ) }) - it('rethrows an unavailable egress route instead of a revoked or failed refresh', async () => { - mocks.refreshOAuthToken.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) - const row = createOAuthAccount(WAITED_GOOGLE_REMAINING_MS) - queueTableRows(account, [row]) - await expect(refreshTokenIfNeeded('test', row, RAW_ACCOUNT_ID)).rejects.toBeInstanceOf( - OutboundRoutingError - ) - }) + it.each(readers)( + '$name rethrows an unavailable egress route instead of a missing, revoked or failed token', + async ({ read }) => { + mocks.refreshOAuthToken.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const row = createOAuthAccount(WAITED_GOOGLE_REMAINING_MS) + queueTableRows(account, [row]) + await expect(read(row)).rejects.toBeInstanceOf(OutboundRoutingError) + expect(dbChainMockFns.set).not.toHaveBeenCalledWith( + expect.objectContaining({ refreshRevokedCode: expect.anything() }) + ) + } + ) it('answers a Microsoft caller from its valid token while refresh-token aging refreshes', async () => { vi.mocked(isMicrosoftProvider).mockReturnValue(true) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 5bd06e7c281..99ebd696943 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -836,6 +836,8 @@ async function resolveMintedServiceAccountToken( return { ...minted.token } } catch (error) { if (error instanceof Error && error.name === 'AbortError') throw error + // Sim's egress route is down: nothing is known about the credential, so nothing is recorded. + if (error instanceof OutboundRoutingError) throw error if (sharedToken && !isCoordinatedReplay(error)) { clientCredentialMintFailureCache.delete(cacheIdentity) throw error @@ -1853,7 +1855,13 @@ async function performCoalescedRefresh({ logger.info('Successfully refreshed access token', logContext) return { kind: 'token', accessToken: result.accessToken } } catch (error) { - if (error instanceof OutboundRoutingError) return { kind: 'egress_unavailable', error } + if (error instanceof OutboundRoutingError) { + logger.warn('Egress route unavailable for token refresh', { + ...logContext, + code: error.code, + }) + return { kind: 'egress_unavailable', error } + } logger.error('Refresh failed inside leader path', { ...logContext, ...(privacyMode === 'selector' ? {} : { error: toError(error).message }), @@ -1963,6 +1971,8 @@ async function resolveStoredAccessToken( }) return { kind: 'token', accessToken: stored.accessToken, refreshed: false } } + // Sim's egress route is down: every caller surfaces it, never a missing or revoked token. + if (fresh?.kind === 'egress_unavailable') throw fresh.error return { kind: 'failed', outcome: fresh } } @@ -2139,7 +2149,6 @@ export async function refreshTokenIfNeeded( refreshed: result.refreshed, } } - if (result.outcome?.kind === 'egress_unavailable') throw result.outcome.error if (result.outcome?.kind === 'revoked') { throw new CredentialRevokedError( `The ${credential.providerId} credential was revoked by the provider (${result.outcome.errorCode}). Reconnect it to continue.`, diff --git a/apps/sim/lib/oauth/shopify-installation.test.ts b/apps/sim/lib/oauth/shopify-installation.test.ts new file mode 100644 index 00000000000..08ea8753dfa --- /dev/null +++ b/apps/sim/lib/oauth/shopify-installation.test.ts @@ -0,0 +1,46 @@ +import { databaseMock, resetDbChainMock } from '@sim/testing' +import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { refreshShopifyInstallation } from '@/lib/oauth/shopify-installation' + +const shopDomain = 'fixture.myshopify.com' +const findFirst = vi.fn() +const redis = { get: vi.fn(async () => null), set: vi.fn(async () => 'OK') } + +Object.assign(databaseMock.db.query, { account: { findFirst } }) +setEnv({ SHOPIFY_CLIENT_ID: 'shopify-client', SHOPIFY_CLIENT_SECRET: 'shopify-secret' }) +afterAll(resetEnvMock) + +beforeEach(() => { + resetDbChainMock() + redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) + findFirst.mockResolvedValue({ + id: 'account-1', + accountId: '123456789012345', + idToken: shopDomain, + accessToken: 'expired-access', + accessTokenExpiresAt: new Date(Date.now() - 60_000), + refreshToken: 'stored-refresh', + refreshTokenExpiresAt: null, + }) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) +}) + +describe('refreshShopifyInstallation', () => { + it('surfaces an unavailable egress route instead of a failed refresh, leaving the chain alive', async () => { + await expect( + runWithOutboundOrganization('organization-1', () => refreshShopifyInstallation('account-1')) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(redis.set).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/oauth/shopify-installation.ts b/apps/sim/lib/oauth/shopify-installation.ts index e2908ce9d35..5f10414c5d5 100644 --- a/apps/sim/lib/oauth/shopify-installation.ts +++ b/apps/sim/lib/oauth/shopify-installation.ts @@ -6,6 +6,7 @@ import { toRecord } from '@sim/utils/object' import { and, eq, sql } from 'drizzle-orm' import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -108,8 +109,9 @@ async function exchangeToken( redirect: 'error', signal: requestSignal, }) - } catch { + } catch (error) { signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error throw new ShopifyOAuthError('Shopify token response unavailable', 'shopify_token_error') } const data = await readShopifyResponse(response, requestSignal, signal) @@ -267,6 +269,7 @@ export async function refreshShopifyInstallation(accountId: string): Promise ({ vi.mock('@/lib/selectors/server/providers/credential-bundle', () => selectorCredentialBundleMock) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' import { harmonicSelectorAttachments } from '@/lib/selectors/server/providers/harmonic' import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' @@ -40,6 +41,14 @@ describe('Harmonic server selector adapter', () => { afterAll(() => vi.unstubAllGlobals()) + it('surfaces an unavailable egress route instead of unavailable selector options', async () => { + mockFetch.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + harmonicSelectorAttachments['harmonic.savedSearches'].execute(detailArgs('1')) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('hydrates a selected saved search after the list projection cap', async () => { const rows = Array.from({ length: 501 }, (_, index) => ({ id: index + 1, diff --git a/apps/sim/lib/selectors/server/providers/harmonic.ts b/apps/sim/lib/selectors/server/providers/harmonic.ts index 678fdcff542..f57bab77c32 100644 --- a/apps/sim/lib/selectors/server/providers/harmonic.ts +++ b/apps/sim/lib/selectors/server/providers/harmonic.ts @@ -1,6 +1,7 @@ import { isPlainRecord } from '@sim/utils/object' import { z } from 'zod' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { ServerSelectorKey } from '@/lib/selectors/manifest' import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' @@ -112,7 +113,7 @@ async function listSavedSearches( signal, }) } catch (error) { - if (args.signal?.aborted) throw error + if (args.signal?.aborted || error instanceof OutboundRoutingError) throw error throw new SelectorOptionsUnavailableError() } if (!response.ok) { diff --git a/apps/sim/lib/selectors/server/providers/http.test.ts b/apps/sim/lib/selectors/server/providers/http.test.ts index b32e09c20f9..37883ec8d33 100644 --- a/apps/sim/lib/selectors/server/providers/http.test.ts +++ b/apps/sim/lib/selectors/server/providers/http.test.ts @@ -1,5 +1,6 @@ import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { SelectorConnectionUnavailableError, SelectorOptionsUnavailableError, @@ -65,6 +66,16 @@ describe('provider HTTP selector boundary', () => { expect(cancel).toHaveBeenCalledOnce() }) + it('surfaces an unavailable egress route instead of unavailable selector options', async () => { + mockFetch.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + fetchProviderJsonWithStatus('https://provider.example/item', undefined, { + passthroughNetworkErrors: true, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('can preserve a generic retry signal without exposing the raw network error', async () => { mockFetch.mockRejectedValueOnce(new Error('fetch failed with provider-secret-canary')) diff --git a/apps/sim/lib/selectors/server/providers/http.ts b/apps/sim/lib/selectors/server/providers/http.ts index 9f0764bd4f0..727bd858811 100644 --- a/apps/sim/lib/selectors/server/providers/http.ts +++ b/apps/sim/lib/selectors/server/providers/http.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { parseRetryAfter } from '@sim/utils/retry' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { SelectorConnectionUnavailableError, SelectorOptionsUnavailableError, @@ -122,7 +123,7 @@ export async function fetchProviderJsonWithStatus( try { response = await outboundFetch(input, { ...init, signal, redirect: init?.redirect ?? 'error' }) } catch (error) { - if (init?.signal?.aborted) throw error + if (init?.signal?.aborted || error instanceof OutboundRoutingError) throw error logProviderFailure(input, 'network') if (options.passthroughNetworkErrors) throw new RetryableProviderNetworkError() throw new SelectorOptionsUnavailableError() diff --git a/apps/sim/lib/webhooks/polling/gmail.test.ts b/apps/sim/lib/webhooks/polling/gmail.test.ts index c1394a9dc82..3dc62d2ad73 100644 --- a/apps/sim/lib/webhooks/polling/gmail.test.ts +++ b/apps/sim/lib/webhooks/polling/gmail.test.ts @@ -10,6 +10,7 @@ import { webhooksProcessorMockFns, } from '@sim/testing/mocks/webhooks-processor.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' vi.mock('@/lib/core/idempotency/service', () => idempotencyServiceMock) @@ -24,8 +25,11 @@ import type { WorkflowRecord, } from '@/lib/webhooks/polling/types' -const { mockUpdateWebhookProviderConfig: mockUpdateConfig, mockResolveOAuthCredential } = - webhooksPollingUtilsMockFns +const { + mockUpdateWebhookProviderConfig: mockUpdateConfig, + mockResolveOAuthCredential, + mockMarkWebhookFailed, +} = webhooksPollingUtilsMockFns const mockProcessEvent = webhooksProcessorMockFns.mockProcessPolledWebhookEvent const EXPIRED_HISTORY_ID = '30000' @@ -191,3 +195,27 @@ describe('gmailPollingHandler history cursor', () => { expect(updates).not.toHaveProperty('historyId') }) }) + +describe('gmailPollingHandler egress outage', () => { + it('skips the poll without counting a failure when the egress route is down', async () => { + mockResolveOAuthCredential.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(gmailPollingHandler.pollWebhook(context({}))).resolves.toBe('skipped') + + expect(mockMarkWebhookFailed).not.toHaveBeenCalled() + }) + + it('leaves the mail for a later poll without counting a failure when the route drops mid-batch', async () => { + mockResolveOAuthCredential.mockResolvedValue('access-token') + mailboxWithExpiredHistory({ + profileHistoryId: '40100', + inboxMessages: [{ id: 'm-new', historyId: '40050' }], + }) + mockProcessEvent.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(gmailPollingHandler.pollWebhook(context({}))).resolves.toBe('skipped') + + expect(mockMarkWebhookFailed).not.toHaveBeenCalled() + expect(mockUpdateConfig).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/webhooks/polling/gmail.ts b/apps/sim/lib/webhooks/polling/gmail.ts index 0c76e27f6a8..52dc437e596 100644 --- a/apps/sim/lib/webhooks/polling/gmail.ts +++ b/apps/sim/lib/webhooks/polling/gmail.ts @@ -9,11 +9,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -124,9 +124,8 @@ export const gmailPollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing Gmail webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -564,7 +563,7 @@ async function processEmails( logger.info(`Successfully processed email ${email.id} for webhook ${webhookData.id}`) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error const errorMessage = getErrorMessage(error, 'Unknown error') logger.error(`Error processing email ${email.id}:`, errorMessage) failedCount++ diff --git a/apps/sim/lib/webhooks/polling/google-calendar.ts b/apps/sim/lib/webhooks/polling/google-calendar.ts index 6aae0073773..38c04d4c58c 100644 --- a/apps/sim/lib/webhooks/polling/google-calendar.ts +++ b/apps/sim/lib/webhooks/polling/google-calendar.ts @@ -10,11 +10,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -186,9 +186,8 @@ export const googleCalendarPollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing Google Calendar webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -371,7 +370,7 @@ async function processEvents( logger.info(`Successfully processed event ${event.id} for webhook ${webhookData.id}`) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error const errorMessage = getErrorMessage(error, 'Unknown error') logger.error(`Error processing event ${event.id}:`, errorMessage) failedCount++ diff --git a/apps/sim/lib/webhooks/polling/google-drive.ts b/apps/sim/lib/webhooks/polling/google-drive.ts index dd2131a8b9c..c463105a2e0 100644 --- a/apps/sim/lib/webhooks/polling/google-drive.ts +++ b/apps/sim/lib/webhooks/polling/google-drive.ts @@ -10,11 +10,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -168,9 +168,8 @@ export const googleDrivePollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped if (error instanceof Error && error.name === 'DrivePageTokenInvalidError') { await updateWebhookProviderConfig(webhookId, { pageToken: undefined }, logger) await markWebhookSuccess(webhookId, logger) @@ -414,7 +413,7 @@ async function processChanges( ) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error const errorMessage = getErrorMessage(error, 'Unknown error') logger.error(`Error processing change for file ${change.fileId}:`, errorMessage) failedCount++ diff --git a/apps/sim/lib/webhooks/polling/google-sheets.ts b/apps/sim/lib/webhooks/polling/google-sheets.ts index 900311a2feb..bf223aca59d 100644 --- a/apps/sim/lib/webhooks/polling/google-sheets.ts +++ b/apps/sim/lib/webhooks/polling/google-sheets.ts @@ -10,11 +10,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -227,9 +227,8 @@ export const googleSheetsPollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing Google Sheets webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -444,7 +443,7 @@ async function processRows( logger.info(`Successfully processed row ${rowNumber} for webhook ${webhookData.id}`) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error const errorMessage = getErrorMessage(error, 'Unknown error') logger.error(`Error processing row ${rowNumber}:`, errorMessage) failedCount++ diff --git a/apps/sim/lib/webhooks/polling/hubspot.ts b/apps/sim/lib/webhooks/polling/hubspot.ts index 543243bb47f..fc742467ee0 100644 --- a/apps/sim/lib/webhooks/polling/hubspot.ts +++ b/apps/sim/lib/webhooks/polling/hubspot.ts @@ -9,11 +9,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -210,9 +210,8 @@ export const hubspotPollingHandler: PollingProviderHandler = { } return await pollSearchBased(ctx, config, accessToken) } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing HubSpot webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -460,7 +459,7 @@ async function pollListMembership( ) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error failedCount++ logger.error( `Error processing HubSpot list membership ${member.recordId}:`, @@ -840,7 +839,7 @@ async function processRecords( snapshot.values.set(record.id, propertyValue ?? null) } } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error failedCount++ cursorFrozen = true logger.error( diff --git a/apps/sim/lib/webhooks/polling/imap.ts b/apps/sim/lib/webhooks/polling/imap.ts index 993fb9db798..fde8a4a6b63 100644 --- a/apps/sim/lib/webhooks/polling/imap.ts +++ b/apps/sim/lib/webhooks/polling/imap.ts @@ -17,10 +17,10 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -203,9 +203,8 @@ export const imapPollingHandler: PollingProviderHandler = { throw innerError } } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing IMAP webhook ${webhookId}`) await markWebhookFailed(webhookId, logger) return 'failure' @@ -608,7 +607,7 @@ async function processEmails( ) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error logger.error(`Error processing email ${email.uid}`) failedCount++ } diff --git a/apps/sim/lib/webhooks/polling/outlook.ts b/apps/sim/lib/webhooks/polling/outlook.ts index 8655ecc53a9..8b48cc22ec0 100644 --- a/apps/sim/lib/webhooks/polling/outlook.ts +++ b/apps/sim/lib/webhooks/polling/outlook.ts @@ -11,11 +11,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -195,9 +195,8 @@ export const outlookPollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing Outlook webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -515,7 +514,7 @@ async function processOutlookEmails( logger.info(`Successfully processed email ${email.id} for webhook ${webhookData.id}`) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error logger.error(`Error processing email ${email.id}:`, error) failedCount++ } diff --git a/apps/sim/lib/webhooks/polling/utils.test.ts b/apps/sim/lib/webhooks/polling/utils.test.ts index 4049ab7d75d..56f084605fb 100644 --- a/apps/sim/lib/webhooks/polling/utils.test.ts +++ b/apps/sim/lib/webhooks/polling/utils.test.ts @@ -6,11 +6,13 @@ vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) vi.mock('@/triggers/constants', () => ({ MAX_CONSECUTIVE_FAILURES: 5 })) import { sql } from 'drizzle-orm' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { isPollBackedOff, PollFetchError, readPollRetryAfterMs, recordPollSourceFailure, + skipUncountedPollFailure, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -113,6 +115,26 @@ describe('poll source backoff', () => { expect(Date.parse(String(stored.pollBackoffUntil))).toBe(Date.now() + minutes(10)) }) + /** Whether any write in this test incremented the webhook's consecutive-failure count. */ + function countedAFailure(): boolean { + return dbChainMockFns.set.mock.calls.some(([values]) => 'failedCount' in Object(values)) + } + + it('counts an ordinary source failure toward the auto-disable', async () => { + await failOnce(0) + expect(countedAFailure()).toBe(true) + }) + + it('backs off an egress outage without counting it against the webhook', async () => { + const outage = new Error('feed fetch failed', { + cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + }) + const stored = await failOnce(0, outage) + + expect(stored.pollSourceFailures).toBe(1) + expect(countedAFailure()).toBe(false) + }) + it('ignores a missing or malformed window', () => { for (const config of [{}, { pollBackoffUntil: 'not-a-date' }, { pollBackoffUntil: 42 }, null]) { expect(isPollBackedOff(config, pollStartedAt)).toBe(false) @@ -131,3 +153,16 @@ describe('readPollRetryAfterMs', () => { expect(readPollRetryAfterMs(header, body)).toBe(expected) }) }) + +describe('skipUncountedPollFailure', () => { + it('skips a poll stopped by an egress outage anywhere in the cause chain', () => { + const outage = new Error('token refresh failed', { + cause: new OutboundRoutingError('ROUTE_BLOCKED'), + }) + expect(skipUncountedPollFailure(outage, logger, 'wh-1')).toBe('skipped') + }) + + it('leaves any other failure to be counted', () => { + expect(skipUncountedPollFailure(new Error('invalid_grant'), logger, 'wh-1')).toBeNull() + }) +}) diff --git a/apps/sim/lib/webhooks/polling/utils.ts b/apps/sim/lib/webhooks/polling/utils.ts index 46ceec0ff9f..18025ad4401 100644 --- a/apps/sim/lib/webhooks/polling/utils.ts +++ b/apps/sim/lib/webhooks/polling/utils.ts @@ -6,6 +6,7 @@ import { toRecord } from '@sim/utils/object' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { and, eq, isNull, ne, or, sql } from 'drizzle-orm' import { getDeterministicAdmissionRejectionCode } from '@/lib/core/admission/rejection' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { getOAuthToken, refreshAccessTokenIfNeeded, @@ -62,12 +63,37 @@ export function throwIfAdmissionRefused(result: PolledWebhookEventResult): void if (getDeterministicAdmissionRejectionCode(result)) throw new PollAdmissionRefusedError(result) } -/** Logs a poll stopped by {@link PollAdmissionRefusedError} and reports it as skipped. */ -export function skipAdmissionRefusedPoll(logger: Logger, webhookId: string): 'skipped' { - logger.info('Execution admission refused the webhook; left its items for a later poll', { - webhookId, - }) - return 'skipped' +/** + * Whether a poll stopped by `error` is stopped by something that is not the webhook's fault: an + * execution admission refusal ({@link PollAdmissionRefusedError}) or an outage of Sim's own egress + * route. A poller rethrows it out of its batch while no item has completed, as it does a refusal. + */ +export function isUncountedPollFailure(error: unknown): boolean { + return error instanceof PollAdmissionRefusedError || isUnavailableFailure(error) +} + +/** + * Reports a poll stopped by a failure that is not the webhook's, so it never counts toward + * `MAX_CONSECUTIVE_FAILURES`: an execution admission refusal ({@link PollAdmissionRefusedError}) + * or an outage of Sim's own egress route. Returns `null` for any other error, which the poller + * records as a failure. + */ +export function skipUncountedPollFailure( + error: unknown, + logger: Logger, + webhookId: string +): 'skipped' | null { + if (error instanceof PollAdmissionRefusedError) { + logger.info('Execution admission refused the webhook; left its items for a later poll', { + webhookId, + }) + return 'skipped' + } + if (isUnavailableFailure(error)) { + logger.warn('Egress route unavailable; left the webhook for a later poll', { webhookId }) + return 'skipped' + } + return null } /** @@ -109,7 +135,8 @@ export function clearPollBackoff(providerConfig: unknown): Record, @@ -144,7 +171,7 @@ export async function recordPollSourceFailure( }, logger ) - await markWebhookFailed(webhookData.id, logger) + if (!isUnavailableFailure(error)) await markWebhookFailed(webhookData.id, logger) } /** diff --git a/apps/sim/lib/webhooks/slack-custom-ingress.test.ts b/apps/sim/lib/webhooks/slack-custom-ingress.test.ts index 44d98649ee7..d1ecd342c8c 100644 --- a/apps/sim/lib/webhooks/slack-custom-ingress.test.ts +++ b/apps/sim/lib/webhooks/slack-custom-ingress.test.ts @@ -1,4 +1,10 @@ import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { + createMockWorkspaceApplicationContext, + workspaceContextMock, + workspaceContextMockFns, +} from '@sim/testing/mocks/workspace-context.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -12,6 +18,8 @@ vi.mock('@/lib/execution/cancel-workflow-execution', () => ({ cancelWorkflowExecution: mocks.cancel, })) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) vi.mock('@/lib/webhooks/slack-agent-api', () => ({ setSlackAgentSessionStatus: mocks.setStatus, })) @@ -22,6 +30,7 @@ vi.mock('@/lib/webhooks/slack-stream-sessions', () => ({ unregisterSlackStreamSession: mocks.unregister, })) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { getLegacySlackCustomBotCredentialId, handleSlackAgentSessionStopped, @@ -51,6 +60,7 @@ describe('getLegacySlackCustomBotCredentialId', () => { describe('handleSlackAgentSessionStopped', () => { beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) mocks.listSessions.mockResolvedValue([]) authOAuthUtilsMockFns.mockGetSlackBotCredential.mockResolvedValue({ botToken: 'xoxb-test' }) }) @@ -111,4 +121,34 @@ describe('handleSlackAgentSessionStopped', () => { expect(mocks.unregister).not.toHaveBeenCalled() expect(mocks.setStatus).not.toHaveBeenCalled() }) + + it("returns the session to active through the bot credential's organization egress route", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + mocks.listSessions.mockResolvedValue([ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + userId: 'user-1', + workspaceId: 'workspace-1', + }, + ]) + mocks.cancel.mockResolvedValue(undefined) + authOAuthUtilsMockFns.mockGetSlackBotCredential.mockResolvedValue({ + botToken: 'xoxb-test', + workspaceId: 'workspace-1', + }) + workspaceContextMockFns.mockLoadWorkspaceApplicationContext.mockResolvedValue( + createMockWorkspaceApplicationContext({ + workspaceId: 'workspace-1', + workspaceOrganizationId: 'org-1', + }) + ) + mocks.setStatus.mockImplementation(async () => { + await resolveCurrentOutboundRoute() + }) + + await handleSlackAgentSessionStopped('credential-1', { kind: 'stop' }) + + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + }) }) diff --git a/apps/sim/lib/webhooks/slack-custom-ingress.ts b/apps/sim/lib/webhooks/slack-custom-ingress.ts index 84dab841f46..2c0b2dff41c 100644 --- a/apps/sim/lib/webhooks/slack-custom-ingress.ts +++ b/apps/sim/lib/webhooks/slack-custom-ingress.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { cancelWorkflowExecution } from '@/lib/execution/cancel-workflow-execution' import { getSlackBotCredential } from '@/lib/oauth/credential-service' import { findWebhooksByRoutingKey, type WebhookDispatchResult } from '@/lib/webhooks/processor' @@ -163,5 +164,7 @@ export async function handleSlackAgentSessionStopped( if (!credential) { throw new Error('Slack agent session stop credential is unavailable') } - await setSlackAgentSessionStatus(credential.botToken, target, 'active') + const { botToken, workspaceId } = credential + const restoreStatus = () => setSlackAgentSessionStatus(botToken, target, 'active') + await (workspaceId ? withResourceOutboundScope({ workspaceId }, restoreStatus) : restoreStatus()) } diff --git a/packages/testing/src/mocks/webhooks-polling-utils.mock.ts b/packages/testing/src/mocks/webhooks-polling-utils.mock.ts index 3e77f839fec..7b3184f53c9 100644 --- a/packages/testing/src/mocks/webhooks-polling-utils.mock.ts +++ b/packages/testing/src/mocks/webhooks-polling-utils.mock.ts @@ -14,6 +14,23 @@ class PollAdmissionRefusedError extends Error { } } +/** + * Faithful copy of the production egress-outage test (`asOrchestrationError(error)?.code === + * 'unavailable'`): an `OutboundRoutingError`, or an `OrchestrationError` coded `unavailable`, + * anywhere in the cause chain. + */ +function isSimOutage(error: unknown): boolean { + let current: unknown = error + while (current instanceof Error) { + if (current.name === 'OutboundRoutingError') return true + if (current.name === 'OrchestrationError') { + return (current as Error & { code?: string }).code === 'unavailable' + } + current = current.cause + } + return false +} + /** Faithful copy of the production `PollFetchError`. */ class PollFetchError extends Error { readonly status: number @@ -32,8 +49,8 @@ class PollFetchError extends Error { * * State writes (`markWebhookFailed`, `markWebhookSuccess`, `updateWebhookProviderConfig`, * `recordPollSourceFailure`) resolve `undefined`; `fetchActiveWebhooks` resolves `[]`; - * `resolveOAuthCredential` is a bare `vi.fn()`. `throwIfAdmissionRefused` and - * `skipAdmissionRefusedPoll` keep production behavior so a poller's refusal path runs as it + * `resolveOAuthCredential` is a bare `vi.fn()`. `throwIfAdmissionRefused`, + * `isUncountedPollFailure`, and `skipUncountedPollFailure` keep production behavior so a poller's refusal path runs as it * does in production. Defaults are `vi.fn(impl)`, so `mockReset()` restores them. * * @example @@ -52,7 +69,12 @@ export const webhooksPollingUtilsMockFns = { } } ), - mockSkipAdmissionRefusedPoll: vi.fn((..._args: unknown[]): 'skipped' => 'skipped'), + mockIsUncountedPollFailure: vi.fn( + (error: unknown): boolean => error instanceof PollAdmissionRefusedError || isSimOutage(error) + ), + mockSkipUncountedPollFailure: vi.fn((error: unknown, ..._args: unknown[]): 'skipped' | null => + error instanceof PollAdmissionRefusedError || isSimOutage(error) ? 'skipped' : null + ), mockReadPollRetryAfterMs: vi.fn((_header: string | null, _body: string): number | null => null), mockClearPollBackoff: vi.fn((_providerConfig: unknown): Record => ({})), mockRecordPollSourceFailure: vi.fn(async (..._args: unknown[]): Promise => {}), @@ -79,7 +101,8 @@ export const webhooksPollingUtilsMock = { PollFetchError, isPollBackedOff: webhooksPollingUtilsMockFns.mockIsPollBackedOff, throwIfAdmissionRefused: webhooksPollingUtilsMockFns.mockThrowIfAdmissionRefused, - skipAdmissionRefusedPoll: webhooksPollingUtilsMockFns.mockSkipAdmissionRefusedPoll, + isUncountedPollFailure: webhooksPollingUtilsMockFns.mockIsUncountedPollFailure, + skipUncountedPollFailure: webhooksPollingUtilsMockFns.mockSkipUncountedPollFailure, readPollRetryAfterMs: webhooksPollingUtilsMockFns.mockReadPollRetryAfterMs, clearPollBackoff: webhooksPollingUtilsMockFns.mockClearPollBackoff, recordPollSourceFailure: webhooksPollingUtilsMockFns.mockRecordPollSourceFailure, From cfd06e41e815a160a95a4e5f160dab7a77adf959 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:41:15 -0800 Subject: [PATCH 31/32] fix(egress): keep a bad routing configuration from failing the auth module's load --- apps/sim/lib/oauth/connect-scope.server.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/oauth/connect-scope.server.ts b/apps/sim/lib/oauth/connect-scope.server.ts index 98a80f3678e..49ff4f69e94 100644 --- a/apps/sim/lib/oauth/connect-scope.server.ts +++ b/apps/sim/lib/oauth/connect-scope.server.ts @@ -59,6 +59,19 @@ async function runInOAuthCallbackDraftScope( return (await inDraftScope)(run) } +/** + * Whether routing is on, read once while the auth config is built. A routing configuration that + * fails to load counts as on, so the connector stays wrapped and the bad configuration surfaces + * on its first request like every other routed call, rather than failing the auth module's load. + */ +function routingConfiguredAtStartup(): boolean { + try { + return isOutboundRoutingEnabled() + } catch { + return true + } +} + /** * Wraps a generic OAuth connector's code exchange and identity lookup so their vendor calls * leave through the connecting organization's egress gateway. A connector without its own @@ -68,7 +81,7 @@ async function runInOAuthCallbackDraftScope( * A deployment without outbound routing keeps the connector exactly as configured. */ export function withConnectDraftOutboundScope(config: GenericOAuthConfig): GenericOAuthConfig { - if (!isOutboundRoutingEnabled()) return config + if (!routingConfiguredAtStartup()) return config const { providerId, getUserInfo } = config const getToken = config.getToken ?? createOutboundCodeExchange(config) return { From 7db9dbcbbd90f22a62ff69c93aee68636afc985c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:56:37 -0800 Subject: [PATCH 32/32] improvement(egress): keep org egress outages from disabling connectors or failing degradable work - slack native formatInput: an outage resolving the bot token degrades the event (no token) instead of failing it - connector content and member sync: an outage retries at the first backoff rung without touching the auto-disable breaker - selector credential bundle: rethrow an outage instead of reporting the connection unavailable - internal LLM Vertex credential: an outage returns 503 with the fixed egress message - webhook provider subscriptions: projected failures keep the unavailable classification; the webhook save route and deploy report it as 503 - gmail/outlook configurePolling: rethrow an outage instead of the permissions rollback message --- apps/sim/app/api/webhooks/route.test.ts | 33 ++++++++++++ apps/sim/app/api/webhooks/route.ts | 18 +++++++ apps/sim/lib/internal/llm/operations.test.ts | 22 ++++++++ apps/sim/lib/internal/llm/operations.ts | 12 +++++ .../connectors/member-sync-engine.test.ts | 15 ++++++ .../connectors/member-sync-engine.ts | 13 ++++- .../knowledge/connectors/sync-engine.test.ts | 15 ++++++ .../lib/knowledge/connectors/sync-engine.ts | 28 ++++++---- .../providers/credential-bundle.test.ts | 16 ++++++ .../server/providers/credential-bundle.ts | 3 +- apps/sim/lib/webhooks/deploy.test.ts | 7 +++ apps/sim/lib/webhooks/deploy.ts | 8 ++- .../webhooks/provider-subscriptions.test.ts | 21 ++++++++ .../lib/webhooks/provider-subscriptions.ts | 8 ++- .../providers/email-polling-outage.test.ts | 33 ++++++++++++ apps/sim/lib/webhooks/providers/gmail.ts | 2 + apps/sim/lib/webhooks/providers/outlook.ts | 2 + apps/sim/lib/webhooks/providers/slack.test.ts | 53 ++++++++++++++++++- apps/sim/lib/webhooks/providers/slack.ts | 15 +++++- apps/sim/lib/webhooks/providers/types.ts | 5 +- 20 files changed, 309 insertions(+), 20 deletions(-) create mode 100644 apps/sim/lib/webhooks/providers/email-polling-outage.test.ts diff --git a/apps/sim/app/api/webhooks/route.test.ts b/apps/sim/app/api/webhooks/route.test.ts index 6417c4382b5..47218919ef4 100644 --- a/apps/sim/app/api/webhooks/route.test.ts +++ b/apps/sim/app/api/webhooks/route.test.ts @@ -50,6 +50,8 @@ vi.mock('@/lib/webhooks/utils.server', () => ({ findConflictingWebhookPathOwner: mocks.findConflictingWebhookPathOwner, })) +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE, OrchestrationError } from '@/lib/core/orchestration/types' import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { POST } from '@/app/api/webhooks/route' @@ -619,3 +621,34 @@ describe('POST /api/webhooks subscription replacement recovery', () => { } ) }) + +describe('POST /api/webhooks egress outage', () => { + beforeEach(setupUpsertMocks) + + it('reports an outage while creating the external subscription as unavailable', async () => { + queueCreatePathRows() + mocks.createExternalWebhookSubscription.mockRejectedValue( + new OrchestrationError('unavailable', EGRESS_UNAVAILABLE_MESSAGE) + ) + + const response = await POST(upsertRequest(), {}) + + expect(response.status).toBe(503) + expect(await response.json()).toEqual({ error: EGRESS_UNAVAILABLE_MESSAGE }) + }) + + it('reports an outage while configuring polling as unavailable', async () => { + queueCreatePathRows() + dbChainMockFns.returning.mockImplementationOnce(async () => [ + { id: 'webhook-1', workflowId: 'workflow-1', path: 'inbound-orders', isActive: true }, + ]) + mocks.activateExternalWebhookSubscription.mockResolvedValue(undefined) + mocks.configurePolling.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + mocks.getProviderHandler.mockReturnValue({ configurePolling: mocks.configurePolling }) + + const response = await POST(upsertRequest(), {}) + + expect(response.status).toBe(503) + expect(await response.json()).toEqual({ error: EGRESS_UNAVAILABLE_MESSAGE }) + }) +}) diff --git a/apps/sim/app/api/webhooks/route.ts b/apps/sim/app/api/webhooks/route.ts index dc93062b21f..4cbc0a8f7e3 100644 --- a/apps/sim/app/api/webhooks/route.ts +++ b/apps/sim/app/api/webhooks/route.ts @@ -17,6 +17,11 @@ import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' import { AuthType } from '@/lib/auth/hybrid' +import { + EGRESS_UNAVAILABLE_MESSAGE, + isUnavailableFailure, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -569,6 +574,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } } } catch (err) { + if (isUnavailableFailure(err)) { + logger.warn('External webhook subscription unavailable during an egress outage') + return NextResponse.json( + { error: EGRESS_UNAVAILABLE_MESSAGE }, + { status: statusForOrchestrationError('unavailable') } + ) + } const isConfigurationError = err instanceof WebhookDeploymentConfigurationError if (isConfigurationError) { logger.warn('Rejected webhook trigger configuration', { @@ -746,6 +758,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => { err ) await revertSavedWebhook(savedWebhook, existingWebhook, requestId) + if (isUnavailableFailure(err)) { + return NextResponse.json( + { error: EGRESS_UNAVAILABLE_MESSAGE }, + { status: statusForOrchestrationError('unavailable') } + ) + } return NextResponse.json( { error: `Failed to configure ${provider} webhook`, diff --git a/apps/sim/lib/internal/llm/operations.test.ts b/apps/sim/lib/internal/llm/operations.test.ts index 93209b0370a..29fa0695f0e 100644 --- a/apps/sim/lib/internal/llm/operations.test.ts +++ b/apps/sim/lib/internal/llm/operations.test.ts @@ -14,6 +14,8 @@ import { permissionCheckMock } from '@sim/testing/mocks/permission-check.mock' import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { providersMock, providersMockFns } from '@sim/testing/mocks/providers.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' const mocks = vi.hoisted(() => ({ authorizeCredential: vi.fn(), @@ -132,6 +134,26 @@ describe('executeLlmProviderOperation', () => { ) }) + it('reports a Vertex credential the egress outage kept from resolving as unavailable', async () => { + mockExecuteProviderRequest.mockClear() + mocks.resolveVertexAccessToken.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + executeLlmProviderOperation( + { + provider: 'vertex', + model: 'vertex/gemini-2.5-pro', + vertexCredential: 'credential-1', + workspaceId: 'workspace-1', + }, + { actorUserId: 'user-1', headers: new Headers(), requestId: 'request-1' } + ) + ).rejects.toMatchObject({ status: 503, body: { error: EGRESS_UNAVAILABLE_MESSAGE } }) + expect(mockExecuteProviderRequest).not.toHaveBeenCalled() + }) + it("signs a Bedrock call with the selected AWS credential's session for the caller", async () => { mocks.authorizeCredential.mockResolvedValueOnce(AUTHORIZED_AWS_CREDENTIAL) mockResolveAuthorizedAwsCredential.mockResolvedValueOnce({ diff --git a/apps/sim/lib/internal/llm/operations.ts b/apps/sim/lib/internal/llm/operations.ts index 528b4954b11..6b642671548 100644 --- a/apps/sim/lib/internal/llm/operations.ts +++ b/apps/sim/lib/internal/llm/operations.ts @@ -8,6 +8,11 @@ import { type BillingAttributionSnapshot, requireBillingAttributionHeader, } from '@/lib/billing/core/attribution' +import { + EGRESS_UNAVAILABLE_MESSAGE, + isUnavailableFailure, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { inspectModelInputProjectionState, inspectModelInputProvenanceRequest, @@ -158,6 +163,13 @@ async function prepareProviderRequest( if (error instanceof BedrockCredentialError && error.reason === 'unavailable') { fail(503, error.message) } + if (isUnavailableFailure(error)) { + logger.warn('Provider credential unavailable during an egress outage', { + requestId: context.requestId, + provider: input.provider, + }) + fail(statusForOrchestrationError('unavailable'), EGRESS_UNAVAILABLE_MESSAGE) + } logger.error('Failed to resolve provider credential', { requestId: context.requestId, provider: input.provider, diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts index a41e08717ee..04c6ec29d2b 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts @@ -4,6 +4,8 @@ import { knowledgeDocumentsServiceMock } from '@sim/testing/mocks/knowledge-docu import { knowledgeMemberAccessMock } from '@sim/testing/mocks/knowledge-member-access.mock' import { DrizzleQueryError } from 'drizzle-orm/errors' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) vi.mock('@/lib/knowledge/documents/service', () => knowledgeDocumentsServiceMock) @@ -273,6 +275,19 @@ describe('member sync engine decisions', () => { }) expect(update.nextMemberSyncAt).not.toBeNull() }) + + it('does not disable a connector one failure from the breaker over an egress outage', async () => { + const update = await resolveMemberSyncFailureUpdate( + new Error('listing failed', { cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE') }), + failure + ) + expect(update).toMatchObject({ + memberSyncStatus: 'error', + lastMemberSyncError: EGRESS_UNAVAILABLE_MESSAGE, + memberSyncConsecutiveFailures: MAX_CONSECUTIVE_FAILURES - 1, + }) + expect(update.nextMemberSyncAt).not.toBeNull() + }) }) describe('nextMemberSyncTime', () => { diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 3e5ffe7088b..aa5a377665f 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -28,6 +28,7 @@ import { type BillingAttributionSnapshot, } from '@/lib/billing/core/attribution' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { EGRESS_UNAVAILABLE_MESSAGE, isUnavailableFailure } from '@/lib/core/orchestration/types' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { @@ -382,8 +383,8 @@ export function memberRunMadeProgress(result: MemberSyncResult): boolean { /** * The connector row a failed members-mode run writes. A deterministic capacity rejection waits for - * an operator, a transient database failure retries without touching the breaker, and anything - * else climbs the ladder toward auto-disable. + * an operator, a transient database failure or an organization egress outage retries without + * touching the breaker, and anything else climbs the ladder toward auto-disable. */ export async function resolveMemberSyncFailureUpdate( error: unknown, @@ -423,6 +424,14 @@ export async function resolveMemberSyncFailureUpdate( }) ) } + if (isUnavailableFailure(error)) { + return buildMemberSyncDatabaseRetryUpdate( + now, + failure.previousFailures, + EGRESS_UNAVAILABLE_MESSAGE, + connectorFailureBackoffMinutes(1) * 60 * 1000 + ) + } return buildMemberSyncFailureUpdate( now, failure.previousFailures, diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index cb218d47599..7e12dc11bf0 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -25,6 +25,8 @@ import { import { generateShortId } from '@sim/utils/id' import { DrizzleQueryError } from 'drizzle-orm/errors' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' import * as connectorTokens from '@/lib/knowledge/connectors/access-token' import { routeWindowTallies, @@ -820,6 +822,19 @@ describe('executeSync database failures', () => { expect.objectContaining({ status: 'failed', databaseFailureClass: 'capacity' }) ) }) + + it('does not disable a connector one failure from the breaker over an egress outage', async () => { + const { terminal, MAX_CONSECUTIVE_FAILURES } = await failSyncWith( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + expect(terminal).toMatchObject({ + status: 'error', + lastSyncError: EGRESS_UNAVAILABLE_MESSAGE, + consecutiveFailures: MAX_CONSECUTIVE_FAILURES - 1, + }) + expect((terminal?.nextSyncAt as Date).getTime()).toBeGreaterThan(NOW.getTime()) + }) }) describe('previous complete listing evidence', () => { diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 2265263d505..dd709a98f86 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -20,6 +20,7 @@ import { type BillingAttributionSnapshot, } from '@/lib/billing/core/attribution' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { EGRESS_UNAVAILABLE_MESSAGE, isUnavailableFailure } from '@/lib/core/orchestration/types' import { type ResourceScope, resourceScopeFields, @@ -1645,19 +1646,26 @@ export async function executeSync( madeProgress: result.docsAdded + result.docsUpdated + result.docsDeleted > 0, }) ) - : rateLimited - ? buildSyncRateLimitUpdate( + : isUnavailableFailure(error) + ? buildSyncDatabaseRetryUpdate( new Date(), connector.consecutiveFailures, - errorMessage, - retryAfterMs - ) - : buildSyncFailureUpdate( - new Date(), - connector.consecutiveFailures, - errorMessage, - retryAfterMs + EGRESS_UNAVAILABLE_MESSAGE, + connectorFailureBackoffMinutes(1) * 60 * 1000 ) + : rateLimited + ? buildSyncRateLimitUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) + : buildSyncFailureUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) if (failureUpdate.status === 'disabled') { logger.warn('Connector disabled after repeated failures', { diff --git a/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts b/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts index a98da4a940e..ba89b458b69 100644 --- a/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts +++ b/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts @@ -1,5 +1,6 @@ import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' import { describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' const mockResolveOrganizationToken = vi.hoisted(() => vi.fn()) vi.mock('@/lib/credentials/application/organization-credentials', () => ({ @@ -36,6 +37,21 @@ describe('selector credential bundles', () => { expect(protectedValues.contains('prefix-cloud-1-suffix')).toBe(false) }) + it('surfaces an egress outage instead of reporting the connection unavailable', async () => { + const outage = new OutboundRoutingError('GATEWAY_UNAVAILABLE') + mockResolveCredentialAccessToken.mockRejectedValueOnce(outage) + + await expect( + resolveSelectorCredentialBundle({ + credential: { + suppliedId: 'credential-1', + access: { ok: true, credentialOwnerUserId: 'owner-1' }, + }, + protectedValues: createSelectorProtectedValues(), + }) + ).rejects.toBe(outage) + }) + it('preserves a selector abort without canceling the shared resolution', async () => { let resolveShared!: (value: { accessToken: string }) => void const sharedResolution = new Promise<{ accessToken: string }>((resolve) => { diff --git a/apps/sim/lib/selectors/server/providers/credential-bundle.ts b/apps/sim/lib/selectors/server/providers/credential-bundle.ts index 109fd5c6750..cc86a43e3f2 100644 --- a/apps/sim/lib/selectors/server/providers/credential-bundle.ts +++ b/apps/sim/lib/selectors/server/providers/credential-bundle.ts @@ -1,3 +1,4 @@ +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { resolveOrganizationCredentialTokenBundle } from '@/lib/credentials/application/organization-credentials' import { resolveCredentialTokenBundle, @@ -61,7 +62,7 @@ export async function resolveSelectorCredentialBundle(input: { ) credential.signal?.throwIfAborted() } catch (error) { - if (credential.signal?.aborted) throw error + if (credential.signal?.aborted || isUnavailableFailure(error)) throw error throw new SelectorConnectionUnavailableError() } if (!bundle?.accessToken) throw new SelectorConnectionUnavailableError() diff --git a/apps/sim/lib/webhooks/deploy.test.ts b/apps/sim/lib/webhooks/deploy.test.ts index 54a4ae39ff3..d84e1d0051a 100644 --- a/apps/sim/lib/webhooks/deploy.test.ts +++ b/apps/sim/lib/webhooks/deploy.test.ts @@ -67,6 +67,7 @@ vi.mock('@/lib/webhooks/quickbooks-credentials', () => ({ getQuickBooksWebhookClientConfigByCredentialId: mockGetQuickBooksWebhookCredential, })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { buildProviderConfig, cleanupInactiveDeploymentWebhooks, @@ -780,4 +781,10 @@ describe('prepareStableTriggerWebhooksForDeploy failure status', () => { const result = await prepare(new AggregateError([rejected, new Error('HTTP 503')])) expect(result.error?.status).toBe(500) }) + + it('reports an egress outage as unavailable', async () => { + const outage = new OutboundRoutingError('GATEWAY_UNAVAILABLE') + const result = await prepare(new AggregateError([rejected, outage])) + expect(result.error?.status).toBe(503) + }) }) diff --git a/apps/sim/lib/webhooks/deploy.ts b/apps/sim/lib/webhooks/deploy.ts index 275c353cc8f..abcd22f20b2 100644 --- a/apps/sim/lib/webhooks/deploy.ts +++ b/apps/sim/lib/webhooks/deploy.ts @@ -7,6 +7,7 @@ import { omit, toRecord } from '@sim/utils/object' import { and, asc, eq, inArray, isNull, ne, or } from 'drizzle-orm' import type { NextRequest } from 'next/server' import { isSlackExtendedScopesEnabled } from '@/lib/core/config/env-flags' +import { isUnavailableFailure, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { getProviderIdFromServiceId } from '@/lib/oauth' import { getSlackBotCredential, @@ -772,6 +773,7 @@ async function configurePollingIfNeeded( */ function webhookConfigFailureStatus(error: unknown): number { const failures = error instanceof AggregateError ? error.errors : [error] + if (failures.some(isUnavailableFailure)) return statusForOrchestrationError('unavailable') const permanent = failures.length > 0 && failures.every((failure) => failure instanceof WebhookDeploymentConfigurationError) @@ -1012,7 +1014,9 @@ export async function saveTriggerWebhooksForDeploy({ success: false, error: { message: getErrorMessage(error, 'Failed to activate webhook subscription'), - status: 500, + status: isUnavailableFailure(error) + ? statusForOrchestrationError('unavailable') + : 500, }, } } @@ -1302,7 +1306,7 @@ export async function saveTriggerWebhooksForDeploy({ (cleanupFailure as Error)?.message || (error as Error)?.message || 'Failed to save webhook records', - status: 500, + status: isUnavailableFailure(error) ? statusForOrchestrationError('unavailable') : 500, }, } } diff --git a/apps/sim/lib/webhooks/provider-subscriptions.test.ts b/apps/sim/lib/webhooks/provider-subscriptions.test.ts index decb331a631..c61cece9c6c 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.test.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.test.ts @@ -5,6 +5,8 @@ import { } from '@sim/testing/mocks/billing-attribution.mock' import type { NextRequest } from 'next/server' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' const { mockGetEffectiveDecryptedEnv, mockGetExecutionEnvironment } = environmentUtilsMockFns @@ -57,6 +59,25 @@ describe('createExternalWebhookSubscription', () => { } ) + it('keeps an egress outage classified as unavailable', async () => { + mockGetProviderHandler.mockReturnValue({ + createSubscription: async () => { + throw new Error('Provider call failed', { + cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + }) + }, + }) + const failure = await createExternalWebhookSubscription( + {} as NextRequest, + { provider: 'ashby', providerConfig: {} }, + { workspaceId: 'ws-1' }, + 'user-1', + 'req-1' + ).catch((error: unknown) => error) + + expect(isUnavailableFailure(failure)).toBe(true) + }) + it('resolves {{ENV_VAR}} references in providerConfig before calling the provider', async () => { const createSubscription = vi.fn().mockResolvedValue({ providerConfigUpdates: { externalId: 'ext-1' }, diff --git a/apps/sim/lib/webhooks/provider-subscriptions.ts b/apps/sim/lib/webhooks/provider-subscriptions.ts index 61f172c6f18..d764c28a9df 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { omit, toRecord } from '@sim/utils/object' import type { NextRequest } from 'next/server' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { isSensitiveKey } from '@/lib/core/security/redaction' import { resolveBackgroundWebhookEnv, @@ -19,12 +20,17 @@ import { OPAQUE_RESOLVED_SECRET_REPLACEMENT } from '@/executor/utils/resolved-se const logger = createLogger('WebhookProviderSubscriptions') -/** Resolving credentials must not make a provider's exception a durable plaintext export. */ +/** + * Resolving credentials must not make a provider's exception a durable plaintext export. An egress + * outage keeps its `unavailable` classification (with fixed text) so callers can report it as such. + */ function projectProviderFailure( error: unknown, secrets: ReadonlyMap, providerConfig: Record ): Error { + const classified = asOrchestrationError(error) + if (classified?.code === 'unavailable') return classified let message = 'Webhook provider request failed' try { const matches = [...secrets].map(([name, plaintext]) => ({ diff --git a/apps/sim/lib/webhooks/providers/email-polling-outage.test.ts b/apps/sim/lib/webhooks/providers/email-polling-outage.test.ts new file mode 100644 index 00000000000..1685f6e8635 --- /dev/null +++ b/apps/sim/lib/webhooks/providers/email-polling-outage.test.ts @@ -0,0 +1,33 @@ +import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' +import { queueTableRows } from '@sim/testing/mocks/database.mock' +import { schemaMock } from '@sim/testing/mocks/schema.mock' +import { describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { gmailHandler } from '@/lib/webhooks/providers/gmail' +import { outlookHandler } from '@/lib/webhooks/providers/outlook' + +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) + +describe.each([ + ['Gmail', gmailHandler], + ['Outlook', outlookHandler], +])('%s polling setup during an egress outage', (_name, handler) => { + it('reports the outage instead of a permissions failure', async () => { + const outage = new OutboundRoutingError('GATEWAY_UNAVAILABLE') + authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce({ accountId: 'acct-1' }) + queueTableRows(schemaMock.account, [{ userId: 'owner-1' }]) + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockRejectedValueOnce(outage) + + const configurePolling = handler.configurePolling + if (!configurePolling) throw new Error('Handler has no configurePolling') + await expect( + configurePolling({ + webhook: { id: 'webhook-1', providerConfig: { credentialId: 'cred-1' } }, + requestId: 'req-1', + userId: 'user-1', + workspaceId: 'ws-1', + deploymentVersionId: null, + }) + ).rejects.toBe(outage) + }) +}) diff --git a/apps/sim/lib/webhooks/providers/gmail.ts b/apps/sim/lib/webhooks/providers/gmail.ts index bc70d25b556..c5e6a452324 100644 --- a/apps/sim/lib/webhooks/providers/gmail.ts +++ b/apps/sim/lib/webhooks/providers/gmail.ts @@ -3,6 +3,7 @@ import { account, webhook } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/lib/oauth/credential-service' import type { FormatInputContext, @@ -112,6 +113,7 @@ export const gmailHandler: WebhookProviderHandler = { }) return true } catch (error: unknown) { + if (isUnavailableFailure(error)) throw error const err = error as Error logger.error('Failed to configure Gmail polling', { requestId, diff --git a/apps/sim/lib/webhooks/providers/outlook.ts b/apps/sim/lib/webhooks/providers/outlook.ts index 58c6374faa9..9d564a31447 100644 --- a/apps/sim/lib/webhooks/providers/outlook.ts +++ b/apps/sim/lib/webhooks/providers/outlook.ts @@ -3,6 +3,7 @@ import { account, webhook } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/lib/oauth/credential-service' import type { FormatInputContext, @@ -108,6 +109,7 @@ export const outlookHandler: WebhookProviderHandler = { }) return true } catch (error: unknown) { + if (isUnavailableFailure(error)) throw error const err = error as Error logger.error('Failed to configure Outlook polling', { requestId, diff --git a/apps/sim/lib/webhooks/providers/slack.test.ts b/apps/sim/lib/webhooks/providers/slack.test.ts index f38a5a0387c..5f7f7047ac0 100644 --- a/apps/sim/lib/webhooks/providers/slack.test.ts +++ b/apps/sim/lib/webhooks/providers/slack.test.ts @@ -1,5 +1,9 @@ import { createHmac } from 'node:crypto' -import { describe, expect, it } from 'vitest' +import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' +import { queueTableRows } from '@sim/testing/mocks/database.mock' +import { schemaMock } from '@sim/testing/mocks/schema.mock' +import { describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { handleSlackChallenge, resolveSlackEventKey, @@ -7,6 +11,8 @@ import { slackHandler, } from '@/lib/webhooks/providers/slack' +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) + const ctx = (body: unknown) => ({ webhook: {}, workflow: { id: 'wf', userId: 'u' }, @@ -146,6 +152,51 @@ describe('slackHandler formatInput - Events API', () => { }) }) +describe('slackHandler formatInput - native app bot token', () => { + async function formatNativeAppEvent(event: Record) { + const formatInput = slackHandler.formatInput + if (!formatInput) throw new Error('Slack handler has no formatInput') + return formatInput({ + ...ctx({ event }), + webhook: { providerConfig: { credentialId: 'cred-1' } }, + query: {}, + method: 'POST', + }) + } + + function resolveCredentialOwner() { + authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce({ accountId: 'acct-1' }) + queueTableRows(schemaMock.account, [{ userId: 'owner-1' }]) + } + + it('still runs the event without a bot token when the egress route is down', async () => { + resolveCredentialOwner() + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + const { input } = await formatNativeAppEvent({ + type: 'message', + channel: 'C1', + user: 'U1', + text: 'hi', + }) + + expect(eventOf(input)).toMatchObject({ event_type: 'message', channel: 'C1', text: 'hi' }) + }) + + it('fails the event on a token error that is not an outage', async () => { + resolveCredentialOwner() + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockRejectedValueOnce( + new Error('decrypt failed') + ) + + await expect(formatNativeAppEvent({ type: 'message', channel: 'C1' })).rejects.toThrow( + 'decrypt failed' + ) + }) +}) + describe('slackHandler formatInput - interactivity (block_actions)', () => { it('carries the full view (state.values + private_metadata) through for a view_submission', async () => { const { input } = await slackHandler.formatInput!( diff --git a/apps/sim/lib/webhooks/providers/slack.ts b/apps/sim/lib/webhooks/providers/slack.ts index a66861f7398..2f0cc3b884f 100644 --- a/apps/sim/lib/webhooks/providers/slack.ts +++ b/apps/sim/lib/webhooks/providers/slack.ts @@ -10,6 +10,7 @@ import { eq } from 'drizzle-orm' import { LRUCache } from 'lru-cache' import { NextResponse } from 'next/server' import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -963,8 +964,18 @@ export const slackHandler: WebhookProviderHandler = { .where(eq(account.id, resolved.accountId)) .limit(1) if (owner?.userId) { - botToken = - (await refreshAccessTokenIfNeeded(credentialId, owner.userId, requestId)) ?? undefined + try { + botToken = + (await refreshAccessTokenIfNeeded(credentialId, owner.userId, requestId)) ?? undefined + } catch (error) { + // An egress outage degrades the event (no reaction text or file downloads) rather + // than failing it, matching a missing token. + if (!isUnavailableFailure(error)) throw error + logger.warn('Slack bot token unavailable during an egress outage', { + credentialId, + requestId, + }) + } } } } diff --git a/apps/sim/lib/webhooks/providers/types.ts b/apps/sim/lib/webhooks/providers/types.ts index 4507c82dabc..dd357a7c893 100644 --- a/apps/sim/lib/webhooks/providers/types.ts +++ b/apps/sim/lib/webhooks/providers/types.ts @@ -222,7 +222,10 @@ export interface WebhookProviderHandler { /** Delete an external webhook subscription during cleanup. Strict outbox cleanup should throw. */ deleteSubscription?(ctx: DeleteSubscriptionContext): Promise - /** Configure polling after webhook creation (gmail, outlook, rss, imap). */ + /** + * Configure polling after webhook creation (gmail, outlook, rss, imap). Resolves false only for a + * verdict on the account or configuration; an unavailable failure (egress outage) must throw. + */ configurePolling?(ctx: PollingConfigContext): Promise /** Handle verification challenges before webhook lookup (Slack url_verification, WhatsApp hub.verify_token, Teams validationToken). */