diff --git a/CLAUDE.md b/CLAUDE.md index dc98057cab4..3a5b7acdf9a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,6 +60,7 @@ Build order **Tools** → **Block** → **Icon** → optional **Trigger**, from - **Tool IDs are `snake_case`** (`service_action`), registered in `tools/registry.ts`; blocks register alphabetically in `blocks/registry-maps.ts`. - **Type coercions go in `tools.config.params`** (runs at execution), never `tools.config.tool` (runs at serialization, where `Number()` destroys `` references). +- **Requests made for a workspace use `outboundFetch`** (`@/lib/core/network/outbound-fetch.server`), never the global `fetch` or another imported `fetch`, so an organization with a dedicated egress gateway keeps its static IPs; `check:organization-egress` enforces this, and a request that rightly goes direct (Sim's own infrastructure, an operator-local server) takes `// egress-direct: `. AWS SDK clients signing with customer credentials take `organizationAwsRequestHandler()`: no audit catches a missing one, so reviewers check it. - A surface listing integrations to a viewer reads `useIntegrationAccess`, never a raw catalog (`check:integration-access`). - Table column types are registry entries in `apps/sim/lib/table/column-types/`; never add a per-type `case` elsewhere except the one documented import-path switch (`coerceValue` in `lib/table/import.ts`), which every new type extends: a missed arm fails silently. Use `/add-column-type`. diff --git a/apps/sim/app/api/auth/oauth/token/route.test.ts b/apps/sim/app/api/auth/oauth/token/route.test.ts deleted file mode 100644 index 9152d3bfa1d..00000000000 --- a/apps/sim/app/api/auth/oauth/token/route.test.ts +++ /dev/null @@ -1,543 +0,0 @@ -/** - * Tests for OAuth token API routes - */ -import { - authOAuthUtilsMock, - authOAuthUtilsMockFns, - createMockRequest, - hybridAuthMockFns, -} from '@sim/testing' -import { createRouteContext } from '@sim/testing/helpers/http' -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockAuthenticateManagedOAuthDelegation, - mockAuthorizeCredentialUse, - mockResolveManagedOAuthCredentialToken, -} = vi.hoisted(() => ({ - mockAuthenticateManagedOAuthDelegation: vi.fn(), - mockAuthorizeCredentialUse: vi.fn(), - mockResolveManagedOAuthCredentialToken: vi.fn(), -})) - -vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) - -vi.mock('@/lib/auth/credential-access', () => ({ - authorizeCredentialUse: mockAuthorizeCredentialUse, - authorizeCredentialUseForAuth: mockAuthorizeCredentialUse, -})) - -vi.mock('@/lib/credentials/application/managed-oauth-delegation', () => ({ - authenticateManagedOAuthDelegation: mockAuthenticateManagedOAuthDelegation, - InvalidManagedOAuthDelegationError: class InvalidManagedOAuthDelegationError extends Error {}, -})) - -vi.mock('@/lib/credentials/application/resolve-managed-oauth-token', () => ({ - resolveManagedOAuthCredentialToken: { execute: mockResolveManagedOAuthCredentialToken }, -})) - -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { createQuickBooksAccountId } from '@/lib/oauth/quickbooks' -import { GET, POST } from '@/app/api/auth/oauth/token/route' -import { getToolMetadata } from '@/tools/metadata' - -const mockGetToolMetadata = vi.mocked(getToolMetadata) -const mockCheckSessionOrInternalAuth = vi.mocked(checkSessionOrInternalAuth) - -describe('OAuth Token API Routes', () => { - beforeEach(() => { - authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValue(null) - }) - - /** - * POST route tests - */ - describe('POST handler', () => { - it('returns realmId only for QuickBooks credentials', async () => { - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - authType: 'session', - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'owner-user-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accountId: createQuickBooksAccountId( - '123456789', - 'intuit-subject-01234567-89ab-4def-8abc-0123456789ab', - { - clientId: 'client-id', - environment: 'sandbox', - } - ), - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId: 'quickbooks', - }) - authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValueOnce({ - accessToken: 'fresh-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest('POST', { - credentialId: 'credential-id', - }) - ) - - expect(response.status).toBe(200) - expect(await response.json()).toEqual({ - accessToken: 'fresh-token', - credentialType: 'oauth', - realmId: '123456789', - quickBooksEnvironment: 'sandbox', - }) - }) - - it('rejects a malformed QuickBooks company identity with reconnect guidance', async () => { - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - authType: 'session', - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'owner-user-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accountId: 'malformed', - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId: 'quickbooks', - }) - - const response = await POST( - createMockRequest('POST', { - credentialId: 'credential-id', - }) - ) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toMatch(/Reconnect the QuickBooks credential/) - expect(authOAuthUtilsMockFns.mockRefreshTokenIfNeeded).not.toHaveBeenCalled() - }) - - it('does not authenticate managed delegation for an ordinary OAuth credential', async () => { - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - authType: 'internal_jwt', - requesterUserId: 'workflow-owner-id', - credentialOwnerUserId: 'workflow-owner-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId: 'google', - }) - authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValueOnce({ - accessToken: 'fresh-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest( - 'POST', - { credentialId: 'credential-id', workflowId: 'workflow-id' }, - { 'x-sim-managed-oauth-delegation': 'Bearer stale-delegation' } - ) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ accessToken: 'fresh-token' }) - expect(mockAuthenticateManagedOAuthDelegation).not.toHaveBeenCalled() - }) - - describe('managed OAuth path', () => { - const managedCredential = { - accountId: '', - credentialId: 'managed-credential-id', - credentialType: 'managed_oauth', - providerId: 'google-email', - workspaceId: 'workspace-id', - usedCredentialTable: true, - } - - beforeEach(() => { - authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce(managedCredential) - mockGetToolMetadata.mockReturnValue({ - oauth: { - required: true, - provider: 'google-email', - requiredScopes: ['https://www.googleapis.com/auth/gmail.readonly'], - }, - }) - }) - - it('fails closed when workflow delegation is missing', async () => { - const response = await POST( - createMockRequest('POST', { - credentialId: 'managed-credential-id', - toolId: 'gmail_read', - }) - ) - - expect(response.status).toBe(403) - await expect(response.json()).resolves.toMatchObject({ - code: 'MANAGED_CREDENTIAL_DELEGATION_REQUIRED', - }) - expect(mockResolveManagedOAuthCredentialToken).not.toHaveBeenCalled() - }) - - it('resolves a manually supplied managed credential ID with scoped delegation', async () => { - const principal = { - kind: 'delegated' as const, - serviceId: 'executor' as const, - subjectUserId: 'user-id', - workspaceId: 'workspace-id', - delegationId: 'delegation-id', - audience: 'sim:managed-oauth-credentials', - issuedAt: new Date(Date.now() - 1_000), - expiresAt: new Date(Date.now() + 60_000), - resourceScope: { credentialId: 'managed-credential-id' }, - delegationContext: { - kind: 'workflow_execution' as const, - workflowId: 'workflow-id', - }, - } - mockAuthenticateManagedOAuthDelegation.mockResolvedValueOnce(principal) - mockResolveManagedOAuthCredentialToken.mockResolvedValueOnce({ - accessToken: 'managed-access-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest( - 'POST', - { credentialId: 'managed-credential-id', toolId: 'gmail_read' }, - { 'x-sim-managed-oauth-delegation': 'Bearer delegated-token' } - ) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - accessToken: 'managed-access-token', - credentialType: 'managed_oauth', - }) - expect(mockResolveManagedOAuthCredentialToken).toHaveBeenCalledWith({ - principal, - input: { - credentialId: 'managed-credential-id', - expectedProviderId: 'google-email', - requiredScopes: ['https://www.googleapis.com/auth/gmail.readonly'], - toolId: 'gmail_read', - }, - request: expect.any(NextRequest), - }) - }) - - it('uses the trusted provider scope policy when a Slack tool omits narrower scopes', async () => { - mockGetToolMetadata.mockReturnValueOnce({ - oauth: { - required: true, - provider: 'slack', - }, - }) - const principal = { - kind: 'delegated' as const, - serviceId: 'executor' as const, - subjectUserId: 'user-id', - workspaceId: 'workspace-id', - delegationId: 'delegation-id', - audience: 'sim:managed-oauth-credentials', - issuedAt: new Date(Date.now() - 1_000), - expiresAt: new Date(Date.now() + 60_000), - resourceScope: { credentialId: 'managed-credential-id' }, - delegationContext: { - kind: 'workflow_execution' as const, - workflowId: 'workflow-id', - }, - } - mockAuthenticateManagedOAuthDelegation.mockResolvedValueOnce(principal) - mockResolveManagedOAuthCredentialToken.mockResolvedValueOnce({ - accessToken: 'managed-slack-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest( - 'POST', - { credentialId: 'managed-credential-id', toolId: 'slack_message' }, - { 'x-sim-managed-oauth-delegation': 'Bearer delegated-token' } - ) - ) - - expect(response.status).toBe(200) - expect(mockResolveManagedOAuthCredentialToken).toHaveBeenCalledWith({ - principal, - input: { - credentialId: 'managed-credential-id', - expectedProviderId: 'slack', - requiredScopes: expect.arrayContaining([ - 'channels:read', - 'channels:history', - 'chat:write', - ]), - toolId: 'slack_message', - }, - request: expect.any(NextRequest), - }) - }) - }) - - describe('saved service-account credentials', () => { - const authorized = { - ok: true, - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'test-user-id', - workspaceId: 'workspace-id', - } - - beforeEach(() => { - authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce({ - accountId: '', - credentialId: 'saved-credential-id', - credentialType: 'service_account', - providerId: 'vanta-service-account', - workspaceId: 'workspace-id', - usedCredentialTable: true, - }) - authOAuthUtilsMockFns.mockResolveServiceAccountToken.mockResolvedValue({ - accessToken: 'saved-secret', - }) - mockGetToolMetadata.mockReturnValue({ - id: 'vanta_list_tests', - params: {}, - oauth: { required: true, provider: 'vanta', credentialKind: 'service-account' }, - }) - }) - - it('refuses to release the secret to a session caller', async () => { - mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: AuthType.SESSION, - userId: 'test-user-id', - }) - mockAuthorizeCredentialUse.mockResolvedValueOnce(authorized) - - const response = await POST( - createMockRequest('POST', { - credentialId: 'saved-credential-id', - toolId: 'vanta_list_tests', - }), - createRouteContext({}) - ) - - expect(response.status).toBe(403) - const body = await response.json() - expect(body).toMatchObject({ code: 'SERVICE_ACCOUNT_SESSION_FORBIDDEN' }) - expect(JSON.stringify(body)).not.toContain('saved-secret') - expect(authOAuthUtilsMockFns.mockResolveServiceAccountToken).not.toHaveBeenCalled() - }) - - it('resolves the secret for an internal-JWT execution caller', async () => { - mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: AuthType.INTERNAL_JWT, - userId: 'test-user-id', - }) - mockAuthorizeCredentialUse.mockResolvedValueOnce(authorized) - - const response = await POST( - createMockRequest('POST', { - credentialId: 'saved-credential-id', - toolId: 'vanta_list_tests', - workflowId: 'workflow-id', - }), - createRouteContext({}) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - accessToken: 'saved-secret', - credentialType: 'service_account', - }) - }) - }) - - it('still returns an OAuth token to a session caller', async () => { - mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: AuthType.SESSION, - userId: 'test-user-id', - }) - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'test-user-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId: 'google', - }) - authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValueOnce({ - accessToken: 'session-oauth-token', - refreshed: false, - }) - - const response = await POST( - createMockRequest('POST', { credentialId: 'credential-id' }), - createRouteContext({}) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - accessToken: 'session-oauth-token', - credentialType: 'oauth', - }) - }) - - describe('credentialAccountUserId + providerId path', () => { - it('should reject internal JWT authentication', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: 'internal_jwt', - userId: 'test-user-id', - }) - - const req = createMockRequest('POST', { - credentialAccountUserId: 'test-user-id', - providerId: 'google', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data).toHaveProperty('error', 'User not authenticated') - expect(authOAuthUtilsMockFns.mockGetOAuthToken).not.toHaveBeenCalled() - }) - - it('should reject requests for other users credentials', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - authType: 'session', - userId: 'attacker-user-id', - }) - - const req = createMockRequest('POST', { - credentialAccountUserId: 'victim-user-id', - providerId: 'google', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data).toHaveProperty('error', 'Unauthorized') - expect(authOAuthUtilsMockFns.mockGetOAuthToken).not.toHaveBeenCalled() - }) - }) - }) - - /** - * GET route tests - */ - describe('GET handler', () => { - it('rejects a malformed QuickBooks identity before reporting a missing token', async () => { - mockAuthorizeCredentialUse.mockResolvedValueOnce({ - ok: true, - authType: 'session', - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'test-user-id', - }) - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - id: 'credential-id', - accountId: 'malformed', - accessToken: null, - refreshToken: 'refresh-token', - providerId: 'quickbooks', - }) - - const response = await GET( - createMockRequest( - 'GET', - undefined, - {}, - 'http://localhost:3000/api/auth/oauth/token?credentialId=credential-id' - ) - ) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toMatch(/Reconnect the QuickBooks credential/) - expect(authOAuthUtilsMockFns.mockRefreshTokenIfNeeded).not.toHaveBeenCalled() - }) - }) -}) - -describe('Salesforce instance URL resolution', () => { - const INSTANCE = 'https://acme--sbx.sandbox.my.salesforce.com' - - beforeEach(() => { - authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValue(null) - mockAuthorizeCredentialUse.mockResolvedValue({ - ok: true, - authType: 'session', - requesterUserId: 'test-user-id', - credentialOwnerUserId: 'owner-user-id', - }) - authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValue({ - accessToken: 'fresh-token', - refreshed: false, - }) - }) - - /** - * The org host is smuggled through `scope` because the token response has - * nowhere to put it; the tools read it back as their `instanceUrl` param. - */ - function credentialForProvider(providerId: string) { - return { - id: 'credential-id', - accessToken: 'test-token', - refreshToken: 'refresh-token', - accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), - providerId, - scope: `__sf_instance__:${INSTANCE} api refresh_token openid`, - } - } - - it.each(['salesforce', 'salesforce-sandbox'])( - 'returns the stored instance URL for a %s credential', - async (providerId) => { - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce( - credentialForProvider(providerId) - ) - - const response = await POST(createMockRequest('POST', { credentialId: 'credential-id' })) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.instanceUrl).toBe(INSTANCE) - } - ) - - it('omits instanceUrl for a non-Salesforce provider carrying a lookalike scope', async () => { - authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ - ...credentialForProvider('google'), - }) - - const response = await POST(createMockRequest('POST', { credentialId: 'credential-id' })) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.instanceUrl).toBeUndefined() - }) -}) diff --git a/apps/sim/app/api/auth/oauth/token/route.ts b/apps/sim/app/api/auth/oauth/token/route.ts deleted file mode 100644 index 99e7dcc986b..00000000000 --- a/apps/sim/app/api/auth/oauth/token/route.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - MANAGED_OAUTH_DELEGATION_HEADER, - oauthTokenGetContract, - oauthTokenPostContract, -} from '@/lib/api/contracts/oauth-connections' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { authenticateManagedOAuthDelegation } from '@/lib/credentials/application/managed-oauth-delegation' -import { getCredential, getOAuthToken } from '@/lib/oauth/credential-service' -import { - completeOAuthCredentialToken, - resolveCredentialAccessToken, - validateOAuthCredentialContext, -} from '@/lib/oauth/token-resolution' -import { captureServerEvent } from '@/lib/posthog/server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OAuthTokenAPI') - -/** - * Get an access token for a specific credential - * Supports both session-based authentication (for client-side requests) - * and workflow-based authentication (for server-side requests) - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - logger.info('OAuth token API POST request received') - - try { - const parsed = await parseRequest( - oauthTokenPostContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn('Invalid token request', { errors: error.issues }) - return NextResponse.json( - { error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const { - credentialId, - credentialAccountUserId, - providerId, - toolId, - workflowId, - scopes, - impersonateEmail, - } = parsed.data.body - const callerUserId = parsed.data.query.userId - - if (credentialAccountUserId && providerId) { - logger.info('Fetching token by credentialAccountUserId + providerId', { - credentialAccountUserId, - providerId, - }) - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || auth.authType !== AuthType.SESSION || !auth.userId) { - logger.warn('Unauthorized request for credentialAccountUserId path', { - success: auth.success, - authType: auth.authType, - }) - return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) - } - - if (auth.userId !== credentialAccountUserId) { - logger.warn( - `User ${auth.userId} attempted to access credentials for ${credentialAccountUserId}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 403 }) - } - - try { - const accessToken = await getOAuthToken(credentialAccountUserId, providerId) - if (!accessToken) { - return NextResponse.json( - { - error: `No credential found for user ${credentialAccountUserId} and provider ${providerId}`, - }, - { status: 404 } - ) - } - - recordAudit({ - actorId: auth.userId, - action: AuditAction.CREDENTIAL_ACCESSED, - resourceType: AuditResourceType.CREDENTIAL, - resourceId: providerId, - description: `Accessed OAuth credential for provider ${providerId}`, - metadata: { - provider: providerId, - credentialType: 'oauth', - credentialAccountUserId, - }, - request, - }) - captureServerEvent(auth.userId, 'credential_used', { - credential_type: 'oauth', - provider_id: providerId, - }) - - return NextResponse.json({ accessToken }, { status: 200 }) - } catch (error) { - const message = getErrorMessage(error, 'Failed to get OAuth token') - logger.warn(`OAuth token error: ${message}`) - return NextResponse.json({ error: message }, { status: 403 }) - } - } - - const managedOAuthDelegation = parsed.data.headers?.[MANAGED_OAUTH_DELEGATION_HEADER] - const result = await resolveCredentialAccessToken({ - requestId, - credentialId, - workflowId: workflowId ?? undefined, - toolId, - scopes, - impersonateEmail, - callerUserId, - auditRequest: request, - authenticate: () => checkSessionOrInternalAuth(request, { requireWorkflowId: false }), - resolveManagedPrincipal: managedOAuthDelegation - ? (managedCredentialId: string) => - authenticateManagedOAuthDelegation(managedOAuthDelegation, managedCredentialId) - : undefined, - }) - - if (!result.ok) { - return NextResponse.json( - { ...(result.code ? { code: result.code } : {}), error: result.error }, - { status: result.status } - ) - } - - return NextResponse.json(result.token, { status: 200 }) - } catch (error) { - logger.error('Error getting access token', error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } -}) - -/** - * Get the access token for a specific credential - */ -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const parsed = await parseRequest( - oauthTokenGetContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn('Invalid query parameters', { errors: error.issues }) - return NextResponse.json( - { error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const { credentialId } = parsed.data.query - - const authz = await authorizeCredentialUse(request, { - credentialId, - requireWorkflowIdForInternal: false, - }) - if (!authz.ok || authz.authType !== AuthType.SESSION || !authz.credentialOwnerUserId) { - return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) - } - - const resolvedCredentialId = authz.resolvedCredentialId || credentialId - const credential = await getCredential( - requestId, - resolvedCredentialId, - authz.credentialOwnerUserId - ) - - if (!credential) { - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) - } - - const contextValidation = validateOAuthCredentialContext(credential) - if (!contextValidation.ok) { - return NextResponse.json({ error: contextValidation.error }, { status: 401 }) - } - - if (!credential.accessToken) { - logger.warn('No access token available for credential') - return NextResponse.json({ error: 'No access token available' }, { status: 400 }) - } - - const result = await completeOAuthCredentialToken({ - requestId, - credential, - resolvedCredentialId, - actorId: authz.requesterUserId, - workspaceId: authz.workspaceId ?? null, - auditRequest: request, - }) - - if (!result.ok) { - return NextResponse.json( - { ...(result.code ? { code: result.code } : {}), error: result.error }, - { status: result.status } - ) - } - - return NextResponse.json(result.token, { status: 200 }) - } catch (error) { - logger.error('Error fetching access token', error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/auth/oauth/utils.test.ts b/apps/sim/app/api/auth/oauth/utils.test.ts index 7ac0342bf58..37b6ee5755f 100644 --- a/apps/sim/app/api/auth/oauth/utils.test.ts +++ b/apps/sim/app/api/auth/oauth/utils.test.ts @@ -7,10 +7,10 @@ import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/oauth/oauth', () => ({ - refreshOAuthToken: vi.fn(), OAUTH_PROVIDERS: {}, TOKEN_REFRESH_TIMEOUT_MS: 15_000, })) +vi.mock('@/lib/oauth/refresh.server', () => ({ refreshOAuthToken: vi.fn() })) vi.mock('@/lib/core/security/encryption', () => encryptionMock) @@ -23,8 +23,8 @@ vi.mock('@/lib/credentials/client-credential-accounts/server', () => ({ import { db } from '@sim/db' import { __resetCoalesceLocallyForTests } from '@/lib/concurrency/singleflight' import { ZOOM_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/credentials/client-credential-accounts/descriptors' -import { refreshOAuthToken } from '@/lib/oauth' import { refreshTokenIfNeeded, resolveServiceAccountToken } from '@/lib/oauth/credential-service' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' import { getOAuthRefreshCoordinationIdentity } from '@/lib/oauth/refresh-coordination' import { GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' diff --git a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts index 62603299bbc..4c294f427e8 100644 --- a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts +++ b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts @@ -9,6 +9,7 @@ import { instagramCallbackContract } from '@/lib/api/contracts/oauth-connections import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -19,6 +20,7 @@ import { isSameOrigin } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processCredentialDraft } from '@/lib/credentials/draft-processor' import { APP_ENTRY_PATH } from '@/lib/navigation/paths' +import { resolveConnectDraftOutboundScope } from '@/lib/oauth/connect-scope.server' import { safeAccountInsert } from '@/lib/oauth/credential-service' import { parseInstagramLongLivedToken, @@ -108,13 +110,21 @@ export const GET = withRouteHandler(async (request: NextRequest) => { code: authorizationCode, }) - const shortLivedResponse = await fetch('https://api.instagram.com/oauth/access_token', { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: tokenForm.toString(), - signal: request.signal, + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId, + userId: session.user.id, + providerId: 'instagram', }) + const shortLivedResponse = await inDraftScope(() => + outboundFetch('https://api.instagram.com/oauth/access_token', { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: tokenForm.toString(), + signal: request.signal, + }) + ) + if (!shortLivedResponse.ok) { const errorText = await readResponseTextWithLimit(shortLivedResponse, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, @@ -148,10 +158,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { exchangeUrl.searchParams.set('client_secret', clientSecret) exchangeUrl.searchParams.set('access_token', shortLived.access_token) - const longLivedResponse = await fetch(exchangeUrl.toString(), { - method: 'GET', - signal: request.signal, - }) + const longLivedResponse = await inDraftScope(() => + outboundFetch(exchangeUrl.toString(), { method: 'GET', signal: request.signal }) + ) if (!longLivedResponse.ok) { const errorText = await readResponseTextWithLimit(longLivedResponse, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, @@ -186,10 +195,12 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const profileUrl = new URL(`${INSTAGRAM_GRAPH_BASE}/me`) profileUrl.searchParams.set('fields', 'user_id,username,name,account_type,profile_picture_url') - const profileResponse = await fetch(profileUrl, { - headers: { Authorization: `Bearer ${longLivedToken}` }, - signal: request.signal, - }) + const profileResponse = await inDraftScope(() => + outboundFetch(profileUrl, { + headers: { Authorization: `Bearer ${longLivedToken}` }, + signal: request.signal, + }) + ) if (!profileResponse.ok) { const errorText = await readResponseTextWithLimit(profileResponse, { diff --git a/apps/sim/app/api/auth/trello/store/route.ts b/apps/sim/app/api/auth/trello/store/route.ts index 22d9a04aefa..16d5720a385 100644 --- a/apps/sim/app/api/auth/trello/store/route.ts +++ b/apps/sim/app/api/auth/trello/store/route.ts @@ -7,8 +7,10 @@ import { storeTrelloTokenContract } from '@/lib/api/contracts/oauth-connections' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processCredentialDraft } from '@/lib/credentials/draft-processor' +import { resolveConnectDraftOutboundScope } from '@/lib/oauth/connect-scope.server' import { safeAccountInsert } from '@/lib/oauth/credential-service' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' @@ -64,9 +66,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const scope = getCanonicalScopesForProvider('trello').join(',') const validationUrl = `https://api.trello.com/1/members/me?key=${apiKey}&token=${token}&fields=id,username,fullName` - const userResponse = await fetch(validationUrl, { - headers: { Accept: 'application/json' }, + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId, + userId: session.user.id, + providerId: 'trello', }) + const userResponse = await inDraftScope(() => + outboundFetch(validationUrl, { headers: { Accept: 'application/json' } }) + ) if (!userResponse.ok) { const errorText = await userResponse.text() diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts index 4caac82f74d..c3c1a7c1ca6 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts @@ -3,17 +3,36 @@ */ import { authOAuthUtilsMock, + authOAuthUtilsMockFns, createMockRequest, dbChainMockFns, + queueTableRows, redisConfigMockFns, resetDbChainMock, + schemaMock, } from '@sim/testing' import { authInternalMock, authInternalMockFns } from '@sim/testing/mocks/auth-internal.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { + createMockWorkspaceApplicationContext, + workspaceContextMock, + workspaceContextMockFns, +} from '@sim/testing/mocks/workspace-context.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -vi.mock('@/lib/auth/internal', () => authInternalMock) +const { mockGetCredentialOwner } = vi.hoisted(() => ({ + mockGetCredentialOwner: vi.fn(), +})) +const { mockLoadWorkspaceApplicationContext } = workspaceContextMockFns +vi.mock('@/lib/auth/internal', () => authInternalMock) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) +vi.mock('@/lib/webhooks/provider-subscription-utils', () => ({ + getCredentialOwner: mockGetCredentialOwner, + getNotificationUrl: vi.fn(() => 'https://sim.test/api/webhooks/trigger/hook-1'), +})) import { GET } from './route' @@ -46,4 +65,64 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { expect(data).toMatchObject({ status: 'skip' }) expect(dbChainMockFns.select).not.toHaveBeenCalled() }) + + describe('with outbound routing enabled', () => { + const graphCalls: Array<{ url: string; organizationId: string | null | undefined }> = [] + let routeOrganizationId: string | null | undefined + + beforeEach(() => { + graphCalls.length = 0 + routeOrganizationId = undefined + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (organizationId) => { + routeOrganizationId = organizationId + return { kind: 'direct' } + }) + mockLoadWorkspaceApplicationContext.mockResolvedValue( + createMockWorkspaceApplicationContext({ + workspaceId: 'workspace-1', + workspaceOrganizationId: 'org-1', + }) + ) + mockGetCredentialOwner.mockResolvedValue({ accountId: 'account-1', userId: 'user-1' }) + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockResolvedValue('graph-token') + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => { + graphCalls.push({ url, organizationId: routeOrganizationId }) + return new Response(JSON.stringify({ expirationDateTime: '2030-01-01T00:00:00Z' })) + }) + ) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + + it("renews each subscription through its workspace organization's outbound route", async () => { + queueTableRows(schemaMock.webhook, [ + { + workspaceId: 'workspace-1', + webhook: { + id: 'hook-1', + providerConfig: { + triggerId: 'microsoftteams_chat_subscription', + subscriptionExpiration: new Date(Date.now() + 60_000).toISOString(), + credentialId: 'credential-1', + externalSubscriptionId: 'subscription-1', + }, + }, + }, + ]) + + const response = await GET(createRequest(), { params: Promise.resolve({}) }) + expect(response.status).toBe(202) + + await vi.waitFor(() => expect(graphCalls).toHaveLength(1)) + expect(graphCalls[0]).toEqual({ + url: 'https://graph.microsoft.com/v1.0/subscriptions/subscription-1', + organizationId: 'org-1', + }) + }) + }) }) diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.ts b/apps/sim/app/api/cron/renew-subscriptions/route.ts index 08ba269b241..c54bdd33de6 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.ts @@ -1,11 +1,13 @@ import { db } from '@sim/db' -import { webhook as webhookTable } from '@sim/db/schema' +import { webhook as webhookTable, workflow as workflowTable } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateShortId } from '@sim/utils/id' import { and, eq, or } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { verifyCronAuth } from '@/lib/auth/internal' import { acquireLock, releaseLock } from '@/lib/core/config/redis' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { runDetached } from '@/lib/core/utils/background' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' @@ -42,7 +44,7 @@ async function recreateSubscription( const notificationUrl = getNotificationUrl(webhook) const expirationDateTime = new Date(Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000).toISOString() - const res = await fetch('https://graph.microsoft.com/v1.0/subscriptions', { + const res = await outboundFetch('https://graph.microsoft.com/v1.0/subscriptions', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -72,6 +74,104 @@ async function recreateSubscription( return { id: payload.id as string, expirationDateTime: payload.expirationDateTime as string } } +/** + * Renews (or, once expired on Microsoft's side, recreates) one webhook's Graph subscription. + * Returns whether the subscription is live again. + */ +async function renewSubscription( + webhook: typeof webhookTable.$inferSelect, + config: Record, + expiresAt: Date +): Promise { + const requestId = `renewal-${webhook.id}` + logger.info( + `Renewing Teams subscription for webhook ${webhook.id} (expires: ${expiresAt.toISOString()})` + ) + + const credentialId = config.credentialId as string | undefined + const externalSubscriptionId = config.externalSubscriptionId as string | undefined + + if (!credentialId || !externalSubscriptionId) { + logger.error(`Missing credentialId or externalSubscriptionId for webhook ${webhook.id}`) + return false + } + + const credentialOwner = await getCredentialOwner(credentialId, requestId) + if (!credentialOwner) { + logger.error(`Credential owner not found for credential ${credentialId}`) + return false + } + + const accessToken = await refreshAccessTokenIfNeeded( + credentialOwner.accountId, + credentialOwner.userId, + requestId + ) + + if (!accessToken) { + logger.error(`Failed to get access token for webhook ${webhook.id}`) + return false + } + + const newExpirationDateTime = new Date( + Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000 + ).toISOString() + + const res = await outboundFetch( + `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, + { + method: 'PATCH', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ expirationDateTime: newExpirationDateTime }), + } + ) + + let newSubscriptionId: string | undefined + let newExpiration: string | undefined + + if (!res.ok) { + const error = await res.json() + logger.error( + `Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`, + { status: res.status, error: error.error } + ) + + if (res.status === 404 || res.status === 410) { + const recreated = await recreateSubscription(webhook, config, accessToken) + if (!recreated) return false + newSubscriptionId = recreated.id + newExpiration = recreated.expirationDateTime + logger.info( + `Recreated Teams subscription for webhook ${webhook.id} after the previous one expired (new id: ${newSubscriptionId})` + ) + } else { + return false + } + } else { + const payload = await res.json() + newExpiration = payload.expirationDateTime as string + } + + const updatedConfig = { + ...config, + ...(newSubscriptionId ? { externalSubscriptionId: newSubscriptionId } : {}), + subscriptionExpiration: newExpiration, + } + + await db + .update(webhookTable) + .set({ providerConfig: updatedConfig, updatedAt: new Date() }) + .where(eq(webhookTable.id, webhook.id)) + + logger.info( + `Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${newExpiration}` + ) + return true +} + /** * Renews Microsoft Teams chat subscriptions that are close to expiring. * @@ -93,8 +193,10 @@ async function renewExpiringSubscriptions(): Promise<{ const webhooksWithWorkflows = await db .select({ webhook: webhookTable, + workspaceId: workflowTable.workspaceId, }) .from(webhookTable) + .innerJoin(workflowTable, eq(webhookTable.workflowId, workflowTable.id)) .where( and( deliverableWebhookPredicate(webhookTable, 'active_only'), @@ -112,7 +214,7 @@ async function renewExpiringSubscriptions(): Promise<{ /** Renew any subscription expiring within the next 48 hours. */ const renewalThreshold = new Date(Date.now() + 48 * 60 * 60 * 1000) - for (const { webhook } of webhooksWithWorkflows) { + for (const { webhook, workspaceId } of webhooksWithWorkflows) { const config = (webhook.providerConfig as Record) || {} if (config.triggerId !== 'microsoftteams_chat_subscription') continue @@ -124,102 +226,14 @@ async function renewExpiringSubscriptions(): Promise<{ if (expiresAt > renewalThreshold) continue totalChecked++ - const requestId = `renewal-${webhook.id}` try { - logger.info( - `Renewing Teams subscription for webhook ${webhook.id} (expires: ${expiresAt.toISOString()})` - ) - - const credentialId = config.credentialId as string | undefined - const externalSubscriptionId = config.externalSubscriptionId as string | undefined - - if (!credentialId || !externalSubscriptionId) { - logger.error(`Missing credentialId or externalSubscriptionId for webhook ${webhook.id}`) - totalFailed++ - continue - } - - const credentialOwner = await getCredentialOwner(credentialId, requestId) - if (!credentialOwner) { - logger.error(`Credential owner not found for credential ${credentialId}`) - totalFailed++ - continue - } - - const accessToken = await refreshAccessTokenIfNeeded( - credentialOwner.accountId, - credentialOwner.userId, - requestId - ) - - if (!accessToken) { - logger.error(`Failed to get access token for webhook ${webhook.id}`) - totalFailed++ - continue - } - - const newExpirationDateTime = new Date( - Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000 - ).toISOString() - - const res = await fetch( - `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, - { - method: 'PATCH', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ expirationDateTime: newExpirationDateTime }), - } - ) - - let newSubscriptionId: string | undefined - let newExpiration: string | undefined - - if (!res.ok) { - const error = await res.json() - logger.error( - `Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`, - { status: res.status, error: error.error } - ) - - if (res.status === 404 || res.status === 410) { - const recreated = await recreateSubscription(webhook, config, accessToken) - if (!recreated) { - totalFailed++ - continue - } - newSubscriptionId = recreated.id - newExpiration = recreated.expirationDateTime - logger.info( - `Recreated Teams subscription for webhook ${webhook.id} after the previous one expired (new id: ${newSubscriptionId})` - ) - } else { - totalFailed++ - continue - } - } else { - const payload = await res.json() - newExpiration = payload.expirationDateTime as string - } - - const updatedConfig = { - ...config, - ...(newSubscriptionId ? { externalSubscriptionId: newSubscriptionId } : {}), - subscriptionExpiration: newExpiration, - } - - await db - .update(webhookTable) - .set({ providerConfig: updatedConfig, updatedAt: new Date() }) - .where(eq(webhookTable.id, webhook.id)) - - logger.info( - `Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${newExpiration}` - ) - totalRenewed++ + const renew = () => renewSubscription(webhook, config, expiresAt) + const renewed = workspaceId + ? await withResourceOutboundScope({ workspaceId }, renew, { includeArchived: true }) + : await renew() + if (renewed) totalRenewed++ + else totalFailed++ } catch (error) { logger.error(`Error renewing subscription for webhook ${webhook.id}:`, error) totalFailed++ diff --git a/apps/sim/app/api/providers/baseten/models/route.test.ts b/apps/sim/app/api/providers/baseten/models/route.test.ts index 2de4e413eb9..f0281e2f13b 100644 --- a/apps/sim/app/api/providers/baseten/models/route.test.ts +++ b/apps/sim/app/api/providers/baseten/models/route.test.ts @@ -1,22 +1,25 @@ import { authMockFns, createMockRequest, resetEnvMock, setEnv } from '@sim/testing' import { jsonResponse } from '@sim/testing/helpers/http' import { apiKeyByokMock, apiKeyByokMockFns } from '@sim/testing/mocks/api-key-byok.mock' -import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock' +import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' +import { workspaceContextMock } from '@sim/testing/mocks/workspace-context.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/providers/utils', () => providersUtilsMock) vi.mock('@/lib/api-key/byok', () => apiKeyByokMock) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) + +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) import { GET } from '@/app/api/providers/baseten/models/route' const mockGetSession = authMockFns.mockGetSession const mockGetBYOKKey = apiKeyByokMockFns.mockGetBYOKKey const { mockFilterBlacklistedModels, mockIsProviderBlacklisted } = providersUtilsMockFns -const { mockGetUserEntityPermissions } = permissionsMockFns +const { mockResolveEffectiveWorkspacePermission } = workspaceAuthzMockFns function setEnvKey(value: string | undefined): void { setEnv({ BASETEN_API_KEY: value }) @@ -38,7 +41,7 @@ describe('GET /api/providers/baseten/models', () => { mockFilterBlacklistedModels.mockImplementation((models: string[]) => models) mockGetBYOKKey.mockResolvedValue(null) mockGetSession.mockResolvedValue(null) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) setEnvKey(undefined) }) @@ -48,8 +51,8 @@ describe('GET /api/providers/baseten/models', () => { it('uses the BYOK key when workspaceId, session, and permission are present', async () => { setEnvKey('env-key') - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue('admin') + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') mockGetBYOKKey.mockResolvedValue({ apiKey: 'byok-key', isBYOK: true }) mockFetch.mockResolvedValueOnce(jsonResponse({ data: [{ id: 'model-a' }] })) @@ -66,8 +69,8 @@ describe('GET /api/providers/baseten/models', () => { it('falls back to the env key when the user lacks workspace permission', async () => { setEnvKey('env-key') - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) mockFetch.mockResolvedValueOnce(jsonResponse({ data: [{ id: 'model-a' }] })) const res = await GET( diff --git a/apps/sim/app/api/providers/baseten/models/route.ts b/apps/sim/app/api/providers/baseten/models/route.ts index b73a6711421..90890c3e47f 100644 --- a/apps/sim/app/api/providers/baseten/models/route.ts +++ b/apps/sim/app/api/providers/baseten/models/route.ts @@ -1,18 +1,11 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { - basetenProviderModelsQuerySchema, - basetenUpstreamResponseSchema, - providerModelsResponseSchema, -} from '@/lib/api/contracts/providers' +import { basetenProviderModelsQuerySchema } from '@/lib/api/contracts/providers' import { validationErrorResponse } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' +import { listProviderModels } from '@/lib/provider-models/application/list-models' +import { isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('BasetenModelsAPI') @@ -22,72 +15,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ models: [] }) } - let apiKey: string | undefined - const queryValidation = basetenProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, }) if (!queryValidation.success) return validationErrorResponse(queryValidation.error) const { workspaceId } = queryValidation.data - if (workspaceId) { - const session = await getSession() - if (session?.user?.id) { - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission) { - const byokResult = await getBYOKKey(workspaceId, 'baseten') - if (byokResult) { - apiKey = byokResult.apiKey - } - } - } - } - - if (!apiKey) { - apiKey = env.BASETEN_API_KEY - } - - if (!apiKey) { - logger.info('No Baseten API key available, returning empty models') - return NextResponse.json({ models: [] }) - } - - try { - const response = await fetch('https://inference.baseten.co/v1/models', { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - }) - if (!response.ok) { - logger.warn('Failed to fetch Baseten models', { - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json({ models: [] }) - } - - const data = basetenUpstreamResponseSchema.parse(await response.json()) - - const allModels: string[] = [] - for (const model of data.data ?? []) { - allModels.push(`baseten/${model.id}`) - } - - const uniqueModels = Array.from(new Set(allModels)) - const models = filterBlacklistedModels(uniqueModels) - - logger.info('Successfully fetched Baseten models', { - count: models.length, - filtered: uniqueModels.length - models.length, - }) - - return NextResponse.json(providerModelsResponseSchema.parse({ models })) - } catch (error) { - logger.error('Error fetching Baseten models', { - error: getErrorMessage(error, 'Unknown error'), - }) - return NextResponse.json({ models: [] }) - } + const principal = workspaceId ? await authenticateOptionalInternalSession() : null + const models = await listProviderModels('baseten', { principal, workspaceId }) + return NextResponse.json({ models }) }) diff --git a/apps/sim/app/api/providers/fireworks/models/route.ts b/apps/sim/app/api/providers/fireworks/models/route.ts index 82754eb53fb..7c04ec1dc97 100644 --- a/apps/sim/app/api/providers/fireworks/models/route.ts +++ b/apps/sim/app/api/providers/fireworks/models/route.ts @@ -1,114 +1,27 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { - fireworksProviderModelsQuerySchema, - fireworksUpstreamResponseSchema, - providerModelsResponseSchema, -} from '@/lib/api/contracts/providers' +import { fireworksProviderModelsQuerySchema } from '@/lib/api/contracts/providers' import { validationErrorResponse } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' -import { isHosted } from '@/lib/core/config/env-flags' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' +import { listProviderModels } from '@/lib/provider-models/application/list-models' +import { isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('FireworksModelsAPI') -interface FireworksModel { - id: string - object?: string - created?: number - owned_by?: string -} - -interface FireworksModelsResponse { - data: FireworksModel[] - object?: string -} - export const GET = withRouteHandler(async (request: NextRequest) => { if (isProviderBlacklisted('fireworks')) { logger.info('Fireworks provider is blacklisted, returning empty models') return NextResponse.json({ models: [] }) } - let apiKey: string | undefined - const queryValidation = fireworksProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, }) if (!queryValidation.success) return validationErrorResponse(queryValidation.error) const { workspaceId } = queryValidation.data - if (workspaceId) { - const session = await getSession() - if (session?.user?.id) { - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission) { - const byokResult = await getBYOKKey(workspaceId, 'fireworks') - if (byokResult) { - apiKey = byokResult.apiKey - } - } - } - } - - /** - * On hosted Sim the platform key is the sim-auto pool's inference key, not a - * workspace credential: enumerating the whole Fireworks catalog from it would - * offer every serverless model as selectable when no key can actually run it - * (`getApiKeyWithBYOK` serves the platform key to catalog models only). - */ - if (!apiKey && !isHosted) { - apiKey = env.FIREWORKS_API_KEY - } - - if (!apiKey) { - logger.info('No Fireworks API key available, returning empty models') - return NextResponse.json({ models: [] }) - } - try { - const response = await fetch('https://api.fireworks.ai/inference/v1/models', { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - }) - - if (!response.ok) { - logger.warn('Failed to fetch Fireworks models', { - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json({ models: [] }) - } - - const data: FireworksModelsResponse = fireworksUpstreamResponseSchema.parse( - await response.json() - ) - - const allModels: string[] = [] - for (const model of data.data ?? []) { - allModels.push(`fireworks/${model.id}`) - } - - const uniqueModels = Array.from(new Set(allModels)) - const models = filterBlacklistedModels(uniqueModels) - - logger.info('Successfully fetched Fireworks models', { - count: models.length, - filtered: uniqueModels.length - models.length, - }) - - return NextResponse.json(providerModelsResponseSchema.parse({ models })) - } catch (error) { - logger.error('Error fetching Fireworks models', { - error: getErrorMessage(error, 'Unknown error'), - }) - return NextResponse.json({ models: [] }) - } + const principal = workspaceId ? await authenticateOptionalInternalSession() : null + const models = await listProviderModels('fireworks', { principal, workspaceId }) + return NextResponse.json({ models }) }) diff --git a/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts b/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts index f30499b2a4a..889f652b7e5 100644 --- a/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts +++ b/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts @@ -1,7 +1,8 @@ import { authMockFns, createMockRequest } from '@sim/testing' import { apiKeyByokMock, apiKeyByokMockFns } from '@sim/testing/mocks/api-key-byok.mock' -import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock' +import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' +import { workspaceContextMock } from '@sim/testing/mocks/workspace-context.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch } = vi.hoisted(() => ({ @@ -12,7 +13,9 @@ vi.mock('@/providers/utils', () => providersUtilsMock) vi.mock('@/lib/api-key/byok', () => apiKeyByokMock) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) + +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) import { GET } from '@/app/api/providers/ollama-cloud/models/route' @@ -20,7 +23,7 @@ const { mockGetBYOKKey } = apiKeyByokMockFns const mockGetSession = authMockFns.mockGetSession const { mockFilterBlacklistedModels, mockIsProviderBlacklisted } = providersUtilsMockFns -const { mockGetUserEntityPermissions } = permissionsMockFns +const { mockResolveEffectiveWorkspacePermission } = workspaceAuthzMockFns /** * Builds a request whose query string carries the given workspaceId. Passing @@ -36,8 +39,8 @@ const requestWithWorkspace = (workspaceId?: string) => { /** Grants a session + workspace permission so the BYOK lookup is reached. */ const grantWorkspaceAccess = () => { - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue('admin') + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') } describe('GET /api/providers/ollama-cloud/models', () => { @@ -48,7 +51,7 @@ describe('GET /api/providers/ollama-cloud/models', () => { mockFilterBlacklistedModels.mockImplementation((models: string[]) => models) mockGetBYOKKey.mockResolvedValue(null) mockGetSession.mockResolvedValue(null) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) }) it('returns empty models when the workspace has no stored BYOK key (never falls back to a hosted key)', async () => { @@ -64,8 +67,8 @@ describe('GET /api/providers/ollama-cloud/models', () => { }) it('does not call getBYOKKey when the session user lacks workspace permission', async () => { - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) const res = await GET(requestWithWorkspace('ws-1')) diff --git a/apps/sim/app/api/providers/ollama-cloud/models/route.ts b/apps/sim/app/api/providers/ollama-cloud/models/route.ts index bd5673e0842..bb8d6006265 100644 --- a/apps/sim/app/api/providers/ollama-cloud/models/route.ts +++ b/apps/sim/app/api/providers/ollama-cloud/models/route.ts @@ -1,17 +1,11 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { - ollamaCloudProviderModelsQuerySchema, - ollamaUpstreamResponseSchema, - providerModelsResponseSchema, -} from '@/lib/api/contracts/providers' +import { ollamaCloudProviderModelsQuerySchema } from '@/lib/api/contracts/providers' import { validationErrorResponse } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' +import { listProviderModels } from '@/lib/provider-models/application/list-models' +import { isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('OllamaCloudModelsAPI') @@ -34,58 +28,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { if (!queryValidation.success) return validationErrorResponse(queryValidation.error) const { workspaceId } = queryValidation.data - let apiKey: string | undefined - if (workspaceId) { - const session = await getSession() - if (session?.user?.id) { - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission) { - const byokResult = await getBYOKKey(workspaceId, 'ollama-cloud') - if (byokResult) { - apiKey = byokResult.apiKey - } - } - } - } - - if (!apiKey) { - logger.info('No Ollama Cloud API key available, returning empty models') - return NextResponse.json({ models: [] }) - } - - try { - const response = await fetch('https://ollama.com/api/tags', { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - }) - - if (!response.ok) { - logger.warn('Failed to fetch Ollama Cloud models', { - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json({ models: [] }) - } - - const data = ollamaUpstreamResponseSchema.parse(await response.json()) - - const allModels = data.models.map((model) => `ollama-cloud/${model.name}`) - const uniqueModels = Array.from(new Set(allModels)) - const models = filterBlacklistedModels(uniqueModels) - - logger.info('Successfully fetched Ollama Cloud models', { - count: models.length, - filtered: uniqueModels.length - models.length, - }) - - return NextResponse.json(providerModelsResponseSchema.parse({ models })) - } catch (error) { - logger.error('Error fetching Ollama Cloud models', { - error: getErrorMessage(error, 'Unknown error'), - }) - return NextResponse.json({ models: [] }) - } + const principal = workspaceId ? await authenticateOptionalInternalSession() : null + const models = await listProviderModels('ollama-cloud', { principal, workspaceId }) + return NextResponse.json({ models }) }) diff --git a/apps/sim/app/api/providers/together/models/route.test.ts b/apps/sim/app/api/providers/together/models/route.test.ts index e92adce54f6..6d3b75dbed4 100644 --- a/apps/sim/app/api/providers/together/models/route.test.ts +++ b/apps/sim/app/api/providers/together/models/route.test.ts @@ -1,8 +1,9 @@ import { authMockFns, createMockRequest, resetEnvMock, setEnv } from '@sim/testing' import { jsonResponse } from '@sim/testing/helpers/http' import { apiKeyByokMock, apiKeyByokMockFns } from '@sim/testing/mocks/api-key-byok.mock' -import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { providersUtilsMock, providersUtilsMockFns } from '@sim/testing/mocks/providers-utils.mock' +import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' +import { workspaceContextMock } from '@sim/testing/mocks/workspace-context.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch } = vi.hoisted(() => ({ @@ -13,14 +14,16 @@ vi.mock('@/providers/utils', () => providersUtilsMock) vi.mock('@/lib/api-key/byok', () => apiKeyByokMock) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) + +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) import { GET } from '@/app/api/providers/together/models/route' const mockGetSession = authMockFns.mockGetSession const mockGetBYOKKey = apiKeyByokMockFns.mockGetBYOKKey const { mockFilterBlacklistedModels, mockIsProviderBlacklisted } = providersUtilsMockFns -const { mockGetUserEntityPermissions } = permissionsMockFns +const { mockResolveEffectiveWorkspacePermission } = workspaceAuthzMockFns /** * Builds a request whose query string carries the given workspaceId. Passing @@ -49,7 +52,7 @@ describe('GET /api/providers/together/models', () => { mockFilterBlacklistedModels.mockImplementation((models: string[]) => models) mockGetBYOKKey.mockResolvedValue(null) mockGetSession.mockResolvedValue(null) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) }) afterAll(() => { @@ -58,8 +61,8 @@ describe('GET /api/providers/together/models', () => { it('uses the BYOK key when a workspace, session, and permission are present', async () => { setEnv({ TOGETHER_API_KEY: 'env-together-key' }) - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue('admin') + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') mockGetBYOKKey.mockResolvedValue({ apiKey: 'byok-together-key' }) mockFetch.mockResolvedValue(jsonResponse([{ id: 'moonshotai/Kimi-K2-Instruct' }])) @@ -74,8 +77,8 @@ describe('GET /api/providers/together/models', () => { it('falls back to the env key when the session user lacks workspace permission', async () => { setEnv({ TOGETHER_API_KEY: 'env-together-key' }) - mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockGetUserEntityPermissions.mockResolvedValue(null) + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) mockFetch.mockResolvedValue(jsonResponse([{ id: 'moonshotai/Kimi-K2-Instruct' }])) const res = await GET(requestWithWorkspace('ws-1')) diff --git a/apps/sim/app/api/providers/together/models/route.ts b/apps/sim/app/api/providers/together/models/route.ts index dcaa0dc0c5a..9e4edde5d74 100644 --- a/apps/sim/app/api/providers/together/models/route.ts +++ b/apps/sim/app/api/providers/together/models/route.ts @@ -1,105 +1,27 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { - providerModelsResponseSchema, - togetherProviderModelsQuerySchema, - togetherUpstreamResponseSchema, -} from '@/lib/api/contracts/providers' +import { togetherProviderModelsQuerySchema } from '@/lib/api/contracts/providers' import { validationErrorResponse } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' +import { listProviderModels } from '@/lib/provider-models/application/list-models' +import { isProviderBlacklisted } from '@/providers/utils' const logger = createLogger('TogetherModelsAPI') -/** Together's catalog includes non-text models; only chat models work with chat completions. */ -const NON_CHAT_MODEL_TYPES = new Set([ - 'image', - 'video', - 'audio', - 'transcribe', - 'embedding', - 'moderation', - 'rerank', -]) - export const GET = withRouteHandler(async (request: NextRequest) => { if (isProviderBlacklisted('together')) { logger.info('Together provider is blacklisted, returning empty models') return NextResponse.json({ models: [] }) } - let apiKey: string | undefined - const queryValidation = togetherProviderModelsQuerySchema.safeParse({ workspaceId: request.nextUrl.searchParams.get('workspaceId') ?? undefined, }) if (!queryValidation.success) return validationErrorResponse(queryValidation.error) const { workspaceId } = queryValidation.data - if (workspaceId) { - const session = await getSession() - if (session?.user?.id) { - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission) { - const byokResult = await getBYOKKey(workspaceId, 'together') - if (byokResult) { - apiKey = byokResult.apiKey - } - } - } - } - - if (!apiKey) { - apiKey = env.TOGETHER_API_KEY - } - - if (!apiKey) { - logger.info('No Together API key available, returning empty models') - return NextResponse.json({ models: [] }) - } - try { - const response = await fetch('https://api.together.ai/v1/models', { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - cache: 'no-store', - }) - - if (!response.ok) { - logger.warn('Failed to fetch Together models', { - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json({ models: [] }) - } - - const data = togetherUpstreamResponseSchema.parse(await response.json()) - - const allModels: string[] = [] - for (const model of data) { - if (model.type && NON_CHAT_MODEL_TYPES.has(model.type)) continue - allModels.push(`together/${model.id}`) - } - - const uniqueModels = Array.from(new Set(allModels)) - const models = filterBlacklistedModels(uniqueModels) - - logger.info('Successfully fetched Together models', { - count: models.length, - filtered: uniqueModels.length - models.length, - }) - - return NextResponse.json(providerModelsResponseSchema.parse({ models })) - } catch (error) { - logger.error('Error fetching Together models', { - error: getErrorMessage(error, 'Unknown error'), - }) - return NextResponse.json({ models: [] }) - } + const principal = workspaceId ? await authenticateOptionalInternalSession() : null + const models = await listProviderModels('together', { principal, workspaceId }) + return NextResponse.json({ models }) }) diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index 11eb1b1df66..375e553473b 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -30,9 +30,15 @@ import { knowledgeTagsServiceMock, knowledgeTagsServiceMockFns, } from '@sim/testing/mocks/knowledge-tags-service.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { v1MiddlewareMock, v1MiddlewareMockFns } from '@sim/testing/mocks/v1-middleware.mock' +import { + createMockWorkspaceApplicationContext, + workspaceContextMock, + workspaceContextMockFns, +} from '@sim/testing/mocks/workspace-context.mock' import { getErrorMessage } from '@sim/utils/errors' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockResolveV1KnowledgeReadAccess, mockExecuteKnowledgeSearch, mockRetrievalStatus } = vi.hoisted(() => ({ @@ -84,7 +90,10 @@ vi.mock('@/app/api/v1/knowledge/utils', () => ({ })) vi.mock('@/lib/knowledge/tags/service', () => knowledgeTagsServiceMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { POST } from '@/app/api/v1/knowledge/search/route' const { mockGetDocumentTagDefinitions } = knowledgeTagsServiceMockFns @@ -143,6 +152,10 @@ describe('v1 knowledge search route — per-KB embedding model', () => { mockRecordSearchEmbeddingUsage.mockResolvedValue(undefined) }) + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + it('fails a search whose retrieval ran out of time instead of returning partial rows', async () => { const access = { kind: 'user' as const, userId: 'user-1', tokens: ['reader-token'] } mockResolveV1KnowledgeReadAccess.mockResolvedValue({ @@ -250,6 +263,32 @@ describe('v1 knowledge search route — per-KB embedding model', () => { ) }) + it("embeds the query through the workspace organization's outbound route", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + workspaceContextMockFns.mockLoadWorkspaceApplicationContext.mockResolvedValueOnce( + createMockWorkspaceApplicationContext({ + workspaceId: 'ws-1', + workspaceOrganizationId: 'org-1', + }) + ) + mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({ + hasAccess: true, + knowledgeBase: baseKb('kb-1', 'text-embedding-3-small'), + }) + mockGenerateSearchEmbedding.mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + return { embedding: [0.1, 0.2, 0.3], isBYOK: false } + }) + + const res = await POST( + createMockRequest('POST', { workspaceId: 'ws-1', knowledgeBaseIds: 'kb-1', query: 'hello' }), + { params: Promise.resolve({}) } + ) + + expect(res.status).toBe(200) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + }) + it('rejects cross-KB queries with mixed vector widths, which store in different columns', async () => { mockCheckKnowledgeBaseAccess .mockResolvedValueOnce({ diff --git a/apps/sim/app/api/v1/knowledge/search/route.ts b/apps/sim/app/api/v1/knowledge/search/route.ts index fd239a218e0..b1b4b1bc333 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.ts @@ -6,6 +6,7 @@ import { resolveSystemBillingAttribution, } from '@/lib/billing/core/attribution' import { checkSearchUsageLimits } from '@/lib/billing/core/usage-gate-cache' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' import { toKbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' @@ -253,10 +254,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { structuredFilters, }) } else if (hasQuery) { - const queryEmbeddingResult = await generateSearchEmbedding( - query!, - queryEmbeddingTarget!, - workspaceId + const queryEmbeddingResult = await withResourceOutboundScope({ workspaceId }, () => + generateSearchEmbedding(query!, queryEmbeddingTarget!, workspaceId) ) queryEmbeddingIsBYOK = queryEmbeddingResult.isBYOK retrieved = await retrieveKnowledgeSearch({ diff --git a/apps/sim/app/api/v1/knowledge/utils.test.ts b/apps/sim/app/api/v1/knowledge/utils.test.ts index b864363be78..6a46dbe25a3 100644 --- a/apps/sim/app/api/v1/knowledge/utils.test.ts +++ b/apps/sim/app/api/v1/knowledge/utils.test.ts @@ -10,7 +10,8 @@ vi.mock('@/lib/knowledge/access/scope', () => knowledgeAccessScopeMock) vi.mock('@/lib/knowledge/service', () => knowledgeServiceMock) vi.mock('@/app/api/v1/middleware', () => v1MiddlewareMock) -import { resolveV1KnowledgeReadAccess } from '@/app/api/v1/knowledge/utils' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { handleError, resolveV1KnowledgeReadAccess } from '@/app/api/v1/knowledge/utils' const createUserProvider = knowledgeAccessScopeMockFns.mockCreateUserKnowledgeAccessProvider @@ -34,3 +35,16 @@ describe('v1 knowledge reader identity', () => { expect(createUserProvider).not.toHaveBeenCalled() }) }) + +describe('v1 knowledge error projection', () => { + it('answers 503 when the organization egress route fails under a wrapping error', async () => { + const error = new Error('Search failed', { + cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + }) + const response = handleError('req-1', error, 'Failed to search') + expect(response.status).toBe(503) + await expect(response.json()).resolves.toEqual({ + error: "The organization's egress route is unavailable", + }) + }) +}) diff --git a/apps/sim/app/api/v1/knowledge/utils.ts b/apps/sim/app/api/v1/knowledge/utils.ts index abeb501818c..83378c16e7e 100644 --- a/apps/sim/app/api/v1/knowledge/utils.ts +++ b/apps/sim/app/api/v1/knowledge/utils.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' import { validationErrorResponseFromError } from '@/lib/api/server' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { createUserKnowledgeAccessProvider, WORKSPACE_ACCESS_SCOPE, @@ -123,5 +124,9 @@ export function handleError( } logger.error(`${defaultMessage}:`, error, { requestId }) + const orchestrationError = asOrchestrationError(error) + if (orchestrationError?.code === 'unavailable') { + return NextResponse.json({ error: orchestrationError.message }, { status: 503 }) + } return NextResponse.json({ error: defaultMessage }, { status: 500 }) } diff --git a/apps/sim/app/api/v2/lib/response.ts b/apps/sim/app/api/v2/lib/response.ts index 1e95d321f71..1e7117e4c72 100644 --- a/apps/sim/app/api/v2/lib/response.ts +++ b/apps/sim/app/api/v2/lib/response.ts @@ -545,6 +545,7 @@ const V2_CODE_BY_ORCHESTRATION_ERROR: Record ({ mockGenerateInWorkspace: vi.fn() })) + +vi.mock('@/lib/wand/application/generate', async (importOriginal) => ({ + ...(await importOriginal()), + generateWandInWorkspace: { execute: mockGenerateInWorkspace }, +})) + +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { WandStreamStartError } from '@/lib/wand/application/generate' +import { POST } from '@/app/api/wand/route' + +const EGRESS_UNAVAILABLE = "The organization's egress route is unavailable" + +function wandRequest(stream: boolean) { + return createMockRequest('POST', { prompt: 'Write a cron', workspaceId: 'ws-1', stream }) +} + +describe('wand route egress failures', () => { + beforeEach(() => { + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: 'user-1' }, + session: { id: 'session-1' }, + }) + }) + + it('answers 503 when the gateway fails a non-streaming generation', async () => { + mockGenerateInWorkspace.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const response = await POST(wandRequest(false), createRouteContext({})) + expect(response.status).toBe(503) + await expect(response.json()).resolves.toEqual({ success: false, error: EGRESS_UNAVAILABLE }) + }) + + it('answers 503 when the gateway fails to open a stream', async () => { + mockGenerateInWorkspace.mockRejectedValueOnce( + new WandStreamStartError(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + ) + const response = await POST(wandRequest(true), createRouteContext({})) + expect(response.status).toBe(503) + await expect(response.json()).resolves.toEqual({ success: false, error: EGRESS_UNAVAILABLE }) + }) +}) diff --git a/apps/sim/app/api/wand/route.ts b/apps/sim/app/api/wand/route.ts index fbe1e60d148..a4b11e4ee0f 100644 --- a/apps/sim/app/api/wand/route.ts +++ b/apps/sim/app/api/wand/route.ts @@ -1,30 +1,22 @@ -import { db } from '@sim/db' -import { workflow } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { wandGenerateContract } from '@/lib/api/contracts' import { parseRequest } from '@/lib/api/server' -import { getBYOKKey } from '@/lib/api-key/byok' -import { getSession } from '@/lib/auth' -import { - type BillingAttributionSnapshot, - checkAttributedBillingBlocks, - checkAttributedUsageLimits, - resolveBillingAttribution, - toBillingContext, -} from '@/lib/billing/core/attribution' -import { recordUsage } from '@/lib/billing/core/usage-log' -import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' -import { env } from '@/lib/core/config/env' -import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' +import { authenticateOptionalInternalSession } from '@/lib/api/server/routes' +import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { enrichSandboxCapabilities } from '@/lib/execution/remote-sandbox/wand-enricher' -import { enrichTableSchema } from '@/lib/table/llm/wand' -import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' -import { extractResponseText, parseResponsesUsage } from '@/providers/openai/utils' -import { getModelPricing } from '@/providers/utils' +import { + generateWandForWorkflow, + generateWandInWorkspace, + WAND_DEPLOYMENT, + WandBillingRefusedError, + WandEmptyResponseError, + WandNotConfiguredError, + WandStreamStartError, + WandWorkflowDetachedError, + WandWorkflowNotFoundError, +} from '@/lib/wand/application/generate' export const dynamic = 'force-dynamic' export const runtime = 'nodejs' @@ -32,639 +24,132 @@ export const maxDuration = 60 const logger = createLogger('WandGenerateAPI') -const azureApiKey = env.AZURE_OPENAI_API_KEY -const azureEndpoint = env.AZURE_OPENAI_ENDPOINT -const azureApiVersion = env.AZURE_OPENAI_API_VERSION -const wandModelName = env.WAND_OPENAI_MODEL_NAME || 'gpt-4o' -const openaiApiKey = env.OPENAI_API_KEY +const WORKSPACE_REQUIRED = 'Workspace context is required.' -const useWandAzure = azureApiKey && azureEndpoint && azureApiVersion - -if (!useWandAzure && !openaiApiKey) { - logger.warn( - 'Neither Azure OpenAI nor OpenAI API key found. Wand generation API will not function.' - ) -} else { - logger.info(`Using ${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} for wand generation`) +function wandError(error: string, status: number, extra: Record = {}) { + return NextResponse.json({ success: false, error, ...extra }, { status }) } -interface ChatMessage { - role: 'user' | 'assistant' | 'system' - content: string -} - -/** - * Wand enricher function type. - * Enrichers add context to the system prompt based on generationType. - */ -type WandEnricher = ( - workspaceId: string | null, - context: Record -) => Promise - -/** - * Registry of wand enrichers by generationType. - * Each enricher returns additional context to append to the system prompt. - */ -const wandEnrichers: Partial> = { - timestamp: async () => { - const now = new Date() - return `Current date and time context for reference: -- Current UTC timestamp: ${now.toISOString()} -- Current Unix timestamp (seconds): ${Math.floor(now.getTime() / 1000)} -- Current Unix timestamp (milliseconds): ${now.getTime()} -- Current date (UTC): ${now.toISOString().split('T')[0]} -- Current year: ${now.getUTCFullYear()} -- Current month: ${now.getUTCMonth() + 1} -- Current day of month: ${now.getUTCDate()} -- Current day of week: ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][now.getUTCDay()]} - -Use this context to calculate relative dates like "yesterday", "last week", "beginning of this month", etc.` - }, - - 'table-schema': enrichTableSchema, - /** JavaScript, Python, and Shell swap prompt text while retaining sandbox enrichment. */ - 'javascript-function-body': enrichSandboxCapabilities, +function errorDetails(error: unknown) { + if (!(error instanceof Error)) return { message: 'Unknown error' } + return { + name: error.name, + message: error.message || 'Unknown error', + code: 'code' in error ? error.code : undefined, + status: 'status' in error ? error.status : undefined, + stack: error.stack, + } } -async function updateUserStatsForWand( - billingAttribution: BillingAttributionSnapshot, - usage: { - prompt_tokens?: number - completion_tokens?: number - total_tokens?: number - }, - requestId: string, - isBYOK = false -): Promise { - if (!isBillingEnabled) { - return +/** Projects a refusal the use case raised, or `null` for a generation failure. */ +function refusalResponse(error: unknown, target: 'workflow' | 'workspace'): NextResponse | null { + if (error instanceof WandWorkflowNotFoundError) return wandError(error.message, 404) + if (error instanceof WandWorkflowDetachedError) return wandError(error.message, 403) + if ( + error instanceof OrchestrationError && + (error.code === 'forbidden' || error.code === 'not_found') + ) { + if (target === 'workspace') return wandError(WORKSPACE_REQUIRED, 400) + logger.warn('Wand caller lacks write access to the workflow workspace') + return wandError('Unauthorized', 403) } - - if (!usage.total_tokens || usage.total_tokens <= 0) { - return + if (error instanceof WandNotConfiguredError) return wandError(error.message, 503) + if (error instanceof WandBillingRefusedError) { + return wandError(error.message, 402, { scope: error.scope }) } + if (error instanceof WandEmptyResponseError) { + return wandError('Failed to generate content. AI response was empty.', 500) + } + if (error instanceof WandStreamStartError) { + logger.error('Failed to create stream', { ...errorDetails(error.cause), ...WAND_DEPLOYMENT }) + return wandError('An error occurred during wand generation streaming.', 500) + } + return null +} - try { - const promptTokens = usage.prompt_tokens || 0 - const completionTokens = usage.completion_tokens || 0 - - const modelName = useWandAzure ? wandModelName : 'gpt-4o' - let costToStore = 0 - - if (!isBYOK) { - const pricing = getModelPricing(modelName) - const costMultiplier = getCostMultiplier() - let modelCost = 0 - - if (pricing) { - const inputCost = (promptTokens / 1000000) * pricing.input - const outputCost = (completionTokens / 1000000) * pricing.output - modelCost = inputCost + outputCost - } else { - modelCost = (promptTokens / 1000000) * 0.005 + (completionTokens / 1000000) * 0.015 - } - - costToStore = modelCost * costMultiplier - } - - await recordUsage({ - userId: billingAttribution.actorUserId, - workspaceId: billingAttribution.workspaceId ?? undefined, - ...toBillingContext(billingAttribution), - entries: [ - { - category: 'model', - source: 'wand', - description: modelName, - cost: costToStore, - sourceReference: `wand:${requestId}`, - metadata: { inputTokens: promptTokens, outputTokens: completionTokens }, - }, - ], - }) - - await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) - } catch (error) { - logger.error('Failed to update user stats for wand usage', error, { requestId }) +function generationFailedResponse(error: unknown): NextResponse { + logger.error('Wand generation failed', { ...errorDetails(error), ...WAND_DEPLOYMENT }) + + let clientErrorMessage = 'Wand generation failed. Please try again later.' + let status = + error instanceof Error && 'status' in error && typeof error.status === 'number' + ? error.status + : 500 + + if ( + WAND_DEPLOYMENT.useWandAzure && + error instanceof Error && + error.message.includes('DeploymentNotFound') + ) { + clientErrorMessage = + 'Azure OpenAI deployment not found. Please check your model deployment configuration.' + status = 404 + } else if (status === 401) { + clientErrorMessage = 'Authentication failed. Please check your API key configuration.' + } else if (status === 429) { + clientErrorMessage = 'Rate limit exceeded. Please try again later.' + } else if (status >= 500) { + clientErrorMessage = + 'The wand generation service is currently unavailable. Please try again later.' } + + return wandError(clientErrorMessage, status) } export const POST = withRouteHandler(async (req: NextRequest) => { const requestId = generateRequestId() logger.info('Received wand generation request') - const session = await getSession() - if (!session?.user?.id) { + const principal = await authenticateOptionalInternalSession() + if (!principal) { logger.warn('Unauthorized wand generation attempt') - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + return wandError('Unauthorized', 401) } + let target: 'workflow' | 'workspace' | undefined try { const parsed = await parseRequest(wandGenerateContract, req, {}) if (!parsed.success) return parsed.response - const { body } = parsed.data - - const { - prompt, - systemPrompt, - stream = false, - history = [], - workflowId, - workspaceId: requestedWorkspaceId, - generationType, - wandContext = {}, - } = body + const { workflowId, workspaceId, ...generation } = parsed.data.body + const request = { ...generation, requestId } - if (!prompt) { - logger.warn('Invalid request: Missing prompt.') - return NextResponse.json( - { success: false, error: 'Missing required field: prompt.' }, - { status: 400 } - ) - } - - let workspaceId: string | null = null + let result if (workflowId) { - const [workflowRecord] = await db - .select({ workspaceId: workflow.workspaceId }) - .from(workflow) - .where(eq(workflow.id, workflowId)) - .limit(1) - - if (!workflowRecord) { - logger.warn(`Workflow not found: ${workflowId}`) - return NextResponse.json({ success: false, error: 'Workflow not found' }, { status: 404 }) - } - - workspaceId = workflowRecord.workspaceId - - if (workflowRecord.workspaceId) { - const permission = await verifyWorkspaceMembership( - session.user.id, - workflowRecord.workspaceId - ) - if (!permission || (permission !== 'admin' && permission !== 'write')) { - logger.warn( - `User ${session.user.id} does not have write access to workspace for workflow ${workflowId}` - ) - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 403 }) - } - } else { - logger.warn(`Workflow ${workflowId} has no workspaceId; wand request blocked`) - return NextResponse.json( - { - success: false, - error: - 'This workflow is not attached to a workspace. Personal workflows are deprecated and cannot be accessed.', - }, - { status: 403 } - ) - } - } else if (requestedWorkspaceId) { - // No workflow entity to resolve from (e.g. table-schema wand or an - // unhydrated editor); attribute to the workspace the wand is running in, - // but only when the caller is a member so usage can't be misattributed. - const permission = await verifyWorkspaceMembership(session.user.id, requestedWorkspaceId) - if (permission) { - workspaceId = requestedWorkspaceId - } - } - - // Per-member usage must be attributable to an org workspace. The editor always - // supplies a workflow or a workspace the caller belongs to; refuse to run rather - // than stamp usage workspace-less (which would silently skip the per-member cap). - if (!workspaceId) { - return NextResponse.json( - { success: false, error: 'Workspace context is required.' }, - { status: 400 } - ) - } - - /** Wand is an interactive action attributed to the authenticated human. */ - const actorUserId = session.user.id - const billingAttribution = await resolveBillingAttribution({ - actorUserId, - workspaceId, - }) - - let isBYOK = false - let activeOpenAIKey = openaiApiKey - - if (workspaceId && !useWandAzure) { - const byokResult = await getBYOKKey(workspaceId, 'openai') - if (byokResult) { - isBYOK = true - activeOpenAIKey = byokResult.apiKey - logger.info('Using BYOK OpenAI key for wand generation') - } - } - - if (!useWandAzure && !activeOpenAIKey) { - logger.error('AI client not initialized. Missing API key.') - return NextResponse.json( - { success: false, error: 'Wand generation service is not configured.' }, - { status: 503 } - ) - } - - /** - * BYOK skips metered caps but still checks the actor and exact workspace - * payer. Non-BYOK applies the complete attributed gate. - */ - if (isBYOK) { - const blocked = await checkAttributedBillingBlocks(billingAttribution) - if (blocked.blocked) { - return NextResponse.json( - { - success: false, - error: blocked.message || 'Account is not in good standing. Please contact support.', - }, - { status: 402 } - ) - } + target = 'workflow' + result = await generateWandForWorkflow.execute({ + principal, + input: { ...request, workflowId }, + }) + } else if (workspaceId) { + target = 'workspace' + result = await generateWandInWorkspace.execute({ + principal, + input: { ...request, workspaceId }, + }) } else { - const usage = await checkAttributedUsageLimits(billingAttribution) - if (usage.isExceeded) { - return NextResponse.json( - { - success: false, - error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.', - scope: usage.scope, - }, - { status: 402 } - ) - } - } - - let finalSystemPrompt = - systemPrompt || - 'You are a helpful AI assistant. Generate content exactly as requested by the user.' - - // Apply enricher if one exists for this generationType - if (generationType) { - const enricher = wandEnrichers[generationType] - if (enricher) { - const enrichment = await enricher(workspaceId, wandContext) - if (enrichment) { - finalSystemPrompt += `\n\n${enrichment}` - } - } - } - - if (generationType === 'cron-expression') { - finalSystemPrompt += - '\n\nIMPORTANT: Return ONLY the raw cron expression (e.g., "0 9 * * 1-5"). Do NOT wrap it in markdown code blocks, backticks, or quotes. Do NOT include any explanation or text before or after the expression.' - } - - // Both the JavaScript and Python function-body prompts share this type, so - // the reinforcement stays language-neutral. - if (generationType === 'javascript-function-body') { - finalSystemPrompt += - '\n\nIMPORTANT: Return ONLY the raw function body. Do NOT wrap it in markdown code blocks (no ```javascript, no ```python, no ```). Do NOT include any explanation before or after the code.' - } - - if (generationType === 'json-object') { - finalSystemPrompt += - '\n\nIMPORTANT: Return ONLY the raw JSON object. Do NOT wrap it in markdown code blocks (no ```json or ```). Do NOT include any explanation or text before or after the JSON. The response must start with { and end with }.' + // Per-member usage must be attributable to an org workspace. The editor always + // supplies a workflow or a workspace the caller belongs to; refuse to run rather + // than stamp usage workspace-less (which would silently skip the per-member cap). + return wandError(WORKSPACE_REQUIRED, 400) } - // Separate from json-object: that reinforcement demands braces, which would - // fight a field whose contract is an array. - if (generationType === 'json-array') { - finalSystemPrompt += - '\n\nIMPORTANT: Return ONLY the raw JSON array. Do NOT wrap it in markdown code blocks (no ```json or ```). Do NOT include any explanation or text before or after the JSON. The response must start with [ and end with ].' + if (result.kind === 'content') { + return NextResponse.json({ success: true, content: result.content }) } - - const messages: ChatMessage[] = [{ role: 'system', content: finalSystemPrompt }] - - messages.push(...history.filter((msg) => msg.role !== 'system')) - - messages.push({ role: 'user', content: prompt }) - - if (stream) { - try { - logger.info(`About to create stream with model: ${useWandAzure ? wandModelName : 'gpt-4o'}`) - - const apiUrl = useWandAzure - ? `${azureEndpoint?.replace(/\/$/, '')}/openai/v1/responses?api-version=${azureApiVersion}` - : 'https://api.openai.com/v1/responses' - - const headers: Record = { - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'responses=v1', - } - - if (useWandAzure) { - headers['api-key'] = azureApiKey! - } else { - headers.Authorization = `Bearer ${activeOpenAIKey}` - } - - const response = await fetch(apiUrl, { - method: 'POST', - headers, - body: JSON.stringify({ - model: useWandAzure ? wandModelName : 'gpt-4o', - input: messages, - temperature: 0.2, - max_output_tokens: 10000, - stream: true, - }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('API request failed', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - throw new Error(`API request failed: ${response.status} ${response.statusText}`) - } - - logger.info('Stream response received, starting processing') - - const encoder = new TextEncoder() - const decoder = new TextDecoder() - - const readable = new ReadableStream({ - async start(controller) { - const reader = response.body?.getReader() - if (!reader) { - controller.close() - return - } - - let finalUsage: any = null - let usageRecorded = false - - const flushUsage = async () => { - if (usageRecorded || !finalUsage) { - return - } - - usageRecorded = true - await updateUserStatsForWand(billingAttribution, finalUsage, requestId, isBYOK) - } - - try { - let buffer = '' - let chunkCount = 0 - let activeEventType: string | undefined - - while (true) { - const { done, value } = await reader.read() - - if (done) { - logger.info(`Stream completed. Total chunks: ${chunkCount}`) - await flushUsage() - controller.enqueue(encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`)) - controller.close() - break - } - - buffer += decoder.decode(value, { stream: true }) - - const lines = buffer.split('\n') - buffer = lines.pop() || '' - - for (const line of lines) { - const trimmed = line.trim() - if (!trimmed) { - continue - } - - if (trimmed.startsWith('event:')) { - activeEventType = trimmed.slice(6).trim() - continue - } - - if (!trimmed.startsWith('data:')) { - continue - } - - const data = trimmed.slice(5).trim() - if (data === '[DONE]') { - logger.info('Received [DONE] signal') - - await flushUsage() - - controller.enqueue( - encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`) - ) - controller.close() - return - } - - let parsed: any - try { - parsed = JSON.parse(data) - } catch { - continue - } - - const eventType = parsed?.type ?? activeEventType - - if ( - eventType === 'response.error' || - eventType === 'error' || - eventType === 'response.failed' - ) { - throw new Error(parsed?.error?.message || 'Responses stream error') - } - - if ( - eventType === 'response.output_text.delta' || - eventType === 'response.output_json.delta' - ) { - let content = '' - if (typeof parsed.delta === 'string') { - content = parsed.delta - } else if (parsed.delta && typeof parsed.delta.text === 'string') { - content = parsed.delta.text - } else if (parsed.delta && parsed.delta.json !== undefined) { - content = JSON.stringify(parsed.delta.json) - } else if (parsed.json !== undefined) { - content = JSON.stringify(parsed.json) - } else if (typeof parsed.text === 'string') { - content = parsed.text - } - - if (content) { - chunkCount++ - if (chunkCount === 1) { - logger.info('Received first content chunk') - } - - controller.enqueue( - encoder.encode(`data: ${JSON.stringify({ chunk: content })}\n\n`) - ) - } - } - - if (eventType === 'response.completed') { - const usage = parseResponsesUsage(parsed?.response?.usage ?? parsed?.usage) - if (usage) { - finalUsage = { - prompt_tokens: usage.promptTokens, - completion_tokens: usage.completionTokens, - total_tokens: usage.totalTokens, - } - logger.info(`Received usage data: ${JSON.stringify(finalUsage)}`) - } - } - } - } - } catch (streamError: any) { - logger.error('Streaming error', { - name: streamError?.name, - message: streamError?.message || 'Unknown error', - stack: streamError?.stack, - }) - - try { - await flushUsage() - } catch (usageError) { - logger.warn('Failed to record usage after stream error', usageError) - } - - const errorData = `data: ${JSON.stringify({ error: 'Streaming failed', done: true })}\n\n` - controller.enqueue(encoder.encode(errorData)) - controller.close() - } finally { - reader.releaseLock() - } - }, - cancel() {}, - }) - - return new Response(readable, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache, no-transform', - Connection: 'keep-alive', - 'X-Accel-Buffering': 'no', - }, - }) - } catch (error: any) { - logger.error('Failed to create stream', { - name: error?.name, - message: error?.message || 'Unknown error', - code: error?.code, - status: error?.status, - stack: error?.stack, - useWandAzure, - model: useWandAzure ? wandModelName : 'gpt-4o', - endpoint: useWandAzure ? azureEndpoint : 'api.openai.com', - apiVersion: useWandAzure ? azureApiVersion : 'N/A', - }) - - return NextResponse.json( - { success: false, error: 'An error occurred during wand generation streaming.' }, - { status: 500 } - ) - } - } - - const apiUrl = useWandAzure - ? `${azureEndpoint?.replace(/\/$/, '')}/openai/v1/responses?api-version=${azureApiVersion}` - : 'https://api.openai.com/v1/responses' - - const headers: Record = { - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'responses=v1', - } - - if (useWandAzure) { - headers['api-key'] = azureApiKey! - } else { - headers.Authorization = `Bearer ${activeOpenAIKey}` - } - - const response = await fetch(apiUrl, { - method: 'POST', - headers, - body: JSON.stringify({ - model: useWandAzure ? wandModelName : 'gpt-4o', - input: messages, - temperature: 0.2, - max_output_tokens: 10000, - }), - }) - - if (!response.ok) { - const errorText = await response.text() - const apiError = new Error( - `API request failed: ${response.status} ${response.statusText} - ${errorText}` - ) - ;(apiError as any).status = response.status - throw apiError - } - - const completion = await response.json() - const generatedContent = extractResponseText(completion.output)?.trim() - - if (!generatedContent) { - logger.error(`${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} response was empty or invalid.`) - return NextResponse.json( - { success: false, error: 'Failed to generate content. AI response was empty.' }, - { status: 500 } - ) - } - - logger.info('Wand generation successful') - - const usage = parseResponsesUsage(completion.usage) - if (usage) { - await updateUserStatsForWand( - billingAttribution, - { - prompt_tokens: usage.promptTokens, - completion_tokens: usage.completionTokens, - total_tokens: usage.totalTokens, - }, - requestId, - isBYOK - ) - } - - return NextResponse.json({ success: true, content: generatedContent }) - } catch (error: any) { - logger.error('Wand generation failed', { - name: error?.name, - message: error?.message || 'Unknown error', - code: error?.code, - status: error?.status, - stack: error?.stack, - useWandAzure, - model: useWandAzure ? wandModelName : 'gpt-4o', - endpoint: useWandAzure ? azureEndpoint : 'api.openai.com', - apiVersion: useWandAzure ? azureApiVersion : 'N/A', + return new Response(result.body, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache, no-transform', + Connection: 'keep-alive', + 'X-Accel-Buffering': 'no', + }, }) - - let clientErrorMessage = 'Wand generation failed. Please try again later.' - let status = typeof (error as any)?.status === 'number' ? (error as any).status : 500 - - if (useWandAzure && error?.message?.includes('DeploymentNotFound')) { - clientErrorMessage = - 'Azure OpenAI deployment not found. Please check your model deployment configuration.' - status = 404 - } else if (status === 401) { - clientErrorMessage = 'Authentication failed. Please check your API key configuration.' - } else if (status === 429) { - clientErrorMessage = 'Rate limit exceeded. Please try again later.' - } else if (status >= 500) { - clientErrorMessage = - 'The wand generation service is currently unavailable. Please try again later.' + } catch (error) { + const orchestrationError = asOrchestrationError(error) + if (orchestrationError?.code === 'unavailable') { + logger.error('Wand generation unavailable', { ...errorDetails(error), ...WAND_DEPLOYMENT }) + return wandError(orchestrationError.message, 503) } - - return NextResponse.json( - { - success: false, - error: clientErrorMessage, - }, - { status } - ) + return (target && refusalResponse(error, target)) || generationFailedResponse(error) } }) diff --git a/apps/sim/app/api/webhooks/route.test.ts b/apps/sim/app/api/webhooks/route.test.ts index 6417c4382b5..47218919ef4 100644 --- a/apps/sim/app/api/webhooks/route.test.ts +++ b/apps/sim/app/api/webhooks/route.test.ts @@ -50,6 +50,8 @@ vi.mock('@/lib/webhooks/utils.server', () => ({ findConflictingWebhookPathOwner: mocks.findConflictingWebhookPathOwner, })) +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE, OrchestrationError } from '@/lib/core/orchestration/types' import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { POST } from '@/app/api/webhooks/route' @@ -619,3 +621,34 @@ describe('POST /api/webhooks subscription replacement recovery', () => { } ) }) + +describe('POST /api/webhooks egress outage', () => { + beforeEach(setupUpsertMocks) + + it('reports an outage while creating the external subscription as unavailable', async () => { + queueCreatePathRows() + mocks.createExternalWebhookSubscription.mockRejectedValue( + new OrchestrationError('unavailable', EGRESS_UNAVAILABLE_MESSAGE) + ) + + const response = await POST(upsertRequest(), {}) + + expect(response.status).toBe(503) + expect(await response.json()).toEqual({ error: EGRESS_UNAVAILABLE_MESSAGE }) + }) + + it('reports an outage while configuring polling as unavailable', async () => { + queueCreatePathRows() + dbChainMockFns.returning.mockImplementationOnce(async () => [ + { id: 'webhook-1', workflowId: 'workflow-1', path: 'inbound-orders', isActive: true }, + ]) + mocks.activateExternalWebhookSubscription.mockResolvedValue(undefined) + mocks.configurePolling.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + mocks.getProviderHandler.mockReturnValue({ configurePolling: mocks.configurePolling }) + + const response = await POST(upsertRequest(), {}) + + expect(response.status).toBe(503) + expect(await response.json()).toEqual({ error: EGRESS_UNAVAILABLE_MESSAGE }) + }) +}) diff --git a/apps/sim/app/api/webhooks/route.ts b/apps/sim/app/api/webhooks/route.ts index dc93062b21f..4cbc0a8f7e3 100644 --- a/apps/sim/app/api/webhooks/route.ts +++ b/apps/sim/app/api/webhooks/route.ts @@ -17,6 +17,11 @@ import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' import { AuthType } from '@/lib/auth/hybrid' +import { + EGRESS_UNAVAILABLE_MESSAGE, + isUnavailableFailure, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -569,6 +574,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } } } catch (err) { + if (isUnavailableFailure(err)) { + logger.warn('External webhook subscription unavailable during an egress outage') + return NextResponse.json( + { error: EGRESS_UNAVAILABLE_MESSAGE }, + { status: statusForOrchestrationError('unavailable') } + ) + } const isConfigurationError = err instanceof WebhookDeploymentConfigurationError if (isConfigurationError) { logger.warn('Rejected webhook trigger configuration', { @@ -746,6 +758,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => { err ) await revertSavedWebhook(savedWebhook, existingWebhook, requestId) + if (isUnavailableFailure(err)) { + return NextResponse.json( + { error: EGRESS_UNAVAILABLE_MESSAGE }, + { status: statusForOrchestrationError('unavailable') } + ) + } return NextResponse.json( { error: `Failed to configure ${provider} webhook`, diff --git a/apps/sim/connectors/fireflies/fireflies.test.ts b/apps/sim/connectors/fireflies/fireflies.test.ts index 88c7f015300..283c7a62044 100644 --- a/apps/sim/connectors/fireflies/fireflies.test.ts +++ b/apps/sim/connectors/fireflies/fireflies.test.ts @@ -1,8 +1,14 @@ +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + +import { OutboundRoutingError } from '@/lib/core/network/routing' import { firefliesConnector } from '@/connectors/fireflies/fireflies' beforeEach(() => { vi.useRealTimers() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) }) interface GraphQLCall { @@ -157,6 +163,26 @@ describe('fireflies listDocuments', () => { }) }) +describe('fireflies outbound scope', () => { + it('refuses to call Fireflies outside a sync scope instead of leaving through the shared egress', async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockImplementationOnce(async (organizationId) => { + if (organizationId === undefined) throw new OutboundRoutingError('MISSING_SCOPE') + return { kind: 'direct' } + }) + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + const error = await firefliesConnector + .listDocuments('key', {}, undefined, {}) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(OutboundRoutingError) + expect(error).toMatchObject({ code: 'MISSING_SCOPE' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) + describe('fireflies getDocument', () => { it.each([null, {}, transcript('different')])( 'rejects malformed or mismatched successful transcript metadata', diff --git a/apps/sim/connectors/fireflies/fireflies.ts b/apps/sim/connectors/fireflies/fireflies.ts index 47a7de32aa2..5fb91073388 100644 --- a/apps/sim/connectors/fireflies/fireflies.ts +++ b/apps/sim/connectors/fireflies/fireflies.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { isRetryableError, @@ -200,7 +201,7 @@ async function firefliesGraphQL( return retryWithExponentialBackoff( async () => { /** One retry layer owns transport, HTTP, and GraphQL semantic failures. */ - const response = await fetch(FIREFLIES_GRAPHQL_URL, { + const response = await scopedOutboundFetch(FIREFLIES_GRAPHQL_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/connectors/google-drive/google-drive-errors.ts b/apps/sim/connectors/google-drive/google-drive-errors.ts index b26749a4155..41ae12d4fbb 100644 --- a/apps/sim/connectors/google-drive/google-drive-errors.ts +++ b/apps/sim/connectors/google-drive/google-drive-errors.ts @@ -1,3 +1,4 @@ +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' import { attachRetryHeaders, isRetryableError, @@ -159,7 +160,8 @@ export async function readGoogleDriveApiError( * Fetches a Google API, retrying errors whose structured body identifies a * transient rejection. Shared by every Google call a connector makes — Drive * and the Admin SDK use the same error envelope and the same rate-limit - * reasons. + * reasons. Requires an outbound scope: a lost scope fails closed instead of + * going direct. */ export async function fetchGoogleDriveWithRetry( url: string, @@ -169,7 +171,7 @@ export async function fetchGoogleDriveWithRetry( ): Promise { return retryWithExponentialBackoff( async () => { - const response = await fetch(url, options) + const response = await scopedOutboundFetch(url, options) if (response.ok) return response const error = await readGoogleDriveApiError(response, operation) diff --git a/apps/sim/connectors/google-drive/workspace-drives.test.ts b/apps/sim/connectors/google-drive/workspace-drives.test.ts index 8a49c131304..d26b1181fa6 100644 --- a/apps/sim/connectors/google-drive/workspace-drives.test.ts +++ b/apps/sim/connectors/google-drive/workspace-drives.test.ts @@ -1,7 +1,11 @@ +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import type { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' import { listGoogleWorkspaceDrives } from '@/connectors/google-drive/workspace-drives' +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + const mockFetch = vi.fn() const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { @@ -16,9 +20,23 @@ beforeEach(() => { afterEach(() => { vi.useRealTimers() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) }) describe('Google Workspace shared-drive enumeration', () => { + it('refuses to reach Google outside an outbound scope when egress routing is on', async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (organizationId) => { + if (organizationId === undefined) throw new OutboundRoutingError('MISSING_SCOPE') + return { kind: 'direct' } + }) + await expect(listGoogleWorkspaceDrives('token')).rejects.toMatchObject({ + code: 'MISSING_SCOPE', + }) + expect(mockFetch).not.toHaveBeenCalled() + }) + it('returns one bounded page using only the delegated user token', async () => { const controller = new AbortController() mockFetch.mockResolvedValueOnce( diff --git a/apps/sim/ee/scim/lib/protocol/errors.ts b/apps/sim/ee/scim/lib/protocol/errors.ts index 3c1841e3acc..3193ab58378 100644 --- a/apps/sim/ee/scim/lib/protocol/errors.ts +++ b/apps/sim/ee/scim/lib/protocol/errors.ts @@ -153,6 +153,7 @@ export function toScimError(error: unknown): ScimError { case 'validation': return new ScimError(400, 'invalidValue', orchestration.message) case 'locked': + case 'unavailable': return new ScimError(503, undefined, orchestration.message, { 'Retry-After': '5' }) default: return new ScimError(500, undefined, 'Internal server error') diff --git a/apps/sim/executor/handlers/pi/cloud/github-pr.ts b/apps/sim/executor/handlers/pi/cloud/github-pr.ts index 290a38da059..8a9b3566472 100644 --- a/apps/sim/executor/handlers/pi/cloud/github-pr.ts +++ b/apps/sim/executor/handlers/pi/cloud/github-pr.ts @@ -7,6 +7,7 @@ */ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { toolResultError } from '@/executor/handlers/pi/cloud/shared' import { executeTool } from '@/tools' import { GITHUB_GRAPHQL_URL, githubGraphQlHeaders, readGraphQlData } from '@/tools/github/graphql' @@ -227,7 +228,7 @@ export async function setPullRequestDraftState( repo: params.repo, pullNumber: params.pullNumber, } - const queryResponse = await fetch(GITHUB_GRAPHQL_URL, { + const queryResponse = await outboundFetch(GITHUB_GRAPHQL_URL, { method: 'POST', headers: githubGraphQlHeaders(params.githubToken), body: JSON.stringify({ @@ -268,7 +269,7 @@ export async function setPullRequestDraftState( if (isDraft === shouldBeDraft) return const mutationName = shouldBeDraft ? 'convertPullRequestToDraft' : 'markPullRequestReadyForReview' - const mutationResponse = await fetch(GITHUB_GRAPHQL_URL, { + const mutationResponse = await outboundFetch(GITHUB_GRAPHQL_URL, { method: 'POST', headers: githubGraphQlHeaders(params.githubToken), body: JSON.stringify({ diff --git a/apps/sim/executor/utils/credential-token.ts b/apps/sim/executor/utils/credential-token.ts index 471c678c1ea..b8351efe2ad 100644 --- a/apps/sim/executor/utils/credential-token.ts +++ b/apps/sim/executor/utils/credential-token.ts @@ -1,5 +1,4 @@ import { createLogger } from '@sim/logger' -import { AuthType } from '@/lib/auth/hybrid' import { createCopilotManagedOAuthPrincipal } from '@/lib/credentials/application/copilot-managed-oauth-delegation' import { bindExecutorManagedOAuthDelegation } from '@/lib/credentials/application/managed-oauth-delegation' import { authorizePersonalCredential } from '@/lib/credentials/application/personal-credentials' @@ -42,10 +41,9 @@ export interface ResolveExecutorCredentialTokenParams { /** * Resolves a credential's access token in-process for server-side workflow - * execution, through the same authorized application dispatch as - * `POST /api/auth/oauth/token` (`resolveCredentialAccessToken`). Both runtimes - * hold the OAuth client config the refresh branch needs, so authorization, - * refresh, and audit run identically to the route. + * execution through the authorized application dispatch + * (`resolveCredentialAccessToken`), so authorization, refresh, and audit run + * identically to every other server surface. */ export async function resolveExecutorCredentialToken( params: ResolveExecutorCredentialTokenParams @@ -119,11 +117,7 @@ export async function resolveExecutorCredentialToken( scopes: params.scopes, impersonateEmail: params.impersonateEmail, callerUserId: userId && params.enforceCredentialAccess ? userId : undefined, - authenticate: () => ({ - success: true, - userId, - authType: AuthType.INTERNAL_JWT, - }), + userId, resolveManagedPrincipal, }) diff --git a/apps/sim/lib/api/contracts/oauth-connections.ts b/apps/sim/lib/api/contracts/oauth-connections.ts index b0555c1ec78..b0869fe6fa1 100644 --- a/apps/sim/lib/api/contracts/oauth-connections.ts +++ b/apps/sim/lib/api/contracts/oauth-connections.ts @@ -7,8 +7,6 @@ import type { } from '@/lib/api/contracts/types' import { defineRouteContract } from '@/lib/api/contracts/types' -export const MANAGED_OAUTH_DELEGATION_HEADER = 'x-sim-managed-oauth-delegation' - export const oauthAccountSummarySchema = z.object({ id: z.string(), name: z.string(), @@ -77,84 +75,9 @@ const trelloCallbackQuerySchema = z }) .passthrough() -/** Google domain-wide-delegation subject. Also applied by in-process credential callers. */ +/** Google domain-wide-delegation subject, validated for in-process credential callers. */ export const impersonateEmailSchema = z.string().email() -export const oauthTokenRequestBodySchema = z - .object({ - credentialId: z.string().min(1).optional(), - credentialAccountUserId: z.string().min(1).optional(), - providerId: z.string().min(1).optional(), - toolId: z.string().min(1).optional(), - workflowId: z.string().min(1).nullish(), - scopes: z.array(z.string()).optional(), - impersonateEmail: impersonateEmailSchema.optional(), - }) - .refine( - (data) => data.credentialId || (data.credentialAccountUserId && data.providerId), - 'Either credentialId or (credentialAccountUserId + providerId) is required' - ) - -export const oauthTokenGetQuerySchema = z.object({ - credentialId: z - .string({ - error: 'Credential ID is required', - }) - .min(1, 'Credential ID is required'), -}) - -export const oauthTokenPostQuerySchema = z.object({ - userId: z.string().min(1).optional(), -}) - -export const oauthTokenPostHeadersSchema = z.object({ - [MANAGED_OAUTH_DELEGATION_HEADER]: z.string().min(1).optional(), -}) - -const oauthTokenResponseSchema = z.object({ - accessToken: z.string(), - credentialType: z.enum(['oauth', 'managed_oauth', 'service_account']).optional(), - idToken: z.string().optional(), - instanceUrl: z.string().optional(), - /** Zoho Desk — the data-center-scoped Desk REST base for this credential. */ - apiDomain: z.string().optional(), - cloudId: z.string().optional(), - domain: z.string().optional(), - providerWorkspaceId: z.string().optional(), - /** Datadog — the application key sent beside the API key (`accessToken`). */ - applicationKey: z.string().optional(), - realmId: z.string().optional(), - quickBooksEnvironment: z.enum(['sandbox', 'production']).optional(), - authStyle: z - .enum(['x-api-token', 'snowflake-workload-identity', 'snowflake-key-pair', 'basic', 'bearer']) - .optional(), -}) - -/** Token material a resolved credential yields, on the wire and in-process alike. */ -export type OAuthTokenResponse = z.output - -export const oauthTokenGetContract = defineRouteContract({ - method: 'GET', - path: '/api/auth/oauth/token', - query: oauthTokenGetQuerySchema, - response: { - mode: 'json', - schema: oauthTokenResponseSchema, - }, -}) - -export const oauthTokenPostContract = defineRouteContract({ - method: 'POST', - path: '/api/auth/oauth/token', - query: oauthTokenPostQuerySchema, - headers: oauthTokenPostHeadersSchema, - body: oauthTokenRequestBodySchema, - response: { - mode: 'json', - schema: oauthTokenResponseSchema, - }, -}) - export const shopifyAuthorizeQuerySchema = z.object({ shop: z.string().optional(), returnUrl: z.string().optional(), diff --git a/apps/sim/lib/api/server/routes/index.ts b/apps/sim/lib/api/server/routes/index.ts index e42507d6cc8..23aefbfdbec 100644 --- a/apps/sim/lib/api/server/routes/index.ts +++ b/apps/sim/lib/api/server/routes/index.ts @@ -1,5 +1,6 @@ export { defineInternalBinaryRoute } from '@/lib/api/server/routes/internal-binary-route' export { + authenticateOptionalInternalSession, createInternalSessionOrExecutorAuth, defineInternalJsonRoute, extendInternalErrorPolicy, diff --git a/apps/sim/lib/api/server/routes/internal-json-route.ts b/apps/sim/lib/api/server/routes/internal-json-route.ts index 4e57e7d7df2..ac9f996b932 100644 --- a/apps/sim/lib/api/server/routes/internal-json-route.ts +++ b/apps/sim/lib/api/server/routes/internal-json-route.ts @@ -62,6 +62,16 @@ export const internalSessionAuth = { }, } as const +/** The session principal when the request is signed in; returns `null` instead of throwing when unauthenticated, so the route answers with its own response. */ +export async function authenticateOptionalInternalSession(): Promise { + try { + return await internalSessionAuth.authenticate() + } catch (error) { + if (error instanceof InternalUnauthenticatedError) return null + throw error + } +} + interface InternalSessionOrExecutorAuthOptions { audience: string resourceScope?( diff --git a/apps/sim/lib/atlassian/discovery.ts b/apps/sim/lib/atlassian/discovery.ts index 33c9aa4f96a..911956baa21 100644 --- a/apps/sim/lib/atlassian/discovery.ts +++ b/apps/sim/lib/atlassian/discovery.ts @@ -1,6 +1,7 @@ import { sha256Hex } from '@sim/security/hash' import { parseRetryAfter } from '@sim/utils/retry' import { LRUCache } from 'lru-cache' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type HTTPError, isRetryableError, @@ -158,7 +159,7 @@ export function fetchAtlassianDiscoveryJson( ): Promise { return retryWithExponentialBackoff( async () => { - const response = await fetch(url, { + const response = await outboundFetch(url, { method: 'GET', headers, signal: AbortSignal.timeout(DISCOVERY_REQUEST_TIMEOUT_MS), diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index 6fb31d22b92..2e959523a29 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -159,6 +159,10 @@ import { isEmailVerificationEffectivelyEnabled } from '@/lib/messaging/email/ver import { scheduleLifecycleEmail } from '@/lib/messaging/lifecycle' import { APP_ENTRY_PATH } from '@/lib/navigation/paths' import { formatApolloAuthorizationUrl } from '@/lib/oauth/apollo' +import { + resolveConnectDraftOutboundScope, + withConnectDraftOutboundScope, +} from '@/lib/oauth/connect-scope.server' import { clearRecordedRevocation } from '@/lib/oauth/credential-service' import { getMicrosoftRefreshTokenExpiry, @@ -170,7 +174,11 @@ import { MICROSOFT_DATAVERSE_PROVIDER_ID, } from '@/lib/oauth/microsoft-dataverse' import { clearOAuthRefreshDeadFlag } from '@/lib/oauth/refresh-coordination' -import { isSalesforceOAuthProviderId, withSalesforceInstanceScope } from '@/lib/oauth/salesforce' +import { + extractSalesforceInstanceUrl, + isSalesforceOAuthProviderId, + withSalesforceInstanceScope, +} from '@/lib/oauth/salesforce' import { fetchSalesforceInstanceUrl } from '@/lib/oauth/salesforce-instance.server' import { extractSlackTeamId, fanOutSlackTokenChain } from '@/lib/oauth/slack' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' @@ -261,6 +269,20 @@ function resolveStripeBilling() { const stripeBilling = resolveStripeBilling() +/** + * Resolves a connecting Salesforce token's instance from the egress route of the workspace or + * organization whose connect draft started the callback. + */ +async function fetchConnectingSalesforceInstanceUrl(params: { + draftId: string | undefined + userId: string + providerId: string + accessToken: string +}): Promise { + const inDraftScope = await resolveConnectDraftOutboundScope(params) + return inDraftScope(() => fetchSalesforceInstanceUrl(params.providerId, params.accessToken)) +} + /** * A fresh Slack connect re-issues the installation's rotating token chain, * invalidating the copies held by sibling account rows for the same team (Slack @@ -508,10 +530,12 @@ export const auth = betterAuth({ } if (account.accessToken && isSalesforceOAuthProviderId(account.providerId)) { - const instanceUrl = await fetchSalesforceInstanceUrl( - account.providerId, - account.accessToken - ) + const instanceUrl = await fetchConnectingSalesforceInstanceUrl({ + draftId: reconnectDraftId, + userId: account.userId, + providerId: account.providerId, + accessToken: account.accessToken, + }) if (instanceUrl) { modifiedAccount.scope = withSalesforceInstanceScope(instanceUrl, account.scope) } @@ -718,11 +742,14 @@ export const auth = betterAuth({ updates.accessTokenExpiresAt = new Date(Date.now() + 2 * 60 * 60 * 1000) } - if (account.accessToken) { - const instanceUrl = await fetchSalesforceInstanceUrl( - account.providerId, - account.accessToken - ) + // The before hook already stored the instance when its lookup succeeded. + if (account.accessToken && !extractSalesforceInstanceUrl(account.scope)) { + const instanceUrl = await fetchConnectingSalesforceInstanceUrl({ + draftId: credentialDraftBinding?.draftId, + userId: account.userId, + providerId: account.providerId, + accessToken: account.accessToken, + }) if (instanceUrl) { updates.scope = withSalesforceInstanceScope(instanceUrl, account.scope) } @@ -1476,7 +1503,7 @@ export const auth = betterAuth({ overrideDefaultEmailVerification: true, }), genericOAuth({ - config: buildConnectorProviders(), + config: buildConnectorProviders().map(withConnectDraftOutboundScope), }), /** * Sim as an OAuth 2.0 authorization server (auth-code + PKCE, refresh diff --git a/apps/sim/lib/auth/connectors/managed-oauth.ts b/apps/sim/lib/auth/connectors/managed-oauth.ts index aad3acd77b4..d6cf2e178ad 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.ts @@ -6,6 +6,7 @@ import type { GenericOAuthConfig } from 'better-auth/plugins' import { OAuth2Client, type TokenPayload } from 'google-auth-library' import { createRemoteJWKSet, type JWTPayload, jwtVerify } from 'jose' import { buildConnectorProviders } from '@/lib/auth/connectors/providers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { boundedAccountSites } from '@/lib/credential-groups/account-sites' import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' @@ -207,7 +208,7 @@ function httpsOrigin(value: string): string | null { */ async function listAtlassianSites(accessToken: string): Promise { try { - const response = await fetch(ATLASSIAN_SITES_URL, { + const response = await outboundFetch(ATLASSIAN_SITES_URL, { headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, signal: AbortSignal.timeout(ATLASSIAN_USER_INFO_TIMEOUT_MS), }) @@ -246,7 +247,7 @@ export function createAtlassianManagedOAuthConnector( throw new Error('Atlassian returned an incomplete authorization') } const sites = listAtlassianSites(tokens.accessToken) - const response = await fetch(ATLASSIAN_USER_INFO_URL, { + const response = await outboundFetch(ATLASSIAN_USER_INFO_URL, { headers: { Accept: 'application/json', Authorization: `Bearer ${tokens.accessToken}`, @@ -354,7 +355,7 @@ function requireMicrosoftIdentityClaims(payload: JWTPayload): MicrosoftIdentityC * not consented to Graph. */ async function fetchMicrosoftAccessTokenSubject(accessToken: string): Promise { - const response = await fetch(MICROSOFT_OIDC_USER_INFO_URL, { + const response = await outboundFetch(MICROSOFT_OIDC_USER_INFO_URL, { headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, signal: AbortSignal.timeout(MICROSOFT_OIDC_USER_INFO_TIMEOUT_MS), }) @@ -500,7 +501,7 @@ async function fetchManagedOAuthProfile( ): Promise { const { providerId, userInfo } = options const url = typeof userInfo.url === 'function' ? userInfo.url(tokens) : userInfo.url - const response = await fetch(url, { + const response = await outboundFetch(url, { method: userInfo.method ?? 'GET', headers: { Accept: 'application/json', @@ -706,7 +707,7 @@ function createAttioManagedOAuthConnector(): ManagedOAuthConnectorConfig { Accept: 'application/json', Authorization: `Bearer ${tokens.accessToken}`, } - const selfResponse = await fetch('https://api.attio.com/v2/self', { + const selfResponse = await outboundFetch('https://api.attio.com/v2/self', { headers, signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), }) @@ -722,7 +723,7 @@ function createAttioManagedOAuthConnector(): ManagedOAuthConnectorConfig { identity.authorized_by_workspace_member_id, 'Attio workspace member id' ) - const memberResponse = await fetch( + const memberResponse = await outboundFetch( `https://api.attio.com/v2/workspace_members/${encodeURIComponent(memberId)}`, { headers, signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS) } ) @@ -791,7 +792,7 @@ function createBitbucketManagedOAuthConnector(): ManagedOAuthConnectorConfig { Accept: 'application/json', Authorization: `Bearer ${tokens.accessToken}`, } - const userResponse = await fetch(`${BITBUCKET_API_BASE}/user`, { + const userResponse = await outboundFetch(`${BITBUCKET_API_BASE}/user`, { headers, signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), }) @@ -804,7 +805,7 @@ function createBitbucketManagedOAuthConnector(): ManagedOAuthConnectorConfig { } const user = asProfileRecord(userBody, 'Bitbucket') const accountId = requireIdentityField(user.account_id, 'Bitbucket account id') - const emailsResponse = await fetch(`${BITBUCKET_API_BASE}/user/emails`, { + const emailsResponse = await outboundFetch(`${BITBUCKET_API_BASE}/user/emails`, { headers, signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), }) diff --git a/apps/sim/lib/auth/connectors/providers.ts b/apps/sim/lib/auth/connectors/providers.ts index 29b8290cb8d..359d29ee43d 100644 --- a/apps/sim/lib/auth/connectors/providers.ts +++ b/apps/sim/lib/auth/connectors/providers.ts @@ -9,6 +9,7 @@ import type { GenericOAuthConfig } from 'better-auth/plugins' import { syntheticConnectorEmail } from '@/lib/auth/connector-email' import { env } from '@/lib/core/config/env' import { inspectConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -144,7 +145,7 @@ function salesforceConnector(providerId: string, loginHost: string): GenericOAut redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/${providerId}`, getUserInfo: async (tokens) => { try { - const response = await fetch(userInfoUrl, { + const response = await outboundFetch(userInfoUrl, { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -218,7 +219,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-email`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -254,7 +255,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-calendar`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -290,7 +291,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-drive`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -326,7 +327,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-docs`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -362,7 +363,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-sheets`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -399,7 +400,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-contacts`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -435,7 +436,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-forms`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -471,7 +472,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-ads`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -506,7 +507,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-bigquery`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -543,7 +544,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-vault`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -580,7 +581,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-groups`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -617,7 +618,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-chat`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -654,7 +655,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-meet`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -690,7 +691,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/google-tasks`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -727,7 +728,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/vertex-ai`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { + const response = await outboundFetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) if (!response.ok) { @@ -984,7 +985,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Wealthbox user profile') - const response = await fetch('https://api.crmworkspace.com/v1/me', { + const response = await outboundFetch('https://api.crmworkspace.com/v1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1065,7 +1066,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Pipedrive user profile') - const response = await fetch('https://api.pipedrive.com/v1/users/me', { + const response = await outboundFetch('https://api.pipedrive.com/v1/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1112,7 +1113,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching HubSpot user profile') - const response = await fetch( + const response = await outboundFetch( `https://api.hubapi.com/oauth/v1/access-tokens/${tokens.accessToken}` ) @@ -1207,7 +1208,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { // pkce:false (which connected) then restoring pkce:true + this verifier. if (codeVerifier) tokenParams.set('code_verifier', codeVerifier) - const response = await fetch('https://accounts.zoho.com/oauth/v2/token', { + const response = await outboundFetch('https://accounts.zoho.com/oauth/v2/token', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: tokenParams, @@ -1254,7 +1255,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { // token response api_domain so every API call targets the correct // host instead of assuming desk.zoho.com. Stored inside the scope // string (survives refreshes, which never rewrite scope) and read - // back in /api/auth/oauth/token as `apiDomain`. + // back during credential token resolution as `apiDomain`. const deskBase = deriveZohoDeskBaseFromApiDomain( typeof data.api_domain === 'string' ? data.api_domain : undefined ) @@ -1275,7 +1276,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, getUserInfo: async (tokens) => { try { - const response = await fetch('https://accounts.zoho.com/oauth/user/info', { + const response = await outboundFetch('https://accounts.zoho.com/oauth/user/info', { headers: { Authorization: `Zoho-oauthtoken ${tokens.accessToken}` }, }) @@ -1355,7 +1356,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { // rejects the request shape (invalid_request). if (codeVerifier) tokenParams.set('code_verifier', codeVerifier) - const response = await fetch('https://accounts.zoho.com/oauth/v2/token', { + const response = await outboundFetch('https://accounts.zoho.com/oauth/v2/token', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: tokenParams.toString(), @@ -1406,7 +1407,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, getUserInfo: async (tokens) => { try { - const response = await fetch('https://accounts.zoho.com/oauth/user/info', { + const response = await outboundFetch('https://accounts.zoho.com/oauth/user/info', { headers: { Authorization: `Zoho-oauthtoken ${tokens.accessToken}` }, }) @@ -1471,7 +1472,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/x`, getUserInfo: async (tokens) => { try { - const response = await fetch( + const response = await outboundFetch( 'https://api.x.com/2/users/me?user.fields=profile_image_url,username,name,verified', { headers: { @@ -1528,7 +1529,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { scope: getCanonicalScopesForProvider('tiktok').join(','), }, getToken: async ({ code, redirectURI }) => { - const response = await fetch('https://open.tiktokapis.com/v2/oauth/token/', { + const response = await outboundFetch('https://open.tiktokapis.com/v2/oauth/token/', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ @@ -1559,7 +1560,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, getUserInfo: async (tokens) => { try { - const response = await fetch( + const response = await outboundFetch( 'https://open.tiktokapis.com/v2/user/info/?fields=open_id,display_name,avatar_url', { headers: { @@ -1643,7 +1644,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }), getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.atlassian.com/me', { + const response = await outboundFetch('https://api.atlassian.com/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1704,7 +1705,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }), getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.atlassian.com/me', { + const response = await outboundFetch('https://api.atlassian.com/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1755,7 +1756,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/airtable`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.airtable.com/v0/meta/whoami', { + const response = await outboundFetch('https://api.airtable.com/v0/meta/whoami', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1805,7 +1806,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { const basicAuth = Buffer.from( `${env.BITBUCKET_CLIENT_ID as string}:${env.BITBUCKET_CLIENT_SECRET as string}` ).toString('base64') - const response = await fetch('https://bitbucket.org/site/oauth2/access_token', { + const response = await outboundFetch('https://bitbucket.org/site/oauth2/access_token', { method: 'POST', headers: { Authorization: `Basic ${basicAuth}`, @@ -1846,7 +1847,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { getUserInfo: async (tokens) => { try { const signal = AbortSignal.timeout(15_000) - const response = await fetch('https://api.bitbucket.org/2.0/user', { + const response = await outboundFetch('https://api.bitbucket.org/2.0/user', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -1909,7 +1910,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/notion`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.notion.com/v1/users/me', { + const response = await outboundFetch('https://api.notion.com/v1/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, 'Notion-Version': '2022-06-28', @@ -1984,7 +1985,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { getUserInfo: async (tokens) => { try { const signal = AbortSignal.timeout(15_000) - const response = await fetch(MONDAY_API_URL, { + const response = await outboundFetch(MONDAY_API_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -2061,7 +2062,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/reddit`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://oauth.reddit.com/api/v1/me', { + const response = await outboundFetch('https://oauth.reddit.com/api/v1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, 'User-Agent': REDDIT_USER_AGENT, @@ -2108,7 +2109,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/clickup`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.clickup.com/api/v2/user', { + const response = await outboundFetch('https://api.clickup.com/api/v2/user', { headers: { Authorization: `Bearer ${tokens.accessToken}`, 'Content-Type': 'application/json', @@ -2159,7 +2160,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { accessType: 'offline', getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.linear.app/graphql', { + const response = await outboundFetch('https://api.linear.app/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -2239,7 +2240,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { * `/v2/self` requires no scope and reports who authorized the token. * @see https://docs.attio.com/rest-api/endpoint-reference/meta/identify */ - const selfResponse = await fetch('https://api.attio.com/v2/self', { + const selfResponse = await outboundFetch('https://api.attio.com/v2/self', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, }) @@ -2269,7 +2270,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { * `user_management:read`, which Sim always requests for Attio. * @see https://docs.attio.com/rest-api/endpoint-reference/workspace-members/get-a-workspace-member */ - const memberResponse = await fetch( + const memberResponse = await outboundFetch( `https://api.attio.com/v2/workspace_members/${encodeURIComponent(memberId)}`, { headers: { Authorization: `Bearer ${tokens.accessToken}` } } ) @@ -2327,7 +2328,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/box`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.box.com/2.0/users/me', { + const response = await outboundFetch('https://api.box.com/2.0/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2378,12 +2379,15 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, getUserInfo: async (tokens) => { try { - const response = await fetch('https://api.dropboxapi.com/2/users/get_current_account', { - method: 'POST', - headers: { - Authorization: `Bearer ${tokens.accessToken}`, - }, - }) + const response = await outboundFetch( + 'https://api.dropboxapi.com/2/users/get_current_account', + { + method: 'POST', + headers: { + Authorization: `Bearer ${tokens.accessToken}`, + }, + } + ) if (!response.ok) { const errorText = await response.text() @@ -2429,7 +2433,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/asana`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://app.asana.com/api/1.0/users/me', { + const response = await outboundFetch('https://app.asana.com/api/1.0/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2479,7 +2483,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { redirectURI: `${getBaseUrl()}/api/auth/oauth2/callback/slack`, getUserInfo: async (tokens) => { try { - const response = await fetch('https://slack.com/api/auth.test', { + const response = await outboundFetch('https://slack.com/api/auth.test', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2558,7 +2562,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Webflow user info') - const response = await fetch('https://api.webflow.com/v2/token/introspect', { + const response = await outboundFetch('https://api.webflow.com/v2/token/introspect', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2609,7 +2613,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching LinkedIn user profile') - const response = await fetch('https://api.linkedin.com/v2/userinfo', { + const response = await outboundFetch('https://api.linkedin.com/v2/userinfo', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2659,7 +2663,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Zoom user profile') - const response = await fetch('https://api.zoom.us/v2/users/me', { + const response = await outboundFetch('https://api.zoom.us/v2/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2707,7 +2711,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Spotify user profile') - const response = await fetch('https://api.spotify.com/v1/me', { + const response = await outboundFetch('https://api.spotify.com/v1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2755,7 +2759,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching WordPress.com user profile') - const response = await fetch('https://public-api.wordpress.com/rest/v1.1/me', { + const response = await outboundFetch('https://public-api.wordpress.com/rest/v1.1/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2807,7 +2811,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching DocuSign user profile') - const response = await fetch(getDocusignOAuthUrl('/oauth/userinfo'), { + const response = await outboundFetch(getDocusignOAuthUrl('/oauth/userinfo'), { headers: { Authorization: `Bearer ${tokens.accessToken}`, }, @@ -2864,7 +2868,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { const signal = AbortSignal.timeout(15_000) - const response = await fetch('https://api.ramp.com/developer/v1/business', { + const response = await outboundFetch('https://api.ramp.com/developer/v1/business', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, redirect: 'error', signal, @@ -2922,7 +2926,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { const signal = AbortSignal.timeout(15_000) - const response = await fetch('https://api.helpscout.net/v2/users/me', { + const response = await outboundFetch('https://api.helpscout.net/v2/users/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, Accept: 'application/json' }, redirect: 'error', signal, @@ -2983,7 +2987,7 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { try { logger.info('Fetching Cal.com user profile') - const response = await fetch('https://api.cal.com/v2/me', { + const response = await outboundFetch('https://api.cal.com/v2/me', { headers: { Authorization: `Bearer ${tokens.accessToken}`, 'cal-api-version': '2024-08-13', diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts index 28ba87ab155..aabbf035890 100644 --- a/apps/sim/lib/core/network/gateway.server.test.ts +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -7,14 +7,22 @@ import { join } from 'node:path' import { getCACertificates, setDefaultCACertificates } from 'node:tls' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' +import { GetCallerIdentityCommand, STSClient } from '@aws-sdk/client-sts' import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import type { OutboundGateway, ResolvedOutboundRoute } from '@/lib/core/network/config.server' vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +/** The synthetic origin's name resolves to the one address the local gateway admits. */ +vi.mock('@sim/security/dns', async (importOriginal) => ({ + ...(await importOriginal()), + resolveHostAddresses: vi.fn(async () => ({ addresses: ['1.1.1.1'], preferred: '1.1.1.1' })), +})) import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { openGatewayTunnel, secureOutboundTunnel } from '@/lib/core/network/gateway.server' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createPinnedFetchWithDispatcher, secureFetchWithPinnedIP, @@ -296,6 +304,55 @@ describe('organization gateways over real TLS CONNECT sockets', () => { await transport.dispatcher.destroy() } }) + it("sends outboundFetch through the scoped organization's gateway", async () => { + const start = admissions.length + await runWithOutboundOrganization('org_b', async () => { + const response = await outboundFetch(url(), { + method: 'POST', + body: 'grant_type=client_credentials', + redirect: 'error', + }) + expect(response.headers.get('x-via-proxy')).toBe('yes') + await response.body?.cancel() + }) + expect(admissions.slice(start)).toEqual([ + { token: 'Bearer bravo', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, + ]) + }) + describe('AWS SDK clients signing with customer credentials', () => { + const sts = new STSClient({ + region: 'us-east-1', + endpoint: url(), + maxAttempts: 1, + credentials: { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'secret' }, + requestHandler: organizationAwsRequestHandler(), + }) + afterAll(() => sts.destroy()) + + /** The origin echoes the signed query body, which STS cannot parse; only the route matters. */ + const viaProxy = async (organizationId: string) => { + const error = await runWithOutboundOrganization(organizationId, () => + sts.send(new GetCallerIdentityCommand({})) + ).then( + () => undefined, + (failure: { $response?: { headers: Record } }) => failure + ) + return error?.$response?.headers['x-via-proxy'] + } + + it('tunnels each organization with its own credential and never shares its pooled socket', async () => { + const start = admissions.length + expect(await viaProxy('org_a')).toBe('yes') + expect(await viaProxy('org_b')).toBe('yes') + expect(await viaProxy('org_a')).toBe('yes') + expect(await viaProxy('org_b')).toBe('yes') + expect(admissions.slice(start)).toEqual([ + { token: 'Bearer alpha', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, + { token: 'Bearer bravo', destination: '1.1.1.1:443', sni: 'gateway.invalid' }, + ]) + }) + }) + it('fails closed on denied proxy credentials without contacting the origin', async () => { state.gateways.set('denied', { ...state.gateways.get('org_a')!, diff --git a/apps/sim/lib/core/network/gateway.server.ts b/apps/sim/lib/core/network/gateway.server.ts index cbe81a57e55..c50ec38031b 100644 --- a/apps/sim/lib/core/network/gateway.server.ts +++ b/apps/sim/lib/core/network/gateway.server.ts @@ -127,6 +127,44 @@ function getErrorCode(error: Error | undefined): string | undefined { return typeof code === 'string' ? code : undefined } +/** + * Opens a TLS connection to `target` through `gateway` after the egress policy approves it: + * HTTPS on 443 only, judged against `profile` (or the already resolved `resolvedIP`). Every + * failure is an {@link OutboundRoutingError}, so no gateway detail reaches the caller. + */ +export async function connectThroughGateway( + gateway: OutboundGateway, + target: { protocol: string; hostname: string; port: number }, + options: { profile: EgressProfile; resolvedIP?: string } +): Promise { + try { + const hostname = target.hostname.replace(/^\[|\]$/g, '') + if (target.protocol !== 'https:' || target.port !== 443) { + throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') + } + const authority = isIP(hostname) === 6 ? `[${hostname}]` : hostname + const url = new URL(`https://${authority}:${target.port}`) + let address = options.resolvedIP + if (address) { + if (!checkResolvedEgress(url, address, options.profile).allowed) { + throw new OutboundRoutingError('ROUTE_BLOCKED') + } + } else { + const result = await validateEgressUrl(url.href, 'url', options.profile, { + logDetails: false, + }) + if (!result.isValid) throw new OutboundRoutingError('ROUTE_BLOCKED') + address = result.resolvedIP + } + const socket = await openGatewayTunnel(gateway, address, target.port) + return await secureOutboundTunnel(socket, hostname, target.port) + } catch (error) { + throw error instanceof OutboundRoutingError + ? error + : new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } +} + /** A dispatcher belongs to one organization and gateway, never a process-wide default. */ export function createGatewayDispatcher( gateway: OutboundGateway, @@ -136,37 +174,17 @@ export function createGatewayDispatcher( allowH2: false, ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), connect(connection, callback) { - const port = Number(connection.port || (connection.protocol === 'https:' ? 443 : 80)) - const hostname = connection.hostname.replace(/^\[|\]$/g, '') - const authority = isIP(hostname) === 6 ? `[${hostname}]` : hostname - const url = new URL(`${connection.protocol}//${authority}:${port}`) - void (async () => { - if (connection.protocol !== 'https:' || port !== 443) { - throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') - } - let address = options.resolvedIP - if (address) { - if (!checkResolvedEgress(url, address, options.profile).allowed) { - throw new OutboundRoutingError('ROUTE_BLOCKED') - } - } else { - const result = await validateEgressUrl(url.href, 'url', options.profile, { - logDetails: false, - }) - if (!result.isValid) throw new OutboundRoutingError('ROUTE_BLOCKED') - address = result.resolvedIP - } - const socket = await openGatewayTunnel(gateway, address, port) - return secureOutboundTunnel(socket, hostname, port) - })().then( + connectThroughGateway( + gateway, + { + protocol: connection.protocol, + hostname: connection.hostname, + port: Number(connection.port || (connection.protocol === 'https:' ? 443 : 80)), + }, + options + ).then( (socket) => callback(null, socket), - (error) => - callback( - error instanceof OutboundRoutingError - ? error - : new OutboundRoutingError('GATEWAY_UNAVAILABLE'), - null - ) + (error: Error) => callback(error, null) ) }, }) diff --git a/apps/sim/lib/core/network/organization-aws-handler.server.ts b/apps/sim/lib/core/network/organization-aws-handler.server.ts new file mode 100644 index 00000000000..d34232863c3 --- /dev/null +++ b/apps/sim/lib/core/network/organization-aws-handler.server.ts @@ -0,0 +1,130 @@ +import type http from 'node:http' +import type { Duplex } from 'node:stream' +import { NodeHttp2Handler, NodeHttpHandler } from '@smithy/node-http-handler' +import { Agent, type AgentConnectOpts } from 'agent-base' +import { LRUCache } from 'lru-cache' +import { isOutboundRoutingEnabled, type OutboundGateway } from '@/lib/core/network/config.server' +import { connectThroughGateway } from '@/lib/core/network/gateway.server' +import { resolveOrganizationRoute } from '@/lib/core/network/outbound-fetch.server' +import { AGENT_OPTIONS, awsRequestHandler } from '@/lib/core/network/proxy-agent.server' + +type AwsHttpHandler = NodeHttpHandler | NodeHttp2Handler +type HandleArguments = Parameters + +/** + * Pools connections tunnelled through one organization's gateway. Its sockets carry only that + * organization's requests, so a keep-alive socket is never reused across organizations or + * for a direct request. + */ +class GatewayAgent extends Agent { + constructor(private readonly gateway: OutboundGateway) { + super(AGENT_OPTIONS) + } + + /** + * The gateway only carries HTTPS. agent-base otherwise infers this from the call stack, which + * differs between opening and freeing a socket, so the socket is never found to reuse. + */ + isSecureEndpoint(): boolean { + return true + } + + connect(req: http.ClientRequest, options: AgentConnectOpts): Promise { + return connectThroughGateway( + this.gateway, + { + protocol: req.protocol, + hostname: options.host ?? '', + port: Number(options.port), + }, + { profile: 'configuredEndpoint' } + ) + } +} + +interface GatewayHandler { + gatewayId: string + agent: GatewayAgent + handler: NodeHttpHandler +} + +/** + * Stops keeping a retired gateway's sockets alive, so Node closes each one as its in-flight + * request finishes and none is handed to a later request. + */ +function retireGatewayHandler({ agent }: GatewayHandler): void { + agent.keepAlive = false + for (const sockets of Object.values(agent.freeSockets)) { + for (const socket of sockets ?? []) socket.destroy() + } +} + +/** One handler per organization with a gateway; bounded like the routing config itself. */ +const gatewayHandlers = new LRUCache({ + max: 1_000, + dispose: retireGatewayHandler, +}) + +function gatewayHandlerFor(gateway: OutboundGateway): NodeHttpHandler { + const current = gatewayHandlers.get(gateway.organizationId) + if (current?.gatewayId === gateway.id) return current.handler + const agent = new GatewayAgent(gateway) + const handler = new NodeHttpHandler({ httpAgent: agent, httpsAgent: agent }) + gatewayHandlers.set(gateway.organizationId, { gatewayId: gateway.id, agent, handler }) + return handler +} + +/** + * Picks each request's handler from the outbound scope it is sent in: an organization with a + * gateway gets that gateway's handler, and every other request goes to `direct`, which is the + * handler the client would use with no organization routing at all. + */ +class OrganizationRoutedHandler { + readonly metadata: AwsHttpHandler['metadata'] + + constructor(private readonly direct: AwsHttpHandler) { + this.metadata = direct.metadata + } + + async handle(...[request, options]: HandleArguments) { + const route = await resolveOrganizationRoute(request.hostname) + const handler = route.kind === 'gateway' ? gatewayHandlerFor(route.gateway) : this.direct + return handler.handle(request, options) + } + + /** The SDK only tunes the direct handler's own options; the gateway handler keeps its own. */ + updateHttpClientConfig(key: never, value: never): void { + this.direct.updateHttpClientConfig(key, value) + } + + httpHandlerConfigs(): ReturnType { + return this.direct.httpHandlerConfigs() + } + + destroy(): void { + this.direct.destroy() + } +} + +/** + * `requestHandler` for an AWS SDK v3 client that signs with a customer's credentials. In the + * outbound scope of an organization with a dedicated egress gateway each request reaches AWS from + * the organization's static IPs (passing `aws:SourceIp` conditions); every other request uses + * the handler {@link awsRequestHandler} gives the client, and routing disabled is exactly that. + * Pass `http2` for a client whose SDK default is HTTP/2 (Bedrock Runtime) so direct requests keep + * it. Never use it for Sim's own AWS usage, which must not leave through a customer's gateway. + */ +export function organizationAwsRequestHandler(options?: { + http2?: boolean +}): ReturnType | OrganizationRoutedHandler { + // `undefined` without an environment proxy, so the client keeps its SDK default handler. + const configured = awsRequestHandler() + if (!isOutboundRoutingEnabled()) return configured + const direct = + configured instanceof NodeHttpHandler + ? configured + : options?.http2 + ? new NodeHttp2Handler({ disableConcurrentStreams: true }) + : new NodeHttpHandler() + return new OrganizationRoutedHandler(direct) +} diff --git a/apps/sim/lib/core/network/outbound-fetch.server.ts b/apps/sim/lib/core/network/outbound-fetch.server.ts new file mode 100644 index 00000000000..9a22828b0d2 --- /dev/null +++ b/apps/sim/lib/core/network/outbound-fetch.server.ts @@ -0,0 +1,63 @@ +import { createLogger } from '@sim/logger' +import type { ResolvedOutboundRoute } from '@/lib/core/network/config.server' +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' + +const logger = createLogger('OutboundFetch') + +let gatewayTransport: ReturnType | undefined + +/** + * The outbound route for a request made on an organization's behalf. Outside any outbound scope + * it is direct, with a warning, unless `requireScope` is set: these requests always went direct, + * and failing closed with `MISSING_SCOPE` (as the SSRF-guarded helpers do) would turn a code path + * that never established a scope into an outage for every organization. A blocked route or an + * unavailable gateway still throws. + */ +export async function resolveOrganizationRoute( + host: string | undefined, + options: { requireScope?: boolean } = {} +): Promise { + try { + return await resolveCurrentOutboundRoute() + } catch (error) { + const missingScope = error instanceof OutboundRoutingError && error.code === 'MISSING_SCOPE' + if (!missingScope || options.requireScope) throw error + logger.warn('Outbound request made without an organization scope', { host }) + return { kind: 'direct' } + } +} + +/** + * `fetch` for a request Sim makes on an organization's behalf to a destination outside Sim (a + * token endpoint, a customer's tenant host, a vendor API). An organization with a dedicated + * egress gateway sends it from its static IPs; every other route is the global `fetch`, + * unchanged. Never use it for Sim's own infrastructure. + */ +export const outboundFetch: typeof fetch = (input, init) => routedFetch(input, init, false) + +/** + * {@link outboundFetch} for a caller that always runs inside an outbound scope, such as a + * knowledge connector sync, so a lost scope fails with `MISSING_SCOPE` instead of going direct. + */ +export const scopedOutboundFetch: typeof fetch = (input, init) => routedFetch(input, init, true) + +async function routedFetch( + input: RequestInfo | URL, + init: RequestInit | undefined, + requireScope: boolean +): Promise { + const route = await resolveOrganizationRoute(requestHost(input), { requireScope }) + if (route.kind === 'direct') return fetch(input, init) + gatewayTransport ??= createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + return gatewayTransport.fetch(input, init) +} + +function requestHost(input: RequestInfo | URL): string | undefined { + try { + return new URL(input instanceof Request ? input.url : input.toString()).host + } catch { + return undefined + } +} diff --git a/apps/sim/lib/core/network/proxy-agent.server.ts b/apps/sim/lib/core/network/proxy-agent.server.ts index de68381bc39..29736a1d5f9 100644 --- a/apps/sim/lib/core/network/proxy-agent.server.ts +++ b/apps/sim/lib/core/network/proxy-agent.server.ts @@ -11,7 +11,7 @@ import { HttpProxyAgent } from 'http-proxy-agent' import { HttpsProxyAgent } from 'https-proxy-agent' /** Matches the AWS SDK's own default agent, so swapping it in changes routing and nothing else. */ -const AGENT_OPTIONS: http.AgentOptions = { keepAlive: true, maxSockets: 50 } +export const AGENT_OPTIONS: http.AgentOptions = { keepAlive: true, maxSockets: 50 } /** * A Node agent for SDKs that bring their own HTTP stack (the AWS SDK, the OTLP exporters) and so @@ -52,7 +52,7 @@ function environmentProxyAgent(): EnvironmentProxyAgent { return sharedAgent } -interface AwsRequestHandlerOptions { +export interface AwsRequestHandlerOptions { connectionTimeout?: number requestTimeout?: number throwOnRequestTimeout?: boolean diff --git a/apps/sim/lib/core/orchestration/types.test.ts b/apps/sim/lib/core/orchestration/types.test.ts index 7cdd3452fe3..addb2c097e3 100644 --- a/apps/sim/lib/core/orchestration/types.test.ts +++ b/apps/sim/lib/core/orchestration/types.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { + asOrchestrationError, messageForOrchestrationError, OrchestrationError, + statusForOrchestrationError, throwOrchestrationFailure, } from '@/lib/core/orchestration/types' @@ -66,3 +69,20 @@ describe('throwOrchestrationFailure', () => { } }) }) + +describe('asOrchestrationError', () => { + it('classifies an egress routing failure deep in a cause chain as a 503 without its routing code', () => { + const wrapped = new Error('Failed query: select ...', { + cause: new Error('token exchange failed', { + cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + }), + }) + + const classified = asOrchestrationError(wrapped) + + expect(classified?.code).toBe('unavailable') + expect(statusForOrchestrationError(classified?.code)).toBe(503) + expect(classified?.message).toBe("The organization's egress route is unavailable") + expect(classified?.message).not.toContain('GATEWAY_UNAVAILABLE') + }) +}) diff --git a/apps/sim/lib/core/orchestration/types.ts b/apps/sim/lib/core/orchestration/types.ts index 47374d0c4bc..6dd21c156d1 100644 --- a/apps/sim/lib/core/orchestration/types.ts +++ b/apps/sim/lib/core/orchestration/types.ts @@ -1,3 +1,4 @@ +import { OutboundRoutingError } from '@/lib/core/network/routing' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' export type OrchestrationErrorCode = @@ -13,6 +14,12 @@ export type OrchestrationErrorCode = | 'conflict' | 'locked' | 'payload_too_large' + /** + * A dependency Sim reaches on the caller's behalf is temporarily unreachable — + * the organization's egress gateway, say. Retryable; nothing about the caller + * or its credentials is wrong. + */ + | 'unavailable' | 'internal' /** @@ -28,6 +35,7 @@ export function statusForOrchestrationError(code: OrchestrationErrorCode | undef if (code === 'conflict') return 409 if (code === 'locked') return 423 if (code === 'payload_too_large') return 413 + if (code === 'unavailable') return 503 return 500 } @@ -93,6 +101,9 @@ export function throwOrchestrationFailure( ) } +/** The fixed caller-facing text for an organization egress outage; the routing code stays in logs. */ +export const EGRESS_UNAVAILABLE_MESSAGE = "The organization's egress route is unavailable" + /** * The {@link OrchestrationError} in `error`'s cause chain, or `null` when the * failure is not a classified one. @@ -101,6 +112,10 @@ export function throwOrchestrationFailure( * raised inside a transaction callback in a `DrizzleQueryError` whose own * message is the failed SQL — the same reason the message-matching this * replaced had to dig for a root cause before it could classify anything. + * + * An {@link OutboundRoutingError} anywhere in the chain is the organization's + * egress route failing, so it classifies as `unavailable` with a fixed message: + * its routing code stays in server logs. */ export function asOrchestrationError(error: unknown): OrchestrationError | null { let current: unknown = error @@ -108,11 +123,22 @@ export function asOrchestrationError(error: unknown): OrchestrationError | null if (current instanceof OrchestrationError) return current if (isPayloadSizeLimitError(current)) return new OrchestrationError('payload_too_large', current.message) + if (current instanceof OutboundRoutingError) + return new OrchestrationError('unavailable', EGRESS_UNAVAILABLE_MESSAGE) current = current.cause } return null } +/** + * Whether `error` is a failure Sim's side cannot currently get past (an `unavailable` failure, + * such as the organization's egress route being down): never a verdict on the caller, its + * credentials, or the resource, so nothing may be recorded against them. + */ +export function isUnavailableFailure(error: unknown): boolean { + return asOrchestrationError(error)?.code === 'unavailable' +} + /** Transport metadata available to an application operation for audit capture. */ export interface OrchestrationRequestContext { headers: { get(name: string): string | null } diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 28274c14bc7..2c45aef509f 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -464,7 +464,8 @@ export const MAX_JSON_API_RESPONSE_BYTES = 10 * 1024 * 1024 */ const FOLLOWED_REDIRECT_STATUSES: ReadonlySet = new Set([301, 302, 303, 307, 308]) -function isRedirectStatus(status: number): boolean { +/** Whether `status` names a redirect target (301, 302, 303, 307 or 308). */ +export function isRedirectStatus(status: number): boolean { return FOLLOWED_REDIRECT_STATUSES.has(status) } diff --git a/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts b/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts index ab422cead40..2a9c5ac637b 100644 --- a/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts +++ b/apps/sim/lib/credential-groups/application/slack-managed-users.test.ts @@ -6,6 +6,7 @@ import { credentialGroupsAvailabilityMockFns, } from '@sim/testing/mocks/credential-groups-availability.mock' import { credentialGroupsOrganizationSetupMock } from '@sim/testing/mocks/credential-groups-organization-setup.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { organizationAuthorizationMock, organizationAuthorizationMockFns, @@ -29,6 +30,7 @@ const hoisted = vi.hoisted(() => ({ vi.mock('@/lib/credential-groups/organization-setup', () => credentialGroupsOrganizationSetupMock) vi.mock('@sim/audit', () => auditMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@/lib/core/application/organization-authorization', () => organizationAuthorizationMock) vi.mock('@/lib/core/application/workspace-authorization', () => workspaceAuthorizationMock) vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) @@ -40,6 +42,7 @@ vi.mock('@/lib/credential-groups/slack-managed-users', () => ({ exchangeAndConfigureSlackManagedUsers: hoisted.exchange, })) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { completeSlackCredentialGroupConfiguration, startSlackCredentialGroupConfiguration, @@ -73,6 +76,7 @@ const attempt = { beforeEach(() => { resetDbChainMock() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) mocks.available.mockResolvedValue(true) organizationAccess.mockResolvedValue({ organizationId: 'organization-1', @@ -194,4 +198,32 @@ describe('scoped Slack setup', () => { }) ) }) + + it("verifies the custom bot through the owning organization's egress route", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + queueTableRows(credentialGroup, [group]) + mocks.create.mockImplementation(async () => { + await resolveCurrentOutboundRoute() + return { state: 'state-1', authorizationUrl: 'https://slack.com/oauth' } + }) + await expect(start()).resolves.toMatchObject({ state: 'state-1' }) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('organization-1') + }) + + it("exchanges the Slack grant through the owning organization's egress route", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + queueTableRows(credentialGroup, [group]) + mocks.exchange.mockImplementation(async () => { + await resolveCurrentOutboundRoute() + return { + credentialGroupId: 'group-1', + credentialGroupName: 'Connected accounts', + appId: 'app-1', + teamId: 'team-1', + requiredScopes: ['search:read'], + } + }) + await expect(finish()).resolves.toMatchObject({ ok: true }) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('organization-1') + }) }) diff --git a/apps/sim/lib/credential-groups/application/slack-managed-users.ts b/apps/sim/lib/credential-groups/application/slack-managed-users.ts index 698b7e8d640..be9431bc820 100644 --- a/apps/sim/lib/credential-groups/application/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/application/slack-managed-users.ts @@ -7,6 +7,7 @@ import { recordProjectedUseCaseAuditEntries } from '@/lib/core/application/autho import { assertOperationPrincipal, type OperationUseCase } from '@/lib/core/application/operation' import { requireOrganizationMembership } from '@/lib/core/application/organization-authorization' import { authorizeWorkspaceOperation } from '@/lib/core/application/workspace-authorization' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceScope, @@ -101,17 +102,20 @@ export const startSlackCredentialGroupConfiguration: OperationUseCase< ) if (scope.kind === 'organization') await requireOrganizationAccountsSetup(scope.organizationId, input.credentialGroupId) - return createSlackManagedUsersAttempt({ - ...resourceScopeFields(scope), - userId: principal.userId, - credentialGroupId: input.credentialGroupId, - slackBotCredentialId: input.slackBotCredentialId, - appId: input.appId, - teamId: input.teamId, - clientId: input.clientId, - clientSecret: input.clientSecret, - requiredScopes: input.requiredScopes, - }) + const owner = resourceScopeFields(scope) + return withResourceOutboundScope(owner, () => + createSlackManagedUsersAttempt({ + ...owner, + userId: principal.userId, + credentialGroupId: input.credentialGroupId, + slackBotCredentialId: input.slackBotCredentialId, + appId: input.appId, + teamId: input.teamId, + clientId: input.clientId, + clientSecret: input.clientSecret, + requiredScopes: input.requiredScopes, + }) + ) }, } @@ -181,8 +185,11 @@ export const completeSlackCredentialGroupConfiguration: OperationUseCase< throw new OrchestrationError('validation', 'Authorization state is invalid or expired') } if (input.providerError) return { ok: false as const, reason: 'provider_error' as const } - if (!input.code) throw new OrchestrationError('validation', 'Authorization code is missing') - const result = await exchangeAndConfigureSlackManagedUsers({ attempt, code: input.code }) + const code = input.code + if (!code) throw new OrchestrationError('validation', 'Authorization code is missing') + const result = await withResourceOutboundScope(resourceScopeFields(scope), () => + exchangeAndConfigureSlackManagedUsers({ attempt, code }) + ) recordProjectedUseCaseAuditEntries( credentialGroupOperations.completeSlackConfiguration, scope.kind === 'workspace' ? scope.workspaceId : undefined, diff --git a/apps/sim/lib/credential-groups/slack-managed-users.test.ts b/apps/sim/lib/credential-groups/slack-managed-users.test.ts index 73eb082630d..4574ac8920a 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.test.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.test.ts @@ -3,6 +3,7 @@ import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' import { resetUrlsMock, urlsMockFns } from '@sim/testing/mocks/urls.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -28,9 +29,12 @@ const { attempts, redis } = vi.hoisted(() => { }) vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@/lib/core/security/encryption', () => encryptionMock) +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { credentialGroupScopePolicyVersion } from '@/lib/credential-groups/provider-adapter' import { SLACK_MANAGED_USER_SCOPES, @@ -42,6 +46,7 @@ import { exchangeAndConfigureSlackManagedUsers, exchangeSlackUserAuthorization, loadSlackManagedUsersAttempt, + verifySlackCustomBotAppIdentity, } from '@/lib/credential-groups/slack-managed-users' const shared = { flag: featureFlagsMockFns.mockIsFeatureEnabled } @@ -84,6 +89,7 @@ function slackResponse(value: Record): Response { describe('Slack managed-user authorization', () => { beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) resetDbChainMock() attempts.clear() setEnv({ SLACK_SEARCH_APP_ID: '' }) @@ -577,6 +583,37 @@ describe('Slack managed-user authorization', () => { expect(dbChainMockFns.transaction).not.toHaveBeenCalled() }) + it('surfaces an unavailable egress route instead of blaming Slack', async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + runWithOutboundOrganization('organization-1', () => + exchangeSlackUserAuthorization({ + clientId: 'client-id', + clientSecret: 'client-secret', + code: 'single-use-code', + redirectUri: 'https://sim.ai/callback', + }) + ) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + + it('surfaces an unavailable egress route while verifying a custom bot instead of blaming Slack', async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + runWithOutboundOrganization('organization-1', () => + verifySlackCustomBotAppIdentity('xoxb-token') + ) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('fails closed when Slack omits the user token type', async () => { vi.stubGlobal( 'fetch', diff --git a/apps/sim/lib/credential-groups/slack-managed-users.ts b/apps/sim/lib/credential-groups/slack-managed-users.ts index 4aefdb93f45..b8986c2a165 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.ts @@ -16,6 +16,8 @@ import { isRecordLike } from '@sim/utils/object' import { compareStrings } from '@sim/utils/string' import { and, eq, inArray, isNull, or } from 'drizzle-orm' import { getRedisClient } from '@/lib/core/config/redis' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' @@ -333,7 +335,7 @@ function parseSlackOAuthResponse(value: unknown): SlackOAuthSuccess { } export async function revokeSlackToken(token: string): Promise { - const response = await fetch('https://slack.com/api/auth.revoke', { + const response = await outboundFetch('https://slack.com/api/auth.revoke', { method: 'POST', headers: { Authorization: `Bearer ${token}`, @@ -359,7 +361,7 @@ export async function revokeSlackToken(token: string): Promise { async function callSlackApi(method: string, accessToken: string, body?: URLSearchParams) { let response: Response try { - response = await fetch(`https://slack.com/api/${method}`, { + response = await outboundFetch(`https://slack.com/api/${method}`, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -369,6 +371,7 @@ async function callSlackApi(method: string, accessToken: string, body?: URLSearc cache: 'no-store', }) } catch (error) { + if (error instanceof OutboundRoutingError) throw error logger.error('Slack API verification failed', { method, error: getErrorMessage(error) }) throw new SlackManagedUsersError('Slack could not verify the authorization.', 'provider_error') } @@ -475,7 +478,7 @@ export async function exchangeSlackUserAuthorization(params: { const body = new URLSearchParams({ code: params.code, redirect_uri: params.redirectUri }) let response: Response try { - response = await fetch('https://slack.com/api/oauth.v2.access', { + response = await outboundFetch('https://slack.com/api/oauth.v2.access', { method: 'POST', headers: { Authorization: `Basic ${basicAuth}`, @@ -485,6 +488,7 @@ export async function exchangeSlackUserAuthorization(params: { cache: 'no-store', }) } catch (error) { + if (error instanceof OutboundRoutingError) throw error logger.error('Slack OAuth exchange failed', { error: getErrorMessage(error) }) throw new SlackManagedUsersError('Slack could not complete authorization.', 'provider_error') } diff --git a/apps/sim/lib/credential-groups/slack-provider.test.ts b/apps/sim/lib/credential-groups/slack-provider.test.ts index 5acbe0394cf..b52aac07a4b 100644 --- a/apps/sim/lib/credential-groups/slack-provider.test.ts +++ b/apps/sim/lib/credential-groups/slack-provider.test.ts @@ -21,6 +21,7 @@ vi.mock('@/lib/credential-groups/slack-managed-users', () => ({ }), })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import type { CredentialGroupOAuthContext } from '@/lib/credential-groups/enrollments' import { SLACK_MANAGED_USER_SCOPES, @@ -95,6 +96,39 @@ describe('Slack member scope policy', () => { } ) + it('passes an unavailable egress route through instead of blaming Slack', async () => { + const current = context(SLACK_SEARCH_USER_SCOPES) + mocks.exchange.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const policy = await adapter.getPolicy(current.option, { + workspaceId: current.workspaceId, + credentialGroupId: current.credentialGroupId, + }) + await expect( + adapter.exchangeAndVerify({ + context: current, + policy, + code: 'code', + attempt: { + state: 'state', + provider: 'slack', + workspaceId: 'workspace-1', + email: current.email, + nonceHash: 'nonce-hash', + enrollmentId: current.enrollmentId, + credentialGroupId: current.credentialGroupId, + optionId: current.option.id, + authorizationAppId: policy.authorizationAppId, + scopeVersion: policy.scopeVersion, + requiredScopes: policy.requiredScopes, + redirectUri: 'https://sim.fixture.test/api/credential-groups/oauth/slack/callback', + invitationToken: 'invitation', + userId: 'user-1', + createdAt: Date.now(), + }, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('accepts a different provider email', async () => { const scopes = SLACK_SEARCH_USER_SCOPES const current = context(scopes) @@ -128,6 +162,7 @@ describe('Slack member scope policy', () => { requiredScopes: policy.requiredScopes, redirectUri: 'https://sim.fixture.test/api/credential-groups/oauth/slack/callback', invitationToken: 'invitation', + userId: 'user-1', createdAt: Date.now(), }, }) @@ -179,6 +214,7 @@ describe('Slack member scope policy', () => { requiredScopes: policy.requiredScopes, redirectUri: 'https://sim.fixture.test/api/credential-groups/oauth/slack/callback', invitationToken: 'invitation', + userId: 'user-1', createdAt: Date.now(), }, }) diff --git a/apps/sim/lib/credential-groups/slack-provider.ts b/apps/sim/lib/credential-groups/slack-provider.ts index f94d443e428..9a8b5630db8 100644 --- a/apps/sim/lib/credential-groups/slack-provider.ts +++ b/apps/sim/lib/credential-groups/slack-provider.ts @@ -2,6 +2,7 @@ import { db } from '@sim/db' import { credentialGroup } from '@sim/db/schema' import { normalizeEmail } from '@sim/utils/string' import { and, eq } from 'drizzle-orm' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { resourceScopeColumns, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { getBaseUrl } from '@/lib/core/utils/urls' @@ -210,7 +211,8 @@ export const slackCredentialGroupProviderAdapter: CredentialGroupProviderAdapter code, redirectUri: attempt.redirectUri, }) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new CredentialGroupOAuthError( 'Slack could not complete authorization. Please try again.', 502 @@ -265,7 +267,9 @@ export const slackCredentialGroupProviderAdapter: CredentialGroupProviderAdapter } } catch (error) { await revokeSlackToken(grant.accessToken) - if (error instanceof CredentialGroupOAuthError) throw error + if (error instanceof CredentialGroupOAuthError || error instanceof OutboundRoutingError) { + throw error + } throw new CredentialGroupOAuthError('Slack could not verify the granted access.', 502) } }, diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts index 12180693dd9..b703d413fb6 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts @@ -1,15 +1,19 @@ import { createHash } from 'node:crypto' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { resetUrlsMock, urlsMockFns } from '@sim/testing/mocks/urls.mock' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { CredentialGroupOAuthContext } from '@/lib/credential-groups/enrollments' import type { CredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth-state' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' -const { mockGetToken, mockVerifyIdentity } = vi.hoisted(() => ({ +const { connectorState, mockGetToken, mockVerifyIdentity } = vi.hoisted(() => ({ + connectorState: { builtInExchange: false }, mockGetToken: vi.fn(), mockVerifyIdentity: vi.fn(), })) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + vi.mock('@/lib/auth/connectors/managed-oauth', () => ({ getManagedOAuthConnectorProviderConfig: (providerId: string) => { if (providerId === 'google-calendar') { @@ -22,7 +26,7 @@ vi.mock('@/lib/auth/connectors/managed-oauth', () => ({ redirectURI: 'https://sim.example.com/api/auth/oauth2/callback/google-calendar', accessType: 'offline', scopes: ['calendar.read', 'profile'], - getToken: mockGetToken, + getToken: connectorState.builtInExchange ? undefined : mockGetToken, managedOAuth: { additionalScopes: ['openid'], requiresRefreshToken: true, @@ -71,6 +75,9 @@ vi.mock('@/lib/auth/connectors/managed-oauth', () => ({ }, })) +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { createStandardOAuthCredentialGroupProviderAdapter } from '@/lib/credential-groups/standard-oauth-provider' urlsMockFns.mockGetBaseUrl.mockReturnValue('https://sim.example.com') @@ -138,6 +145,12 @@ async function exchange() { } describe('standard OAuth Credential Group provider', () => { + afterEach(() => { + connectorState.builtInExchange = false + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + beforeEach(() => { mockGetToken.mockResolvedValue({ tokenType: 'Bearer', @@ -232,6 +245,12 @@ describe('standard OAuth Credential Group provider', () => { }) }) + it('classifies an egress outage during identity verification as unavailable, not an invalid identity', async () => { + mockVerifyIdentity.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const failure = await exchange().catch((error: unknown) => error) + expect(statusForOrchestrationError(asOrchestrationError(failure)?.code)).toBe(503) + }) + it('uses the existing Atlassian callback and state-bound identity verification', async () => { const requiredScopes = ['read:me', 'read:jira-work', 'offline_access'] const context: CredentialGroupOAuthContext = { @@ -312,4 +331,23 @@ describe('standard OAuth Credential Group provider', () => { }) await expect(exchange()).rejects.toMatchObject({ statusCode: 502 }) }) + + it("fails a connector's built-in code exchange closed with 503 when its organization's gateway is unavailable", async () => { + connectorState.builtInExchange = true + const sharedIpFetch = vi.fn(async () => + Response.json({ token_type: 'Bearer', access_token: 'access-1' }) + ) + vi.stubGlobal('fetch', sharedIpFetch) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + const failure = await runWithOutboundOrganization('org-1', exchange).catch( + (error: unknown) => error + ) + expect(statusForOrchestrationError(asOrchestrationError(failure)?.code)).toBe(503) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + expect(sharedIpFetch).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index 575aef31263..e18587e4ada 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -4,7 +4,6 @@ import { applyDefaultAccessTokenExpiry, createAuthorizationURL, type OAuth2Tokens, - validateAuthorizationCode, } from 'better-auth/oauth2' import { type ConnectorProviderConfig, @@ -12,6 +11,8 @@ import { getManagedOAuthConnectorProviderConfig, type ManagedOAuthConnectorConfig, } from '@/lib/auth/connectors/managed-oauth' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { credentialGroupOAuthNonceMatches } from '@/lib/credential-groups/oauth-state' import type { @@ -25,8 +26,9 @@ import { } from '@/lib/credential-groups/provider-adapter' import type { CredentialGroupStandardOAuthProvider } from '@/lib/credential-groups/providers' import { getCredentialGroupProviderService } from '@/lib/credential-groups/providers' -import { refreshOAuthToken } from '@/lib/oauth' +import { exchangeAuthorizationCode } from '@/lib/oauth/authorization-code.server' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' const OAUTH_DISCOVERY_TIMEOUT_MS = 10_000 const OAUTH_DISCOVERY_MAX_BYTES = 256 * 1024 @@ -81,7 +83,7 @@ async function resolveOAuthEndpoints( if (connector.discoveryUrl) { let response: Response try { - response = await fetch(connector.discoveryUrl, { + response = await outboundFetch(connector.discoveryUrl, { headers: connector.discoveryHeaders, signal: AbortSignal.timeout(OAUTH_DISCOVERY_TIMEOUT_MS), }) @@ -206,7 +208,7 @@ function generatePkceVerifier(): string { return randomBytes(64).toString('base64url') } -async function exchangeAuthorizationCode(params: { +async function exchangeConnectorCode(params: { connector: ConnectorProviderConfig code: string codeVerifier?: string @@ -216,7 +218,7 @@ async function exchangeAuthorizationCode(params: { const { connector, code, codeVerifier, redirectUri, tokenEndpoint } = params const tokens = connector.getToken ? await connector.getToken({ code, redirectURI: redirectUri, codeVerifier }) - : await validateAuthorizationCode({ + : await exchangeAuthorizationCode({ headers: connector.authorizationHeaders, code, codeVerifier, @@ -306,14 +308,16 @@ export function createStandardOAuthCredentialGroupProviderAdapter( const endpoints = await resolveOAuthEndpoints(current.connector, service.name) let tokens: OAuth2Tokens try { - tokens = await exchangeAuthorizationCode({ + tokens = await exchangeConnectorCode({ connector: current.connector, code, ...(attempt.codeVerifier ? { codeVerifier: attempt.codeVerifier } : {}), redirectUri: attempt.redirectUri, tokenEndpoint: endpoints.tokenEndpoint, }) - } catch { + } catch (error) { + // An egress outage is classified once, by asOrchestrationError, not as a provider failure. + if (error instanceof OutboundRoutingError) throw error throw new CredentialGroupOAuthError( `${service.name} could not complete authorization. Please try again.`, 502 @@ -332,6 +336,7 @@ export function createStandardOAuthCredentialGroupProviderAdapter( clientId: current.connector.clientId, }) } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new CredentialGroupOAuthError( `${service.name} returned an invalid identity token.`, 502, diff --git a/apps/sim/lib/credentials/application/managed-oauth-delegation.ts b/apps/sim/lib/credentials/application/managed-oauth-delegation.ts index d5c68994745..3c1e0e9b039 100644 --- a/apps/sim/lib/credentials/application/managed-oauth-delegation.ts +++ b/apps/sim/lib/credentials/application/managed-oauth-delegation.ts @@ -1,12 +1,5 @@ import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' -import { - InvalidInternalDelegationTokenError, - verifyInternalDelegationToken, -} from '@/lib/auth/internal' -import { - bindInternalExecutorDelegation, - InvalidInternalDelegationBindingError, -} from '@/lib/auth/internal-delegation' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' import { MANAGED_OAUTH_DELEGATION_AUDIENCE } from '@/lib/credentials/application/authorization' import { createExecutorPrincipalFromDelegationOrigin } from '@/lib/internal/principals/executor' import type { ExecutorDelegationOrigin } from '@/executor/types' @@ -18,35 +11,10 @@ export class InvalidManagedOAuthDelegationError extends Error { } } -/** Authenticates and binds an executor delegation to one managed credential ID. */ -export async function authenticateManagedOAuthDelegation( - authorization: string, - credentialId: string -): Promise { - if (!authorization.startsWith('Bearer ')) throw new InvalidManagedOAuthDelegationError() - - try { - const claims = await verifyInternalDelegationToken(authorization.slice('Bearer '.length)) - return await bindInternalExecutorDelegation(claims, { - audience: MANAGED_OAUTH_DELEGATION_AUDIENCE, - resourceScope: { credentialId }, - }) - } catch (error) { - if ( - error instanceof InvalidInternalDelegationTokenError || - error instanceof InvalidInternalDelegationBindingError - ) { - throw new InvalidManagedOAuthDelegationError() - } - throw error - } -} - /** - * In-process sibling of {@link authenticateManagedOAuthDelegation}: binds the - * executor's own delegation origin to one managed credential without minting and - * re-verifying a delegation JWT — see {@link createExecutorPrincipalFromDelegationOrigin} - * for why that loses nothing. + * Binds the executor's own delegation origin to one managed credential without + * minting and re-verifying a delegation JWT — see + * {@link createExecutorPrincipalFromDelegationOrigin} for why that loses nothing. */ export async function bindExecutorManagedOAuthDelegation( origin: ExecutorDelegationOrigin, diff --git a/apps/sim/lib/credentials/atlassian-service-account.ts b/apps/sim/lib/credentials/atlassian-service-account.ts index e5a737cf531..97fdbd524ab 100644 --- a/apps/sim/lib/credentials/atlassian-service-account.ts +++ b/apps/sim/lib/credentials/atlassian-service-account.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { AtlassianProduct } from '@/lib/credentials/service-account-fields' import { parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -94,7 +95,7 @@ export async function validateAtlassianServiceAccount( }> { assertAtlassianCloudHost(domain) - const tenantInfoRes = await fetch(`https://${domain}/_edge/tenant_info`, { + const tenantInfoRes = await outboundFetch(`https://${domain}/_edge/tenant_info`, { headers: { Accept: 'application/json' }, }) if (tenantInfoRes.status === 404) { @@ -112,7 +113,7 @@ export async function validateAtlassianServiceAccount( const cloudId = tenantInfo.cloudId const identityPath = product === 'confluence' ? 'wiki/rest/api/user/current' : 'rest/api/3/myself' - const myselfRes = await fetch( + const myselfRes = await outboundFetch( `https://api.atlassian.com/ex/${product}/${cloudId}/${identityPath}`, { headers: { Authorization: `Bearer ${apiToken}`, Accept: 'application/json' }, diff --git a/apps/sim/lib/credentials/aws-service-account.server.test.ts b/apps/sim/lib/credentials/aws-service-account.server.test.ts index f99f068f03d..0a81e34d879 100644 --- a/apps/sim/lib/credentials/aws-service-account.server.test.ts +++ b/apps/sim/lib/credentials/aws-service-account.server.test.ts @@ -24,6 +24,7 @@ vi.mock('@aws-sdk/client-sts', async (importActual) => ({ import { AssumeRoleWithWebIdentityCommand, GetCallerIdentityCommand } from '@aws-sdk/client-sts' import { credential } from '@sim/db/schema' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { getAwsCredentialConfig, resolveAwsSessionCredentials, @@ -134,6 +135,23 @@ describe('saving a keyless AWS role', () => { ).rejects.toThrow() expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() }) + + it('surfaces an unavailable egress route during the any-subject check instead of asking to retry', async () => { + roleTrusting(true) + const pinned = hoisted.send.getMockImplementation() + hoisted.send.mockImplementation(async (command: unknown) => { + const probesUnboundSubject = + command instanceof AssumeRoleWithWebIdentityCommand && + command.input.WebIdentityToken !== `token-for:${SUBJECT}` + if (probesUnboundSubject) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + return pinned?.(command) + }) + + await expect( + verifyAndEncryptAwsCredential(keylessReconnect.serialized, keylessReconnect.owner) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) }) describe('choosing the audience of a keyless AWS role', () => { diff --git a/apps/sim/lib/credentials/aws-service-account.server.ts b/apps/sim/lib/credentials/aws-service-account.server.ts index 513bd7ab74b..1b10fc7defa 100644 --- a/apps/sim/lib/credentials/aws-service-account.server.ts +++ b/apps/sim/lib/credentials/aws-service-account.server.ts @@ -9,7 +9,8 @@ import { db } from '@sim/db' import { credential } from '@sim/db/schema' import { and, eq, isNull, or } from 'drizzle-orm' import { z } from 'zod' -import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { failureClause, parseServiceAccountConfig } from '@/lib/credentials/service-account-config' import { @@ -195,7 +196,7 @@ export async function resolveAwsSessionCredentials( const client = new STSClient({ region, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey }, - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), }) const response = await client.send( new AssumeRoleCommand({ @@ -221,7 +222,7 @@ export async function resolveAwsSessionCredentials( const client = new STSClient({ region, credentials: { accessKeyId: 'anonymous', secretAccessKey: 'anonymous' }, - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), }) const response = await client.send( new AssumeRoleWithWebIdentityCommand({ @@ -280,6 +281,8 @@ function bindSubject( function verificationFailure(config: AwsCredentialConfig, error: unknown): Error { // A keyless refusal already names its cause and code; wrapping it would lose both. if (error instanceof KeylessCredentialRefusedError) return error + // A refused or unavailable egress route says nothing about the credential. + if (error instanceof OutboundRoutingError) return error const detail = failureClause(error) if (config.authType === 'oidc_role') { return new AwsCredentialError( @@ -357,7 +360,7 @@ async function verifyCallerArn( const identity = await new STSClient({ region: stsRegionFor(config), credentials: session, - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), }).send(new GetCallerIdentityCommand({}), { abortSignal: signal }) if (!identity.Arn) throw new Error('AWS returned no caller identity') return identity.Arn @@ -382,6 +385,7 @@ async function assertRolePinsSubject( } catch (error) { if (error instanceof Error && error.name === 'AccessDenied') return if (error instanceof KeylessCredentialRefusedError) throw error + if (error instanceof OutboundRoutingError) throw error throw new AwsCredentialError( `Sim could not confirm the role's trust policy limits it to this credential: ${failureClause(error)}. Try again.` ) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts index 795bdc4ab3f..21e47062c73 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.test.ts @@ -1,5 +1,6 @@ import { jsonResponse } from '@sim/testing/helpers/http' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { mintCrowdStrikeServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/crowdstrike' const FIELDS = { @@ -162,6 +163,14 @@ describe('CrowdStrike API client token exchange', () => { }) }) + it('surfaces an unavailable egress route instead of a provider failure', async () => { + mockFetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(mintCrowdStrikeServiceAccountToken(FIELDS)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + }) + it('refuses a token response without an access token', async () => { mockFetch.mockResolvedValue(jsonResponse({ expires_in: 1799 })) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts index 85474bf2121..e026c2629d1 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/crowdstrike.ts @@ -1,3 +1,4 @@ +import { OutboundRoutingError } from '@/lib/core/network/routing' import type { ClientCredentialAccountFields, ClientCredentialAccountMintOptions, @@ -68,6 +69,7 @@ export async function mintCrowdStrikeServiceAccountToken( ) } catch (error) { options?.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error if (error instanceof CrowdStrikeTokenRedirectError) { throw new TokenServiceAccountValidationError('invalid_credentials', 400, { step: STEP, diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts index 57ffd1443f7..aabe7b394ec 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts @@ -2,6 +2,7 @@ import { generateKeyPairSync } from 'node:crypto' import { jsonResponse } from '@sim/testing/helpers/http' import { jwtVerify } from 'jose' import { describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { mintNetSuiteServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/netsuite' const ORIGIN = 'https://1234567-sb1.suitetalk.api.netsuite.com' @@ -257,4 +258,12 @@ describe('mintNetSuiteServiceAccountToken', () => { logDetail: { step: 'netsuite_token_mint', reason: 'network error reaching provider' }, }) }) + + it('surfaces an unavailable egress route instead of a provider outage', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new OutboundRoutingError('ROUTE_BLOCKED'))) + + await expect(mintNetSuiteServiceAccountToken(FIELDS)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + }) }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts index 9e3738e5970..ec3b5995b45 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts @@ -2,6 +2,8 @@ import { createPrivateKey, type KeyObject } from 'node:crypto' import { generateId } from '@sim/utils/id' import { truncate } from '@sim/utils/string' import { SignJWT } from 'jose' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -109,7 +111,7 @@ async function exchangeNetSuiteToken( const signal = AbortSignal.timeout(TOKEN_EXCHANGE_TIMEOUT_MS) let response: Response try { - response = await fetch(tokenUrl, { + response = await outboundFetch(tokenUrl, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ @@ -120,7 +122,8 @@ async function exchangeNetSuiteToken( redirect: 'error', signal, }) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new TokenServiceAccountValidationError('provider_unavailable', 502, { step: TOKEN_MINT_STEP, reason: 'network error reaching provider', diff --git a/apps/sim/lib/credentials/coordinated-client-token.test.ts b/apps/sim/lib/credentials/coordinated-client-token.test.ts new file mode 100644 index 00000000000..12ad2dc109d --- /dev/null +++ b/apps/sim/lib/credentials/coordinated-client-token.test.ts @@ -0,0 +1,24 @@ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { resolveCoordinatedClientToken } from '@/lib/credentials/coordinated-client-token' + +beforeEach(resetDbChainMock) + +describe('resolveCoordinatedClientToken', () => { + it("surfaces an unavailable egress route as Sim's outage and remembers no failure", async () => { + const exchange = vi.fn().mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + resolveCoordinatedClientToken({ + namespace: 'fixture', + application: ['fixture-client'], + clientSecret: 'fixture-secret', + step: 'fixture_token', + exchange, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(exchange).toHaveBeenCalledOnce() + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/credentials/coordinated-client-token.ts b/apps/sim/lib/credentials/coordinated-client-token.ts index c61d1eabe3e..3d8dc25dfbc 100644 --- a/apps/sim/lib/credentials/coordinated-client-token.ts +++ b/apps/sim/lib/credentials/coordinated-client-token.ts @@ -4,6 +4,7 @@ import { clientCredentialToken } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { and, eq, inArray, lt, sql } from 'drizzle-orm' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { assessTokenRequestFailure, @@ -521,6 +522,7 @@ export async function resolveCoordinatedClientToken( } catch (error) { signal?.throwIfAborted() if (error instanceof TokenServiceAccountValidationError) throw error + if (error instanceof OutboundRoutingError) throw error throw unavailable(step, 'token coordination or exchange failed') } } diff --git a/apps/sim/lib/credentials/databricks-service-account.server.test.ts b/apps/sim/lib/credentials/databricks-service-account.server.test.ts index 5659f3cfb28..ee1d40d5c92 100644 --- a/apps/sim/lib/credentials/databricks-service-account.server.test.ts +++ b/apps/sim/lib/credentials/databricks-service-account.server.test.ts @@ -25,6 +25,7 @@ vi.mock('@/lib/credentials/workload-identity/issuer', () => ({ })) import { credential } from '@sim/db/schema' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { EgressRefusedError } from '@/lib/core/network/transport.server' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { @@ -108,6 +109,14 @@ describe('saving a Databricks connection', () => { } ) + it('surfaces an unavailable egress route instead of failing verification', async () => { + secureFetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + verifyAndEncryptDatabricksCredential(patConfig(HOST), owner) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('reports a host the egress policy refuses as not allowed', async () => { secureFetch.mockRejectedValue(new EgressRefusedError('url resolves to a private address')) @@ -223,6 +232,22 @@ describe('saving a keyless Databricks connection', () => { verifyAndEncryptDatabricksCredential(JSON.stringify(config), reconnectOwner) ).rejects.toThrow(/this credential: Could not reach the Databricks workspace\. Try again\.$/) }) + + it('surfaces an unavailable egress route during the any-subject check instead of asking to retry', async () => { + principalTrusting(true) + const trusted = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit) => { + const body = new URLSearchParams(String(init?.body)) + if (url.endsWith('/oidc/v1/token') && body.get('subject_token') !== `token-for:${SUBJECT}`) { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + return trusted?.(url, init) + }) + + await expect( + verifyAndEncryptDatabricksCredential(JSON.stringify(config), reconnectOwner) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) }) describe('a keyless Databricks connection with its own audience', () => { @@ -436,6 +461,14 @@ describe('getting a Databricks token for a run', () => { }) }) + it('surfaces an unavailable egress route instead of an unreachable workspace', async () => { + fetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(resolveDatabricksCredentialToken('cred-1')).rejects.toBeInstanceOf( + OutboundRoutingError + ) + }) + it('reports a token reply over the size cap as oversized, not as a refused request', async () => { fetch.mockResolvedValue(jsonResponse({ access_token: 't'.repeat(1024 * 1024) }, 200)) diff --git a/apps/sim/lib/credentials/databricks-service-account.server.ts b/apps/sim/lib/credentials/databricks-service-account.server.ts index be27b8670d2..ef2f6419496 100644 --- a/apps/sim/lib/credentials/databricks-service-account.server.ts +++ b/apps/sim/lib/credentials/databricks-service-account.server.ts @@ -3,6 +3,8 @@ import { credential } from '@sim/db/schema' import { getErrorMessage } from '@sim/utils/errors' import { eq } from 'drizzle-orm' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { EgressRefusedError } from '@/lib/core/network/transport.server' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation' @@ -162,7 +164,7 @@ async function requestWorkspaceToken( const url = `${workspaceOrigin(host)}/oidc/v1/token` let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', ...headers }, body, @@ -170,8 +172,9 @@ async function requestWorkspaceToken( redirect: 'error', signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }) - } catch { + } catch (error) { signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error // A network failure, timeout, or refused redirect says nothing about the credential. throw new DatabricksCredentialError('Could not reach the Databricks workspace', 502) } @@ -308,10 +311,10 @@ function databricksRefusalReason(body: string): string | undefined { } /** - * Proves a configuration by calling the workspace's SCIM `Me` with the resolved - * token. The call goes through the egress guard, which also routes it through - * the organization's outbound gateway, so a workspace whose IP access list - * allows only the organization's dedicated egress addresses accepts it. + * Proves a configuration by minting a workspace token and calling the workspace's + * SCIM `Me` with it. Both requests leave through the organization's outbound + * gateway when it has one, so a workspace whose IP access list allows only the + * organization's dedicated egress addresses accepts them. */ async function verifyWorkspaceIdentity( config: DatabricksCredentialConfig, @@ -352,6 +355,7 @@ async function verifyWorkspaceIdentity( } catch (error) { // A keyless refusal already names its cause and code; wrapping it would lose both. if (error instanceof KeylessCredentialRefusedError) throw error + if (error instanceof OutboundRoutingError) throw error if (error instanceof EgressRefusedError) { throw new DatabricksCredentialError( 'Could not verify the Databricks credential: the workspace host is not allowed' @@ -388,6 +392,7 @@ async function assertPolicyPinsSubject( providerFailureCode(error.status) === 'invalid_credentials' if (refused) return if (error instanceof KeylessCredentialRefusedError) throw error + if (error instanceof OutboundRoutingError) throw error throw new DatabricksCredentialError( `Sim could not confirm the federation policy limits this service principal to this credential: ${failureClause(error)}. Try again.` ) diff --git a/apps/sim/lib/credentials/google-workload-identity.server.test.ts b/apps/sim/lib/credentials/google-workload-identity.server.test.ts index 4fcfbc78b84..e7c70b2f24f 100644 --- a/apps/sim/lib/credentials/google-workload-identity.server.test.ts +++ b/apps/sim/lib/credentials/google-workload-identity.server.test.ts @@ -17,6 +17,7 @@ vi.mock('@/lib/credentials/workload-identity/issuer', () => ({ `sim-oidc|${audience}|${subject}`, })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { resolveGoogleWorkloadIdentityToken, verifyAndEncryptGoogleWorkloadIdentityCredential, @@ -305,6 +306,32 @@ describe('saving a keyless Google Cloud credential', () => { expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() }) + it.each([ + { + label: 'exchanging this credential’s token', + fails: (url: string, body: string) => + url === 'https://sts.googleapis.com/v1/token' && body.includes(SUBJECT), + }, + { + label: 'confirming the subject is pinned', + fails: (url: string, _body: string, auth: string) => + url.startsWith('https://iamcredentials.') && auth !== `Bearer fed:${SUBJECT}`, + }, + ])('surfaces an unavailable egress route while $label', async ({ fails }) => { + fakeGoogle(fetch, { canImpersonate: (subject) => subject === SUBJECT }) + const google = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit = {}) => { + const auth = new Headers(init.headers).get('authorization') ?? '' + if (fails(url, String(init.body ?? ''), auth)) { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + return google?.(url, init) + }) + + await expect(save({})).rejects.toBeInstanceOf(OutboundRoutingError) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) + it('saves when the provider’s attribute condition refuses every other subject', async () => { const google = fakeGoogle(fetch, { acceptSubject: (subject) => subject === SUBJECT }) @@ -671,6 +698,21 @@ describe('saving a keyless credential with domain-wide delegation', () => { expect(error.message).not.toContain('Manage Domain Wide Delegation') }) + it.each([ + { label: 'exchanging the delegated assertion', url: 'https://oauth2.googleapis.com/token' }, + { label: 'reading as the delegated user', url: GMAIL_PROFILE }, + ])('surfaces an unavailable egress route while $label', async ({ url: failing }) => { + fakeGoogle(fetch, { canImpersonate: (subject) => subject === SUBJECT }) + const google = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit = {}) => { + if (url === failing) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + return google?.(url, init) + }) + + await expect(save({ delegation })).rejects.toBeInstanceOf(OutboundRoutingError) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) + it('names the API to enable when a delegated read is refused', async () => { fakeGoogle(fetch, { canImpersonate: (subject) => subject === SUBJECT, diff --git a/apps/sim/lib/credentials/google-workload-identity.server.ts b/apps/sim/lib/credentials/google-workload-identity.server.ts index 31af85ea02f..9b9b6183ffe 100644 --- a/apps/sim/lib/credentials/google-workload-identity.server.ts +++ b/apps/sim/lib/credentials/google-workload-identity.server.ts @@ -3,6 +3,8 @@ import { toRecord } from '@sim/utils/object' import { LRUCache } from 'lru-cache' import { z } from 'zod' import { coalesceLocally } from '@/lib/concurrency/singleflight' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { encryptSecret } from '@/lib/core/security/encryption' import { GOOGLE_DELEGATION_SERVICE_NAMES, @@ -273,7 +275,7 @@ async function postJson( ): Promise<{ status: number; payload: Record }> { let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json', ...headers }, body: JSON.stringify(body), @@ -281,7 +283,8 @@ async function postJson( redirect: 'error', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), }) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new GoogleWorkloadIdentityError('Could not reach Google Cloud. Try again.') } const payload = (await response.json().catch(() => ({}))) as Record @@ -291,18 +294,19 @@ async function postJson( /** * The HTTP status of a bearer-authorized GET, with the same redirect and * timeout guards as {@link postJson}. The body is discarded unread. - * Undefined when Google could not be reached. + * Undefined when Google could not be reached; an unavailable egress route throws. */ async function getStatus(url: string, bearer: string): Promise { try { - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${bearer}`, Accept: 'application/json' }, redirect: 'error', signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), }) void response.body?.cancel().catch(() => undefined) return response.status - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error return undefined } } @@ -545,7 +549,8 @@ async function exchangeDelegationAssertion( let response: Awaited> try { response = await exchangeGoogleServiceAccountJwt(OAUTH_TOKEN_URL, assertion) - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new GoogleWorkloadIdentityError('Could not reach Google. Try again.') } const payload = parsePayload(response.body) @@ -645,6 +650,7 @@ async function assertSubjectPinned( await step() } catch (error) { if (isRefusal(error)) return true + if (error instanceof OutboundRoutingError) throw error throw unconfirmed() } return false @@ -817,7 +823,12 @@ export async function verifyAndEncryptGoogleWorkloadIdentityCredential( assertSubjectPinned(config, owner), ]) if (verified.status === 'rejected') { - if (verified.reason instanceof GoogleWorkloadIdentityError) throw verified.reason + if ( + verified.reason instanceof GoogleWorkloadIdentityError || + verified.reason instanceof OutboundRoutingError + ) { + throw verified.reason + } throw new GoogleWorkloadIdentityError( `Could not verify the Google Cloud keyless credential: ${getErrorMessage(verified.reason, 'Unknown error')}` ) diff --git a/apps/sim/lib/credentials/managed-oauth.test.ts b/apps/sim/lib/credentials/managed-oauth.test.ts index 74b8427d31d..f3407e18e8f 100644 --- a/apps/sim/lib/credentials/managed-oauth.test.ts +++ b/apps/sim/lib/credentials/managed-oauth.test.ts @@ -34,6 +34,7 @@ vi.mock('@/lib/credential-groups/slack-managed-users', () => ({ verifySlackUserIdentity: vi.fn(), })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { credentialGroupScopePolicyVersion } from '@/lib/credential-groups/provider-adapter' import { SLACK_MANAGED_USER_SCOPES, @@ -519,4 +520,33 @@ describe('managed OAuth token resolution', () => { ) expect(mocks.encryptSecret).not.toHaveBeenCalled() }) + + it('surfaces an unavailable egress route during refresh without reauthorizing or writing', async () => { + const row = mondayCredentialRow() + dbChainMockFns.limit.mockResolvedValueOnce([row]).mockResolvedValueOnce([row]) + mocks.decryptSecret.mockResolvedValue({ + decrypted: JSON.stringify({ + type: 'managed-oauth-token-set', + version: 1, + tokenType: 'Bearer', + accessToken: 'expired-access-token', + refreshToken: 'old-refresh-token', + }), + }) + mocks.getAdapter.mockReturnValue({ + getPolicy: vi.fn().mockResolvedValue({ + authorizationAppId: row.authorizationAppId, + scopeVersion: 1, + requiredScopes: row.grantedScopes, + }), + hasRequiredScopes: vi.fn().mockReturnValue(true), + refreshToken: vi.fn().mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')), + isTerminalRefreshError: vi.fn().mockReturnValue(true), + }) + + await expect(resolveManagedOAuthToken(mondayTokenResolutionParams())).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(dbChainMockFns.set).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/credentials/microsoft-app-service-account.server.test.ts b/apps/sim/lib/credentials/microsoft-app-service-account.server.test.ts index 7b2c5b34075..fd86f262baf 100644 --- a/apps/sim/lib/credentials/microsoft-app-service-account.server.test.ts +++ b/apps/sim/lib/credentials/microsoft-app-service-account.server.test.ts @@ -24,6 +24,7 @@ vi.mock('@/lib/credentials/workload-identity/issuer', () => ({ })) import { credential } from '@sim/db/schema' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { MicrosoftAppCredentialError, resolveMicrosoftAppCredentialToken, @@ -205,6 +206,26 @@ describe('saving a Microsoft app credential', () => { ) }) + it.each([ + { label: 'Entra', failing: (url: string) => url === TOKEN_URL }, + { + label: 'Microsoft Graph', + failing: (url: string) => url.includes('/v1.0/servicePrincipals('), + }, + ])('surfaces an unavailable egress route while reaching $label', async ({ failing }) => { + fakeMicrosoft(fetch) + const microsoft = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit) => { + if (failing(url)) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + return microsoft?.(url, init) + }) + + await expect(save({ authType: 'client_secret', clientSecret: SECRET })).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) + it('signs a PS256 certificate assertion Entra can verify with the uploaded certificate', async () => { const tokens = fakeMicrosoft(fetch) @@ -306,6 +327,23 @@ describe('saving a Microsoft app credential', () => { expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() }) + it('surfaces an unavailable egress route while confirming the subject is pinned', async () => { + fakeMicrosoft(fetch, { acceptAssertion: (assertion) => assertion.endsWith(`|${SUBJECT}`) }) + const microsoft = fetch.getMockImplementation() + fetch.mockImplementation(async (url: string, init: RequestInit) => { + const assertion = new URLSearchParams(String(init?.body)).get('client_assertion') ?? '' + if (url === TOKEN_URL && !assertion.endsWith(`|${SUBJECT}`)) { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + return microsoft?.(url, init) + }) + + await expect(save({ authType: 'federated' }, reconnectOwner)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(encryptionMockFns.mockEncryptSecret).not.toHaveBeenCalled() + }) + it('refuses a subject issued to another member, without contacting Entra', async () => { fakeMicrosoft(fetch) const grant = issueWorkloadIdentitySubject('microsoft', { workspaceId: 'ws-1' }, 'user-2') diff --git a/apps/sim/lib/credentials/microsoft-app-service-account.server.ts b/apps/sim/lib/credentials/microsoft-app-service-account.server.ts index 1a37df45dcd..fdb616aca10 100644 --- a/apps/sim/lib/credentials/microsoft-app-service-account.server.ts +++ b/apps/sim/lib/credentials/microsoft-app-service-account.server.ts @@ -11,6 +11,8 @@ import { SignJWT } from 'jose' import { LRUCache } from 'lru-cache' import { z } from 'zod' import { coalesceLocally } from '@/lib/concurrency/singleflight' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' import { MICROSOFT_CLOUD_IDS, @@ -414,7 +416,7 @@ async function requestEntraToken( const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS) let response: Response try { - response = await fetch(tokenEndpoint(config), { + response = await outboundFetch(tokenEndpoint(config), { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', Accept: 'application/json' }, body, @@ -422,8 +424,9 @@ async function requestEntraToken( redirect: 'error', signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }) - } catch { + } catch (error) { signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error throw new MicrosoftAppCredentialError('Could not reach Microsoft Entra. Try again.', 502) } const payload = (await response.json().catch(() => ({}))) as { @@ -453,7 +456,7 @@ async function requestEntraToken( } async function getJson(url: string, token: string): Promise<{ status: number; body: unknown }> { - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${token}`, Accept: 'application/json', @@ -569,6 +572,7 @@ async function assertFederationPinsSubject( error.status !== undefined && providerFailureCode(error.status) === 'invalid_credentials' if (refused) return + if (error instanceof OutboundRoutingError) throw error throw new MicrosoftAppCredentialError( 'Sim could not confirm this app trusts only this credential. Try again.' ) @@ -660,7 +664,12 @@ export async function verifyAndEncryptMicrosoftAppCredential( config.authType === 'federated' ? assertFederationPinsSubject(config, owner) : undefined, ]) if (verified.status === 'rejected') { - if (verified.reason instanceof MicrosoftAppCredentialError) throw verified.reason + if ( + verified.reason instanceof MicrosoftAppCredentialError || + verified.reason instanceof OutboundRoutingError + ) { + throw verified.reason + } throw new MicrosoftAppCredentialError( `Could not verify the Microsoft app credential: ${getErrorMessage(verified.reason, 'Unknown error')}` ) diff --git a/apps/sim/lib/credentials/orchestration/credential-create.ts b/apps/sim/lib/credentials/orchestration/credential-create.ts index 222b505c8ee..6e7c6064c43 100644 --- a/apps/sim/lib/credentials/orchestration/credential-create.ts +++ b/apps/sim/lib/credentials/orchestration/credential-create.ts @@ -8,7 +8,10 @@ import { generateId } from '@sim/utils/id' import { and, eq } from 'drizzle-orm' import { normalizeCredentialEnvKey } from '@/lib/api/contracts/credentials' import { acquireOrganizationUserMutationLocks } from '@/lib/billing/organizations/membership' -import type { OrchestrationRequestContext } from '@/lib/core/orchestration/types' +import { + isUnavailableFailure, + type OrchestrationRequestContext, +} from '@/lib/core/orchestration/types' import { type ResourceOwner, resourceScopeColumns, @@ -664,6 +667,8 @@ export async function createCredentialRecord( return failure('Credential source data failed validation checks', 'validation') } + // Sim's egress route is down: the surface answers 503, never a verdict on the credential. + if (isUnavailableFailure(error)) throw error const errAsRecord = typeof error === 'object' && error !== null ? (error as Record) : {} logger.error('Credential create failure details', { diff --git a/apps/sim/lib/credentials/orchestration/index.test.ts b/apps/sim/lib/credentials/orchestration/index.test.ts index 0178be1a078..fc4da856e9c 100644 --- a/apps/sim/lib/credentials/orchestration/index.test.ts +++ b/apps/sim/lib/credentials/orchestration/index.test.ts @@ -84,6 +84,7 @@ vi.mock('@/lib/credentials/token-service-accounts/errors', () => ({ })) vi.mock('@/lib/posthog/server', () => posthogServerMock) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { createServiceAccountCredential, deleteCredentialRecord, @@ -179,6 +180,23 @@ describe('performUpdateCredential — service-account secret rotation', () => { expect(result.previousDisplayName).toBe(OLD_EMAIL) }) + it('surfaces an unavailable egress route instead of an internal failure', async () => { + mockCredential() + mockStoredBlob({ type: 'service_account', client_email: OLD_EMAIL }) + mockVerifyAndBuildServiceAccountSecret.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + performUpdateCredential({ + credentialId: 'cred-1', + userId: 'user-1', + serviceAccountJson: NEW_GOOGLE_KEY, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(dbChainMockFns.set).not.toHaveBeenCalled() + }) + it('requires the complete OCI tuple and treats an omitted passphrase as clearing it', async () => { mockCredential({ providerId: 'oci-api-key-service-account', @@ -696,6 +714,23 @@ describe('createServiceAccountCredential', () => { } ) + it('surfaces an unavailable egress route instead of an internal failure', async () => { + mockVerifyAndBuildServiceAccountSecret.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + createServiceAccountCredential({ + workspaceId: 'workspace-1', + userId: 'user-1', + providerId: 'google-service-account', + displayName: 'Google signer', + serviceAccountJson: NEW_GOOGLE_KEY, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) + it('rejects an existing service-account source instead of discarding the submitted secret', async () => { mockVerifyAndBuildServiceAccountSecret.mockResolvedValue({ providerId: 'zoom-service-account', diff --git a/apps/sim/lib/credentials/orchestration/index.ts b/apps/sim/lib/credentials/orchestration/index.ts index 70ab4e4d3d6..e09985c55f8 100644 --- a/apps/sim/lib/credentials/orchestration/index.ts +++ b/apps/sim/lib/credentials/orchestration/index.ts @@ -11,7 +11,7 @@ import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { and, eq, sql } from 'drizzle-orm' import type { NextRequest } from 'next/server' -import { OrchestrationError } from '@/lib/core/orchestration/types' +import { isUnavailableFailure, OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeColumns, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' @@ -618,6 +618,8 @@ export async function updateCredentialRecord( errorCode: 'conflict', } } + // Sim's egress route is down: the surface answers 503, never a verdict on the credential. + if (isUnavailableFailure(error)) throw error logger.error('Failed to update credential', { error }) return { success: false, error: 'Internal server error', errorCode: 'internal' } } diff --git a/apps/sim/lib/credentials/service-account-secret.test.ts b/apps/sim/lib/credentials/service-account-secret.test.ts index 3bf33e761d1..b85f8a1323a 100644 --- a/apps/sim/lib/credentials/service-account-secret.test.ts +++ b/apps/sim/lib/credentials/service-account-secret.test.ts @@ -91,6 +91,7 @@ vi.mock('@/lib/credentials/client-credential-accounts/server', () => ({ : undefined, })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { AtlassianValidationError } from '@/lib/credentials/atlassian-service-account' import { ServiceAccountSecretError, @@ -98,6 +99,7 @@ import { } from '@/lib/credentials/service-account-secret' import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, + DATABRICKS_SERVICE_ACCOUNT_PROVIDER_ID, POSTGRESQL_SERVICE_ACCOUNT_PROVIDER_ID, SLACK_CUSTOM_BOT_PROVIDER_ID, } from '@/lib/oauth/types' @@ -112,6 +114,34 @@ describe('verifyAndBuildServiceAccountSecret', () => { mockNormalizeDomain.mockImplementation((raw: string) => raw.trim().toLowerCase()) }) + it('surfaces an unavailable egress route while verifying a Slack bot token', async () => { + mockFetchSlackTeamId.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + verifyAndBuildServiceAccountSecret(SLACK_CUSTOM_BOT_PROVIDER_ID, { + signingSecret: 'sec', + botToken: 'xoxb-1', + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + + it('surfaces an unavailable egress route while verifying a Databricks connection', async () => { + inputValidationMockFns.mockSecureFetchWithValidation.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + verifyAndBuildServiceAccountSecret(DATABRICKS_SERVICE_ACCOUNT_PROVIDER_ID, { + serviceAccountJson: JSON.stringify({ + authType: 'personal_access_token', + host: 'dbc-1234abcd-5678.cloud.databricks.com', + token: 'dapi-secret', + }), + owner: { workspaceId: 'ws-1', actorUserId: 'user-1' }, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('verifies a Slack bot token and encrypts the derived blob', async () => { mockFetchSlackTeamId.mockResolvedValue({ teamId: 'T1', userId: 'U_BOT', teamName: 'Acme' }) const result = await verifyAndBuildServiceAccountSecret(SLACK_CUSTOM_BOT_PROVIDER_ID, { diff --git a/apps/sim/lib/credentials/service-account-secret.ts b/apps/sim/lib/credentials/service-account-secret.ts index cb2bbea5176..e8c127f1e2e 100644 --- a/apps/sim/lib/credentials/service-account-secret.ts +++ b/apps/sim/lib/credentials/service-account-secret.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { serviceAccountJsonSchema } from '@/lib/api/contracts/credentials' import { getValidationErrorMessage } from '@/lib/api/server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { encryptSecret } from '@/lib/core/security/encryption' import { mintAtlassianServiceAccountToken } from '@/lib/credentials/atlassian-client-credentials' import { @@ -320,6 +321,7 @@ async function buildSlackCustomBotSecret( botUserId = auth.userId teamName = auth.teamName } catch (error) { + if (error instanceof OutboundRoutingError) throw error throw new ServiceAccountSecretError( `Could not verify the Slack bot token: ${getErrorMessage(error)}` ) @@ -392,6 +394,25 @@ async function buildGoogleServiceAccountSecret( } } +/** + * Rethrows a keyless-capable provider's verification failure as the save reports it: a keyless + * refusal and an unavailable egress route keep their own class (the surface maps both), the + * provider's own error becomes its message, and anything else is named for `label`. + */ +function throwVerificationFailure( + error: unknown, + ProviderError: abstract new (...args: never[]) => Error, + label: string +): never { + if (error instanceof KeylessCredentialRefusedError || error instanceof OutboundRoutingError) { + throw error + } + if (error instanceof ProviderError) throw new ServiceAccountSecretError(error.message) + throw new ServiceAccountSecretError( + `Could not verify the ${label}: ${getErrorMessage(error, 'Unknown error')}` + ) +} + /** * A keyless Google credential: Workload Identity Federation, no stored key, * optionally with domain-wide delegation to one Workspace user. @@ -405,13 +426,7 @@ async function buildGoogleWorkloadIdentitySecret( try { result = await verifyAndEncryptGoogleWorkloadIdentityCredential(serviceAccountJson, owner) } catch (error) { - if (error instanceof KeylessCredentialRefusedError) throw error - if (error instanceof GoogleWorkloadIdentityError) { - throw new ServiceAccountSecretError(error.message) - } - throw new ServiceAccountSecretError( - `Could not verify the Google Cloud keyless credential: ${getErrorMessage(error, 'Unknown error')}` - ) + throwVerificationFailure(error, GoogleWorkloadIdentityError, 'Google Cloud keyless credential') } const principal: ServiceAccountPrincipal = { kind: 'user', id: result.serviceAccountEmail } return { @@ -444,11 +459,7 @@ async function buildAwsServiceAccountSecret( try { result = await verifyAndEncryptAwsCredential(fields.serviceAccountJson, fields.owner) } catch (error) { - if (error instanceof KeylessCredentialRefusedError) throw error - if (error instanceof AwsCredentialError) throw new ServiceAccountSecretError(error.message) - throw new ServiceAccountSecretError( - `Could not verify the AWS credential: ${getErrorMessage(error, 'Unknown error')}` - ) + throwVerificationFailure(error, AwsCredentialError, 'AWS credential') } const principal: ServiceAccountPrincipal = { kind: 'user', id: result.callerArn } return { @@ -471,13 +482,7 @@ async function buildDatabricksServiceAccountSecret( try { result = await verifyAndEncryptDatabricksCredential(fields.serviceAccountJson, fields.owner) } catch (error) { - if (error instanceof KeylessCredentialRefusedError) throw error - if (error instanceof DatabricksCredentialError) { - throw new ServiceAccountSecretError(error.message) - } - throw new ServiceAccountSecretError( - `Could not verify the Databricks credential: ${getErrorMessage(error, 'Unknown error')}` - ) + throwVerificationFailure(error, DatabricksCredentialError, 'Databricks credential') } const principal: ServiceAccountPrincipal = { kind: 'user', id: result.identity } return { @@ -501,13 +506,7 @@ async function buildMicrosoftAppServiceAccountSecret( try { result = await verifyAndEncryptMicrosoftAppCredential(fields.serviceAccountJson, fields.owner) } catch (error) { - if (error instanceof KeylessCredentialRefusedError) throw error - if (error instanceof MicrosoftAppCredentialError) { - throw new ServiceAccountSecretError(error.message) - } - throw new ServiceAccountSecretError( - `Could not verify the Microsoft app credential: ${getErrorMessage(error, 'Unknown error')}` - ) + throwVerificationFailure(error, MicrosoftAppCredentialError, 'Microsoft app credential') } // The app's service principal is the identity every app-only token acts as. const principal: ServiceAccountPrincipal = { diff --git a/apps/sim/lib/credentials/token-service-accounts/descriptors.ts b/apps/sim/lib/credentials/token-service-accounts/descriptors.ts index 40e9e464a84..6e1d2ca1c3a 100644 --- a/apps/sim/lib/credentials/token-service-accounts/descriptors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/descriptors.ts @@ -110,7 +110,7 @@ export interface TokenServiceAccountDescriptor { * HTTP auth scheme the pasted token requires at execution time. Defaults to * `bearer` (`Authorization: Bearer `); `x-api-token` providers (e.g. * Pipedrive) send the token in an `x-api-token` header instead, and the - * token route surfaces this so tool header builders can switch schemes. + * resolved credential token carries this so tool header builders can switch schemes. */ authStyle?: 'bearer' | 'x-api-token' /** diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts index 5c5f9dc205d..cc6178c68f3 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { fetchProvider, isHostNotFound, @@ -55,6 +56,14 @@ describe('token service-account error helpers', () => { expect(error.logDetail).toEqual({ step: 'self', reason: 'network error reaching provider' }) }) + it("surfaces an unavailable egress route as Sim's outage, not the provider's", async () => { + mockFetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(fetchProvider(PROVIDER_URL, {}, 'self')).rejects.toBeInstanceOf( + OutboundRoutingError + ) + }) + it('never includes the URL in logDetail', async () => { mockFetch.mockRejectedValue(new Error(`ECONNREFUSED ${PROVIDER_URL}`)) diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.ts b/apps/sim/lib/credentials/token-service-accounts/errors.ts index a4ff91cfc10..6d483e87b56 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.ts @@ -1,6 +1,8 @@ import { getErrorMessage } from '@sim/utils/errors' import { parseRetryAfter } from '@sim/utils/retry' import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -206,7 +208,9 @@ export function isHostNotFound(error: unknown): boolean { * (DNS, TLS, connection reset) to `provider_unavailable` so they never escape * as raw undici errors — whose `cause` can carry connection details — and are * never blamed on the pasted token. DNS-resolution failures can optionally be - * mapped to a different code via {@link FetchProviderOptions}. + * mapped to a different code via {@link FetchProviderOptions}. A blocked or unavailable + * organization egress route is Sim's outage, not the provider's, so its + * {@link OutboundRoutingError} propagates unchanged. * * Redirects are refused (`redirect: 'error'`) unless `init` names another mode: * every caller sends a secret, and fetch replays custom headers and 307/308 @@ -222,9 +226,10 @@ export async function fetchProvider( try { const timeout = AbortSignal.timeout(PROVIDER_FETCH_TIMEOUT_MS) const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout - return await fetch(url, { ...init, redirect: init.redirect ?? 'error', signal }) + return await outboundFetch(url, { ...init, redirect: init.redirect ?? 'error', signal }) } catch (error) { init.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error // Only ENOTFOUND proves the host doesn't exist; EAI_AGAIN is a transient // resolver failure and stays provider_unavailable. if (options?.dnsFailureCode && isHostNotFound(error)) { diff --git a/apps/sim/lib/data-drains/destinations/bigquery.ts b/apps/sim/lib/data-drains/destinations/bigquery.ts index b4dc41b841a..80d4a174516 100644 --- a/apps/sim/lib/data-drains/destinations/bigquery.ts +++ b/apps/sim/lib/data-drains/destinations/bigquery.ts @@ -2,10 +2,11 @@ import { createHash } from 'node:crypto' import { createLogger } from '@sim/logger' import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' -import { JWT } from 'google-auth-library' +import type { JWT } from 'google-auth-library' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { createServiceAccountJwt } from '@/lib/data-drains/destinations/google-jwt' import { - type ParsedServiceAccount, parseNdjsonObjects, parseServiceAccount, refineServiceAccountJson, @@ -74,10 +75,6 @@ const bigqueryCredentialsSchema = z export type BigQueryDestinationConfig = z.infer export type BigQueryDestinationCredentials = z.infer -function buildJwt(account: ParsedServiceAccount): JWT { - return new JWT({ email: account.clientEmail, key: account.privateKey, scopes: SCOPES }) -} - async function getAccessToken(jwt: JWT, forceRefresh = false): Promise { if (forceRefresh) { /** Clearing `credentials` forces `getAccessToken` to mint a new token instead of returning the cached one. */ @@ -110,7 +107,7 @@ async function postInsertAll( const token = await getAccessToken(input.jwt, forceRefresh) const perAttempt = AbortSignal.any([input.signal, AbortSignal.timeout(PER_ATTEMPT_TIMEOUT_MS)]) try { - return await fetch(url, { + return await outboundFetch(url, { method: 'POST', headers: { Authorization: `Bearer ${token}`, @@ -288,11 +285,11 @@ export const bigqueryDestination: DrainDestination< */ async test({ config, credentials, signal }) { const account = parseServiceAccount(credentials.serviceAccountJson) - const jwt = buildJwt(account) + const jwt = createServiceAccountJwt(account, SCOPES) const token = await getAccessToken(jwt) const url = `https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(config.projectId)}/datasets/${encodeURIComponent(config.datasetId)}/tables/${encodeURIComponent(config.tableId)}?fields=id` const perAttempt = AbortSignal.any([signal, AbortSignal.timeout(PER_ATTEMPT_TIMEOUT_MS)]) - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${token}` }, signal: perAttempt, }) @@ -306,7 +303,7 @@ export const bigqueryDestination: DrainDestination< openSession({ config, credentials }) { const account = parseServiceAccount(credentials.serviceAccountJson) - const jwt = buildJwt(account) + const jwt = createServiceAccountJwt(account, SCOPES) return { async deliver({ body, metadata, signal }) { const rows = parseNdjsonObjects(body, { requireObject: true }) as Record[] diff --git a/apps/sim/lib/data-drains/destinations/datadog.ts b/apps/sim/lib/data-drains/destinations/datadog.ts index edb89210d12..d300282fcaa 100644 --- a/apps/sim/lib/data-drains/destinations/datadog.ts +++ b/apps/sim/lib/data-drains/destinations/datadog.ts @@ -4,6 +4,7 @@ import { toError } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { redactExactSensitiveValues } from '@/lib/core/security/redaction' import { parseNdjsonObjects } from '@/lib/data-drains/destinations/utils' import type { DeliveryMetadata, DrainDestination } from '@/lib/data-drains/types' @@ -160,7 +161,7 @@ async function postWithRetries(input: PostInput): Promise { let retryAfterMs: number | null = null let response: Response | undefined try { - response = await fetch(input.url, { + response = await outboundFetch(input.url, { method: 'POST', // double-cast-allowed: Uint8Array is a valid runtime BodyInit but the DOM lib types only enumerate Blob/FormData/string/etc. body: body as unknown as BodyInit, diff --git a/apps/sim/lib/data-drains/destinations/gcs.ts b/apps/sim/lib/data-drains/destinations/gcs.ts index e7cfe734536..62056470309 100644 --- a/apps/sim/lib/data-drains/destinations/gcs.ts +++ b/apps/sim/lib/data-drains/destinations/gcs.ts @@ -2,12 +2,13 @@ import { createLogger } from '@sim/logger' import { interruptibleSleep } from '@sim/utils/helpers' import { generateShortId } from '@sim/utils/id' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' -import { JWT } from 'google-auth-library' +import type { JWT } from 'google-auth-library' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { createServiceAccountJwt } from '@/lib/data-drains/destinations/google-jwt' import { buildObjectKey, normalizePrefix, - type ParsedServiceAccount, parseServiceAccount, refineServiceAccountJson, } from '@/lib/data-drains/destinations/utils' @@ -84,10 +85,6 @@ const gcsCredentialsSchema = z export type GCSDestinationConfig = z.infer export type GCSDestinationCredentials = z.infer -function buildJwt(account: ParsedServiceAccount): JWT { - return new JWT({ email: account.clientEmail, key: account.privateKey, scopes: [SCOPE] }) -} - async function getAccessToken(jwt: JWT): Promise { const { token } = await jwt.getAccessToken() if (!token) throw new Error('Failed to obtain GCS access token') @@ -139,7 +136,7 @@ async function fetchWithRetry(input: RetryRequestInput): Promise { let response: Response try { const headers = await input.buildHeaders() - response = await fetch(input.url, { + response = await outboundFetch(input.url, { method: input.method, body: input.body as BodyInit | undefined, headers, @@ -262,7 +259,7 @@ export const gcsDestination: DrainDestination networkConfigMock) + +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { asOrchestrationError } from '@/lib/core/orchestration/types' +import { createServiceAccountJwt } from '@/lib/data-drains/destinations/google-jwt' + +const mockFetch = vi.fn() +const { privateKey } = generateKeyPairSync('rsa', { + modulusLength: 2048, + privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, + publicKeyEncoding: { type: 'spki', format: 'pem' }, +}) +const account = { clientEmail: 'drain@project.iam.gserviceaccount.com', privateKey } + +describe('createServiceAccountJwt', () => { + beforeEach(() => { + vi.stubGlobal('fetch', mockFetch) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + + it("mints the access token on the drain organization's egress route", async () => { + mockFetch.mockResolvedValue( + Response.json({ access_token: 'minted', token_type: 'Bearer', expires_in: 3600 }) + ) + const jwt = createServiceAccountJwt(account, ['https://www.googleapis.com/auth/scope']) + const { token } = await runWithOutboundOrganization('org-1', () => jwt.getAccessToken()) + expect(token).toBe('minted') + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + expect(String(mockFetch.mock.calls[0][0])).toBe('https://oauth2.googleapis.com/token') + }) + + it('surfaces an unavailable egress route as Sim unavailability', async () => { + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + const jwt = createServiceAccountJwt(account, ['https://www.googleapis.com/auth/scope']) + const error = await runWithOutboundOrganization('org-1', () => jwt.getAccessToken()).catch( + (caught: unknown) => caught + ) + expect(asOrchestrationError(error)?.code).toBe('unavailable') + expect(mockFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/data-drains/destinations/google-jwt.ts b/apps/sim/lib/data-drains/destinations/google-jwt.ts new file mode 100644 index 00000000000..3c5c2dd3500 --- /dev/null +++ b/apps/sim/lib/data-drains/destinations/google-jwt.ts @@ -0,0 +1,17 @@ +import { JWT } from 'google-auth-library' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import type { ParsedServiceAccount } from '@/lib/data-drains/destinations/utils' + +/** + * A service-account JWT client whose token mint leaves through `outboundFetch`, so an + * organization with a dedicated egress gateway reaches Google's token endpoint from its own + * static IPs. gaxios otherwise mints with its own fetch, outside the organization's route. + */ +export function createServiceAccountJwt(account: ParsedServiceAccount, scopes: string[]): JWT { + return new JWT({ + email: account.clientEmail, + key: account.privateKey, + scopes, + transporterOptions: { fetchImplementation: outboundFetch }, + }) +} diff --git a/apps/sim/lib/data-drains/destinations/snowflake.ts b/apps/sim/lib/data-drains/destinations/snowflake.ts index b5550495bec..9dfdbb88501 100644 --- a/apps/sim/lib/data-drains/destinations/snowflake.ts +++ b/apps/sim/lib/data-drains/destinations/snowflake.ts @@ -6,6 +6,7 @@ import { generateId } from '@sim/utils/id' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { importPKCS8, SignJWT } from 'jose' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { DrainDestination } from '@/lib/data-drains/types' const logger = createLogger('DataDrainSnowflakeDestination') @@ -245,7 +246,7 @@ async function executeStatement(input: ExecuteInput): Promise { const url = `${baseUrl}?${params.toString()}` let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { Authorization: `Bearer ${jwt}`, @@ -347,7 +348,7 @@ async function pollStatement(input: PollInput): Promise { const perAttempt = AbortSignal.any([input.signal, AbortSignal.timeout(PER_ATTEMPT_TIMEOUT_MS)]) let response: Response try { - response = await fetch(url, { + response = await outboundFetch(url, { headers: { Authorization: `Bearer ${jwt}`, Accept: 'application/json', diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index 4596d14750e..9b255e95b1c 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -12,6 +12,7 @@ import { getRotatingApiKey } from '@/lib/core/config/api-keys' import { env, envNumber } from '@/lib/core/config/env' import { isHosted } from '@/lib/core/config/env-flags' import { isQuotaExhaustionBody } from '@/lib/core/errors/provider-quota' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { ProviderQuotaExhaustedError, recordProviderCooldown, @@ -543,6 +544,9 @@ async function callEmbeddingAPI( ): Promise<{ embeddings: number[][]; totalTokens: number; dimensions: number }> { const admissionWaitMs = bulk ? KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS : EMBEDDING_RETRY_BUDGET_MS const admissionIdentity = embeddingAdmissionIdentity({ providerId, quotaCircuitIdentity, isBYOK }) + const operatorOwned = providerId === 'ollama' || providerId === 'azure-openai' + // egress-direct: Ollama and the Azure OpenAI override are the operator's own deployments + const requestFetch = operatorOwned ? fetch : outboundFetch return retryWithExponentialBackoff( async (operationSignal, deadlineAt) => { if (await isEmbeddingQuotaCircuitOpen(admissionIdentity)) { @@ -581,7 +585,7 @@ async function callEmbeddingAPI( const timeout = setTimeout(() => controller.abort(), EMBEDDING_REQUEST_TIMEOUT_MS) try { - const response = await fetch(request.apiUrl, { + const response = await requestFetch(request.apiUrl, { method: 'POST', headers: request.headers, body: JSON.stringify(request.body), diff --git a/apps/sim/lib/embeddings/ollama-model-catalog.server.ts b/apps/sim/lib/embeddings/ollama-model-catalog.server.ts index 75008e1f4be..6424236a002 100644 --- a/apps/sim/lib/embeddings/ollama-model-catalog.server.ts +++ b/apps/sim/lib/embeddings/ollama-model-catalog.server.ts @@ -70,6 +70,7 @@ export class OllamaEmbeddingWidthUnknownError extends Error { } async function fetchOllamaJson(path: string, init: RequestInit, signal?: AbortSignal) { + // egress-direct: operator-local Ollama server named by the deployment const response = await fetch(`${getOllamaUrl().replace(/\/+$/, '')}${path}`, { headers: { 'Content-Type': 'application/json' }, signal, diff --git a/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts b/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts index b65f499b892..d7f5d93d262 100644 --- a/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts +++ b/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts @@ -24,6 +24,7 @@ export class OpenRouterEmbeddingModelNotFoundError extends Error { export async function fetchOpenRouterEmbeddingModelCatalog( signal?: AbortSignal ): Promise { + // egress-direct: public model catalog, read without a workspace const response = await fetch(OPENROUTER_EMBEDDING_MODELS_URL, { headers: { 'Content-Type': 'application/json' }, next: { revalidate: 300 }, diff --git a/apps/sim/lib/folders/status.ts b/apps/sim/lib/folders/status.ts index 9c5762c3a1d..17a1617febb 100644 --- a/apps/sim/lib/folders/status.ts +++ b/apps/sim/lib/folders/status.ts @@ -24,5 +24,6 @@ export function folderMutationStatus(errorCode: FolderMutationErrorCode | undefi // A folder collection too large to materialize reaches these routes as a classified // failure too; without this arm it rendered as an unexplained 500. if (errorCode === 'payload_too_large') return 413 + if (errorCode === 'unavailable') return 503 return 500 } diff --git a/apps/sim/lib/internal/asana/client.ts b/apps/sim/lib/internal/asana/client.ts index d5b8e9b3f74..818f4983d8c 100644 --- a/apps/sim/lib/internal/asana/client.ts +++ b/apps/sim/lib/internal/asana/client.ts @@ -1,4 +1,5 @@ import { toArray, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { AsanaOperationError } from '@/lib/internal/asana/errors' @@ -33,7 +34,7 @@ export class AsanaClient { private async fetch(path: string, init: RequestInit, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(this.url(path), { + return outboundFetch(this.url(path), { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/ashby/operations.ts b/apps/sim/lib/internal/ashby/operations.ts index b0547b52309..60e1129761f 100644 --- a/apps/sim/lib/internal/ashby/operations.ts +++ b/apps/sim/lib/internal/ashby/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -40,7 +41,7 @@ async function ashbyPost( onBehalfOfUserId: string | null | undefined, signal?: AbortSignal ): Promise { - const response = await fetch(`https://api.ashbyhq.com/${path}`, { + const response = await outboundFetch(`https://api.ashbyhq.com/${path}`, { method: 'POST', headers: ashbyAuthHeaders(apiKey, onBehalfOfUserId ?? undefined), body: JSON.stringify(body), diff --git a/apps/sim/lib/internal/aws/credentials.test.ts b/apps/sim/lib/internal/aws/credentials.test.ts index 4c35cb52b77..64307c1d26d 100644 --- a/apps/sim/lib/internal/aws/credentials.test.ts +++ b/apps/sim/lib/internal/aws/credentials.test.ts @@ -24,6 +24,8 @@ import { IDPRejectedClaimException, InvalidIdentityTokenException, } from '@aws-sdk/client-sts' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' import { awsClientCredentials, withAwsCredential } from '@/lib/internal/aws/credentials' function call(overrides: Partial = {}): InternalToolOperationCall { @@ -212,4 +214,24 @@ describe('withAwsCredential', () => { expect(response.status).toBe(status) expect(handler).not.toHaveBeenCalled() }) + + it("reports an egress outage as Sim's, without naming its routing code", async () => { + mockGetAwsCredentialConfig.mockResolvedValue({ config: {}, owner: { workspaceId: 'ws-1' } }) + mockResolveAwsSessionCredentials.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + const response = (await withAwsCredential(vi.fn())( + call({ + context: { + workflowId: 'wf-1', + workspaceId: 'ws-1', + authorizedServiceAccountCredential: { credentialId: 'cred-1', service: 'aws' }, + }, + }) + )) as Response + + expect(response.status).toBe(503) + await expect(response.json()).resolves.toEqual({ error: EGRESS_UNAVAILABLE_MESSAGE }) + }) }) diff --git a/apps/sim/lib/internal/aws/credentials.ts b/apps/sim/lib/internal/aws/credentials.ts index 1d24f170217..acf68a0c0f1 100644 --- a/apps/sim/lib/internal/aws/credentials.ts +++ b/apps/sim/lib/internal/aws/credentials.ts @@ -1,6 +1,6 @@ import { AsyncLocalStorage } from 'node:async_hooks' import { isRecordLike } from '@sim/utils/object' -import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' import { type AwsSessionCredentials, buildAwsRoleSessionName, @@ -70,13 +70,14 @@ export function awsSessionLifetime( /** * SDK client settings for a client built from an operation's key fields: its region, - * {@link awsClientCredentials}, and the request handler that honors `HTTPS_PROXY`. + * {@link awsClientCredentials}, and the request handler that sends through the organization's + * egress route (and honors `HTTPS_PROXY` on a direct one). */ export function awsClientConfig(region: string, accessKeyId: string, secretAccessKey: string) { return { region, credentials: awsClientCredentials(accessKeyId, secretAccessKey), - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), } } diff --git a/apps/sim/lib/internal/aws/failure.ts b/apps/sim/lib/internal/aws/failure.ts index ce5188ac2fc..241a712a19b 100644 --- a/apps/sim/lib/internal/aws/failure.ts +++ b/apps/sim/lib/internal/aws/failure.ts @@ -1,6 +1,8 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { isRetryableNetworkError } from '@/lib/core/errors/retryable-infrastructure' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' const THROTTLING_ERROR_NAMES = new Set(['Throttling', 'ThrottlingException']) @@ -8,9 +10,11 @@ const THROTTLING_ERROR_NAMES = new Set(['Throttling', 'ThrottlingException']) * Whether resolving a saved AWS credential failed because AWS was briefly unavailable rather * than because it denied the credential: an error STS models as retryable * (IDPCommunicationError, when STS could not reach Sim's OIDC issuer), throttling, an STS 5xx, - * or a network failure. The SDK has already retried these. + * a network failure, or the organization's egress route refusing or failing the connection. + * The SDK has already retried these. */ export function isTransientAwsFailure(error: unknown): boolean { + if (error instanceof OutboundRoutingError) return true if (!isRecordLike(error)) return false const status = isRecordLike(error.$metadata) ? error.$metadata.httpStatusCode : undefined return ( @@ -23,7 +27,11 @@ export function isTransientAwsFailure(error: unknown): boolean { ) } -/** The caller-facing reason a saved AWS credential could not be used: the AWS message, never a key. */ +/** + * The caller-facing reason a saved AWS credential could not be used: the AWS message, never a key. + * An egress outage is Sim's, not the credential's, so it reads as that and never names its routing code. + */ export function awsCredentialFailureMessage(error: unknown): string { + if (error instanceof OutboundRoutingError) return EGRESS_UNAVAILABLE_MESSAGE return `Could not use the AWS credential: ${getErrorMessage(error, 'Unknown error')}` } diff --git a/apps/sim/lib/internal/box/client.ts b/apps/sim/lib/internal/box/client.ts index be7cc2150f1..735aba86019 100644 --- a/apps/sim/lib/internal/box/client.ts +++ b/apps/sim/lib/internal/box/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -46,7 +47,7 @@ export class BoxClient { new Blob([new Uint8Array(buffer)], { type: 'application/octet-stream' }), fileName ) - const response = await fetch('https://upload.box.com/api/2.0/files/content', { + const response = await outboundFetch('https://upload.box.com/api/2.0/files/content', { method: 'POST', headers: { Authorization: `Bearer ${this.accessToken}` }, body: formData, diff --git a/apps/sim/lib/internal/brex/client.ts b/apps/sim/lib/internal/brex/client.ts index 82f1b0fdca9..7a7b9fa9fce 100644 --- a/apps/sim/lib/internal/brex/client.ts +++ b/apps/sim/lib/internal/brex/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -49,7 +50,7 @@ export class BrexReceiptClient { const endpoint = expenseId ? `${BREX_API_BASE}/v1/expenses/card/${encodeURIComponent(expenseId)}/receipt_upload` : `${BREX_API_BASE}/v1/expenses/card/receipt_match` - const response = await fetch(endpoint, { + const response = await outboundFetch(endpoint, { method: 'POST', headers: buildBrexHeaders(this.apiKey), body: JSON.stringify({ receipt_name: receiptName }), diff --git a/apps/sim/lib/internal/browser-use/operations/run-task.ts b/apps/sim/lib/internal/browser-use/operations/run-task.ts index 4b9bf78b10a..4146c3a5425 100644 --- a/apps/sim/lib/internal/browser-use/operations/run-task.ts +++ b/apps/sim/lib/internal/browser-use/operations/run-task.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { z } from 'zod' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { BrowserUseRunTaskParams, @@ -86,7 +87,7 @@ async function fetchBrowserUse( ): Promise { options.signal?.throwIfAborted() const hasBody = options.body !== undefined - const response = await fetch(`${API_BASE}${path}`, { + const response = await outboundFetch(`${API_BASE}${path}`, { method: options.method ?? 'GET', headers: { ...(hasBody ? { 'Content-Type': 'application/json' } : {}), diff --git a/apps/sim/lib/internal/buffer/execute-tool.ts b/apps/sim/lib/internal/buffer/execute-tool.ts index 7018e177aef..d2baf8a3983 100644 --- a/apps/sim/lib/internal/buffer/execute-tool.ts +++ b/apps/sim/lib/internal/buffer/execute-tool.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '@sim/utils/errors' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { BufferOperationError } from '@/lib/internal/buffer/errors' import { bufferCreatePostInputSchema, bufferEditPostInputSchema } from '@/lib/internal/buffer/input' import { createBufferPost, editBufferPost } from '@/lib/internal/buffer/operations' @@ -33,6 +34,7 @@ export const executeBufferTool: InternalToolOperationHandler = async (request) = ) } catch (error) { request.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error if (error instanceof BufferOperationError) { return Response.json({ success: false, error: error.message }, { status: error.status }) } diff --git a/apps/sim/lib/internal/buffer/operations.test.ts b/apps/sim/lib/internal/buffer/operations.test.ts index 22cf5ca9d1f..cd632b503c9 100644 --- a/apps/sim/lib/internal/buffer/operations.test.ts +++ b/apps/sim/lib/internal/buffer/operations.test.ts @@ -8,11 +8,31 @@ vi.mock('@/lib/uploads/utils/file.server', () => fileUtilsServerMock) const { mockResolveFileInputToUrl } = fileUtilsServerMockFns +import { OutboundRoutingError } from '@/lib/core/network/routing' import { executeBufferTool } from '@/lib/internal/buffer/execute-tool' import { createBufferPost } from '@/lib/internal/buffer/operations' import bufferPostFixture from '@/tools/buffer/__fixtures__/post.json' describe('Buffer operations', () => { + it('surfaces an egress routing failure instead of a provider error', async () => { + vi.spyOn(globalThis, 'fetch').mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + executeBufferTool({ + toolId: 'buffer_create_post', + input: { + apiKey: 'buffer-key', + channelId: 'channel-1', + mode: 'addToQueue', + text: 'Caption', + }, + headers: new Headers(), + context: { workflowId: 'workflow-1', userId: 'user-1' }, + requestId: 'request-1', + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('creates a text post when optional assets are null', async () => { const providerInputs: Record[] = [] vi.spyOn(globalThis, 'fetch').mockImplementation(async (_input, init) => { diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts index 8c9aac638cf..f98af4f57fd 100644 --- a/apps/sim/lib/internal/buffer/operations.ts +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -1,6 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, @@ -160,7 +162,7 @@ async function executePostMutation(args: { const { apiKey, mutation, input, context } = args let result: Record try { - const response = await fetch(BUFFER_API_URL, { + const response = await outboundFetch(BUFFER_API_URL, { method: 'POST', headers: bufferHeaders(apiKey), body: JSON.stringify({ query: mutation, variables: { input } }), @@ -171,6 +173,7 @@ async function executePostMutation(args: { result = toRecord(candidate) } catch (error) { context.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error const message = getErrorMessage(error, 'Buffer API request failed') logger.error('Buffer post mutation failed', { requestId: context.requestId, error: message }) throw new BufferOperationError(message, 502) diff --git a/apps/sim/lib/internal/clickup/client.ts b/apps/sim/lib/internal/clickup/client.ts index 78da954d554..60c391154db 100644 --- a/apps/sim/lib/internal/clickup/client.ts +++ b/apps/sim/lib/internal/clickup/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ClickUpOperationError } from '@/lib/internal/clickup/errors' @@ -14,7 +15,7 @@ export async function uploadClickUpAttachment( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch( + const response = await outboundFetch( `${CLICKUP_API_BASE_URL}/task/${encodeURIComponent(taskId)}/attachment`, { method: 'POST', diff --git a/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts b/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts index 4722472c8eb..9dd1c7b5713 100644 --- a/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts +++ b/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import { mapZoneSetting, zoneSettingUrl } from '@/tools/cloudflare/get_zone_settings' @@ -32,7 +33,7 @@ export const executeGetZoneSettingsOperation: InternalToolOperationImplementatio const reads = await Promise.all( settingIds.map(async (settingId) => { try { - const response = await fetch(zoneSettingUrl(zoneId, settingId), { + const response = await outboundFetch(zoneSettingUrl(zoneId, settingId), { method: 'GET', headers, signal, diff --git a/apps/sim/lib/internal/confluence/client.ts b/apps/sim/lib/internal/confluence/client.ts index fb9d5791443..ce148d6f823 100644 --- a/apps/sim/lib/internal/confluence/client.ts +++ b/apps/sim/lib/internal/confluence/client.ts @@ -1,4 +1,5 @@ import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { @@ -46,7 +47,7 @@ export class ConfluenceClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return outboundFetch(path, { ...init, headers: { Accept: 'application/json', diff --git a/apps/sim/lib/internal/crowdstrike/client.ts b/apps/sim/lib/internal/crowdstrike/client.ts index 4ab2896711e..8acd635f5a0 100644 --- a/apps/sim/lib/internal/crowdstrike/client.ts +++ b/apps/sim/lib/internal/crowdstrike/client.ts @@ -1,5 +1,6 @@ import { toNumberOrNull, toStringOrNull } from '@sim/utils/coerce' import { isRecordLike, toRecordOrNull } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { throttledRetryAfterMs } from '@/lib/credentials/token-service-accounts/errors' @@ -244,7 +245,7 @@ export async function requestCrowdStrikeToken( }) if (request.memberCid) body.set('member_cid', request.memberCid) - const response = await fetch(`${getCloudBaseUrl(request.cloud)}/oauth2/token`, { + const response = await outboundFetch(`${getCloudBaseUrl(request.cloud)}/oauth2/token`, { method: 'POST', headers: { Accept: 'application/json', @@ -347,7 +348,7 @@ export async function callCrowdStrike( headers['Content-Type'] = 'application/json' } - const response = await fetch(buildUrl(baseUrl, options), { + const response = await outboundFetch(buildUrl(baseUrl, options), { method: options.method, headers, body: options.body === undefined ? undefined : JSON.stringify(options.body), diff --git a/apps/sim/lib/internal/cursor/operations.ts b/apps/sim/lib/internal/cursor/operations.ts index 132cb3bdbd1..5d72ddbde7e 100644 --- a/apps/sim/lib/internal/cursor/operations.ts +++ b/apps/sim/lib/internal/cursor/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -57,7 +58,7 @@ export async function downloadCursorArtifact( ): Promise { context.signal?.throwIfAborted() const authHeader = `Basic ${Buffer.from(`${input.apiKey}:`).toString('base64')}` - const artifactResponse = await fetch( + const artifactResponse = await outboundFetch( `https://api.cursor.com/v0/agents/${encodeURIComponent(input.agentId)}/artifacts/download?path=${encodeURIComponent(input.path)}`, { method: 'GET', diff --git a/apps/sim/lib/internal/datadog/operations/update-slo.ts b/apps/sim/lib/internal/datadog/operations/update-slo.ts index 074fb4c3f5e..eb33cdc5eda 100644 --- a/apps/sim/lib/internal/datadog/operations/update-slo.ts +++ b/apps/sim/lib/internal/datadog/operations/update-slo.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { UpdateSloParams } from '@/tools/datadog/types' import { @@ -14,7 +15,7 @@ export const executeUpdateSloOperation: InternalToolOperationImplementation< const url = datadogApiUrl(params.site, `/api/v1/slo/${datadogPathSegment(params.sloId)}`) const headers = datadogHeaders(params) - const existingResponse = await fetch(url, { method: 'GET', headers, signal }) + const existingResponse = await outboundFetch(url, { method: 'GET', headers, signal }) if (!existingResponse.ok) { return { success: false, @@ -33,7 +34,7 @@ export const executeUpdateSloOperation: InternalToolOperationImplementation< } } - const response = await fetch(url, { + const response = await outboundFetch(url, { method: 'PUT', headers, body: JSON.stringify(mergeSloUpdatePayload(stored, params)), diff --git a/apps/sim/lib/internal/daytona/operations.ts b/apps/sim/lib/internal/daytona/operations.ts index 9338689c3a0..04ebe7019dd 100644 --- a/apps/sim/lib/internal/daytona/operations.ts +++ b/apps/sim/lib/internal/daytona/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { DaytonaOperationError } from '@/lib/internal/daytona/errors' import { isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' @@ -96,7 +97,7 @@ export async function uploadDaytonaFile( new Blob([new Uint8Array(fileBuffer)], { type: 'application/octet-stream' }), fileName ) - const response = await fetch( + const response = await outboundFetch( daytonaToolboxUrl( input.sandboxId, `/files/upload-v2?path=${encodeURIComponent(destinationPath)}` diff --git a/apps/sim/lib/internal/discord/client.ts b/apps/sim/lib/internal/discord/client.ts index 30959f99873..c24c14f3f35 100644 --- a/apps/sim/lib/internal/discord/client.ts +++ b/apps/sim/lib/internal/discord/client.ts @@ -1,4 +1,5 @@ import { isRecordLike, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { DiscordOperationError } from '@/lib/internal/discord/errors' @@ -11,15 +12,18 @@ export async function sendDiscordMessage( signal?: AbortSignal ): Promise> { signal?.throwIfAborted() - const response = await fetch(`https://discord.com/api/v10/channels/${channelId}/messages`, { - method: 'POST', - headers: { - Authorization: `Bot ${botToken}`, - ...(contentType === 'json' ? { 'Content-Type': 'application/json' } : {}), - }, - body, - signal, - }) + const response = await outboundFetch( + `https://discord.com/api/v10/channels/${channelId}/messages`, + { + method: 'POST', + headers: { + Authorization: `Bot ${botToken}`, + ...(contentType === 'json' ? { 'Content-Type': 'application/json' } : {}), + }, + body, + signal, + } + ) let data: unknown try { data = await readResponseJsonWithLimit(response, { diff --git a/apps/sim/lib/internal/docusign/client.ts b/apps/sim/lib/internal/docusign/client.ts index 372762da549..48a3f3941ca 100644 --- a/apps/sim/lib/internal/docusign/client.ts +++ b/apps/sim/lib/internal/docusign/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -35,7 +36,7 @@ async function fetchDocusign( const abort = () => controller.abort(parentSignal?.reason ?? new Error('Request aborted')) parentSignal?.addEventListener('abort', abort, { once: true }) try { - return await fetch(input, { ...init, signal: controller.signal }) + return await outboundFetch(input, { ...init, signal: controller.signal }) } finally { clearTimeout(timeout) parentSignal?.removeEventListener('abort', abort) diff --git a/apps/sim/lib/internal/dropbox/client.ts b/apps/sim/lib/internal/dropbox/client.ts index 5e0f66d6a0a..da87a9c6fa5 100644 --- a/apps/sim/lib/internal/dropbox/client.ts +++ b/apps/sim/lib/internal/dropbox/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -35,7 +36,7 @@ export class DropboxClient { } ): Promise> { this.signal?.throwIfAborted() - const response = await fetch('https://content.dropboxapi.com/2/files/upload', { + const response = await outboundFetch('https://content.dropboxapi.com/2/files/upload', { method: 'POST', headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/elevenlabs/client.ts b/apps/sim/lib/internal/elevenlabs/client.ts index 9fc0e862db8..47331db61ac 100644 --- a/apps/sim/lib/internal/elevenlabs/client.ts +++ b/apps/sim/lib/internal/elevenlabs/client.ts @@ -2,6 +2,7 @@ import { createTimeoutAbortController, DEFAULT_EXECUTION_TIMEOUT_MS, } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { consumeOrCancelBody, readResponseToBufferWithLimit } from '@/lib/core/utils/stream-limits' import { ElevenLabsOperationError } from '@/lib/internal/elevenlabs/errors' import type { @@ -90,7 +91,7 @@ export async function generateElevenLabsAudio( const timeout = createTimeoutAbortController(DEFAULT_EXECUTION_TIMEOUT_MS, signal) try { const { url, init } = buildRequest(args) - const response = await fetch(url, { ...init, signal: timeout.signal }) + const response = await outboundFetch(url, { ...init, signal: timeout.signal }) timeout.signal.throwIfAborted() if (!response.ok) { await consumeOrCancelBody(response) diff --git a/apps/sim/lib/internal/firecrawl/operations.test.ts b/apps/sim/lib/internal/firecrawl/operations.test.ts index aca3d12c46f..4749977f319 100644 --- a/apps/sim/lib/internal/firecrawl/operations.test.ts +++ b/apps/sim/lib/internal/firecrawl/operations.test.ts @@ -11,6 +11,7 @@ import { workspaceFileSecretProvenanceMockFns, } from '@sim/testing/mocks/workspace-file-secret-provenance.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' import { RESOLVED_SECRET_PROVENANCE_FIELD, @@ -91,4 +92,15 @@ describe('executeFirecrawlParse', () => { expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() expect(fetch).not.toHaveBeenCalled() }) + + it('surfaces an egress routing failure instead of a parse error', async () => { + vi.mocked(fetch).mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + executeFirecrawlParse( + { apiKey: 'firecrawl-key', file: FILE, options: { formats: ['markdown'] } }, + createContext() + ) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) }) diff --git a/apps/sim/lib/internal/firecrawl/operations.ts b/apps/sim/lib/internal/firecrawl/operations.ts index 3b4faa306e0..d016e90a5b9 100644 --- a/apps/sim/lib/internal/firecrawl/operations.ts +++ b/apps/sim/lib/internal/firecrawl/operations.ts @@ -1,5 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES, isPayloadSizeLimitError, @@ -85,7 +87,7 @@ export async function executeFirecrawlParse( formData.append('options', JSON.stringify(input.options)) } - const response = await fetch('https://api.firecrawl.dev/v2/parse', { + const response = await outboundFetch('https://api.firecrawl.dev/v2/parse', { method: 'POST', headers: { Authorization: `Bearer ${input.apiKey}` }, body: formData, @@ -119,6 +121,7 @@ export async function executeFirecrawlParse( }) } catch (error) { context.signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error const notReady = docNotReadyResponse(error) if (notReady) return notReady logger.error('Firecrawl parse failed', { diff --git a/apps/sim/lib/internal/fireflies/operations.test.ts b/apps/sim/lib/internal/fireflies/operations.test.ts index b4d305fe1d1..49b66bc1a00 100644 --- a/apps/sim/lib/internal/fireflies/operations.test.ts +++ b/apps/sim/lib/internal/fireflies/operations.test.ts @@ -3,6 +3,7 @@ import { fileUtilsServerMockFns, } from '@sim/testing/mocks/file-utils-server.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' import { RESOLVED_SECRET_PROVENANCE_FIELD, @@ -71,4 +72,15 @@ describe('executeFirefliesUploadAudio', () => { }) ) }) + + it('surfaces an egress routing failure instead of an upload error', async () => { + vi.mocked(fetch).mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + executeFirefliesUploadAudio( + { apiKey: 'fireflies-key', audioUrl: 'https://media.example.com/audio.mp3' }, + createContext() + ) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) }) diff --git a/apps/sim/lib/internal/fireflies/operations.ts b/apps/sim/lib/internal/fireflies/operations.ts index 8b0dff6a229..f4ca58d64f9 100644 --- a/apps/sim/lib/internal/fireflies/operations.ts +++ b/apps/sim/lib/internal/fireflies/operations.ts @@ -1,5 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' import type { FirefliesUploadAudioInput } from '@/lib/internal/fireflies/schema' @@ -96,7 +98,7 @@ export async function executeFirefliesUploadAudio( if (body.clientReferenceId) input.client_reference_id = body.clientReferenceId if (body.attendees !== undefined) input.attendees = body.attendees - const response = await fetch(FIREFLIES_API_URL, { + const response = await outboundFetch(FIREFLIES_API_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -117,7 +119,7 @@ export async function executeFirefliesUploadAudio( }) return Response.json(data, { status: response.status }) } catch (error) { - if (context.signal?.aborted) throw error + if (context.signal?.aborted || error instanceof OutboundRoutingError) throw error logger.error('Fireflies upload failed', { requestId: context.requestId, error: getErrorMessage(error, 'Unknown error'), diff --git a/apps/sim/lib/internal/gmail/client.ts b/apps/sim/lib/internal/gmail/client.ts index 9d77de93ddd..94b8678f6f2 100644 --- a/apps/sim/lib/internal/gmail/client.ts +++ b/apps/sim/lib/internal/gmail/client.ts @@ -1,4 +1,5 @@ import { toArray, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type ReadResponseWithLimitOptions, readResponseJsonWithLimit, @@ -29,7 +30,7 @@ export class GmailClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return outboundFetch(path, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/image/operations.ts b/apps/sim/lib/internal/image/operations.ts index bc3071472a9..3707791eaa0 100644 --- a/apps/sim/lib/internal/image/operations.ts +++ b/apps/sim/lib/internal/image/operations.ts @@ -4,6 +4,7 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike, omit } from '@sim/utils/object' import { getUserFacingErrorMessage } from '@/lib/core/errors/database-query-error' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -264,7 +265,7 @@ async function generateWithOpenAI( const requestBody = prepared.body const outputFormat = typeof requestBody.output_format === 'string' ? requestBody.output_format : 'png' - const openaiResponse = await fetch(prepared.endpoint, { + const openaiResponse = await outboundFetch(prepared.endpoint, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -331,7 +332,7 @@ async function generateWithGemini( ): Promise { const model = prepared.model.id const requestBody = prepared.body - const geminiResponse = await fetch(prepared.endpoint, { + const geminiResponse = await outboundFetch(prepared.endpoint, { method: 'POST', headers: { 'x-goog-api-key': apiKey, @@ -437,7 +438,7 @@ async function generateWithFalAI( const endpoint = prepared.model.endpoint if (!endpoint) throw new Error('Missing Fal.ai image endpoint') const requestBody = prepared.body - const createResponse = await fetch(prepared.endpoint, { + const createResponse = await outboundFetch(prepared.endpoint, { redirect: 'error', method: 'POST', headers: { @@ -648,7 +649,7 @@ async function generateWithHiggsfield( const model = prepared.model.id const headers = higgsfieldAuthHeaders(apiKey) let data = await readHiggsfieldRequest( - await fetch(prepared.endpoint, { + await outboundFetch(prepared.endpoint, { method: 'POST', redirect: 'error', headers: { ...headers, 'Content-Type': 'application/json' }, @@ -687,7 +688,7 @@ async function generateWithHiggsfield( await interruptibleSleep(pollIntervalMs, signal) signal?.throwIfAborted() data = await readHiggsfieldRequest( - await fetch(higgsfieldStatusUrl(higgsfieldRequestId), { + await outboundFetch(higgsfieldStatusUrl(higgsfieldRequestId), { redirect: 'error', headers, signal, diff --git a/apps/sim/lib/internal/instagram/operations.ts b/apps/sim/lib/internal/instagram/operations.ts index 07e71152dad..a4e6e1f11e7 100644 --- a/apps/sim/lib/internal/instagram/operations.ts +++ b/apps/sim/lib/internal/instagram/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { createMediaContainer, @@ -123,7 +124,7 @@ async function fetchMediaMetadata({ fields: string signal?: AbortSignal }): Promise { - const response = await fetch(graphUrl(`/${encodeURIComponent(mediaId)}`, { fields }), { + const response = await outboundFetch(graphUrl(`/${encodeURIComponent(mediaId)}`, { fields }), { headers: bearerHeaders(accessToken), signal, }) diff --git a/apps/sim/lib/internal/instagram/publishing.ts b/apps/sim/lib/internal/instagram/publishing.ts index cb00babd95b..f82c7ae152b 100644 --- a/apps/sim/lib/internal/instagram/publishing.ts +++ b/apps/sim/lib/internal/instagram/publishing.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { hasCloudStorage } from '@/lib/uploads/core/storage-service' import { getFileExtension, @@ -291,7 +292,7 @@ export async function resolveIgUserId( ): Promise { if (igUserId?.trim()) return igUserId.trim() - const response = await fetch(graphUrl('/me', { fields: 'user_id' }), { + const response = await outboundFetch(graphUrl('/me', { fields: 'user_id' }), { headers: bearerHeaders(accessToken), signal, }) @@ -316,10 +317,13 @@ async function getContainerStatus( containerId: string, signal?: AbortSignal ): Promise<{ statusCode: ContainerStatusCode | null; status: string | null }> { - const response = await fetch(graphUrl(`/${containerId}`, { fields: 'status_code,status' }), { - headers: bearerHeaders(accessToken), - signal, - }) + const response = await outboundFetch( + graphUrl(`/${containerId}`, { fields: 'status_code,status' }), + { + headers: bearerHeaders(accessToken), + signal, + } + ) if (!response.ok) { throw new Error(`Failed to get container status: ${await readGraphError(response)}`) } @@ -393,7 +397,7 @@ async function postGraphForm( if (value !== undefined && value !== null) form.set(key, String(value)) } - return fetch(graphUrl(path), { + return outboundFetch(graphUrl(path), { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/internal/jira/client.ts b/apps/sim/lib/internal/jira/client.ts index 62243e95e08..a9ac47b1cb5 100644 --- a/apps/sim/lib/internal/jira/client.ts +++ b/apps/sim/lib/internal/jira/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' @@ -56,7 +57,7 @@ export class JiraClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(url, { + const response = await outboundFetch(url, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/jsm/client.ts b/apps/sim/lib/internal/jsm/client.ts index 39ea6730377..404b5dd61b4 100644 --- a/apps/sim/lib/internal/jsm/client.ts +++ b/apps/sim/lib/internal/jsm/client.ts @@ -1,4 +1,5 @@ import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { JsmOperationError } from '@/lib/internal/jsm/errors' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -67,7 +68,7 @@ export class JsmClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return outboundFetch(path, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/latex/operations.ts b/apps/sim/lib/internal/latex/operations.ts index 6baae21f1e3..1f21974790d 100644 --- a/apps/sim/lib/internal/latex/operations.ts +++ b/apps/sim/lib/internal/latex/operations.ts @@ -1,4 +1,5 @@ import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseToBufferWithLimit, @@ -80,7 +81,7 @@ export async function compileLatexDocument( const signal = context.signal ? AbortSignal.any([context.signal, timeoutSignal]) : timeoutSignal let response: Response try { - response = await fetch(LATEX_COMPILE_URL, { + response = await outboundFetch(LATEX_COMPILE_URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ diff --git a/apps/sim/lib/internal/linq/client.ts b/apps/sim/lib/internal/linq/client.ts index 7ab0fd5e45f..f47a3ca86e1 100644 --- a/apps/sim/lib/internal/linq/client.ts +++ b/apps/sim/lib/internal/linq/client.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES, secureFetchWithPinnedIP, @@ -38,7 +39,7 @@ export async function registerLinqAttachment( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${LINQ_API_BASE}/attachments`, { + const response = await outboundFetch(`${LINQ_API_BASE}/attachments`, { method: 'POST', headers: linqHeaders(input.apiKey), body: JSON.stringify({ diff --git a/apps/sim/lib/internal/llm/operations.test.ts b/apps/sim/lib/internal/llm/operations.test.ts index 93209b0370a..29fa0695f0e 100644 --- a/apps/sim/lib/internal/llm/operations.test.ts +++ b/apps/sim/lib/internal/llm/operations.test.ts @@ -14,6 +14,8 @@ import { permissionCheckMock } from '@sim/testing/mocks/permission-check.mock' import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock' import { providersMock, providersMockFns } from '@sim/testing/mocks/providers.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' const mocks = vi.hoisted(() => ({ authorizeCredential: vi.fn(), @@ -132,6 +134,26 @@ describe('executeLlmProviderOperation', () => { ) }) + it('reports a Vertex credential the egress outage kept from resolving as unavailable', async () => { + mockExecuteProviderRequest.mockClear() + mocks.resolveVertexAccessToken.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + await expect( + executeLlmProviderOperation( + { + provider: 'vertex', + model: 'vertex/gemini-2.5-pro', + vertexCredential: 'credential-1', + workspaceId: 'workspace-1', + }, + { actorUserId: 'user-1', headers: new Headers(), requestId: 'request-1' } + ) + ).rejects.toMatchObject({ status: 503, body: { error: EGRESS_UNAVAILABLE_MESSAGE } }) + expect(mockExecuteProviderRequest).not.toHaveBeenCalled() + }) + it("signs a Bedrock call with the selected AWS credential's session for the caller", async () => { mocks.authorizeCredential.mockResolvedValueOnce(AUTHORIZED_AWS_CREDENTIAL) mockResolveAuthorizedAwsCredential.mockResolvedValueOnce({ diff --git a/apps/sim/lib/internal/llm/operations.ts b/apps/sim/lib/internal/llm/operations.ts index 528b4954b11..6b642671548 100644 --- a/apps/sim/lib/internal/llm/operations.ts +++ b/apps/sim/lib/internal/llm/operations.ts @@ -8,6 +8,11 @@ import { type BillingAttributionSnapshot, requireBillingAttributionHeader, } from '@/lib/billing/core/attribution' +import { + EGRESS_UNAVAILABLE_MESSAGE, + isUnavailableFailure, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' import { inspectModelInputProjectionState, inspectModelInputProvenanceRequest, @@ -158,6 +163,13 @@ async function prepareProviderRequest( if (error instanceof BedrockCredentialError && error.reason === 'unavailable') { fail(503, error.message) } + if (isUnavailableFailure(error)) { + logger.warn('Provider credential unavailable during an egress outage', { + requestId: context.requestId, + provider: input.provider, + }) + fail(statusForOrchestrationError('unavailable'), EGRESS_UNAVAILABLE_MESSAGE) + } logger.error('Failed to resolve provider credential', { requestId: context.requestId, provider: input.provider, diff --git a/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts b/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts index fd0e7ff4d45..d00266b2907 100644 --- a/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts +++ b/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import { mapAppRoleAssignment, @@ -17,7 +18,7 @@ export const executeAddUserAppRoleAssignmentOperation: InternalToolOperationImpl const { userId, resourceId, appRoleId } = readIdentifiers(params) const principalId = await resolveGraphUserObjectId(userId, params.accessToken, signal) - const response = await fetch( + const response = await outboundFetch( `https://graph.microsoft.com/v1.0/users/${safeUrlPathSegment(userId, 'userId')}/appRoleAssignments`, { method: 'POST', diff --git a/apps/sim/lib/internal/microsoft-teams/client.ts b/apps/sim/lib/internal/microsoft-teams/client.ts index e2eed2148f3..6ef48da97c2 100644 --- a/apps/sim/lib/internal/microsoft-teams/client.ts +++ b/apps/sim/lib/internal/microsoft-teams/client.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { MicrosoftTeamsOperationError } from '@/lib/internal/microsoft-teams/errors' @@ -23,7 +24,7 @@ export class MicrosoftTeamsClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${MICROSOFT_GRAPH_BASE_URL}${path}`, { + const response = await outboundFetch(`${MICROSOFT_GRAPH_BASE_URL}${path}`, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/microsoft-teams/operations.ts b/apps/sim/lib/internal/microsoft-teams/operations.ts index 7dad3cef30f..a64b540543a 100644 --- a/apps/sim/lib/internal/microsoft-teams/operations.ts +++ b/apps/sim/lib/internal/microsoft-teams/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { MicrosoftTeamsClient, @@ -465,7 +466,7 @@ export async function deleteMicrosoftTeamsChatMessage( throw new MicrosoftTeamsOperationError('Chat ID and Message ID are required', 400) } - const meResponse = await fetch('https://graph.microsoft.com/v1.0/me', { + const meResponse = await outboundFetch('https://graph.microsoft.com/v1.0/me', { headers: { Authorization: `Bearer ${input.accessToken}` }, signal: context.signal, }) @@ -477,7 +478,7 @@ export async function deleteMicrosoftTeamsChatMessage( ) } - const response = await fetch( + const response = await outboundFetch( `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(me.id)}/chats/${encodeURIComponent(chatId)}/messages/${encodeURIComponent(messageId)}/softDelete`, { method: 'POST', diff --git a/apps/sim/lib/internal/okta/operations/update-group.ts b/apps/sim/lib/internal/okta/operations/update-group.ts index dfcf829dbf5..ae2f63ef237 100644 --- a/apps/sim/lib/internal/okta/operations/update-group.ts +++ b/apps/sim/lib/internal/okta/operations/update-group.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateOktaDomain } from '@/lib/core/security/input-validation' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { OktaGroup, OktaUpdateGroupParams, OktaUpdateGroupResponse } from '@/tools/okta/types' @@ -34,13 +35,13 @@ export const executeOktaUpdateGroupOperation: InternalToolOperationImplementatio const url = `https://${domain}/api/v1/groups/${encodeURIComponent(params.groupId.trim())}` const headers = oktaHeaders(params.apiKey) - const readResponse = await fetch(url, { headers, signal }) + const readResponse = await outboundFetch(url, { headers, signal }) if (!readResponse.ok) { await throwOktaError(readResponse, logger, 'Failed to load group for update in Okta') } const existing: OktaGroup = await readResponse.json() - const writeResponse = await fetch(url, { + const writeResponse = await outboundFetch(url, { method: 'PUT', headers, body: JSON.stringify({ profile: mergeOktaGroupProfile(existing.profile, params) }), diff --git a/apps/sim/lib/internal/outlook/client.ts b/apps/sim/lib/internal/outlook/client.ts index 22427779c73..4a0f6c9173c 100644 --- a/apps/sim/lib/internal/outlook/client.ts +++ b/apps/sim/lib/internal/outlook/client.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -36,7 +37,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await outboundFetch(this.url(path), { ...init, headers: { 'Content-Type': 'application/json', @@ -72,7 +73,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await outboundFetch(this.url(path), { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, @@ -105,7 +106,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise<{ buffer: Buffer; contentType: string | null }> { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await outboundFetch(this.url(path), { method: 'GET', headers: { Authorization: `Bearer ${this.accessToken}` }, signal, diff --git a/apps/sim/lib/internal/persona/operations.ts b/apps/sim/lib/internal/persona/operations.ts index b29d19ef1da..4446b4da701 100644 --- a/apps/sim/lib/internal/persona/operations.ts +++ b/apps/sim/lib/internal/persona/operations.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { PersonaOperationError } from '@/lib/internal/persona/errors' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' @@ -46,7 +47,7 @@ export async function importPersonaAccounts( }) const buffer = resolved.buffer context.signal?.throwIfAborted() - const response = await fetch(`${PERSONA_API_BASE}/importer/accounts`, { + const response = await outboundFetch(`${PERSONA_API_BASE}/importer/accounts`, { method: 'POST', headers: buildPersonaHeaders(input.apiKey), body: JSON.stringify({ diff --git a/apps/sim/lib/internal/quickbooks/operations.ts b/apps/sim/lib/internal/quickbooks/operations.ts index 2629edf413c..3e5af8a459c 100644 --- a/apps/sim/lib/internal/quickbooks/operations.ts +++ b/apps/sim/lib/internal/quickbooks/operations.ts @@ -4,6 +4,7 @@ import type { QuickBooksAddAttachmentBody, QuickBooksDownloadDocumentBody, } from '@/lib/api/contracts/tools/quickbooks' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { assertContentLengthWithinLimit, @@ -93,7 +94,7 @@ async function downloadQuickBooksAttachment( body.quickBooksEnvironment ) const metadataSignal = quickBooksDocumentSignal(signal, QUICKBOOKS_DOCUMENT_METADATA_TIMEOUT_MS) - const downloadUrlResponse = await fetch(downloadUrl, { + const downloadUrlResponse = await outboundFetch(downloadUrl, { method: 'GET', headers: { ...buildQuickBooksHeaders(body.accessToken), Accept: '*/*' }, signal: metadataSignal, @@ -183,7 +184,7 @@ async function downloadQuickBooksTransactionPdf( body.quickBooksEnvironment ) const transferSignal = quickBooksDocumentSignal(signal, QUICKBOOKS_DOCUMENT_TRANSFER_TIMEOUT_MS) - const response = await fetch(url, { + const response = await outboundFetch(url, { method: 'GET', headers: { ...buildQuickBooksHeaders(body.accessToken), Accept: 'application/pdf' }, signal: transferSignal, @@ -221,7 +222,7 @@ export async function executeQuickBooksAddAttachment( const metadata = buildQuickBooksAttachableMetadata(data.targetType, data.targetId, { note: data.note!, }) - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { ...buildQuickBooksHeaders(data.accessToken), @@ -301,7 +302,7 @@ export async function executeQuickBooksAddAttachment( ), resolvedName ) - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: buildQuickBooksHeaders(data.accessToken), body: formData, diff --git a/apps/sim/lib/internal/quickbooks/provider-operations.ts b/apps/sim/lib/internal/quickbooks/provider-operations.ts index c6a2a644904..1e3b2341367 100644 --- a/apps/sim/lib/internal/quickbooks/provider-operations.ts +++ b/apps/sim/lib/internal/quickbooks/provider-operations.ts @@ -1,4 +1,5 @@ import { filterUndefined } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { buildQuickBooksCreateBillPaymentBody, buildQuickBooksUpdateBillBody, @@ -178,7 +179,7 @@ export async function executeQuickBooksCreateBillPaymentOperation( const paymentAccountId = params.paymentAccountId.trim() if (!paymentAccountId) throw new Error('paymentAccountId is required') - const accountResponse = await fetch( + const accountResponse = await outboundFetch( buildQuickBooksEntityUrl(params, 'account', paymentAccountId), { method: 'GET', @@ -197,7 +198,7 @@ export async function executeQuickBooksCreateBillPaymentOperation( assertCompatiblePaymentAccount(account, params.paymentType, paymentAccountId) signal?.throwIfAborted() - const paymentResponse = await fetch( + const paymentResponse = await outboundFetch( addQuickBooksRequestId(buildQuickBooksEntityUrl(params, 'billpayment'), params.requestId), { method: 'POST', @@ -273,7 +274,7 @@ export async function executeQuickBooksUpdateRefundReceiptOperation( params: QuickBooksUpdateRefundReceiptParams, signal?: AbortSignal ) { - const response = await fetch(buildQuickBooksEntityUrl(params, 'refundreceipt'), { + const response = await outboundFetch(buildQuickBooksEntityUrl(params, 'refundreceipt'), { method: 'POST', headers: getQuickBooksToolHeaders(params.accessToken, 'application/json'), body: JSON.stringify(buildQuickBooksUpdateSalesDocumentBody(params)), @@ -301,7 +302,7 @@ export async function executeQuickBooksUpdateCustomerPaymentOperation( const syncToken = params.syncToken?.trim() if (!syncToken) throw new Error('syncToken is required') - const readResponse = await fetch(buildQuickBooksEntityUrl(params, 'payment', paymentId), { + const readResponse = await outboundFetch(buildQuickBooksEntityUrl(params, 'payment', paymentId), { method: 'GET', headers: getQuickBooksToolHeaders(params.accessToken), signal, @@ -335,7 +336,7 @@ export async function executeQuickBooksUpdateCustomerPaymentOperation( paymentId, syncToken ) - const updateResponse = await fetch(buildQuickBooksEntityUrl(params, 'payment'), { + const updateResponse = await outboundFetch(buildQuickBooksEntityUrl(params, 'payment'), { method: 'POST', headers: getQuickBooksToolHeaders(params.accessToken, 'application/json'), body: JSON.stringify(fullBody), diff --git a/apps/sim/lib/internal/quiver/client.ts b/apps/sim/lib/internal/quiver/client.ts index acb7d512020..d52223e0750 100644 --- a/apps/sim/lib/internal/quiver/client.ts +++ b/apps/sim/lib/internal/quiver/client.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit, @@ -19,7 +20,7 @@ export async function requestQuiverSvg( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${QUIVER_API_BASE_URL}/${path}`, { + const response = await outboundFetch(`${QUIVER_API_BASE_URL}/${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/resend/client.ts b/apps/sim/lib/internal/resend/client.ts index 2527689b487..4ab78b519a2 100644 --- a/apps/sim/lib/internal/resend/client.ts +++ b/apps/sim/lib/internal/resend/client.ts @@ -1,4 +1,5 @@ import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ResendOperationError } from '@/lib/internal/resend/errors' @@ -20,7 +21,7 @@ export async function sendResendEmail( signal?: AbortSignal ): Promise> { signal?.throwIfAborted() - const response = await fetch('https://api.resend.com/emails', { + const response = await outboundFetch('https://api.resend.com/emails', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/internal/sailpoint/client.test.ts b/apps/sim/lib/internal/sailpoint/client.test.ts index a7b3adac435..68fdf122f0e 100644 --- a/apps/sim/lib/internal/sailpoint/client.test.ts +++ b/apps/sim/lib/internal/sailpoint/client.test.ts @@ -58,9 +58,10 @@ describe('SailPoint client', () => { const credentials = credentialsFor('single-flight') const first = getSailPointAccessToken(credentials) const second = getSailPointAccessToken(credentials) - expect(mockFetch).toHaveBeenCalledTimes(1) + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(1)) release?.(tokenResponse('shared')) await expect(Promise.all([first, second])).resolves.toEqual(['shared', 'shared']) + expect(mockFetch).toHaveBeenCalledTimes(1) }) it('lets one token waiter abort without cancelling the shared exchange', async () => { @@ -98,7 +99,7 @@ describe('SailPoint client', () => { await expect(getSailPointAccessToken(credentialsFor('in-flight-overflow'))).rejects.toThrow( 'Too many concurrent SailPoint token exchanges' ) - expect(mockFetch).toHaveBeenCalledTimes(100) + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(100)) for (const release of releases) release(tokenResponse('token')) await Promise.all(pending) diff --git a/apps/sim/lib/internal/sailpoint/client.ts b/apps/sim/lib/internal/sailpoint/client.ts index ec0840142c7..56f4e4c98e7 100644 --- a/apps/sim/lib/internal/sailpoint/client.ts +++ b/apps/sim/lib/internal/sailpoint/client.ts @@ -3,6 +3,7 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { LRUCache } from 'lru-cache' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { consumeOrCancelBody, @@ -205,7 +206,7 @@ export async function requestSailPointToken( while (true) { signal?.throwIfAborted() - const response = await fetch(tokenUrl, { + const response = await outboundFetch(tokenUrl, { method: 'POST', headers: { Accept: 'application/json', @@ -317,7 +318,7 @@ export async function sailpointFetch( headers.set('Authorization', `Bearer ${token}`) if (!headers.has('Accept')) headers.set('Accept', 'application/json') - const response = await fetch(url, { + const response = await outboundFetch(url, { ...init, cache: 'no-store', headers, diff --git a/apps/sim/lib/internal/sendgrid/client.ts b/apps/sim/lib/internal/sendgrid/client.ts index 7075e1d41a2..f910b446485 100644 --- a/apps/sim/lib/internal/sendgrid/client.ts +++ b/apps/sim/lib/internal/sendgrid/client.ts @@ -1,4 +1,5 @@ import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { SendGridOperationError } from '@/lib/internal/sendgrid/errors' import { SENDGRID_REGIONS } from '@/tools/sendgrid/utils' @@ -26,7 +27,7 @@ export async function sendSendGridMail( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${SENDGRID_REGIONS.apiBase(region)}/mail/send`, { + const response = await outboundFetch(`${SENDGRID_REGIONS.apiBase(region)}/mail/send`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/internal/slack/client.ts b/apps/sim/lib/internal/slack/client.ts index 23700aec6f8..95a2226067a 100644 --- a/apps/sim/lib/internal/slack/client.ts +++ b/apps/sim/lib/internal/slack/client.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' const MAX_SLACK_JSON_BYTES = 2 * 1024 * 1024 @@ -72,7 +73,7 @@ export async function requestSlackApi({ } const isForm = body instanceof URLSearchParams - const response = await fetch(url, { + const response = await outboundFetch(url, { method: httpMethod, headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/internal/slack/oauth.ts b/apps/sim/lib/internal/slack/oauth.ts index 8d1f1dcccb1..bcdfb2b95d9 100644 --- a/apps/sim/lib/internal/slack/oauth.ts +++ b/apps/sim/lib/internal/slack/oauth.ts @@ -1,5 +1,6 @@ import { Buffer } from 'node:buffer' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { requestSlackApi } from '@/lib/internal/slack/client' @@ -25,7 +26,7 @@ export async function exchangeSlackBotAuthorization(input: { code: string redirectUri?: string }) { - const response = await fetch('https://slack.com/api/oauth.v2.access', { + const response = await outboundFetch('https://slack.com/api/oauth.v2.access', { method: 'POST', headers: { Authorization: `Basic ${Buffer.from(`${input.clientId}:${input.clientSecret}`).toString('base64')}`, diff --git a/apps/sim/lib/internal/square/operations.ts b/apps/sim/lib/internal/square/operations.ts index 46fad0ec68b..03f6c84faac 100644 --- a/apps/sim/lib/internal/square/operations.ts +++ b/apps/sim/lib/internal/square/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -71,7 +72,7 @@ export async function executeSquareCreateCatalogImage( input.fileName || userFile.name ) - const response = await fetch( + const response = await outboundFetch( `${SQUARE_ENVIRONMENTS.apiBase(input.environment)}/v2/catalog/images`, { method: 'POST', diff --git a/apps/sim/lib/internal/stagehand/operations.ts b/apps/sim/lib/internal/stagehand/operations.ts index 7cafab86303..f8ccd994121 100644 --- a/apps/sim/lib/internal/stagehand/operations.ts +++ b/apps/sim/lib/internal/stagehand/operations.ts @@ -98,6 +98,7 @@ async function getLiveViewUrl(sessionId: string, signal?: AbortSignal): Promise< const browserbaseApiKey = getBrowserbaseApiKey() if (!browserbaseApiKey) return null try { + // egress-direct: Sim's own Browserbase project, like the Stagehand session itself const response = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/debug`, { method: 'GET', headers: { 'X-BB-API-Key': browserbaseApiKey }, diff --git a/apps/sim/lib/internal/sts/client.ts b/apps/sim/lib/internal/sts/client.ts index aaf30eb5a6d..a3f0a30cd68 100644 --- a/apps/sim/lib/internal/sts/client.ts +++ b/apps/sim/lib/internal/sts/client.ts @@ -9,7 +9,7 @@ import { STSClient, type Tag, } from '@aws-sdk/client-sts' -import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' import { awsClientConfig } from '@/lib/internal/aws/credentials' import type { STSConnectionConfig } from '@/tools/sts/types' @@ -35,7 +35,7 @@ export function createUnauthenticatedSTSClient(region: string): STSClient { return new STSClient({ region, credentials: { accessKeyId: 'anonymous', secretAccessKey: 'anonymous' }, - requestHandler: awsRequestHandler(), + requestHandler: organizationAwsRequestHandler(), }) } diff --git a/apps/sim/lib/internal/stt/operations.ts b/apps/sim/lib/internal/stt/operations.ts index a68c15750ea..0f7f297fb36 100644 --- a/apps/sim/lib/internal/stt/operations.ts +++ b/apps/sim/lib/internal/stt/operations.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { secureFetchWithPinnedIP, @@ -492,7 +493,7 @@ async function transcribeWithWhisper( } const endpoint = translate ? 'translations' : 'transcriptions' - const response = await fetch(`https://api.openai.com/v1/audio/${endpoint}`, { + const response = await outboundFetch(`https://api.openai.com/v1/audio/${endpoint}`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -562,7 +563,7 @@ async function transcribeWithDeepgram( params.append('diarize', 'true') } - const response = await fetch(`https://api.deepgram.com/v1/listen?${params.toString()}`, { + const response = await outboundFetch(`https://api.deepgram.com/v1/listen?${params.toString()}`, { method: 'POST', headers: { Authorization: `Token ${apiKey}`, @@ -645,7 +646,7 @@ async function transcribeWithElevenLabs( formData.append('timestamps_granularity', 'word') } - const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', { + const response = await outboundFetch('https://api.elevenlabs.io/v1/speech-to-text', { method: 'POST', headers: { 'xi-api-key': apiKey, @@ -705,7 +706,7 @@ async function transcribeWithAssemblyAI( entities?: Record[] summary?: string }> { - const uploadResponse = await fetch('https://api.assemblyai.com/v2/upload', { + const uploadResponse = await outboundFetch('https://api.assemblyai.com/v2/upload', { method: 'POST', headers: { authorization: apiKey, @@ -764,7 +765,7 @@ async function transcribeWithAssemblyAI( transcriptRequest.summary_type = 'bullets' } - const transcriptResponse = await fetch('https://api.assemblyai.com/v2/transcript', { + const transcriptResponse = await outboundFetch('https://api.assemblyai.com/v2/transcript', { method: 'POST', headers: { authorization: apiKey, @@ -788,7 +789,7 @@ async function transcribeWithAssemblyAI( const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) while (attempts < maxAttempts) { - const statusResponse = await fetch(`https://api.assemblyai.com/v2/transcript/${id}`, { + const statusResponse = await outboundFetch(`https://api.assemblyai.com/v2/transcript/${id}`, { headers: { authorization: apiKey, }, @@ -902,7 +903,7 @@ async function transcribeWithGemini( ], } - const response = await fetch( + const response = await outboundFetch( `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:generateContent?key=${apiKey}`, { method: 'POST', diff --git a/apps/sim/lib/internal/supabase/operations.ts b/apps/sim/lib/internal/supabase/operations.ts index 05f95de0b83..4f4e80d8dbf 100644 --- a/apps/sim/lib/internal/supabase/operations.ts +++ b/apps/sim/lib/internal/supabase/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateSupabaseProjectId } from '@/lib/core/security/input-validation' import { assertKnownSizeWithinLimit, @@ -136,7 +137,7 @@ export async function executeSupabaseStorageUpload( } if (input.upsert) headers['x-upsert'] = 'true' - const response = await fetch(`${baseUrl}/${encodedBucket}/${encodedPath}`, { + const response = await outboundFetch(`${baseUrl}/${encodedBucket}/${encodedPath}`, { method: 'POST', headers, body: new Uint8Array(upload.body), diff --git a/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts b/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts index f2f6734d8d4..97eba56c257 100644 --- a/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts +++ b/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { filterUndefined } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { SupabaseStorageUpdateBucketParams, @@ -43,7 +44,7 @@ export const executeStorageUpdateBucketOperation: InternalToolOperationImplement }) if (Object.keys(payload).length === 0) { - const currentResponse = await fetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { + const currentResponse = await outboundFetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { method: 'GET', headers, redirect: 'error', @@ -65,7 +66,7 @@ export const executeStorageUpdateBucketOperation: InternalToolOperationImplement } } - const updateResponse = await fetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { + const updateResponse = await outboundFetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { method: 'PUT', headers, body: JSON.stringify(payload), diff --git a/apps/sim/lib/internal/telegram/operations.ts b/apps/sim/lib/internal/telegram/operations.ts index 4fea2e14764..7caf93653ba 100644 --- a/apps/sim/lib/internal/telegram/operations.ts +++ b/apps/sim/lib/internal/telegram/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { TelegramOperationError } from '@/lib/internal/telegram/errors' import { @@ -100,7 +101,7 @@ export async function sendTelegramDocument( form.append('parse_mode', 'HTML') } - const response = await fetch( + const response = await outboundFetch( `https://api.telegram.org/bot${encodeURIComponent(input.botToken)}/sendDocument`, { method: 'POST', body: form, signal: context.signal } ) diff --git a/apps/sim/lib/internal/tiktok/operations.ts b/apps/sim/lib/internal/tiktok/operations.ts index 0ecdd7eb1a1..44a49389807 100644 --- a/apps/sim/lib/internal/tiktok/operations.ts +++ b/apps/sim/lib/internal/tiktok/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import type { TikTokUploadVideoDraftInput } from '@/lib/internal/tiktok/schema' import { @@ -77,7 +78,7 @@ export async function executeTikTokUploadVideoDraft( if (videoSize === 0) return failureResponse('The video file is empty.', 400) const { chunkSize, totalChunkCount } = computeTikTokChunkPlan(videoSize) - const initResponse = await fetch( + const initResponse = await outboundFetch( 'https://open.tiktokapis.com/v2/post/publish/inbox/video/init/', { method: 'POST', diff --git a/apps/sim/lib/internal/tiktok/upload.ts b/apps/sim/lib/internal/tiktok/upload.ts index 4fc1c72cb23..b3da1d9525f 100644 --- a/apps/sim/lib/internal/tiktok/upload.ts +++ b/apps/sim/lib/internal/tiktok/upload.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { assertKnownSizeWithinLimit, readResponseTextWithLimit, @@ -235,7 +236,7 @@ async function uploadChunk(options: { for (let attempt = 1; attempt <= MAX_UPLOAD_ATTEMPTS; attempt++) { throwIfAborted(options.signal) - const response = await fetch(options.uploadUrl, { + const response = await outboundFetch(options.uploadUrl, { method: 'PUT', headers: { 'Content-Type': options.mimeType, diff --git a/apps/sim/lib/internal/tts/client.ts b/apps/sim/lib/internal/tts/client.ts index c88a9b4b23c..f5240a8e160 100644 --- a/apps/sim/lib/internal/tts/client.ts +++ b/apps/sim/lib/internal/tts/client.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { assertKnownSizeWithinLimit, @@ -51,14 +52,14 @@ async function providerFetch( timeoutMs?: number ): Promise { signal?.throwIfAborted() - if (!timeoutMs) return fetch(input, { ...init, signal }) + if (!timeoutMs) return outboundFetch(input, { ...init, signal }) const controller = new AbortController() const timeout = setTimeout(() => controller.abort(new Error('TTS request timed out')), timeoutMs) const abort = () => controller.abort(signal?.reason ?? new Error('Request aborted')) signal?.addEventListener('abort', abort, { once: true }) try { - return await fetch(input, { ...init, signal: controller.signal }) + return await outboundFetch(input, { ...init, signal: controller.signal }) } finally { clearTimeout(timeout) signal?.removeEventListener('abort', abort) diff --git a/apps/sim/lib/internal/uptimerobot/client.ts b/apps/sim/lib/internal/uptimerobot/client.ts index cfa44d61505..9f299281e89 100644 --- a/apps/sim/lib/internal/uptimerobot/client.ts +++ b/apps/sim/lib/internal/uptimerobot/client.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { UptimeRobotOperationError } from '@/lib/internal/uptimerobot/errors' @@ -24,7 +25,7 @@ export async function requestUptimeRobotPsp(args: { }): Promise { const { apiKey, method, path, form, signal } = args signal?.throwIfAborted() - const response = await fetch(`${UPTIMEROBOT_API_BASE}${path}`, { + const response = await outboundFetch(`${UPTIMEROBOT_API_BASE}${path}`, { method, headers: { Authorization: `Bearer ${apiKey}`, Accept: 'application/json' }, body: form, diff --git a/apps/sim/lib/internal/vanta/operations.ts b/apps/sim/lib/internal/vanta/operations.ts index 3aa546769a7..67de8842f49 100644 --- a/apps/sim/lib/internal/vanta/operations.ts +++ b/apps/sim/lib/internal/vanta/operations.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, @@ -415,7 +416,7 @@ export async function executeVantaQuery( const response = await fetchVantaWithAuth( params, (accessToken) => - fetch(apiRequest.url, { + outboundFetch(apiRequest.url, { method: apiRequest.method, headers: { Accept: 'application/json', @@ -471,7 +472,7 @@ export async function executeVantaUploadDocumentFile( const response = await fetchVantaWithAuth( input, (accessToken) => - fetch(uploadUrl, { + outboundFetch(uploadUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}` }, body: formData, @@ -508,7 +509,7 @@ export async function executeVantaDownloadDocumentFile( const response = await fetchVantaWithAuth( input, (accessToken) => - fetch(mediaUrl, { + outboundFetch(mediaUrl, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, cache: 'no-store', diff --git a/apps/sim/lib/internal/video/client.ts b/apps/sim/lib/internal/video/client.ts index 1972d909937..be728beb479 100644 --- a/apps/sim/lib/internal/video/client.ts +++ b/apps/sim/lib/internal/video/client.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -179,7 +180,7 @@ async function generateWithDirectProvider( return value } let data = await readProviderResponse( - await fetch(prepared.url, { + await outboundFetch(prepared.url, { method: 'POST', redirect: 'error', headers, @@ -232,7 +233,7 @@ async function generateWithDirectProvider( const fileId = getStringProperty(task, 'file_id') if (!fileId) throw new Error('MiniMax response missing file_id') const fileData = await readProviderResponse( - await fetch( + await outboundFetch( `https://api.minimax.io/v1/files/retrieve?file_id=${encodeURIComponent(fileId)}`, { headers, redirect: 'error', signal } ) @@ -274,7 +275,7 @@ async function generateWithDirectProvider( const metadataUrl = `https://generativelanguage.googleapis.com/v1beta/files/${fileId}` for (let fileAttempt = 0; ; fileAttempt++) { const file = await readProviderResponse( - await fetch(metadataUrl, { headers, redirect: 'error', signal }) + await outboundFetch(metadataUrl, { headers, redirect: 'error', signal }) ) const state = getStringProperty(file, 'state') if (state === 'ACTIVE') break @@ -359,7 +360,9 @@ async function generateWithDirectProvider( throw new Error('Google synchronous video generation did not complete') if (attempt === maxAttempts) break await waitForProvider(context) - data = await readProviderResponse(await fetch(pollUrl, { headers, redirect: 'error', signal })) + data = await readProviderResponse( + await outboundFetch(pollUrl, { headers, redirect: 'error', signal }) + ) } throw new Error(`${input.provider} video generation timed out`) } @@ -410,7 +413,7 @@ async function generateWithFalAI( ) signal?.throwIfAborted() } - const createResponse = await fetch(`https://queue.fal.run/${endpoint}`, { + const createResponse = await outboundFetch(`https://queue.fal.run/${endpoint}`, { method: 'POST', redirect: 'error', headers: { @@ -456,7 +459,7 @@ async function generateWithFalAI( while (attempts < maxAttempts) { await waitForProvider(context) - const statusResponse = await fetch(statusUrl, { + const statusResponse = await outboundFetch(statusUrl, { redirect: 'error', headers: { Authorization: `Key ${apiKey}`, @@ -486,11 +489,14 @@ async function generateWithFalAI( onCost: context.onFalCost, signal, fetchResponse: (resultSignal) => - fetch(falQueueUrl(getStringProperty(statusData, 'response_url') || responseUrl), { - redirect: 'error', - headers: { Authorization: `Key ${apiKey}` }, - signal: resultSignal, - }), + outboundFetch( + falQueueUrl(getStringProperty(statusData, 'response_url') || responseUrl), + { + redirect: 'error', + headers: { Authorization: `Key ${apiKey}` }, + signal: resultSignal, + } + ), }) if (!isRecordLike(resultData)) { diff --git a/apps/sim/lib/internal/vision/client.ts b/apps/sim/lib/internal/vision/client.ts index 0a09ae97322..8d7dcb53330 100644 --- a/apps/sim/lib/internal/vision/client.ts +++ b/apps/sim/lib/internal/vision/client.ts @@ -1,6 +1,7 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { MAX_JSON_API_RESPONSE_BYTES, @@ -121,7 +122,7 @@ async function analyzeWithGemini( signal?.throwIfAborted() const base64Payload = await fetchGeminiImage(input, signal) const { mediaType, base64Data } = parseDataImage(base64Payload) - const ai = new GoogleGenAI({ apiKey: input.apiKey }) + const ai = new GoogleGenAI({ apiKey: input.apiKey, httpOptions: { fetch: outboundFetch } }) const response = await ai.models.generateContent({ model: input.model, contents: [ @@ -197,7 +198,7 @@ async function analyzeWithHttpProvider( } signal?.throwIfAborted() - const response = await fetch(apiUrl, { + const response = await outboundFetch(apiUrl, { method: 'POST', headers, body: JSON.stringify(isClaude ? anthropicRequest(input) : openAiRequest(input)), diff --git a/apps/sim/lib/internal/whatsapp/operations.ts b/apps/sim/lib/internal/whatsapp/operations.ts index e9f68bc86e7..e9c0d87af0e 100644 --- a/apps/sim/lib/internal/whatsapp/operations.ts +++ b/apps/sim/lib/internal/whatsapp/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -149,7 +150,7 @@ export async function executeWhatsAppSendMedia( caption: input.caption ?? undefined, filename, }) - const response = await fetch(buildMessagesUrl(input.phoneNumberId), { + const response = await outboundFetch(buildMessagesUrl(input.phoneNumberId), { method: 'POST', headers: buildAuthHeaders(input.accessToken), body: JSON.stringify(messageBody), @@ -183,10 +184,13 @@ export async function executeWhatsAppGetMedia( context.signal?.throwIfAborted() const authorization = `Bearer ${input.accessToken.trim()}` try { - const metadataResponse = await fetch(buildMediaUrl(input.mediaId, input.phoneNumberId), { - headers: { Authorization: authorization }, - signal: context.signal, - }) + const metadataResponse = await outboundFetch( + buildMediaUrl(input.mediaId, input.phoneNumberId), + { + headers: { Authorization: authorization }, + signal: context.signal, + } + ) const metadataBody = await readWhatsAppGraphResponse( metadataResponse, `WhatsApp media ${input.mediaId} metadata`, diff --git a/apps/sim/lib/internal/whatsapp/upload.ts b/apps/sim/lib/internal/whatsapp/upload.ts index 9c9f733bb38..580036c13dd 100644 --- a/apps/sim/lib/internal/whatsapp/upload.ts +++ b/apps/sim/lib/internal/whatsapp/upload.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readWhatsAppGraphResponse } from '@/lib/internal/whatsapp/client' import type { RawFileInput } from '@/lib/uploads/utils/file' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file' @@ -93,7 +94,7 @@ export async function uploadWhatsAppMedia({ userFile.name ) - const response = await fetch(buildMediaUploadUrl(phoneNumberId), { + const response = await outboundFetch(buildMediaUploadUrl(phoneNumberId), { method: 'POST', headers: { Authorization: `Bearer ${accessToken.trim()}` }, body: formData, diff --git a/apps/sim/lib/internal/wordpress/operations.ts b/apps/sim/lib/internal/wordpress/operations.ts index 06559b77abc..8f2840bed43 100644 --- a/apps/sim/lib/internal/wordpress/operations.ts +++ b/apps/sim/lib/internal/wordpress/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -96,7 +97,7 @@ export async function uploadWordPressMedia( if (input.altText) formData.append('alt_text', input.altText) if (input.description) formData.append('description', input.description) - const response = await fetch(`${WORDPRESS_COM_API_BASE}/${input.siteId}/media`, { + const response = await outboundFetch(`${WORDPRESS_COM_API_BASE}/${input.siteId}/media`, { method: 'POST', headers: { Authorization: `Bearer ${input.accessToken}` }, body: formData, diff --git a/apps/sim/lib/internal/workday/client.ts b/apps/sim/lib/internal/workday/client.ts index 2e578c8f658..43021343cde 100644 --- a/apps/sim/lib/internal/workday/client.ts +++ b/apps/sim/lib/internal/workday/client.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateWorkdayTenantUrl } from '@/lib/core/security/input-validation' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' @@ -566,7 +567,7 @@ async function callOperation( signal?.throwIfAborted() const envelope = buildEnvelope(operation, args, username, password) - const response = await fetch(endpoint, { + const response = await outboundFetch(endpoint, { method: 'POST', headers: { 'Content-Type': 'text/xml; charset=utf-8', diff --git a/apps/sim/lib/knowledge/access/confluence-site.test.ts b/apps/sim/lib/knowledge/access/confluence-site.test.ts index fc51b7ab720..fe5f1cc6059 100644 --- a/apps/sim/lib/knowledge/access/confluence-site.test.ts +++ b/apps/sim/lib/knowledge/access/confluence-site.test.ts @@ -4,7 +4,9 @@ import { credentialsManagedOauthMockFns, } from '@sim/testing/mocks/credentials-managed-oauth.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { CONFLUENCE_READ_CREDENTIAL_ALTERNATIVES, CONFLUENCE_READ_RESPONSE_MAX_BYTES, @@ -14,6 +16,7 @@ import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' vi.mock('@/lib/credentials/managed-oauth', () => credentialsManagedOauthMock) vi.mock('@/lib/core/security/encryption', () => encryptionMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) const mocks = { token: credentialsManagedOauthMockFns.mockResolveManagedOAuthToken, fetch: vi.fn() } @@ -90,6 +93,31 @@ describe('current Confluence site access', () => { expect(JSON.stringify(mocks.fetch.mock.calls)).not.toContain(binding.apiToken) }) + describe('with organization egress routing', () => { + beforeEach(() => networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true)) + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + + it("reaches Confluence from the knowledge base owner's route when the caller set none", async () => { + queueSources() + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([grant]) + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + }) + + it('fails the admission on an unavailable route instead of denying the site', async () => { + queueSources() + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + await expect(resolveConfluenceSiteReadGrants(input)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + }) + it('denies newly revoked site access even if a direct space grant remains indexed', async () => { queueSources() await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([grant]) diff --git a/apps/sim/lib/knowledge/access/confluence-site.ts b/apps/sim/lib/knowledge/access/confluence-site.ts index dbb0b65a08d..4bfc224422e 100644 --- a/apps/sim/lib/knowledge/access/confluence-site.ts +++ b/apps/sim/lib/knowledge/access/confluence-site.ts @@ -4,6 +4,9 @@ import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' @@ -91,6 +94,11 @@ async function readSiteBinding(key: string): Promise { return { cloudId: value.cloudId, domain: value.domain } } +/** An unavailable egress route is Sim's outage, so it fails the admission instead of denying a source. */ +function isUnavailable(error: unknown): boolean { + return asOrchestrationError(error)?.code === 'unavailable' +} + /** Space grants do not prove that the reader still has Confluence's site-level Can use permission. */ async function verifySite( binding: SiteBinding, @@ -99,7 +107,7 @@ async function verifySite( signal: AbortSignal ): Promise { signal.throwIfAborted() - const response = await fetch( + const response = await scopedOutboundFetch( `https://api.atlassian.com/ex/confluence/${binding.cloudId}/wiki/rest/api/user/current`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, @@ -200,8 +208,12 @@ export async function resolveConfluenceSiteReadGrants(input: { ) ) .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) - const timeout = AbortSignal.timeout(CONFLUENCE_READ_TIMEOUT_MS) - const admissionSignal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout + const outage = new AbortController() + const admissionSignal = AbortSignal.any([ + outage.signal, + AbortSignal.timeout(CONFLUENCE_READ_TIMEOUT_MS), + ...(input.signal ? [input.signal] : []), + ]) const bindings = new Map() for (const content of credentials) { if (!content.key || admissionSignal.aborted) continue @@ -309,7 +321,15 @@ export async function resolveConfluenceSiteReadGrants(input: { .slice(offset, offset + CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY) .map((credentialId) => attempt(binding, identity, credentialId, cancellation.signal)) ) - } catch { + } catch (error) { + const unavailable = + error instanceof AggregateError + ? error.errors.find((attemptError) => isUnavailable(attemptError)) + : undefined + if (unavailable) { + outage.abort(unavailable) + throw unavailable + } if (requests >= MAX_KNOWLEDGE_ACCESS_CANDIDATES) return undefined } finally { cancellation.abort() @@ -332,15 +352,17 @@ export async function resolveConfluenceSiteReadGrants(input: { } } } - await Promise.all( - Array.from( - { - length: Math.min( - proofCount, - CONFLUENCE_READ_CONCURRENCY / CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY - ), - }, - worker + await withResourceOutboundScope(resourceScopeFields(input.scope), () => + Promise.all( + Array.from( + { + length: Math.min( + proofCount, + CONFLUENCE_READ_CONCURRENCY / CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY + ), + }, + worker + ) ) ) input.signal?.throwIfAborted() diff --git a/apps/sim/lib/knowledge/access/github-installation.test.ts b/apps/sim/lib/knowledge/access/github-installation.test.ts index dd8ef7fb4df..289deb1f2ae 100644 --- a/apps/sim/lib/knowledge/access/github-installation.test.ts +++ b/apps/sim/lib/knowledge/access/github-installation.test.ts @@ -8,13 +8,16 @@ import { githubInstallationMock, githubInstallationMockFns, } from '@sim/testing/mocks/github-installation.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { resolveGitHubInstallationReadGrants } from '@/lib/knowledge/access/github-installation' import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' vi.mock('@/lib/credentials/managed-oauth', () => credentialsManagedOauthMock) vi.mock('@/lib/core/security/encryption', () => encryptionMock) vi.mock('@/lib/oauth/github-installation', () => githubInstallationMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) const mocks = { token: credentialsManagedOauthMockFns.mockResolveManagedOAuthToken, @@ -192,6 +195,34 @@ describe('live GitHub installation reader access', () => { expect(mocks.token).toHaveBeenCalledTimes(1) }) + describe('with organization egress routing', () => { + beforeEach(() => networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true)) + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + + it("reaches GitHub from the knowledge base owner's route when the caller set none", async () => { + queueSources() + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([grant]) + expect(networkConfigMockFns.mockResolveOutboundRoute.mock.calls).toEqual([ + ['org-1'], + ['org-1'], + ]) + }) + + it('fails the admission on an unavailable route instead of denying the source', async () => { + queueSources() + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + await expect(resolveGitHubInstallationReadGrants(input)).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + }) + it('bounds one candidate batch without enumerating all organization sources', async () => { await expect( resolveGitHubInstallationReadGrants({ diff --git a/apps/sim/lib/knowledge/access/github-installation.ts b/apps/sim/lib/knowledge/access/github-installation.ts index ac182449158..4488c1b73a4 100644 --- a/apps/sim/lib/knowledge/access/github-installation.ts +++ b/apps/sim/lib/knowledge/access/github-installation.ts @@ -9,6 +9,9 @@ import { import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' @@ -71,7 +74,7 @@ function positiveId(value: unknown): string | null { async function readGitHubJson(path: string, accessToken: string, signal: AbortSignal) { signal.throwIfAborted() - const response = await fetch(`https://api.github.com${path}`, { + const response = await scopedOutboundFetch(`https://api.github.com${path}`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/vnd.github+json', @@ -138,7 +141,9 @@ async function verifyRepository( /** * Proves current reader access before any installation-backed indexed content is selected. * The caller supplies credentials already bound to the verified current member. All positive - * evidence and token reuse are local to this admission; failures grant nothing for that source. + * evidence and token reuse are local to this admission; failures grant nothing for that source, + * except an unavailable egress route, which is Sim's outage and fails the admission. GitHub is + * reached from the knowledge base owner's egress route whether or not the caller set one. */ export async function resolveGitHubInstallationReadGrants(input: { scope: ResourceScope @@ -223,8 +228,12 @@ export async function resolveGitHubInstallationReadGrants(input: { ) .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) const contentById = new Map(credentials.map((entry) => [entry.id, entry])) - const timeout = AbortSignal.timeout(GITHUB_READ_TIMEOUT_MS) - const admissionSignal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout + const outage = new AbortController() + const admissionSignal = AbortSignal.any([ + outage.signal, + AbortSignal.timeout(GITHUB_READ_TIMEOUT_MS), + ...(input.signal ? [input.signal] : []), + ]) const installations = new Map>() const tokens = new Map>() const proofs = new Map>() @@ -300,15 +309,19 @@ export async function resolveGitHubInstallationReadGrants(input: { readerSubjectToken: source.subjectToken, repositoryId: source.repositoryId, }) - } catch { + } catch (error) { + if (asOrchestrationError(error)?.code === 'unavailable') { + outage.abort(error) + throw error + } logger.warn('GitHub did not confirm current Search access', { connectorId: source.connectorId, }) } } } - await Promise.all( - Array.from({ length: Math.min(GITHUB_READ_CONCURRENCY, sources.length) }, worker) + await withResourceOutboundScope(resourceScopeFields(input.scope), () => + Promise.all(Array.from({ length: Math.min(GITHUB_READ_CONCURRENCY, sources.length) }, worker)) ) input.signal?.throwIfAborted() return [...grants.values()] diff --git a/apps/sim/lib/knowledge/application/slack-search/authorization.ts b/apps/sim/lib/knowledge/application/slack-search/authorization.ts index e322b5d0299..352c9480708 100644 --- a/apps/sim/lib/knowledge/application/slack-search/authorization.ts +++ b/apps/sim/lib/knowledge/application/slack-search/authorization.ts @@ -1,4 +1,7 @@ import type { Principal, SlackInstallationPrincipal } from '@sim/auth/principal' +import type { ApplicationOperation, OperationUseCase } from '@/lib/core/application/operation' +import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability' import { @@ -25,6 +28,26 @@ export function requireSlackInstallationPrincipal( throw new OrchestrationError('forbidden', 'Slack delivery has expired') } +/** + * Runs a Slack installation use case in the outbound scope of the organization that owns the + * principal's installation, so its Slack API calls leave through that organization's egress. + * An unknown credential runs unscoped; the use case's own authorization rejects it. + */ +export function withSlackSearchOutboundScope( + useCase: OperationUseCase +): OperationUseCase { + return { + ...useCase, + async execute(args) { + requireSlackInstallationPrincipal(args.principal) + if (!isOutboundRoutingEnabled()) return useCase.execute(args) + const installation = await findSlackSearchInstallation(args.principal.credentialId) + if (!installation) return useCase.execute(args) + return runWithOutboundOrganization(installation.organizationId, () => useCase.execute(args)) + }, + } +} + /** Rechecks the binding and exact authenticated secret on every lifecycle boundary. */ export async function authorizeSlackSearchInstallation( principal: Principal, diff --git a/apps/sim/lib/knowledge/application/slack-search/home.test.ts b/apps/sim/lib/knowledge/application/slack-search/home.test.ts index af290e206aa..6fb36abd101 100644 --- a/apps/sim/lib/knowledge/application/slack-search/home.test.ts +++ b/apps/sim/lib/knowledge/application/slack-search/home.test.ts @@ -1,10 +1,12 @@ import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' import { asyncJobsMock, asyncJobsMockFns } from '@sim/testing/mocks/async-jobs.mock' import { knowledgeAvailabilityMock } from '@sim/testing/mocks/knowledge-availability.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' import { urlsMockFns } from '@sim/testing/mocks/urls.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' const hoisted = vi.hoisted(() => ({ installation: vi.fn(), publish: vi.fn() })) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@/lib/core/async-jobs', () => asyncJobsMock) vi.mock('@/lib/internal/slack/client', () => ({ requestSlackApi: hoisted.publish })) vi.mock('@/lib/knowledge/application/slack-search/repository', () => ({ @@ -13,6 +15,7 @@ vi.mock('@/lib/knowledge/application/slack-search/repository', () => ({ })) vi.mock('@/lib/knowledge/access/availability', () => knowledgeAvailabilityMock) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { publishSlackSearchHome, receiveSlackSearchHome, @@ -66,6 +69,7 @@ function publish(signal = new AbortController().signal) { }) } beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) mocks.installation.mockResolvedValue(installation) mocks.publish.mockResolvedValue({ status: 200, data: { ok: true } }) }) @@ -131,6 +135,15 @@ describe('static Home delivery', () => { expect(view.blocks).toHaveLength(3) expect(view.blocks[2].elements[0].url).toBe('https://sim.test/o/org1/integrations') }) + it("calls Slack from the installation's organization outbound scope", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + mocks.publish.mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + return { status: 200, data: { ok: true } } + }) + await publish() + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org1') + }) it.each([ { enabled: false }, { revision: 'r2' }, diff --git a/apps/sim/lib/knowledge/application/slack-search/home.ts b/apps/sim/lib/knowledge/application/slack-search/home.ts index fdc56d80678..3cec3930ecb 100644 --- a/apps/sim/lib/knowledge/application/slack-search/home.ts +++ b/apps/sim/lib/knowledge/application/slack-search/home.ts @@ -7,6 +7,7 @@ import { requestSlackApi } from '@/lib/internal/slack/client' import { authorizeSlackSearchInstallation, requireSlackInstallationPrincipal, + withSlackSearchOutboundScope, } from '@/lib/knowledge/application/slack-search/authorization' import { organizationRoutes } from '@/lib/navigation/paths' import { @@ -89,7 +90,7 @@ export const publishSlackSearchHome: OperationUseCase< typeof operation, { job: SlackSearchHomeJob; signal: AbortSignal }, void -> = { +> = withSlackSearchOutboundScope({ operation, async execute({ principal, input: { job, signal } }) { requireSlackInstallationPrincipal(principal) @@ -121,4 +122,4 @@ export const publishSlackSearchHome: OperationUseCase< if (response.status !== 200 || response.data.ok !== true) throw new Error('Could not publish the Slack Search Home tab') }, -} +}) diff --git a/apps/sim/lib/knowledge/application/slack-search/onboarding.ts b/apps/sim/lib/knowledge/application/slack-search/onboarding.ts index 84602d24549..ac27128544b 100644 --- a/apps/sim/lib/knowledge/application/slack-search/onboarding.ts +++ b/apps/sim/lib/knowledge/application/slack-search/onboarding.ts @@ -10,6 +10,7 @@ import { type OperationUseCase, } from '@/lib/core/application/operation' import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { postSlackMessage, requestSlackApi, slackString } from '@/lib/internal/slack/client' import { getSlackSearchSender } from '@/lib/internal/slack/search-client' @@ -232,11 +233,13 @@ async function resolveOnboarding(principal: Principal, token: string) { 'forbidden', 'Slack Search is disabled. Ask your administrator to reconnect it.' ) - const sender = await getSlackSearchSender( - context.secret.botToken, - job.message.userId, - job.message.teamId, - AbortSignal.timeout(10_000) + const sender = await runWithOutboundOrganization(context.installation.organizationId, () => + getSlackSearchSender( + context.secret.botToken, + job.message.userId, + job.message.teamId, + AbortSignal.timeout(10_000) + ) ) if (!sender || sender.email !== state.email) return { view: { status: 'identity_conflict' } as OnboardingBlocked } diff --git a/apps/sim/lib/knowledge/application/slack-search/process-message.test.ts b/apps/sim/lib/knowledge/application/slack-search/process-message.test.ts index 9e505352ddf..0e652265f03 100644 --- a/apps/sim/lib/knowledge/application/slack-search/process-message.test.ts +++ b/apps/sim/lib/knowledge/application/slack-search/process-message.test.ts @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ })) vi.mock('@/lib/knowledge/application/slack-search/authorization', () => ({ requireSlackInstallationPrincipal: vi.fn(), + withSlackSearchOutboundScope: (useCase: T) => useCase, authorizeSlackSearchInstallation: mocks.authorize, authorizeSlackSearchRedirect: mocks.redirect, })) diff --git a/apps/sim/lib/knowledge/application/slack-search/process-message.ts b/apps/sim/lib/knowledge/application/slack-search/process-message.ts index 76848dd8729..352226c8922 100644 --- a/apps/sim/lib/knowledge/application/slack-search/process-message.ts +++ b/apps/sim/lib/knowledge/application/slack-search/process-message.ts @@ -7,6 +7,7 @@ import { authorizeSlackSearchInstallation, authorizeSlackSearchRedirect, requireSlackInstallationPrincipal, + withSlackSearchOutboundScope, } from '@/lib/knowledge/application/slack-search/authorization' import { routeSlackSearchMentionToDm } from '@/lib/knowledge/application/slack-search/mention' import { dispatchSlackSearchTurn } from '@/lib/knowledge/application/slack-search/outbox' @@ -98,7 +99,7 @@ export const respondToSlackSearchMessage: OperationUseCase< typeof respondOperation, RespondInput, void -> = { +> = withSlackSearchOutboundScope({ operation: respondOperation, async execute({ principal, input }) { requireSlackInstallationPrincipal(principal) @@ -151,4 +152,4 @@ export const respondToSlackSearchMessage: OperationUseCase< } await runSlackSearchAssistant(principal, { ...input, job }) }, -} +}) diff --git a/apps/sim/lib/knowledge/application/slack-search/stop.test.ts b/apps/sim/lib/knowledge/application/slack-search/stop.test.ts index 835d7403779..f49674cf596 100644 --- a/apps/sim/lib/knowledge/application/slack-search/stop.test.ts +++ b/apps/sim/lib/knowledge/application/slack-search/stop.test.ts @@ -18,6 +18,7 @@ const mocks = vi.hoisted(() => ({ })) vi.mock('@/lib/knowledge/application/slack-search/authorization', () => ({ requireSlackInstallationPrincipal: vi.fn(), + withSlackSearchOutboundScope: (useCase: T) => useCase, authorizeSlackSearchInstallation: mocks.authorize, })) vi.mock('@/lib/internal/slack/search-client', () => ({ getSlackSearchSender: mocks.sender })) diff --git a/apps/sim/lib/knowledge/application/slack-search/stop.ts b/apps/sim/lib/knowledge/application/slack-search/stop.ts index a5284743f30..91572374859 100644 --- a/apps/sim/lib/knowledge/application/slack-search/stop.ts +++ b/apps/sim/lib/knowledge/application/slack-search/stop.ts @@ -9,6 +9,7 @@ import { getSlackSearchSender } from '@/lib/internal/slack/search-client' import { authorizeSlackSearchInstallation, requireSlackInstallationPrincipal, + withSlackSearchOutboundScope, } from '@/lib/knowledge/application/slack-search/authorization' import { resolveSlackSearchChatRecord, @@ -51,7 +52,7 @@ export const stopSlackSearchThread: OperationUseCase< typeof operation, z.infer, void -> = { +> = withSlackSearchOutboundScope({ operation, async execute({ principal, input }) { requireSlackInstallationPrincipal(principal) @@ -168,4 +169,4 @@ export const stopSlackSearchThread: OperationUseCase< AbortSignal.timeout(10_000) ) }, -} +}) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts index a41e08717ee..04c6ec29d2b 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts @@ -4,6 +4,8 @@ import { knowledgeDocumentsServiceMock } from '@sim/testing/mocks/knowledge-docu import { knowledgeMemberAccessMock } from '@sim/testing/mocks/knowledge-member-access.mock' import { DrizzleQueryError } from 'drizzle-orm/errors' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) vi.mock('@/lib/knowledge/documents/service', () => knowledgeDocumentsServiceMock) @@ -273,6 +275,19 @@ describe('member sync engine decisions', () => { }) expect(update.nextMemberSyncAt).not.toBeNull() }) + + it('does not disable a connector one failure from the breaker over an egress outage', async () => { + const update = await resolveMemberSyncFailureUpdate( + new Error('listing failed', { cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE') }), + failure + ) + expect(update).toMatchObject({ + memberSyncStatus: 'error', + lastMemberSyncError: EGRESS_UNAVAILABLE_MESSAGE, + memberSyncConsecutiveFailures: MAX_CONSECUTIVE_FAILURES - 1, + }) + expect(update.nextMemberSyncAt).not.toBeNull() + }) }) describe('nextMemberSyncTime', () => { diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 3e5ffe7088b..aa5a377665f 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -28,6 +28,7 @@ import { type BillingAttributionSnapshot, } from '@/lib/billing/core/attribution' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { EGRESS_UNAVAILABLE_MESSAGE, isUnavailableFailure } from '@/lib/core/orchestration/types' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { @@ -382,8 +383,8 @@ export function memberRunMadeProgress(result: MemberSyncResult): boolean { /** * The connector row a failed members-mode run writes. A deterministic capacity rejection waits for - * an operator, a transient database failure retries without touching the breaker, and anything - * else climbs the ladder toward auto-disable. + * an operator, a transient database failure or an organization egress outage retries without + * touching the breaker, and anything else climbs the ladder toward auto-disable. */ export async function resolveMemberSyncFailureUpdate( error: unknown, @@ -423,6 +424,14 @@ export async function resolveMemberSyncFailureUpdate( }) ) } + if (isUnavailableFailure(error)) { + return buildMemberSyncDatabaseRetryUpdate( + now, + failure.previousFailures, + EGRESS_UNAVAILABLE_MESSAGE, + connectorFailureBackoffMinutes(1) * 60 * 1000 + ) + } return buildMemberSyncFailureUpdate( now, failure.previousFailures, diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index cb218d47599..7e12dc11bf0 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -25,6 +25,8 @@ import { import { generateShortId } from '@sim/utils/id' import { DrizzleQueryError } from 'drizzle-orm/errors' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types' import * as connectorTokens from '@/lib/knowledge/connectors/access-token' import { routeWindowTallies, @@ -820,6 +822,19 @@ describe('executeSync database failures', () => { expect.objectContaining({ status: 'failed', databaseFailureClass: 'capacity' }) ) }) + + it('does not disable a connector one failure from the breaker over an egress outage', async () => { + const { terminal, MAX_CONSECUTIVE_FAILURES } = await failSyncWith( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + expect(terminal).toMatchObject({ + status: 'error', + lastSyncError: EGRESS_UNAVAILABLE_MESSAGE, + consecutiveFailures: MAX_CONSECUTIVE_FAILURES - 1, + }) + expect((terminal?.nextSyncAt as Date).getTime()).toBeGreaterThan(NOW.getTime()) + }) }) describe('previous complete listing evidence', () => { diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 2265263d505..dd709a98f86 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -20,6 +20,7 @@ import { type BillingAttributionSnapshot, } from '@/lib/billing/core/attribution' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { EGRESS_UNAVAILABLE_MESSAGE, isUnavailableFailure } from '@/lib/core/orchestration/types' import { type ResourceScope, resourceScopeFields, @@ -1645,19 +1646,26 @@ export async function executeSync( madeProgress: result.docsAdded + result.docsUpdated + result.docsDeleted > 0, }) ) - : rateLimited - ? buildSyncRateLimitUpdate( + : isUnavailableFailure(error) + ? buildSyncDatabaseRetryUpdate( new Date(), connector.consecutiveFailures, - errorMessage, - retryAfterMs - ) - : buildSyncFailureUpdate( - new Date(), - connector.consecutiveFailures, - errorMessage, - retryAfterMs + EGRESS_UNAVAILABLE_MESSAGE, + connectorFailureBackoffMinutes(1) * 60 * 1000 ) + : rateLimited + ? buildSyncRateLimitUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) + : buildSyncFailureUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) if (failureUpdate.status === 'disabled') { logger.warn('Connector disabled after repeated failures', { diff --git a/apps/sim/lib/knowledge/documents/image-extraction-gemini.ts b/apps/sim/lib/knowledge/documents/image-extraction-gemini.ts index a36c981d1f8..521e2d4c507 100644 --- a/apps/sim/lib/knowledge/documents/image-extraction-gemini.ts +++ b/apps/sim/lib/knowledge/documents/image-extraction-gemini.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { parseRetryAfter } from '@sim/utils/retry' import { z } from 'zod' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { EmbeddingConfigurationError } from '@/lib/embeddings/configuration-error' @@ -219,7 +220,7 @@ export async function requestGeminiImageExtraction(input: { maxOutputTokens: number signal: AbortSignal }): Promise { - const response = await fetch( + const response = await outboundFetch( `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(input.model)}:generateContent`, { method: 'POST', diff --git a/apps/sim/lib/knowledge/documents/processor.ts b/apps/sim/lib/knowledge/documents/processor.ts index ad1c1db6016..c97522d6905 100644 --- a/apps/sim/lib/knowledge/documents/processor.ts +++ b/apps/sim/lib/knowledge/documents/processor.ts @@ -669,6 +669,7 @@ async function makeOCRRequest( const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal try { + // egress-direct: Sim's own Azure OCR deployment, configured by the operator const response = await fetch(endpoint, { method: 'POST', headers, diff --git a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts index 27302ee9442..385e441836d 100644 --- a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts +++ b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts @@ -1,6 +1,5 @@ -import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' import { - createSsrfGuardedFetchWithDispatcher, type SecureFetchOptions, type SecureFetchResponse, secureFetchWithValidation, @@ -61,7 +60,6 @@ export async function secureFetchWithRetry( } const DEFAULT_FETCH_RETRY_BUDGET_MS = 150_000 -let connectorTransport: ReturnType | undefined /** * Bounds requests and response bodies within one retry budget. @@ -84,18 +82,10 @@ export async function fetchWithRetry( signal, AbortSignal.timeout(Math.max(0, Math.ceil(deadlineAt - Date.now()))), ]) - const transport: typeof fetch = async (input, init) => { - const route = await resolveCurrentOutboundRoute() - if (route.kind === 'direct') return fetch(input, init) - connectorTransport ??= createSsrfGuardedFetchWithDispatcher({ - profile: 'configuredEndpoint', - }) - return connectorTransport.fetch(input, init) - } const init = { ...options, signal: requestSignal } const response = retryOptions.fetcher - ? await retryOptions.fetcher(url, init, transport) - : await transport(url, init) + ? await retryOptions.fetcher(url, init, scopedOutboundFetch) + : await scopedOutboundFetch(url, init) if ( !response.ok && diff --git a/apps/sim/lib/knowledge/reranker.ts b/apps/sim/lib/knowledge/reranker.ts index 2a28d1a4ce5..9f0085d33ff 100644 --- a/apps/sim/lib/knowledge/reranker.ts +++ b/apps/sim/lib/knowledge/reranker.ts @@ -5,6 +5,7 @@ import { getBYOKKey } from '@/lib/api-key/byok' import { getRotatingApiKey, hasRotatingApiKey } from '@/lib/core/config/api-keys' import { env } from '@/lib/core/config/env' import { isHosted } from '@/lib/core/config/env-flags' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { type ProviderIdentity, recordProviderCooldown, @@ -177,7 +178,7 @@ export async function rerank( maxWaitMs: Math.max(0, deadlineAt - Date.now()), }) attempt += 1 - const res = await fetch('https://api.cohere.com/v2/rerank', { + const res = await outboundFetch('https://api.cohere.com/v2/rerank', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/managed-agents/session-client.test.ts b/apps/sim/lib/managed-agents/session-client.test.ts index e26034d5ec6..ae52a346652 100644 --- a/apps/sim/lib/managed-agents/session-client.test.ts +++ b/apps/sim/lib/managed-agents/session-client.test.ts @@ -131,6 +131,7 @@ describe('managedAgentsList — selector collection bounds', () => { global.fetch = vi.fn( (_url: string, init?: RequestInit) => new Promise((_resolve, reject) => { + if (init?.signal?.aborted) reject(cancelled) init?.signal?.addEventListener('abort', () => reject(cancelled), { once: true }) }) ) as unknown as typeof fetch diff --git a/apps/sim/lib/managed-agents/session-client.ts b/apps/sim/lib/managed-agents/session-client.ts index f7166ae847e..9252d9290e4 100644 --- a/apps/sim/lib/managed-agents/session-client.ts +++ b/apps/sim/lib/managed-agents/session-client.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' /** * Provider-neutral HTTP client for the Claude Platform Managed Agents API. * @@ -203,7 +204,7 @@ export function buildSessionCreatePayload(input: CreateSessionInput): Record { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions`, { method: 'POST', headers: managedAgentsHeaders(input.apiKey, { json: true }), body: JSON.stringify(buildSessionCreatePayload(input)), @@ -278,7 +279,7 @@ export async function sendSessionEvents( input: SessionAuth & { sessionId: string; events: OutboundSessionEvent[] } ): Promise { if (input.events.length === 0) return - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/events`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/events`, { method: 'POST', headers: managedAgentsHeaders(input.apiKey, { json: true }), body: JSON.stringify({ events: input.events }), @@ -467,11 +468,14 @@ export async function sendCustomToolResults( export async function openSessionStream( input: SessionAuth & { sessionId: string } ): Promise { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/events/stream`, { - method: 'GET', - headers: managedAgentsHeaders(input.apiKey, { accept: 'text/event-stream' }), - signal: input.signal, - }) + const resp = await outboundFetch( + `${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/events/stream`, + { + method: 'GET', + headers: managedAgentsHeaders(input.apiKey, { accept: 'text/event-stream' }), + signal: input.signal, + } + ) if (!resp.ok) { const detail = await resp.text().catch(() => '') throw new Error(`Anthropic events.stream failed (${resp.status}): ${detail.slice(0, 400)}`) @@ -559,7 +563,7 @@ async function fetchSelectorListPage( let response: Response try { - response = await fetch(url.toString(), { + response = await outboundFetch(url.toString(), { method: 'GET', headers: managedAgentsHeaders(input.apiKey, { beta: input.beta }), redirect: 'error', @@ -621,7 +625,7 @@ async function listPaginated( // `append`, not `set` — `types[]` is repeatable and each value must survive. for (const [key, value] of input.searchParams ?? []) url.searchParams.append(key, value) if (page) url.searchParams.set('page', page) - const resp = await fetch(url.toString(), { + const resp = await outboundFetch(url.toString(), { method: 'GET', headers: managedAgentsHeaders(input.apiKey, { beta: input.beta }), redirect: 'error', @@ -779,11 +783,14 @@ export async function getEnvironmentType( input: SessionAuth & { environmentId: string } ): Promise { try { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/environments/${input.environmentId}`, { - method: 'GET', - headers: managedAgentsHeaders(input.apiKey), - signal: input.signal, - }) + const resp = await outboundFetch( + `${ANTHROPIC_API_BASE}/v1/environments/${input.environmentId}`, + { + method: 'GET', + headers: managedAgentsHeaders(input.apiKey), + signal: input.signal, + } + ) if (!resp.ok) return undefined const body = (await resp.json()) as { config?: { type?: unknown } } const type = body.config?.type @@ -803,7 +810,7 @@ export async function getSession( input: SessionAuth & { sessionId: string } ): Promise { try { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { method: 'GET', headers: managedAgentsHeaders(input.apiKey), signal: input.signal, @@ -878,7 +885,7 @@ export function parseSessionSnapshot(raw: unknown): SessionSnapshot { export async function retrieveSession( input: SessionAuth & { sessionId: string } ): Promise { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { method: 'GET', headers: managedAgentsHeaders(input.apiKey), signal: input.signal, @@ -912,7 +919,7 @@ export async function updateSession( if (Object.keys(payload).length === 0) { throw new Error('Update session requires a title or metadata to change.') } - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { method: 'POST', headers: managedAgentsHeaders(input.apiKey, { json: true }), body: JSON.stringify(payload), @@ -931,7 +938,7 @@ export async function updateSession( * it first), and archiving is NOT reversible. */ export async function archiveSession(input: SessionAuth & { sessionId: string }): Promise { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/archive`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}/archive`, { method: 'POST', headers: managedAgentsHeaders(input.apiKey), signal: input.signal, @@ -949,7 +956,7 @@ export async function archiveSession(input: SessionAuth & { sessionId: string }) * independent resources and are NOT affected. */ export async function deleteSession(input: SessionAuth & { sessionId: string }): Promise { - const resp = await fetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { + const resp = await outboundFetch(`${ANTHROPIC_API_BASE}/v1/sessions/${input.sessionId}`, { method: 'DELETE', headers: managedAgentsHeaders(input.apiKey), signal: input.signal, diff --git a/apps/sim/lib/media/falai.ts b/apps/sim/lib/media/falai.ts index 28ed99a0687..51c57da2bf1 100644 --- a/apps/sim/lib/media/falai.ts +++ b/apps/sim/lib/media/falai.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -140,7 +141,7 @@ async function submitAndPollFalQueue( ): Promise { // Only a submit already under way outlives its caller; a stopped call never starts one. queueSignal.throwIfAborted() - const createResponse = await fetch(`https://queue.fal.run/${endpoint}`, { + const createResponse = await outboundFetch(`https://queue.fal.run/${endpoint}`, { method: 'POST', headers: { Authorization: `Key ${apiKey}`, @@ -201,7 +202,7 @@ async function pollFalQueue( await interruptibleSleep(POLL_INTERVAL_MS, signal) signal.throwIfAborted() - const statusResponse = await fetch(statusUrl, { + const statusResponse = await outboundFetch(statusUrl, { headers: { Authorization: `Key ${apiKey}` }, signal, }) @@ -226,10 +227,13 @@ async function pollFalQueue( if (statusData.error) { throw new Error(`Fal.ai generation failed: ${falErrorMessage(statusData.error)}`) } - const resultResponse = await fetch(getStringProp(statusData, 'response_url') || responseUrl, { - headers: { Authorization: `Key ${apiKey}` }, - signal, - }) + const resultResponse = await outboundFetch( + getStringProp(statusData, 'response_url') || responseUrl, + { + headers: { Authorization: `Key ${apiKey}` }, + signal, + } + ) if (!resultResponse.ok) { const body = await readResponseTextWithLimit(resultResponse, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, @@ -260,7 +264,7 @@ async function pollFalQueue( export async function cancelFalRequest(cancelUrl: string, apiKey: string, requestId: string) { try { // boundary-raw-fetch: cancels a request on the external Fal.ai queue - const response = await fetch(cancelUrl, { + const response = await outboundFetch(cancelUrl, { method: 'PUT', headers: { Authorization: `Key ${apiKey}` }, signal: AbortSignal.timeout(FAL_CANCEL_TIMEOUT_MS), diff --git a/apps/sim/lib/oauth/apollo.ts b/apps/sim/lib/oauth/apollo.ts index 90b88fe2b6b..a84b522957e 100644 --- a/apps/sim/lib/oauth/apollo.ts +++ b/apps/sim/lib/oauth/apollo.ts @@ -1,6 +1,7 @@ import { generateId } from '@sim/utils/id' import type { GenericOAuthConfig } from 'better-auth/plugins' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' export const APOLLO_TOKEN_URL = 'https://app.apollo.io/api/v1/oauth/token' @@ -39,8 +40,7 @@ export function createApolloProvider( tokenUrl: APOLLO_TOKEN_URL, authentication: 'post', getUserInfo: async (tokens) => { - // boundary-raw-fetch: Apollo's external current-user endpoint verifies the OAuth token's identity. - const response = await fetch('https://app.apollo.io/api/v1/users/api_profile', { + const response = await outboundFetch('https://app.apollo.io/api/v1/users/api_profile', { headers: { Authorization: `Bearer ${tokens.accessToken}` }, redirect: 'error', signal: AbortSignal.timeout(15_000), diff --git a/apps/sim/lib/oauth/atlassian-token.ts b/apps/sim/lib/oauth/atlassian-token.ts index dd2e4403979..bb13b1c5a1c 100644 --- a/apps/sim/lib/oauth/atlassian-token.ts +++ b/apps/sim/lib/oauth/atlassian-token.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { getOAuth2Tokens } from 'better-auth/oauth2' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' const logger = createLogger('AtlassianOAuth') @@ -24,7 +25,7 @@ export async function exchangeAtlassianAuthorizationCode({ redirectUri, codeVerifier, }: ExchangeAtlassianAuthorizationCodeInput) { - const response = await fetch(ATLASSIAN_TOKEN_URL, { + const response = await outboundFetch(ATLASSIAN_TOKEN_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify({ diff --git a/apps/sim/lib/oauth/authorization-code.server.ts b/apps/sim/lib/oauth/authorization-code.server.ts new file mode 100644 index 00000000000..7d1fe569d46 --- /dev/null +++ b/apps/sim/lib/oauth/authorization-code.server.ts @@ -0,0 +1,120 @@ +import { toRecord } from '@sim/utils/object' +import { + authorizationCodeRequest, + getOAuth2Tokens, + type OAuth2Tokens, + type validateAuthorizationCode, +} from 'better-auth/oauth2' +import type { GenericOAuthConfig } from 'better-auth/plugins' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { isRedirectStatus } from '@/lib/core/security/input-validation.server' + +type AuthorizationCodeExchange = Parameters[0] +type GetToken = NonNullable + +const JSON_MEDIA_TYPE = /^application\/(?:[\w!#$%&*.^`~-]*\+)?json$/i +const TEXT_MEDIA_TYPES = new Set([ + 'image/svg', + 'application/xml', + 'application/xhtml', + 'application/html', +]) + +/** + * Reads a successful response the way Better Auth's `betterFetch` does: JSON and text bodies are + * JSON-parsed when they parse (else kept as text), anything else is a Blob. + */ +async function readResponseData(response: Response): Promise { + const mediaType = response.headers.get('content-type')?.split(';')[0] ?? '' + if ( + !mediaType || + JSON_MEDIA_TYPE.test(mediaType) || + TEXT_MEDIA_TYPES.has(mediaType) || + mediaType.startsWith('text/') + ) { + const text = await response.text() + try { + return JSON.parse(text) + } catch { + return text + } + } + return response.blob() +} + +/** + * Better Auth's `validateAuthorizationCode`, sent through {@link outboundFetch} so the code + * exchange leaves through the current outbound scope's egress route. The request is built by + * Better Auth's own `authorizationCodeRequest` and the response parsed by its `getOAuth2Tokens`; + * like Better Auth, it refuses redirects and throws on a non-2xx answer. + */ +export async function exchangeAuthorizationCode({ + tokenEndpoint, + ...request +}: AuthorizationCodeExchange): Promise { + const { body, headers } = await authorizationCodeRequest(request) + const response = await outboundFetch(tokenEndpoint, { + method: 'POST', + body, + headers, + redirect: 'manual', + }) + const redirected = response.type === 'opaqueredirect' || isRedirectStatus(response.status) + if (redirected || !response.ok) { + await response.body?.cancel().catch(() => {}) + throw new Error( + redirected + ? `The OAuth endpoint "${tokenEndpoint}" returned an HTTP redirect` + : `OAuth token endpoint responded with ${response.status}` + ) + } + return getOAuth2Tokens(toRecord(await readResponseData(response))) +} + +/** The plugin's token endpoint: the discovery document's when it answers, else `tokenUrl`. */ +async function resolveTokenEndpoint(config: GenericOAuthConfig): Promise { + let tokenEndpoint = config.tokenUrl + if (config.discoveryUrl) { + const response = await outboundFetch(config.discoveryUrl, { + method: 'GET', + headers: config.discoveryHeaders, + }) + if (response.ok) { + const discovery = await readResponseData(response) + if (discovery) { + const endpoint = toRecord(discovery).token_endpoint + tokenEndpoint = typeof endpoint === 'string' ? endpoint : undefined + } + } else { + await response.body?.cancel().catch(() => {}) + } + } + if (!tokenEndpoint) throw new Error(`OAuth token endpoint not found for ${config.providerId}`) + return tokenEndpoint +} + +/** + * The code exchange Better Auth's generic OAuth callback performs for a connector without its + * own `getToken` (discovery, client authentication, PKCE verifier, static `tokenUrlParams`), sent + * through {@link outboundFetch}. Undefined when `tokenUrlParams` reads the callback request, + * which a `getToken` never sees. + */ +export function createOutboundCodeExchange(config: GenericOAuthConfig): GetToken | undefined { + const { tokenUrlParams } = config + if (typeof tokenUrlParams === 'function') return undefined + return async ({ code, redirectURI, codeVerifier }) => + exchangeAuthorizationCode({ + headers: config.authorizationHeaders, + code, + codeVerifier, + redirectURI, + options: { + clientId: config.clientId, + clientSecret: config.clientSecret, + redirectURI: config.redirectURI, + }, + tokenEndpoint: await resolveTokenEndpoint(config), + authentication: config.authentication, + additionalParams: tokenUrlParams, + }) +} diff --git a/apps/sim/lib/oauth/connect-scope.server.test.ts b/apps/sim/lib/oauth/connect-scope.server.test.ts new file mode 100644 index 00000000000..200acc2600b --- /dev/null +++ b/apps/sim/lib/oauth/connect-scope.server.test.ts @@ -0,0 +1,332 @@ +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { + createMockWorkspaceApplicationContext, + workspaceContextMock, + workspaceContextMockFns, +} from '@sim/testing/mocks/workspace-context.mock' +import { validateAuthorizationCode } from 'better-auth/oauth2' +import { type GenericOAuthConfig, genericOAuth } from 'better-auth/plugins' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetActiveConnectDraft, mockGetOAuthState } = vi.hoisted(() => ({ + mockGetActiveConnectDraft: vi.fn(), + mockGetOAuthState: vi.fn(), +})) +const { mockLoadWorkspaceApplicationContext } = workspaceContextMockFns + +vi.mock('better-auth/api', () => ({ getOAuthState: mockGetOAuthState })) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +vi.mock('@/lib/credentials/connect-draft', () => ({ + getActiveConnectDraft: mockGetActiveConnectDraft, +})) +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) + +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { + resolveConnectDraftOutboundScope, + withConnectDraftOutboundScope, +} from '@/lib/oauth/connect-scope.server' + +async function organizationSeenByVendorCall( + draftOwner: { workspaceId: string | null; organizationId: string | null }, + requestedProviderId: string +): Promise { + mockGetActiveConnectDraft.mockResolvedValue({ + id: 'draft-1', + providerId: 'instagram', + ...draftOwner, + }) + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId: 'draft-1', + userId: 'user-1', + providerId: requestedProviderId, + }) + let organizationId: string | null | undefined + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (id) => { + organizationId = id + return { kind: 'direct' } + }) + await inDraftScope(() => resolveCurrentOutboundRoute()) + return organizationId +} + +describe('resolveConnectDraftOutboundScope', () => { + beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + mockLoadWorkspaceApplicationContext.mockResolvedValue( + createMockWorkspaceApplicationContext({ + workspaceId: 'workspace-1', + workspaceOrganizationId: 'org-1', + }) + ) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + + it("routes vendor calls through the draft workspace's organization", async () => { + await expect( + organizationSeenByVendorCall( + { workspaceId: 'workspace-1', organizationId: null }, + 'instagram' + ) + ).resolves.toBe('org-1') + expect(mockGetActiveConnectDraft).toHaveBeenCalledWith('draft-1', 'user-1') + }) + + it('leaves vendor calls unscoped when the draft belongs to another provider', async () => { + await expect( + organizationSeenByVendorCall({ workspaceId: 'workspace-1', organizationId: null }, 'trello') + ).resolves.toBeUndefined() + }) +}) + +describe('withConnectDraftOutboundScope', () => { + beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + + it('keeps a connector exactly as configured when the deployment has no outbound routing', () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + const config: GenericOAuthConfig = { + providerId: 'idp', + clientId: 'client-1', + tokenUrl: 'https://idp.test/token', + } + + expect(withConnectDraftOutboundScope(config)).toBe(config) + }) + + it("runs a generic connector's code exchange in its connect draft's organization", async () => { + mockGetOAuthState.mockResolvedValue({ + callbackURL: '/workspace/ws-1/integrations?credentialDraftId=draft-1', + link: { userId: 'user-1', email: 'user@example.com' }, + }) + mockGetActiveConnectDraft.mockResolvedValue({ + id: 'draft-1', + providerId: 'jira', + workspaceId: null, + organizationId: 'org-1', + }) + let organizationId: string | null | undefined + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (id) => { + organizationId = id + return { kind: 'direct' } + }) + const connector = withConnectDraftOutboundScope({ + providerId: 'jira', + clientId: 'client', + getToken: async () => { + await resolveCurrentOutboundRoute() + return { accessToken: 'token' } + }, + }) + + await connector.getToken?.({ code: 'code', redirectURI: 'https://sim.test/callback' }) + + expect(organizationId).toBe('org-1') + expect(mockGetActiveConnectDraft).toHaveBeenCalledWith('draft-1', 'user-1') + }) +}) + +const TOKEN_RESPONSE = { + token_type: 'Bearer', + access_token: 'access-1', + refresh_token: 'refresh-1', + expires_in: 3600, + scope: 'openid email', +} +const CODE = { code: 'code-1', redirectURI: 'https://sim.test/api/auth/oauth2/callback/idp' } +type CodeData = typeof CODE & { codeVerifier?: string } + +interface RecordedRequest { + url: string + method: string + redirect: string + headers: Record + body: string +} + +/** Records every request sent through the global `fetch`, the transport of both exchanges. */ +function stubVendor(respond: (url: string) => Response = () => Response.json(TOKEN_RESPONSE)) { + const requests: RecordedRequest[] = [] + vi.stubGlobal('fetch', async (input: RequestInfo | URL, init?: RequestInit) => { + const request = new Request(input, init) + requests.push({ + url: request.url, + method: request.method, + redirect: request.redirect, + headers: Object.fromEntries(request.headers), + body: await request.text(), + }) + return respond(request.url) + }) + return requests +} + +/** The exchange Better Auth's generic OAuth plugin performs for a configured connector. */ +async function pluginExchange(config: GenericOAuthConfig, data: CodeData) { + const initialized = await genericOAuth({ config: [config] }).init?.({ + socialProviders: [], + } as never) + const provider = initialized?.context?.socialProviders?.[0] + if (!provider) throw new Error('generic OAuth provider was not registered') + return provider.validateAuthorizationCode(data) +} + +async function compareExchanges( + config: GenericOAuthConfig, + data: CodeData, + respond?: (url: string) => Response +) { + const betterAuthRequests = stubVendor(respond) + const betterAuthTokens = await pluginExchange(config, data) + const routedRequests = stubVendor(respond) + const routedTokens = await pluginExchange(withConnectDraftOutboundScope(config), data) + return { betterAuthRequests, betterAuthTokens, routedRequests, routedTokens } +} + +describe('default code exchange for connectors without getToken', () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(new Date('2026-11-13T12:00:00Z')) + mockGetOAuthState.mockResolvedValue(null) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + }) + + afterEach(() => { + vi.useRealTimers() + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + }) + + const base = { + clientId: 'client-1', + clientSecret: 'secret-1', + redirectURI: 'https://sim.test/api/auth/oauth2/callback/idp', + } + + it.each<[string, GenericOAuthConfig, CodeData]>([ + [ + 'basic client authentication with PKCE', + { + ...base, + providerId: 'microsoft-teams', + tokenUrl: 'https://login.idp.test/common/oauth2/v2.0/token', + authentication: 'basic', + pkce: true, + }, + { ...CODE, codeVerifier: 'verifier-1' }, + ], + [ + 'post client authentication with custom headers and a default expiry', + { + ...base, + providerId: 'hubspot', + tokenUrl: 'https://api.idp.test/oauth/v1/token', + authentication: 'post', + authorizationHeaders: { 'x-tenant': 'tenant-1' }, + accessTokenExpiresIn: 7200, + }, + CODE, + ], + [ + 'a token endpoint resolved from discovery', + { + ...base, + providerId: 'google-email', + discoveryUrl: 'https://accounts.idp.test/.well-known/openid-configuration', + discoveryHeaders: { 'x-discovery': '1' }, + }, + CODE, + ], + ])('sends the request Better Auth sends for %s', async (_label, config, data) => { + const respond = (url: string) => + url.endsWith('openid-configuration') + ? Response.json({ token_endpoint: 'https://oauth2.idp.test/token' }) + : Response.json( + config.accessTokenExpiresIn + ? { ...TOKEN_RESPONSE, expires_in: undefined } + : TOKEN_RESPONSE + ) + const result = await compareExchanges(config, data, respond) + + expect(result.routedRequests.length).toBeGreaterThan(0) + expect(result.routedRequests).toEqual(result.betterAuthRequests) + expect(result.routedTokens).toEqual(result.betterAuthTokens) + }) + + it("sends a connector's static token parameters the way Better Auth's callback does", async () => { + const tokenUrl = 'https://idp.test/token' + const tokenUrlParams = { audience: 'api.idp.test', grant_type: 'ignored' } + const config: GenericOAuthConfig = { ...base, providerId: 'idp', tokenUrl, tokenUrlParams } + const betterAuthRequests = stubVendor() + await validateAuthorizationCode({ + headers: config.authorizationHeaders, + ...CODE, + options: { + clientId: config.clientId, + clientSecret: config.clientSecret, + redirectURI: config.redirectURI, + }, + tokenEndpoint: tokenUrl, + authentication: config.authentication, + additionalParams: tokenUrlParams, + }) + const routedRequests = stubVendor() + await withConnectDraftOutboundScope(config).getToken?.(CODE) + + expect(routedRequests).toEqual(betterAuthRequests) + }) + + it.each([ + ['an error status', () => Response.json({ error: 'invalid_grant' }, { status: 400 })], + [ + 'a redirect', + () => new Response(null, { status: 302, headers: { location: 'http://169.254.169.254/' } }), + ], + ])('fails the exchange on %s like Better Auth does', async (_label, respond) => { + const config: GenericOAuthConfig = { + ...base, + providerId: 'idp', + tokenUrl: 'https://idp.test/token', + } + stubVendor(respond) + await expect(pluginExchange(config, CODE)).rejects.toThrow() + stubVendor(respond) + await expect(pluginExchange(withConnectDraftOutboundScope(config), CODE)).rejects.toThrow() + }) + + it("runs the default exchange in the connect draft's organization", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + mockGetOAuthState.mockResolvedValue({ + callbackURL: '/workspace/ws-1/integrations?credentialDraftId=draft-1', + link: { userId: 'user-1', email: 'user@example.com' }, + }) + mockGetActiveConnectDraft.mockResolvedValue({ + id: 'draft-1', + providerId: 'microsoft-teams', + workspaceId: null, + organizationId: 'org-1', + }) + const routedOrganizations: Array = [] + networkConfigMockFns.mockResolveOutboundRoute.mockImplementation(async (id) => { + routedOrganizations.push(id) + return { kind: 'direct' } + }) + stubVendor() + const connector = withConnectDraftOutboundScope({ + ...base, + providerId: 'microsoft-teams', + tokenUrl: 'https://login.idp.test/common/oauth2/v2.0/token', + authentication: 'basic', + }) + + await expect(connector.getToken?.(CODE)).resolves.toMatchObject({ accessToken: 'access-1' }) + expect(routedOrganizations).toEqual(['org-1']) + }) +}) diff --git a/apps/sim/lib/oauth/connect-scope.server.ts b/apps/sim/lib/oauth/connect-scope.server.ts new file mode 100644 index 00000000000..49ff4f69e94 --- /dev/null +++ b/apps/sim/lib/oauth/connect-scope.server.ts @@ -0,0 +1,95 @@ +import { getOAuthState } from 'better-auth/api' +import type { GenericOAuthConfig } from 'better-auth/plugins' +import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' +import { captureOutboundScope } from '@/lib/core/network/context.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { getActiveConnectDraft } from '@/lib/credentials/connect-draft' +import { parseCredentialDraftIdFromCallbackUrl } from '@/lib/credentials/draft-processor' +import { createOutboundCodeExchange } from '@/lib/oauth/authorization-code.server' + +type OutboundScopeRunner = (run: () => Promise) => Promise + +/** + * Resolves the outbound scope for a custom OAuth callback's vendor calls (code exchange, + * identity lookup) from the workspace or organization that owns its connect draft, so an + * organization with a dedicated egress gateway connects from its own static IPs. + * + * Ownership comes from the caller's own unexpired draft row, never from the request, and + * is resolved once so every vendor call in the callback reuses it. Without a draft bound + * to this user and provider, the runner leaves calls unscoped. + */ +export async function resolveConnectDraftOutboundScope(params: { + draftId: string | undefined + userId: string + providerId: string +}): Promise { + if (!isOutboundRoutingEnabled() || !params.draftId) return (run) => run() + const draft = await getActiveConnectDraft(params.draftId, params.userId) + if (!draft || draft.providerId !== params.providerId) return (run) => run() + return withResourceOutboundScope( + { workspaceId: draft.workspaceId, organizationId: draft.organizationId }, + async () => captureOutboundScope() + ) +} + +/** One draft lookup per callback, shared by its code exchange and identity lookup. */ +const callbackDraftScopes = new WeakMap>() + +/** + * Runs a generic OAuth callback step in the scope of the connect draft named by Better Auth's + * server-verified OAuth state (`link.userId` started the link, `callbackURL` carries the draft + * id). A sign-in callback, which has no `link`, runs unscoped. + */ +async function runInOAuthCallbackDraftScope( + providerId: string, + run: () => Promise +): Promise { + const state = await getOAuthState() + const userId = state?.link?.userId + if (!userId) return run() + let inDraftScope = callbackDraftScopes.get(state) + if (!inDraftScope) { + inDraftScope = resolveConnectDraftOutboundScope({ + draftId: parseCredentialDraftIdFromCallbackUrl(state.callbackURL), + userId, + providerId, + }) + callbackDraftScopes.set(state, inDraftScope) + } + return (await inDraftScope)(run) +} + +/** + * Whether routing is on, read once while the auth config is built. A routing configuration that + * fails to load counts as on, so the connector stays wrapped and the bad configuration surfaces + * on its first request like every other routed call, rather than failing the auth module's load. + */ +function routingConfiguredAtStartup(): boolean { + try { + return isOutboundRoutingEnabled() + } catch { + return true + } +} + +/** + * Wraps a generic OAuth connector's code exchange and identity lookup so their vendor calls + * leave through the connecting organization's egress gateway. A connector without its own + * `getToken` gets Better Auth's exchange sent through `outboundFetch`, since the plugin's + * built-in exchange uses its own fetch. Better Auth's own discovery reads for the authorize URL + * and callback stay unrouted; the exchange resolves its token endpoint through `outboundFetch`. + * A deployment without outbound routing keeps the connector exactly as configured. + */ +export function withConnectDraftOutboundScope(config: GenericOAuthConfig): GenericOAuthConfig { + if (!routingConfiguredAtStartup()) return config + const { providerId, getUserInfo } = config + const getToken = config.getToken ?? createOutboundCodeExchange(config) + return { + ...config, + getToken: + getToken && ((data) => runInOAuthCallbackDraftScope(providerId, () => getToken(data))), + getUserInfo: + getUserInfo && + ((tokens) => runInOAuthCallbackDraftScope(providerId, () => getUserInfo(tokens))), + } +} diff --git a/apps/sim/lib/oauth/credential-service-cancellation.test.ts b/apps/sim/lib/oauth/credential-service-cancellation.test.ts index 3fe7db1a82f..18c4cb1ee1d 100644 --- a/apps/sim/lib/oauth/credential-service-cancellation.test.ts +++ b/apps/sim/lib/oauth/credential-service-cancellation.test.ts @@ -3,6 +3,7 @@ import { createDeferred } from '@sim/testing/helpers/deferred' import { queueTableRows, resetDbChainMock } from '@sim/testing/mocks/database.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { CLIENT_CREDENTIAL_ACCOUNT_SECRET_TYPE, RAMP_SERVICE_ACCOUNT_PROVIDER_ID, @@ -286,6 +287,22 @@ describe('client credential token refresh after a failed mint', () => { }) }) + it("records nothing about the credential when Sim's egress route is down, so it asks again at once", async () => { + const mint = minter() + mint.mockResolvedValueOnce({ accessToken: 'current-token', expiresInSeconds: 600 }) + await resolveFor('ramp-egress-outage') + mint.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + vi.advanceTimersByTime(6 * 60_000) + + await expect(resolveFor('ramp-egress-outage')).resolves.toEqual({ + error: expect.any(OutboundRoutingError), + }) + mint.mockResolvedValueOnce({ accessToken: 'next-token', expiresInSeconds: 600 }) + await expect(resolveFor('ramp-egress-outage')).resolves.toEqual({ + value: { accessToken: 'next-token' }, + }) + }) + it('retries a quick transient failure once inside the mint', async () => { const mint = minter() mint.mockRejectedValueOnce(unavailable(503)).mockResolvedValueOnce(token) diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts index 572d3ecf8ec..4fe8913dde0 100644 --- a/apps/sim/lib/oauth/credential-service.test.ts +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -60,10 +60,13 @@ vi.mock('@/lib/oauth/microsoft', () => ({ vi.mock('@/lib/oauth/oauth', () => ({ OAUTH_PROVIDERS: {}, - refreshOAuthToken: hoisted.refreshOAuthToken, TOKEN_REFRESH_TIMEOUT_MS: 15_000, })) +vi.mock('@/lib/oauth/refresh.server', () => ({ + refreshOAuthToken: hoisted.refreshOAuthToken, +})) + vi.mock('@/lib/oauth/quickbooks-client-config', () => ({ decryptQuickBooksOAuthClientConfig: hoisted.decryptQuickBooksOAuthClientConfig, })) @@ -88,6 +91,7 @@ vi.mock('@/lib/oauth/refresh-error-codes', () => ({ isTerminalRefreshError: vi.fn(() => false), })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { CredentialRevokedError } from '@/lib/oauth/credential-revoked' import { getCredentialTerminalRefreshError, @@ -903,6 +907,19 @@ describe('OAuth access-token refresh headroom', () => { ) }) + it.each(readers)( + '$name rethrows an unavailable egress route instead of a missing, revoked or failed token', + async ({ read }) => { + mocks.refreshOAuthToken.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + const row = createOAuthAccount(WAITED_GOOGLE_REMAINING_MS) + queueTableRows(account, [row]) + await expect(read(row)).rejects.toBeInstanceOf(OutboundRoutingError) + expect(dbChainMockFns.set).not.toHaveBeenCalledWith( + expect.objectContaining({ refreshRevokedCode: expect.anything() }) + ) + } + ) + it('answers a Microsoft caller from its valid token while refresh-token aging refreshes', async () => { vi.mocked(isMicrosoftProvider).mockReturnValue(true) const provider = createDeferred() diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 846076c444d..99ebd696943 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -9,6 +9,7 @@ import { and, desc, eq, isNotNull, sql } from 'drizzle-orm' import { withLeaderLock } from '@/lib/concurrency/leader-lock' import { coalesceLocally } from '@/lib/concurrency/singleflight' import { env } from '@/lib/core/config/env' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { decryptSecret } from '@/lib/core/security/encryption' import { waitWithAbort } from '@/lib/core/utils/concurrency' import { mintAtlassianServiceAccountToken } from '@/lib/credentials/atlassian-client-credentials' @@ -65,8 +66,9 @@ import { isMicrosoftProvider, PROACTIVE_REFRESH_THRESHOLD_DAYS, } from '@/lib/oauth/microsoft' -import { refreshOAuthToken, TOKEN_REFRESH_TIMEOUT_MS } from '@/lib/oauth/oauth' +import { TOKEN_REFRESH_TIMEOUT_MS } from '@/lib/oauth/oauth' import { decryptQuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' import { clearOAuthRefreshDeadFlag, getOAuthRefreshCoordinationIdentity, @@ -834,6 +836,8 @@ async function resolveMintedServiceAccountToken( return { ...minted.token } } catch (error) { if (error instanceof Error && error.name === 'AbortError') throw error + // Sim's egress route is down: nothing is known about the credential, so nothing is recorded. + if (error instanceof OutboundRoutingError) throw error if (sharedToken && !isCoordinatedReplay(error)) { clientCredentialMintFailureCache.delete(cacheIdentity) throw error @@ -1086,8 +1090,8 @@ const SERVICE_ACCOUNT_TOKEN_RESOLVERS: Record encryptionMock) vi.mock('@/lib/oauth/github-installation', () => githubInstallationMock) vi.mock('@/lib/oauth/oauth', () => ({ OAUTH_PROVIDERS: {}, - refreshOAuthToken: vi.fn(), TOKEN_REFRESH_TIMEOUT_MS: 15_000, })) +vi.mock('@/lib/oauth/refresh.server', () => ({ refreshOAuthToken: vi.fn() })) import { resolveServiceAccountToken } from '@/lib/oauth/credential-service' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' diff --git a/apps/sim/lib/oauth/github-installation.ts b/apps/sim/lib/oauth/github-installation.ts index 839ea1d48fe..0806c86f744 100644 --- a/apps/sim/lib/oauth/github-installation.ts +++ b/apps/sim/lib/oauth/github-installation.ts @@ -1,6 +1,7 @@ import { createHash, createPrivateKey, createSign } from 'node:crypto' import { z } from 'zod' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { GitHubInstallationBinding, @@ -166,7 +167,7 @@ async function request( operation: GitHubInstallationOperation = 'installation' ): Promise { const requestSignal = AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]) - const response = await fetch(`${API_URL}${path}`, { + const response = await outboundFetch(`${API_URL}${path}`, { method: body === undefined ? 'GET' : 'POST', headers: { Accept: 'application/vnd.github+json', diff --git a/apps/sim/lib/oauth/github-repositories.test.ts b/apps/sim/lib/oauth/github-repositories.test.ts index 13db90f5d98..85baf77cf67 100644 --- a/apps/sim/lib/oauth/github-repositories.test.ts +++ b/apps/sim/lib/oauth/github-repositories.test.ts @@ -8,7 +8,7 @@ import { verifyGitHubRepositoriesIdentity, } from '@/lib/oauth/github-repositories' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' -import { refreshOAuthToken } from '@/lib/oauth/oauth' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' const tokenResponse = { access_token: 'ghu_access', diff --git a/apps/sim/lib/oauth/github-repositories.ts b/apps/sim/lib/oauth/github-repositories.ts index 320c2658bd0..37398dea7d8 100644 --- a/apps/sim/lib/oauth/github-repositories.ts +++ b/apps/sim/lib/oauth/github-repositories.ts @@ -2,6 +2,7 @@ import { generateId } from '@sim/utils/id' import { getOAuth2Tokens, type OAuth2Tokens } from 'better-auth/oauth2' import type { GenericOAuthConfig } from 'better-auth/plugins' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { OAuthIdentityVerificationError, @@ -60,7 +61,7 @@ export async function verifyGitHubRepositoriesIdentity(accessToken: string) { const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS) let response: Response try { - response = await fetch(`${API_BASE_URL}${path}`, { + response = await outboundFetch(`${API_BASE_URL}${path}`, { headers, signal, redirect: 'error', @@ -170,7 +171,7 @@ export function createGitHubRepositoriesProvider({ getToken: async ({ code, codeVerifier, redirectURI: callbackURI }) => { if (!codeVerifier) throw new Error('GitHub authorization requires a PKCE verifier') const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS) - const response = await fetch(GITHUB_TOKEN_URL, { + const response = await outboundFetch(GITHUB_TOKEN_URL, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/google-service-account-transport.ts b/apps/sim/lib/oauth/google-service-account-transport.ts index 85bd48c1849..1f3001ccf52 100644 --- a/apps/sim/lib/oauth/google-service-account-transport.ts +++ b/apps/sim/lib/oauth/google-service-account-transport.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { parseRetryAfter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isRetryableError, readBoundedHttpErrorPayload, @@ -61,7 +62,7 @@ export async function exchangeGoogleServiceAccountJwt( stage = 'awaiting_response' currentStatus = undefined lastResponse = undefined - const response = await fetch(tokenUri, { + const response = await outboundFetch(tokenUri, { method: 'POST', redirect: 'error', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, diff --git a/apps/sim/lib/oauth/greenhouse.ts b/apps/sim/lib/oauth/greenhouse.ts index 13b4e05e2e8..0c8990d52b2 100644 --- a/apps/sim/lib/oauth/greenhouse.ts +++ b/apps/sim/lib/oauth/greenhouse.ts @@ -3,6 +3,7 @@ import type { OAuth2Tokens } from 'better-auth/oauth2' import type { GenericOAuthConfig } from 'better-auth/plugins' import { z } from 'zod' import { syntheticConnectorEmail } from '@/lib/auth/connector-email' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -90,8 +91,7 @@ export function createGreenhouseProvider({ const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS) let response: Response try { - // boundary-raw-fetch: Greenhouse OAuth token exchange and introspection use external form requests. - response = await fetch(url, { + response = await outboundFetch(url, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/monday.ts b/apps/sim/lib/oauth/monday.ts index ace87d76433..e563c91023f 100644 --- a/apps/sim/lib/oauth/monday.ts +++ b/apps/sim/lib/oauth/monday.ts @@ -1,6 +1,7 @@ import type { OAuth2Tokens } from 'better-auth/oauth2' import { decodeJwt } from 'jose' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -73,7 +74,7 @@ export async function exchangeMondayAuthorizationCode({ redirectUri, }: ExchangeMondayAuthorizationCodeParams): Promise { const signal = AbortSignal.timeout(MONDAY_OAUTH_TOKEN_TIMEOUT_MS) - const response = await fetch(MONDAY_OAUTH_TOKEN_URL, { + const response = await outboundFetch(MONDAY_OAUTH_TOKEN_URL, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index 1554aab2b6e..ee37f34fdc7 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -36,8 +36,10 @@ beforeAll(() => { afterAll(resetEnvMock) import { buildConnectorProviders } from '@/lib/auth/connectors/providers' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES } from '@/lib/core/utils/stream-limits' -import { getPerRequestOAuthLinkScopes, OAUTH_PROVIDERS, refreshOAuthToken } from '@/lib/oauth' +import { getPerRequestOAuthLinkScopes, OAUTH_PROVIDERS } from '@/lib/oauth' +import { refreshOAuthToken } from '@/lib/oauth/refresh.server' import { getMissingRequiredScopes } from '@/lib/oauth/utils' import { PowerBIBlock } from '@/blocks/blocks/powerbi' @@ -286,7 +288,7 @@ describe('Bitbucket OAuth Connector', () => { describe('OAuth Token Refresh', () => { describe('Basic Auth Providers', () => { - it.concurrent('sends Basic Auth credentials and keeps them out of the body', async () => { + it('sends Basic Auth credentials and keeps them out of the body', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -418,7 +420,7 @@ describe('OAuth Token Refresh', () => { } ) - it.concurrent('should refresh TikTok with client_key instead of client_id', async () => { + it('should refresh TikTok with client_key instead of client_id', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -434,7 +436,7 @@ describe('OAuth Token Refresh', () => { expect(bodyParams.get('client_id')).toBeNull() }) - it.concurrent('should preserve Cal.com bearer refresh authentication', async () => { + it('should preserve Cal.com bearer refresh authentication', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -454,7 +456,7 @@ describe('OAuth Token Refresh', () => { expect(bodyParams.get('refresh_token')).toBeNull() }) - it.concurrent('should send Notion request with Basic Auth header and JSON body', async () => { + it('should send Notion request with Basic Auth header and JSON body', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -494,7 +496,7 @@ describe('OAuth Token Refresh', () => { }) describe('Error Handling', () => { - it.concurrent('should return the canonical error for partial OAuth configuration', async () => { + it('should return the canonical error for partial OAuth configuration', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const result = await withMockFetch(mockFetch, () => @@ -508,7 +510,14 @@ describe('OAuth Token Refresh', () => { }) expect(mockFetch).not.toHaveBeenCalled() }) - it.concurrent('should return failure with errorCode for HTTP error responses', async () => { + it('throws an unavailable egress route instead of reporting a failed refresh', async () => { + const mockFetch = vi.fn().mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + withMockFetch(mockFetch, () => refreshOAuthToken('google', 'test_refresh_token')) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('should return failure with errorCode for HTTP error responses', async () => { const mockFetch = vi.fn().mockResolvedValue({ ok: false, status: 400, @@ -528,7 +537,7 @@ describe('OAuth Token Refresh', () => { } }) - it.concurrent('should return failure for Slack-style body errors', async () => { + it('should return failure for Slack-style body errors', async () => { const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, @@ -545,37 +554,32 @@ describe('OAuth Token Refresh', () => { } }) - it.concurrent( - 'should redact literal and encoded credentials echoed by a provider', - async () => { - const refreshToken = 'refresh/with space' - const formEncodedRefreshToken = new URLSearchParams({ value: refreshToken }) - .toString() - .slice('value='.length) - const mockFetch = vi - .fn() - .mockResolvedValue( - new Response( - `provider echo: ${refreshToken}, ${encodeURIComponent(refreshToken)}, ${formEncodedRefreshToken}, and google_client_secret`, - { status: 400 } - ) + it('should redact literal and encoded credentials echoed by a provider', async () => { + const refreshToken = 'refresh/with space' + const formEncodedRefreshToken = new URLSearchParams({ value: refreshToken }) + .toString() + .slice('value='.length) + const mockFetch = vi + .fn() + .mockResolvedValue( + new Response( + `provider echo: ${refreshToken}, ${encodeURIComponent(refreshToken)}, ${formEncodedRefreshToken}, and google_client_secret`, + { status: 400 } ) - - const result = await withMockFetch(mockFetch, () => - refreshOAuthToken('google', refreshToken) ) - expect(result.ok).toBe(false) - if (!result.ok) { - expect(result.message).not.toContain(refreshToken) - expect(result.message).not.toContain(encodeURIComponent(refreshToken)) - expect(result.message).not.toContain(formEncodedRefreshToken) - expect(result.message).not.toContain('google_client_secret') - } + const result = await withMockFetch(mockFetch, () => refreshOAuthToken('google', refreshToken)) + + expect(result.ok).toBe(false) + if (!result.ok) { + expect(result.message).not.toContain(refreshToken) + expect(result.message).not.toContain(encodeURIComponent(refreshToken)) + expect(result.message).not.toContain(formEncodedRefreshToken) + expect(result.message).not.toContain('google_client_secret') } - ) + }) - it.concurrent('should redact a secret from a successful HTTP body error', async () => { + it('should redact a secret from a successful HTTP body error', async () => { const refreshToken = 'slack-refresh-secret' const mockFetch = vi .fn() @@ -590,7 +594,7 @@ describe('OAuth Token Refresh', () => { } }) - it.concurrent('uses canonical endpoints without following credential redirects', async () => { + it('uses canonical endpoints without following credential redirects', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) await withMockFetch(mockFetch, () => refreshOAuthToken('google', 'test_refresh_token')) @@ -601,23 +605,20 @@ describe('OAuth Token Refresh', () => { ) }) - it.concurrent( - 'should reject oversized OAuth error responses without materializing them', - async () => { - const mockFetch = vi - .fn() - .mockResolvedValue( - new Response('x'.repeat(DEFAULT_MAX_ERROR_BODY_BYTES + 1), { status: 400 }) - ) - - const result = await withMockFetch(mockFetch, () => - refreshOAuthToken('google', 'test_refresh_token') + it('should reject oversized OAuth error responses without materializing them', async () => { + const mockFetch = vi + .fn() + .mockResolvedValue( + new Response('x'.repeat(DEFAULT_MAX_ERROR_BODY_BYTES + 1), { status: 400 }) ) - expect(result.ok).toBe(false) - if (!result.ok) expect(result.message).toContain('exceeds maximum size') - } - ) + const result = await withMockFetch(mockFetch, () => + refreshOAuthToken('google', 'test_refresh_token') + ) + + expect(result.ok).toBe(false) + if (!result.ok) expect(result.message).toContain('exceeds maximum size') + }) }) describe('Token Response Handling', () => { @@ -647,7 +648,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('should bound successful token responses before parsing them', async () => { + it('should bound successful token responses before parsing them', async () => { const mockFetch = vi .fn() .mockResolvedValue(new Response('x'.repeat(DEFAULT_MAX_ERROR_BODY_BYTES + 1))) @@ -660,7 +661,7 @@ describe('OAuth Token Refresh', () => { if (!result.ok) expect(result.message).toContain('exceeds maximum size') }) - it.concurrent('refreshes Monday with JSON body credentials and rotates its token', async () => { + it('refreshes Monday with JSON body credentials and rotates its token', async () => { const expiresAtSeconds = Math.floor(Date.now() / 1000) + 2700 const mockFetch = createMockFetch({ json: { @@ -695,7 +696,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('rejects a Monday refresh response that omits token rotation', async () => { + it('rejects a Monday refresh response that omits token rotation', async () => { const mockFetch = createMockFetch({ json: { access_token: oauthTestJwt({ exp: Math.floor(Date.now() / 1000) + 3600 }), @@ -713,7 +714,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('should use original refresh token when new one is not provided', async () => { + it('should use original refresh token when new one is not provided', async () => { const refreshToken = 'original_refresh_token' const mockFetch = vi.fn().mockResolvedValue({ @@ -734,7 +735,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('should return failure when access token is missing', async () => { + it('should return failure when access token is missing', async () => { const refreshToken = 'test_refresh_token' const mockFetch = vi.fn().mockResolvedValue({ @@ -751,7 +752,7 @@ describe('OAuth Token Refresh', () => { }) describe('Instagram Token Refresh', () => { - it.concurrent('validates and rotates the long-lived token response', async () => { + it('validates and rotates the long-lived token response', async () => { const mockFetch = vi.fn().mockResolvedValue( Response.json({ access_token: 'new_instagram_access_token', @@ -776,7 +777,7 @@ describe('OAuth Token Refresh', () => { ) }) - it.concurrent('rejects malformed long-lived token response fields', async () => { + it('rejects malformed long-lived token response fields', async () => { const mockFetch = vi.fn().mockResolvedValue( Response.json({ access_token: 'new_instagram_access_token', @@ -794,7 +795,7 @@ describe('OAuth Token Refresh', () => { }) }) - it.concurrent('extracts nested Meta error codes from a bounded response', async () => { + it('extracts nested Meta error codes from a bounded response', async () => { const mockFetch = vi .fn() .mockResolvedValue( diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index ed881ddac3d..a38a7d12554 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -1,10 +1,3 @@ -import { - type OAuthClientCapabilityField, - type OAuthClientCapabilityId, - requireOAuthClientCapability, -} from '@sim/deployment-config/env-capabilities' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { AffinityIcon, AgentMailIcon, @@ -201,40 +194,14 @@ import { ZoomIcon, ZoomInfoIcon, } from '@/components/icons' -import { env } from '@/lib/core/config/env' import { isSlackExtendedScopesEnabled } from '@/lib/core/config/env-flags' -import { redactExactSensitiveValues } from '@/lib/core/security/redaction' -import { - DEFAULT_MAX_ERROR_BODY_BYTES, - readResponseTextWithLimit, -} from '@/lib/core/utils/stream-limits' -import { APOLLO_TOKEN_URL } from '@/lib/oauth/apollo' -import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' -import { - GITHUB_TOKEN_URL, - parseGitHubRepositoriesTokenResponse, -} from '@/lib/oauth/github-repositories' -import { - GREENHOUSE_TOKEN_URL, - getGreenhouseRefreshErrorCode, - parseGreenhouseTokens, -} from '@/lib/oauth/greenhouse' -import { parseInstagramLongLivedToken } from '@/lib/oauth/instagram' -import { MONDAY_OAUTH_TOKEN_URL, resolveMondayAccessTokenExpiresAt } from '@/lib/oauth/monday' -import type { QuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' -import { QUICKBOOKS_TOKEN_URL } from '@/lib/oauth/quickbooks-constants' -import { isCredentialRevocationError } from '@/lib/oauth/refresh-error-codes' import { SALESFORCE_ADDITIONAL_PROVIDER_IDS, - SALESFORCE_LOGIN_HOSTS, SALESFORCE_PROVIDER_ID_LABELS, } from '@/lib/oauth/salesforce' -import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' import type { OAuthProviderConfig } from './types' -const logger = createLogger('OAuth') - /** * Slack scopes requested only where the app is approved for them, gated by * {@link isSlackExtendedScopesEnabled}. Slack rejects the entire authorization @@ -4053,720 +4020,6 @@ export const OAUTH_PROVIDERS: Record = { }, } -interface ProviderAuthConfig { - tokenEndpoint: string - clientId: string - clientSecret: string - useBasicAuth: boolean - additionalHeaders?: Record - supportsRefreshTokenRotation?: boolean - /** - * If true, the refresh token is sent in the Authorization header as Bearer token - * instead of in the request body. Used by Cal.com. - */ - refreshTokenInAuthHeader?: boolean - /** - * If true, the token endpoint expects a JSON body with Content-Type: application/json - * instead of the default application/x-www-form-urlencoded. Used by Notion. - */ - useJsonBody?: boolean - /** - * Token refresh strategy. `instagram_long_lived` uses Meta's GET - * `refresh_access_token?grant_type=ig_refresh_token` flow instead of a - * standard OAuth refresh_token POST. - */ - refreshStrategy?: 'standard' | 'instagram_long_lived' - /** - * Body param name to use for the client identifier instead of the standard `client_id`. - * TikTok requires `client_key` instead. - */ - clientIdParamName?: string - /** Greenhouse requires grant parameters in the query string, with credentials in Basic auth. */ - refreshParamsInQuery?: boolean -} - -function getConfiguredClientCredentials( - providerId: TCapabilityId, - clientIdField: NoInfer>, - clientSecretField?: NoInfer> -): Pick { - const { values } = requireOAuthClientCapability(providerId, env) - return { - clientId: values[clientIdField], - clientSecret: clientSecretField ? values[clientSecretField] : '', - } -} - -/** - * Get OAuth provider configuration for token refresh - */ -function getProviderAuthConfig( - provider: string, - clientOverride?: Pick -): ProviderAuthConfig { - if (clientOverride && provider !== 'quickbooks') { - throw new Error(`OAuth client override is not supported for provider ${provider}`) - } - switch (provider) { - case 'greenhouse': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'greenhouse', - 'GREENHOUSE_CLIENT_ID', - 'GREENHOUSE_CLIENT_SECRET' - ) - return { - tokenEndpoint: GREENHOUSE_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - refreshParamsInQuery: true, - } - } - case 'google': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'google', - 'GOOGLE_CLIENT_ID', - 'GOOGLE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://oauth2.googleapis.com/token', - clientId, - clientSecret, - useBasicAuth: false, - } - } - case 'x': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'x', - 'X_CLIENT_ID', - 'X_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.x.com/2/oauth2/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'tiktok': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'tiktok', - 'TIKTOK_CLIENT_ID', - 'TIKTOK_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://open.tiktokapis.com/v2/oauth/token/', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - // TikTok requires `client_key` in the token request body instead of `client_id`. - clientIdParamName: 'client_key', - } - } - case 'confluence': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'confluence', - 'CONFLUENCE_CLIENT_ID', - 'CONFLUENCE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://auth.atlassian.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - useJsonBody: true, - supportsRefreshTokenRotation: true, - } - } - case 'jira': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'jira', - 'JIRA_CLIENT_ID', - 'JIRA_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://auth.atlassian.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - useJsonBody: true, - supportsRefreshTokenRotation: true, - } - } - case 'calcom': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'calcom', - 'CALCOM_CLIENT_ID' - ) - return { - tokenEndpoint: 'https://app.cal.com/api/auth/oauth/refreshToken', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - // Cal.com requires refresh token in Authorization header, not body - refreshTokenInAuthHeader: true, - } - } - case 'airtable': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'airtable', - 'AIRTABLE_CLIENT_ID', - 'AIRTABLE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://airtable.com/oauth2/v1/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'bitbucket': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'bitbucket', - 'BITBUCKET_CLIENT_ID', - 'BITBUCKET_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://bitbucket.org/site/oauth2/access_token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'github-repositories': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'github-repositories', - 'GITHUB_APP_CLIENT_ID', - 'GITHUB_APP_CLIENT_SECRET' - ) - return { - tokenEndpoint: GITHUB_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: false, - additionalHeaders: { Accept: 'application/json' }, - supportsRefreshTokenRotation: true, - } - } - case 'notion': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'notion', - 'NOTION_CLIENT_ID', - 'NOTION_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.notion.com/v1/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - useJsonBody: true, - } - } - case 'microsoft': - case 'outlook': - case 'onedrive': - case 'sharepoint': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'microsoft', - 'MICROSOFT_CLIENT_ID', - 'MICROSOFT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://login.microsoftonline.com/common/oauth2/v2.0/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'clickup': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'clickup', - 'CLICKUP_CLIENT_ID', - 'CLICKUP_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.clickup.com/api/v2/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'linear': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'linear', - 'LINEAR_CLIENT_ID', - 'LINEAR_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.linear.app/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'attio': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'attio', - 'ATTIO_CLIENT_ID', - 'ATTIO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.attio.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - } - } - case 'box': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'box', - 'BOX_CLIENT_ID', - 'BOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.box.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - // Box refresh tokens are single-use: "the Refresh Token is invalidated and a - // new Refresh Token is returned" and "A Refresh Token is valid for 60 days and - // can be used to obtain a new Access Token and Refresh Token only once." - // (developer.box.com/guides/authentication/tokens/refresh). Without rotation the - // new token is discarded and the credential dies on the second refresh. - supportsRefreshTokenRotation: true, - } - } - case 'docusign': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'docusign', - 'DOCUSIGN_CLIENT_ID', - 'DOCUSIGN_CLIENT_SECRET' - ) - return { - tokenEndpoint: getDocusignOAuthUrl('/oauth/token'), - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'dropbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'dropbox', - 'DROPBOX_CLIENT_ID', - 'DROPBOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.dropboxapi.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'slack': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'slack', - 'SLACK_CLIENT_ID', - 'SLACK_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://slack.com/api/oauth.v2.access', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'reddit': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'reddit', - 'REDDIT_CLIENT_ID', - 'REDDIT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://www.reddit.com/api/v1/access_token', - clientId, - clientSecret, - useBasicAuth: true, - additionalHeaders: { - 'User-Agent': REDDIT_USER_AGENT, - }, - } - } - case 'ramp': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'ramp', - 'RAMP_CLIENT_ID', - 'RAMP_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.ramp.com/developer/v1/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: false, - } - } - case 'helpscout': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'helpscout', - 'HELPSCOUT_CLIENT_ID', - 'HELPSCOUT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.helpscout.net/v2/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'wealthbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'wealthbox', - 'WEALTHBOX_CLIENT_ID', - 'WEALTHBOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.crmworkspace.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'webflow': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'webflow', - 'WEBFLOW_CLIENT_ID', - 'WEBFLOW_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.webflow.com/oauth/access_token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'asana': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'asana', - 'ASANA_CLIENT_ID', - 'ASANA_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.asana.com/-/oauth_token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'apollo': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'apollo', - 'APOLLO_CLIENT_ID', - 'APOLLO_CLIENT_SECRET' - ) - return { - tokenEndpoint: APOLLO_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'pipedrive': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'pipedrive', - 'PIPEDRIVE_CLIENT_ID', - 'PIPEDRIVE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://oauth.pipedrive.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'quickbooks': { - if (!clientOverride) { - throw new Error('QuickBooks OAuth client configuration is missing') - } - return { - tokenEndpoint: QUICKBOOKS_TOKEN_URL, - clientId: clientOverride.clientId, - clientSecret: clientOverride.clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'hubspot': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'hubspot', - 'HUBSPOT_CLIENT_ID', - 'HUBSPOT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.hubapi.com/oauth/v1/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'linkedin': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'linkedin', - 'LINKEDIN_CLIENT_ID', - 'LINKEDIN_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://www.linkedin.com/oauth/v2/accessToken', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'instagram': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'instagram', - 'INSTAGRAM_CLIENT_ID', - 'INSTAGRAM_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://graph.instagram.com/refresh_access_token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - refreshStrategy: 'instagram_long_lived', - } - } - case 'salesforce': - case 'salesforce-sandbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'salesforce', - 'SALESFORCE_CLIENT_ID', - 'SALESFORCE_CLIENT_SECRET' - ) - // A refresh token is only redeemable at the authorization server that - // issued it: a sandbox token posted to login.salesforce.com fails with - // `invalid_grant`. One Connected App's consumer key is valid at both - // hosts, so only the endpoint differs. - return { - tokenEndpoint: `https://${SALESFORCE_LOGIN_HOSTS[provider]}/services/oauth2/token`, - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'shopify': - throw new Error('Shopify refresh requires its persisted installation context') - case 'zoom': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoom', - 'ZOOM_CLIENT_ID', - 'ZOOM_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://zoom.us/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'wordpress': { - // WordPress.com does NOT support refresh tokens - // Users will need to re-authorize when tokens expire (~2 weeks) - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'wordpress', - 'WORDPRESS_CLIENT_ID', - 'WORDPRESS_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://public-api.wordpress.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'spotify': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'spotify', - 'SPOTIFY_CLIENT_ID', - 'SPOTIFY_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.spotify.com/api/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: false, - } - } - case 'monday': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'monday', - 'MONDAY_CLIENT_ID', - 'MONDAY_CLIENT_SECRET' - ) - return { - tokenEndpoint: MONDAY_OAUTH_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: false, - useJsonBody: true, - supportsRefreshTokenRotation: true, - } - } - case 'manageengine-sdp': { - // ServiceDesk Plus Cloud authenticates through Zoho, so the grant is the - // same one Zoho Desk uses and shares its client credentials: scopes are - // chosen per authorization request, not per API-console client, so one - // registered client serves both products. - // - // Rotation stays off for the same reason as zoho-desk below - Zoho's - // refresh_token grant returns a new access token but no new refresh token. - // accounts.zoho.com is correct because the authorize and code-exchange - // legs in lib/auth/connectors/providers.ts are pinned to the US accounts - // server, so every refresh token in the system is US-issued. Data - // residency for API calls is honored separately, via the block's data - // center selector. - // Keyed on the `zoho-desk` capability, which is what - // `resolveOAuthClientCapabilityId('manageengine-sdp')` aliases to — the - // capability names the env pair, not the product. - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoho-desk', - 'ZOHO_CLIENT_ID', - 'ZOHO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'zoho-desk': { - // Zoho's refresh_token grant returns a new access token but no new refresh - // token, so rotation stays off (the existing refresh token is preserved). - // The refresh must target the accounts server of the data center that issued - // the token - "if location=eu, you will need to make access token request to - // https://accounts.zoho.eu" (zoho.com/accounts/protocol/oauth/multi-dc.html). - // accounts.zoho.com is correct here because the authorize and code-exchange - // legs in lib/auth/connectors/providers.ts are also pinned to the US accounts - // server, so every refresh token in the system is US-issued. Making refresh - // DC-aware requires making the grant DC-aware first (read the `accounts-server` - // callback param) and threading the credential's persisted `__zoho_domain__` - // marker into refreshOAuthToken, which today only receives the token string. - // Data residency for API calls is already honored via that persisted Desk base. - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoho-desk', - 'ZOHO_CLIENT_ID', - 'ZOHO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - default: - throw new Error(`Unsupported provider: ${provider}`) - } -} - -/** - * Build the authentication request headers and body for OAuth token refresh - */ -function buildAuthRequest( - config: ProviderAuthConfig, - refreshToken: string -): { headers: Record; bodyParams: Record; useJsonBody?: boolean } { - const headers: Record = { - 'Content-Type': config.useJsonBody ? 'application/json' : 'application/x-www-form-urlencoded', - ...config.additionalHeaders, - } - - const bodyParams: Record = { - grant_type: 'refresh_token', - } - - // Handle refresh token placement - if (config.refreshTokenInAuthHeader) { - // Cal.com style: refresh token in Authorization header as Bearer token - headers.Authorization = `Bearer ${refreshToken}` - } else { - // Standard OAuth: refresh token in request body - bodyParams.refresh_token = refreshToken - } - - if (config.useBasicAuth) { - // Use Basic Authentication - credentials in Authorization header only - const basicAuth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64') - headers.Authorization = `Basic ${basicAuth}` - } else { - // Use body credentials - include client credentials in request body - bodyParams[config.clientIdParamName || 'client_id'] = config.clientId - if (config.clientSecret) { - bodyParams.client_secret = config.clientSecret - } - } - - return { headers, bodyParams, useJsonBody: config.useJsonBody } -} - -/** - * Resolves the key {@link getProviderAuthConfig} is switched on for a stored - * credential's provider id. - * - * Normally that is the base provider, because every service in a family - * refreshes against the same endpoint with the same client. A provider id - * listed in a service's `additionalProviderIds` is the exception: it names a - * *different* authorization server for the same service, so it must reach - * `getProviderAuthConfig` intact — collapsing it to the base would silently - * refresh a sandbox token against the production endpoint. - */ -function getBaseProviderForService(providerId: string): string { - if (providerId in OAUTH_PROVIDERS) { - return providerId - } - - for (const [baseProvider, config] of Object.entries(OAUTH_PROVIDERS)) { - for (const service of Object.values(config.services)) { - if (service.providerId === providerId) { - return baseProvider - } - if (service.additionalProviderIds?.includes(providerId)) { - return providerId - } - } - } - - throw new Error(`Unknown OAuth provider: ${providerId}`) -} - export interface RefreshTokenSuccess { ok: true accessToken: string @@ -4783,322 +4036,11 @@ export interface RefreshTokenFailure { export type RefreshTokenResult = RefreshTokenSuccess | RefreshTokenFailure -function extractErrorCode(value: unknown): string | undefined { - if (value && typeof value === 'object' && 'error' in value) { - const error = (value as { error: unknown }).error - if (typeof error === 'string') return error - if (error && typeof error === 'object' && 'code' in error) { - const code = (error as { code: unknown }).code - if (typeof code === 'string' || typeof code === 'number') return String(code) - } - } - return undefined -} - -function safeOAuthErrorCode(value: unknown, secrets: string[]): string | undefined { - const errorCode = extractErrorCode(value) - if (!errorCode) return undefined - const safeCode = redactExactSensitiveValues(errorCode, secrets).trim().toLowerCase() - return /^[a-z0-9][a-z0-9._:-]{0,127}$/.test(safeCode) ? safeCode : undefined -} - /** - * Hard deadline on the token-endpoint exchange. This function does not coalesce + * Hard deadline on the token-endpoint exchange. `refreshOAuthToken` does not coalesce * on its own; its sole production caller (`performCoalescedRefresh` in the OAuth * utils) shares one in-flight refresh across concurrent callers for a credential. * Without this bound a hung endpoint would wedge every joiner on that key until * the undici socket defaults (~5 min) gave up. */ export const TOKEN_REFRESH_TIMEOUT_MS = 15_000 - -function parseOAuthResponse(responseText: string): unknown { - try { - return JSON.parse(responseText) - } catch { - return responseText - } -} - -function oauthResponseRecord(value: unknown): Record | undefined { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : undefined -} - -const OAUTH_RESPONSE_OMITTED = '[token endpoint response omitted]' - -/** A revoked grant is its owner's to reconnect, not a fault of ours, so it logs at WARN. */ -function logRefreshRejection(errorCode: string | undefined, providerId: string) { - return isCredentialRevocationError(errorCode, providerId) - ? logger.warn.bind(logger) - : logger.error.bind(logger) -} - -async function refreshInstagramLongLivedToken( - config: ProviderAuthConfig, - longLivedToken: string, - providerId: string -): Promise { - const url = new URL(config.tokenEndpoint) - url.searchParams.set('grant_type', 'ig_refresh_token') - url.searchParams.set('access_token', longLivedToken) - - const response = await fetch(url.toString(), { - method: 'GET', - redirect: 'error', - signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), - }) - - const responseText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Instagram token refresh response', - }) - const responseData = parseOAuthResponse(responseText) - - if (!response.ok) { - const exactSecrets = [longLivedToken, config.clientSecret ?? ''] - const errorCode = safeOAuthErrorCode(responseData, exactSecrets) - logRefreshRejection(errorCode, providerId)('Instagram long-lived token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, - } - } - - const payload = parseInstagramLongLivedToken(responseData) - if (!payload) { - logger.warn('Invalid Instagram refresh response', { providerId }) - return { ok: false, message: 'Invalid Instagram token refresh response' } - } - - logger.info('Instagram long-lived token refreshed successfully', { - expiresIn: payload.expires_in, - providerId, - }) - - // Instagram returns a new long-lived token; store it as both access and refresh. - return { - ok: true, - accessToken: payload.access_token, - expiresIn: payload.expires_in, - refreshToken: payload.access_token, - } -} - -export async function refreshOAuthToken( - providerId: string, - refreshToken: string, - clientOverride?: Pick -): Promise { - const exactSecrets = [refreshToken] - try { - const provider = getBaseProviderForService(providerId) - - const config = getProviderAuthConfig(provider, clientOverride) - if (config.clientSecret) exactSecrets.push(config.clientSecret) - - if (config.refreshStrategy === 'instagram_long_lived') { - return await refreshInstagramLongLivedToken(config, refreshToken, providerId) - } - - const { headers, bodyParams, useJsonBody } = buildAuthRequest(config, refreshToken) - - // Microsoft refresh tokens are resource-independent. Keep the Power BI audience explicit. - if (providerId === 'microsoft-powerbi') { - bodyParams.scope = OAUTH_PROVIDERS.microsoft.services['microsoft-powerbi'].scopes.join(' ') - } - - const tokenUrl = new URL(config.tokenEndpoint) - if (config.refreshParamsInQuery) { - tokenUrl.search = new URLSearchParams(bodyParams).toString() - } - const response = await fetch(tokenUrl.toString(), { - method: 'POST', - headers, - body: config.refreshParamsInQuery - ? undefined - : useJsonBody - ? JSON.stringify(bodyParams) - : new URLSearchParams(bodyParams).toString(), - redirect: 'error', - signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), - }) - - const responseText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'OAuth token refresh response', - }) - const responseData = parseOAuthResponse(responseText) - - if (!response.ok) { - const errorCode = - (provider === 'greenhouse' ? getGreenhouseRefreshErrorCode(responseData) : undefined) ?? - safeOAuthErrorCode(responseData, exactSecrets) - - logRefreshRejection(errorCode, providerId)('Token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - hasClientId: !!config.clientId, - hasClientSecret: !!config.clientSecret, - hasRefreshToken: !!refreshToken, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, - } - } - - const data = oauthResponseRecord(responseData) - if (!data) { - logger.warn('Invalid OAuth token refresh response', { providerId }) - return { ok: false, message: 'Invalid OAuth token refresh response' } - } - - if ( - data.ok === false || - (provider === 'github-repositories' && typeof data.error === 'string') - ) { - const errorCode = safeOAuthErrorCode(data, exactSecrets) - logRefreshRejection(errorCode, providerId)('Token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - hasClientId: !!config.clientId, - hasClientSecret: !!config.clientSecret, - hasRefreshToken: !!refreshToken, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${OAUTH_RESPONSE_OMITTED}`, - } - } - - if (provider === 'greenhouse') { - const tokens = parseGreenhouseTokens(data) - return { - ok: true, - accessToken: tokens.accessToken, - refreshToken: tokens.refreshToken, - expiresIn: Math.max( - 1, - Math.ceil((tokens.accessTokenExpiresAt.getTime() - Date.now()) / 1000) - ), - refreshTokenExpiresIn: Math.max( - 1, - Math.ceil((tokens.refreshTokenExpiresAt.getTime() - Date.now()) / 1000) - ), - } - } - - if (provider === 'github-repositories') { - const tokens = parseGitHubRepositoriesTokenResponse(data) - return { - ok: true, - accessToken: tokens.access_token, - refreshToken: tokens.refresh_token, - expiresIn: tokens.expires_in, - refreshTokenExpiresIn: tokens.refresh_token_expires_in, - } - } - - const accessToken = - typeof data.access_token === 'string' && data.access_token.length > 0 - ? data.access_token - : undefined - - let newRefreshToken: string | undefined - if ( - config.supportsRefreshTokenRotation && - typeof data.refresh_token === 'string' && - data.refresh_token.length > 0 - ) { - newRefreshToken = data.refresh_token - logger.info(`Received new refresh token from ${provider}`) - } - if (provider === 'monday' && !newRefreshToken) { - logger.warn('Monday token refresh response omitted its rotating refresh token') - return { ok: false, message: 'Invalid Monday token refresh response' } - } - if (provider === 'quickbooks' && !newRefreshToken) { - logger.warn('QuickBooks token refresh response omitted its rotating refresh token') - return { ok: false, message: 'Invalid QuickBooks token refresh response' } - } - - const rawExpiresIn = data.expires_in ?? data.expiresIn - const parsedExpiresIn = - typeof rawExpiresIn === 'number' || typeof rawExpiresIn === 'string' - ? Number(rawExpiresIn) - : Number.NaN - const responseExpiresIn = - Number.isFinite(parsedExpiresIn) && parsedExpiresIn > 0 ? parsedExpiresIn : undefined - const expiresIn = - provider === 'monday' && accessToken - ? Math.max( - 1, - Math.ceil( - (resolveMondayAccessTokenExpiresAt(accessToken, responseExpiresIn).getTime() - - Date.now()) / - 1000 - ) - ) - : (responseExpiresIn ?? 3600) - - const rawRefreshTokenExpiresIn = data.x_refresh_token_expires_in - const parsedRefreshTokenExpiresIn = - typeof rawRefreshTokenExpiresIn === 'number' || typeof rawRefreshTokenExpiresIn === 'string' - ? Number(rawRefreshTokenExpiresIn) - : Number.NaN - const refreshTokenExpiresIn = - provider === 'quickbooks' && - Number.isSafeInteger(parsedRefreshTokenExpiresIn) && - parsedRefreshTokenExpiresIn > 0 - ? parsedRefreshTokenExpiresIn - : undefined - - if (!accessToken) { - // Log only the shape, never `data` itself - on a partial success it can - // carry live tokens. - logger.warn('No access token found in refresh response', { - providerId, - responseKeys: Object.keys(data ?? {}), - }) - return { ok: false, message: 'No access token in refresh response' } - } - - logger.info('Token refreshed successfully with expiration', { - expiresIn, - hasNewRefreshToken: !!newRefreshToken, - provider, - }) - - return { - ok: true, - accessToken, - expiresIn, - refreshToken: newRefreshToken ?? refreshToken, - ...(refreshTokenExpiresIn ? { refreshTokenExpiresIn } : {}), - } - } catch (error) { - const normalized = toError(error) - const message = - normalized.name === 'PayloadSizeLimitError' || normalized.message.startsWith('OAuth client ') - ? normalized.message - : 'Token refresh failed' - logger.error('Error refreshing token', { errorType: normalized.name }) - return { ok: false, message } - } -} diff --git a/apps/sim/lib/oauth/quickbooks.ts b/apps/sim/lib/oauth/quickbooks.ts index 2f4f9b6d840..9f1990d4480 100644 --- a/apps/sim/lib/oauth/quickbooks.ts +++ b/apps/sim/lib/oauth/quickbooks.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -150,7 +151,7 @@ export async function revokeQuickBooksToken( let response: Response try { - response = await fetch(QUICKBOOKS_REVOCATION_URL, { + response = await outboundFetch(QUICKBOOKS_REVOCATION_URL, { method: 'POST', headers: { Accept: 'application/json', @@ -183,7 +184,7 @@ export async function fetchQuickBooksConnectionProfile( clientConfig: Pick ): Promise { const realmId = normalizeQuickBooksRealmId(callbackRealmId) - const response = await fetch(getQuickBooksUserInfoUrl(clientConfig.environment), { + const response = await outboundFetch(getQuickBooksUserInfoUrl(clientConfig.environment), { headers: buildQuickBooksHeaders(accessToken), signal: AbortSignal.timeout(QUICKBOOKS_OAUTH_REQUEST_TIMEOUT_MS), }) @@ -254,7 +255,7 @@ export async function exchangeQuickBooksAuthorizationCode(params: { signal?: AbortSignal }): Promise { const clientConfig = normalizeQuickBooksOAuthClientConfig(params.clientConfig) - const response = await fetch(QUICKBOOKS_TOKEN_URL, { + const response = await outboundFetch(QUICKBOOKS_TOKEN_URL, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/refresh.server.ts b/apps/sim/lib/oauth/refresh.server.ts new file mode 100644 index 00000000000..862d1d90d67 --- /dev/null +++ b/apps/sim/lib/oauth/refresh.server.ts @@ -0,0 +1,1073 @@ +import { + type OAuthClientCapabilityField, + type OAuthClientCapabilityId, + requireOAuthClientCapability, +} from '@sim/deployment-config/env-capabilities' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { redactExactSensitiveValues } from '@/lib/core/security/redaction' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { APOLLO_TOKEN_URL } from '@/lib/oauth/apollo' +import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' +import { + GITHUB_TOKEN_URL, + parseGitHubRepositoriesTokenResponse, +} from '@/lib/oauth/github-repositories' +import { + GREENHOUSE_TOKEN_URL, + getGreenhouseRefreshErrorCode, + parseGreenhouseTokens, +} from '@/lib/oauth/greenhouse' +import { parseInstagramLongLivedToken } from '@/lib/oauth/instagram' +import { MONDAY_OAUTH_TOKEN_URL, resolveMondayAccessTokenExpiresAt } from '@/lib/oauth/monday' +import { + OAUTH_PROVIDERS, + type RefreshTokenResult, + TOKEN_REFRESH_TIMEOUT_MS, +} from '@/lib/oauth/oauth' +import type { QuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' +import { QUICKBOOKS_TOKEN_URL } from '@/lib/oauth/quickbooks-constants' +import { isCredentialRevocationError } from '@/lib/oauth/refresh-error-codes' +import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' +import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' + +const logger = createLogger('OAuth') + +interface ProviderAuthConfig { + tokenEndpoint: string + clientId: string + clientSecret: string + useBasicAuth: boolean + additionalHeaders?: Record + supportsRefreshTokenRotation?: boolean + /** + * If true, the refresh token is sent in the Authorization header as Bearer token + * instead of in the request body. Used by Cal.com. + */ + refreshTokenInAuthHeader?: boolean + /** + * If true, the token endpoint expects a JSON body with Content-Type: application/json + * instead of the default application/x-www-form-urlencoded. Used by Notion. + */ + useJsonBody?: boolean + /** + * Token refresh strategy. `instagram_long_lived` uses Meta's GET + * `refresh_access_token?grant_type=ig_refresh_token` flow instead of a + * standard OAuth refresh_token POST. + */ + refreshStrategy?: 'standard' | 'instagram_long_lived' + /** + * Body param name to use for the client identifier instead of the standard `client_id`. + * TikTok requires `client_key` instead. + */ + clientIdParamName?: string + /** Greenhouse requires grant parameters in the query string, with credentials in Basic auth. */ + refreshParamsInQuery?: boolean +} + +function getConfiguredClientCredentials( + providerId: TCapabilityId, + clientIdField: NoInfer>, + clientSecretField?: NoInfer> +): Pick { + const { values } = requireOAuthClientCapability(providerId, env) + return { + clientId: values[clientIdField], + clientSecret: clientSecretField ? values[clientSecretField] : '', + } +} + +/** + * Get OAuth provider configuration for token refresh + */ +function getProviderAuthConfig( + provider: string, + clientOverride?: Pick +): ProviderAuthConfig { + if (clientOverride && provider !== 'quickbooks') { + throw new Error(`OAuth client override is not supported for provider ${provider}`) + } + switch (provider) { + case 'greenhouse': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'greenhouse', + 'GREENHOUSE_CLIENT_ID', + 'GREENHOUSE_CLIENT_SECRET' + ) + return { + tokenEndpoint: GREENHOUSE_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + refreshParamsInQuery: true, + } + } + case 'google': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'google', + 'GOOGLE_CLIENT_ID', + 'GOOGLE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://oauth2.googleapis.com/token', + clientId, + clientSecret, + useBasicAuth: false, + } + } + case 'x': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'x', + 'X_CLIENT_ID', + 'X_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.x.com/2/oauth2/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'tiktok': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'tiktok', + 'TIKTOK_CLIENT_ID', + 'TIKTOK_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://open.tiktokapis.com/v2/oauth/token/', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + // TikTok requires `client_key` in the token request body instead of `client_id`. + clientIdParamName: 'client_key', + } + } + case 'confluence': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'confluence', + 'CONFLUENCE_CLIENT_ID', + 'CONFLUENCE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://auth.atlassian.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + useJsonBody: true, + supportsRefreshTokenRotation: true, + } + } + case 'jira': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'jira', + 'JIRA_CLIENT_ID', + 'JIRA_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://auth.atlassian.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + useJsonBody: true, + supportsRefreshTokenRotation: true, + } + } + case 'calcom': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'calcom', + 'CALCOM_CLIENT_ID' + ) + return { + tokenEndpoint: 'https://app.cal.com/api/auth/oauth/refreshToken', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + // Cal.com requires refresh token in Authorization header, not body + refreshTokenInAuthHeader: true, + } + } + case 'airtable': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'airtable', + 'AIRTABLE_CLIENT_ID', + 'AIRTABLE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://airtable.com/oauth2/v1/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'bitbucket': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'bitbucket', + 'BITBUCKET_CLIENT_ID', + 'BITBUCKET_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://bitbucket.org/site/oauth2/access_token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'github-repositories': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'github-repositories', + 'GITHUB_APP_CLIENT_ID', + 'GITHUB_APP_CLIENT_SECRET' + ) + return { + tokenEndpoint: GITHUB_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: false, + additionalHeaders: { Accept: 'application/json' }, + supportsRefreshTokenRotation: true, + } + } + case 'notion': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'notion', + 'NOTION_CLIENT_ID', + 'NOTION_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.notion.com/v1/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + useJsonBody: true, + } + } + case 'microsoft': + case 'outlook': + case 'onedrive': + case 'sharepoint': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'microsoft', + 'MICROSOFT_CLIENT_ID', + 'MICROSOFT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://login.microsoftonline.com/common/oauth2/v2.0/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'clickup': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'clickup', + 'CLICKUP_CLIENT_ID', + 'CLICKUP_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.clickup.com/api/v2/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'linear': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'linear', + 'LINEAR_CLIENT_ID', + 'LINEAR_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.linear.app/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'attio': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'attio', + 'ATTIO_CLIENT_ID', + 'ATTIO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.attio.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + } + } + case 'box': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'box', + 'BOX_CLIENT_ID', + 'BOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.box.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + // Box refresh tokens are single-use: "the Refresh Token is invalidated and a + // new Refresh Token is returned" and "A Refresh Token is valid for 60 days and + // can be used to obtain a new Access Token and Refresh Token only once." + // (developer.box.com/guides/authentication/tokens/refresh). Without rotation the + // new token is discarded and the credential dies on the second refresh. + supportsRefreshTokenRotation: true, + } + } + case 'docusign': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'docusign', + 'DOCUSIGN_CLIENT_ID', + 'DOCUSIGN_CLIENT_SECRET' + ) + return { + tokenEndpoint: getDocusignOAuthUrl('/oauth/token'), + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'dropbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'dropbox', + 'DROPBOX_CLIENT_ID', + 'DROPBOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.dropboxapi.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'slack': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'slack', + 'SLACK_CLIENT_ID', + 'SLACK_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://slack.com/api/oauth.v2.access', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'reddit': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'reddit', + 'REDDIT_CLIENT_ID', + 'REDDIT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://www.reddit.com/api/v1/access_token', + clientId, + clientSecret, + useBasicAuth: true, + additionalHeaders: { + 'User-Agent': REDDIT_USER_AGENT, + }, + } + } + case 'ramp': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'ramp', + 'RAMP_CLIENT_ID', + 'RAMP_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.ramp.com/developer/v1/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: false, + } + } + case 'helpscout': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'helpscout', + 'HELPSCOUT_CLIENT_ID', + 'HELPSCOUT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.helpscout.net/v2/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'wealthbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'wealthbox', + 'WEALTHBOX_CLIENT_ID', + 'WEALTHBOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.crmworkspace.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'webflow': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'webflow', + 'WEBFLOW_CLIENT_ID', + 'WEBFLOW_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.webflow.com/oauth/access_token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'asana': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'asana', + 'ASANA_CLIENT_ID', + 'ASANA_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.asana.com/-/oauth_token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'apollo': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'apollo', + 'APOLLO_CLIENT_ID', + 'APOLLO_CLIENT_SECRET' + ) + return { + tokenEndpoint: APOLLO_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'pipedrive': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'pipedrive', + 'PIPEDRIVE_CLIENT_ID', + 'PIPEDRIVE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://oauth.pipedrive.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'quickbooks': { + if (!clientOverride) { + throw new Error('QuickBooks OAuth client configuration is missing') + } + return { + tokenEndpoint: QUICKBOOKS_TOKEN_URL, + clientId: clientOverride.clientId, + clientSecret: clientOverride.clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'hubspot': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'hubspot', + 'HUBSPOT_CLIENT_ID', + 'HUBSPOT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.hubapi.com/oauth/v1/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'linkedin': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'linkedin', + 'LINKEDIN_CLIENT_ID', + 'LINKEDIN_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://www.linkedin.com/oauth/v2/accessToken', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'instagram': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'instagram', + 'INSTAGRAM_CLIENT_ID', + 'INSTAGRAM_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://graph.instagram.com/refresh_access_token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + refreshStrategy: 'instagram_long_lived', + } + } + case 'salesforce': + case 'salesforce-sandbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'salesforce', + 'SALESFORCE_CLIENT_ID', + 'SALESFORCE_CLIENT_SECRET' + ) + // A refresh token is only redeemable at the authorization server that + // issued it: a sandbox token posted to login.salesforce.com fails with + // `invalid_grant`. One Connected App's consumer key is valid at both + // hosts, so only the endpoint differs. + return { + tokenEndpoint: `https://${SALESFORCE_LOGIN_HOSTS[provider]}/services/oauth2/token`, + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'shopify': + throw new Error('Shopify refresh requires its persisted installation context') + case 'zoom': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoom', + 'ZOOM_CLIENT_ID', + 'ZOOM_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://zoom.us/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'wordpress': { + // WordPress.com does NOT support refresh tokens + // Users will need to re-authorize when tokens expire (~2 weeks) + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'wordpress', + 'WORDPRESS_CLIENT_ID', + 'WORDPRESS_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://public-api.wordpress.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'spotify': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'spotify', + 'SPOTIFY_CLIENT_ID', + 'SPOTIFY_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.spotify.com/api/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: false, + } + } + case 'monday': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'monday', + 'MONDAY_CLIENT_ID', + 'MONDAY_CLIENT_SECRET' + ) + return { + tokenEndpoint: MONDAY_OAUTH_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: false, + useJsonBody: true, + supportsRefreshTokenRotation: true, + } + } + case 'manageengine-sdp': { + // ServiceDesk Plus Cloud authenticates through Zoho, so the grant is the + // same one Zoho Desk uses and shares its client credentials: scopes are + // chosen per authorization request, not per API-console client, so one + // registered client serves both products. + // + // Rotation stays off for the same reason as zoho-desk below - Zoho's + // refresh_token grant returns a new access token but no new refresh token. + // accounts.zoho.com is correct because the authorize and code-exchange + // legs in lib/auth/connectors/providers.ts are pinned to the US accounts + // server, so every refresh token in the system is US-issued. Data + // residency for API calls is honored separately, via the block's data + // center selector. + // Keyed on the `zoho-desk` capability, which is what + // `resolveOAuthClientCapabilityId('manageengine-sdp')` aliases to — the + // capability names the env pair, not the product. + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoho-desk', + 'ZOHO_CLIENT_ID', + 'ZOHO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'zoho-desk': { + // Zoho's refresh_token grant returns a new access token but no new refresh + // token, so rotation stays off (the existing refresh token is preserved). + // The refresh must target the accounts server of the data center that issued + // the token - "if location=eu, you will need to make access token request to + // https://accounts.zoho.eu" (zoho.com/accounts/protocol/oauth/multi-dc.html). + // accounts.zoho.com is correct here because the authorize and code-exchange + // legs in lib/auth/connectors/providers.ts are also pinned to the US accounts + // server, so every refresh token in the system is US-issued. Making refresh + // DC-aware requires making the grant DC-aware first (read the `accounts-server` + // callback param) and threading the credential's persisted `__zoho_domain__` + // marker into refreshOAuthToken, which today only receives the token string. + // Data residency for API calls is already honored via that persisted Desk base. + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoho-desk', + 'ZOHO_CLIENT_ID', + 'ZOHO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + default: + throw new Error(`Unsupported provider: ${provider}`) + } +} + +/** + * Build the authentication request headers and body for OAuth token refresh + */ +function buildAuthRequest( + config: ProviderAuthConfig, + refreshToken: string +): { headers: Record; bodyParams: Record; useJsonBody?: boolean } { + const headers: Record = { + 'Content-Type': config.useJsonBody ? 'application/json' : 'application/x-www-form-urlencoded', + ...config.additionalHeaders, + } + + const bodyParams: Record = { + grant_type: 'refresh_token', + } + + // Handle refresh token placement + if (config.refreshTokenInAuthHeader) { + // Cal.com style: refresh token in Authorization header as Bearer token + headers.Authorization = `Bearer ${refreshToken}` + } else { + // Standard OAuth: refresh token in request body + bodyParams.refresh_token = refreshToken + } + + if (config.useBasicAuth) { + // Use Basic Authentication - credentials in Authorization header only + const basicAuth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64') + headers.Authorization = `Basic ${basicAuth}` + } else { + // Use body credentials - include client credentials in request body + bodyParams[config.clientIdParamName || 'client_id'] = config.clientId + if (config.clientSecret) { + bodyParams.client_secret = config.clientSecret + } + } + + return { headers, bodyParams, useJsonBody: config.useJsonBody } +} + +/** + * Resolves the key {@link getProviderAuthConfig} is switched on for a stored + * credential's provider id. + * + * Normally that is the base provider, because every service in a family + * refreshes against the same endpoint with the same client. A provider id + * listed in a service's `additionalProviderIds` is the exception: it names a + * *different* authorization server for the same service, so it must reach + * `getProviderAuthConfig` intact — collapsing it to the base would silently + * refresh a sandbox token against the production endpoint. + */ +function getBaseProviderForService(providerId: string): string { + if (providerId in OAUTH_PROVIDERS) { + return providerId + } + + for (const [baseProvider, config] of Object.entries(OAUTH_PROVIDERS)) { + for (const service of Object.values(config.services)) { + if (service.providerId === providerId) { + return baseProvider + } + if (service.additionalProviderIds?.includes(providerId)) { + return providerId + } + } + } + + throw new Error(`Unknown OAuth provider: ${providerId}`) +} + +function extractErrorCode(value: unknown): string | undefined { + if (value && typeof value === 'object' && 'error' in value) { + const error = (value as { error: unknown }).error + if (typeof error === 'string') return error + if (error && typeof error === 'object' && 'code' in error) { + const code = (error as { code: unknown }).code + if (typeof code === 'string' || typeof code === 'number') return String(code) + } + } + return undefined +} + +function safeOAuthErrorCode(value: unknown, secrets: string[]): string | undefined { + const errorCode = extractErrorCode(value) + if (!errorCode) return undefined + const safeCode = redactExactSensitiveValues(errorCode, secrets).trim().toLowerCase() + return /^[a-z0-9][a-z0-9._:-]{0,127}$/.test(safeCode) ? safeCode : undefined +} + +function parseOAuthResponse(responseText: string): unknown { + try { + return JSON.parse(responseText) + } catch { + return responseText + } +} + +function oauthResponseRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined +} + +const OAUTH_RESPONSE_OMITTED = '[token endpoint response omitted]' + +/** A revoked grant is its owner's to reconnect, not a fault of ours, so it logs at WARN. */ +function logRefreshRejection(errorCode: string | undefined, providerId: string) { + return isCredentialRevocationError(errorCode, providerId) + ? logger.warn.bind(logger) + : logger.error.bind(logger) +} + +async function refreshInstagramLongLivedToken( + config: ProviderAuthConfig, + longLivedToken: string, + providerId: string +): Promise { + const url = new URL(config.tokenEndpoint) + url.searchParams.set('grant_type', 'ig_refresh_token') + url.searchParams.set('access_token', longLivedToken) + + const response = await outboundFetch(url.toString(), { + method: 'GET', + redirect: 'error', + signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), + }) + + const responseText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Instagram token refresh response', + }) + const responseData = parseOAuthResponse(responseText) + + if (!response.ok) { + const exactSecrets = [longLivedToken, config.clientSecret ?? ''] + const errorCode = safeOAuthErrorCode(responseData, exactSecrets) + logRefreshRejection(errorCode, providerId)('Instagram long-lived token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, + } + } + + const payload = parseInstagramLongLivedToken(responseData) + if (!payload) { + logger.warn('Invalid Instagram refresh response', { providerId }) + return { ok: false, message: 'Invalid Instagram token refresh response' } + } + + logger.info('Instagram long-lived token refreshed successfully', { + expiresIn: payload.expires_in, + providerId, + }) + + // Instagram returns a new long-lived token; store it as both access and refresh. + return { + ok: true, + accessToken: payload.access_token, + expiresIn: payload.expires_in, + refreshToken: payload.access_token, + } +} + +/** + * Exchanges a stored refresh token at its provider's token endpoint. The request + * leaves through the current outbound scope, so an organization with a dedicated + * egress gateway refreshes from its own static IPs. An unavailable egress route + * throws its `OutboundRoutingError` rather than reporting a failed refresh, so it + * is never read as the provider rejecting the token. + */ +export async function refreshOAuthToken( + providerId: string, + refreshToken: string, + clientOverride?: Pick +): Promise { + const exactSecrets = [refreshToken] + try { + const provider = getBaseProviderForService(providerId) + + const config = getProviderAuthConfig(provider, clientOverride) + if (config.clientSecret) exactSecrets.push(config.clientSecret) + + if (config.refreshStrategy === 'instagram_long_lived') { + return await refreshInstagramLongLivedToken(config, refreshToken, providerId) + } + + const { headers, bodyParams, useJsonBody } = buildAuthRequest(config, refreshToken) + + // Microsoft refresh tokens are resource-independent. Keep the Power BI audience explicit. + if (providerId === 'microsoft-powerbi') { + bodyParams.scope = OAUTH_PROVIDERS.microsoft.services['microsoft-powerbi'].scopes.join(' ') + } + + const tokenUrl = new URL(config.tokenEndpoint) + if (config.refreshParamsInQuery) { + tokenUrl.search = new URLSearchParams(bodyParams).toString() + } + const response = await outboundFetch(tokenUrl.toString(), { + method: 'POST', + headers, + body: config.refreshParamsInQuery + ? undefined + : useJsonBody + ? JSON.stringify(bodyParams) + : new URLSearchParams(bodyParams).toString(), + redirect: 'error', + signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), + }) + + const responseText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'OAuth token refresh response', + }) + const responseData = parseOAuthResponse(responseText) + + if (!response.ok) { + const errorCode = + (provider === 'greenhouse' ? getGreenhouseRefreshErrorCode(responseData) : undefined) ?? + safeOAuthErrorCode(responseData, exactSecrets) + + logRefreshRejection(errorCode, providerId)('Token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + hasClientId: !!config.clientId, + hasClientSecret: !!config.clientSecret, + hasRefreshToken: !!refreshToken, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, + } + } + + const data = oauthResponseRecord(responseData) + if (!data) { + logger.warn('Invalid OAuth token refresh response', { providerId }) + return { ok: false, message: 'Invalid OAuth token refresh response' } + } + + if ( + data.ok === false || + (provider === 'github-repositories' && typeof data.error === 'string') + ) { + const errorCode = safeOAuthErrorCode(data, exactSecrets) + logRefreshRejection(errorCode, providerId)('Token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + hasClientId: !!config.clientId, + hasClientSecret: !!config.clientSecret, + hasRefreshToken: !!refreshToken, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${OAUTH_RESPONSE_OMITTED}`, + } + } + + if (provider === 'greenhouse') { + const tokens = parseGreenhouseTokens(data) + return { + ok: true, + accessToken: tokens.accessToken, + refreshToken: tokens.refreshToken, + expiresIn: Math.max( + 1, + Math.ceil((tokens.accessTokenExpiresAt.getTime() - Date.now()) / 1000) + ), + refreshTokenExpiresIn: Math.max( + 1, + Math.ceil((tokens.refreshTokenExpiresAt.getTime() - Date.now()) / 1000) + ), + } + } + + if (provider === 'github-repositories') { + const tokens = parseGitHubRepositoriesTokenResponse(data) + return { + ok: true, + accessToken: tokens.access_token, + refreshToken: tokens.refresh_token, + expiresIn: tokens.expires_in, + refreshTokenExpiresIn: tokens.refresh_token_expires_in, + } + } + + const accessToken = + typeof data.access_token === 'string' && data.access_token.length > 0 + ? data.access_token + : undefined + + let newRefreshToken: string | undefined + if ( + config.supportsRefreshTokenRotation && + typeof data.refresh_token === 'string' && + data.refresh_token.length > 0 + ) { + newRefreshToken = data.refresh_token + logger.info(`Received new refresh token from ${provider}`) + } + if (provider === 'monday' && !newRefreshToken) { + logger.warn('Monday token refresh response omitted its rotating refresh token') + return { ok: false, message: 'Invalid Monday token refresh response' } + } + if (provider === 'quickbooks' && !newRefreshToken) { + logger.warn('QuickBooks token refresh response omitted its rotating refresh token') + return { ok: false, message: 'Invalid QuickBooks token refresh response' } + } + + const rawExpiresIn = data.expires_in ?? data.expiresIn + const parsedExpiresIn = + typeof rawExpiresIn === 'number' || typeof rawExpiresIn === 'string' + ? Number(rawExpiresIn) + : Number.NaN + const responseExpiresIn = + Number.isFinite(parsedExpiresIn) && parsedExpiresIn > 0 ? parsedExpiresIn : undefined + const expiresIn = + provider === 'monday' && accessToken + ? Math.max( + 1, + Math.ceil( + (resolveMondayAccessTokenExpiresAt(accessToken, responseExpiresIn).getTime() - + Date.now()) / + 1000 + ) + ) + : (responseExpiresIn ?? 3600) + + const rawRefreshTokenExpiresIn = data.x_refresh_token_expires_in + const parsedRefreshTokenExpiresIn = + typeof rawRefreshTokenExpiresIn === 'number' || typeof rawRefreshTokenExpiresIn === 'string' + ? Number(rawRefreshTokenExpiresIn) + : Number.NaN + const refreshTokenExpiresIn = + provider === 'quickbooks' && + Number.isSafeInteger(parsedRefreshTokenExpiresIn) && + parsedRefreshTokenExpiresIn > 0 + ? parsedRefreshTokenExpiresIn + : undefined + + if (!accessToken) { + // Log only the shape, never `data` itself - on a partial success it can + // carry live tokens. + logger.warn('No access token found in refresh response', { + providerId, + responseKeys: Object.keys(data ?? {}), + }) + return { ok: false, message: 'No access token in refresh response' } + } + + logger.info('Token refreshed successfully with expiration', { + expiresIn, + hasNewRefreshToken: !!newRefreshToken, + provider, + }) + + return { + ok: true, + accessToken, + expiresIn, + refreshToken: newRefreshToken ?? refreshToken, + ...(refreshTokenExpiresIn ? { refreshTokenExpiresIn } : {}), + } + } catch (error) { + if (error instanceof OutboundRoutingError) throw error + const normalized = toError(error) + const message = + normalized.name === 'PayloadSizeLimitError' || normalized.message.startsWith('OAuth client ') + ? normalized.message + : 'Token refresh failed' + logger.error('Error refreshing token', { errorType: normalized.name }) + return { ok: false, message } + } +} diff --git a/apps/sim/lib/oauth/salesforce-instance.server.test.ts b/apps/sim/lib/oauth/salesforce-instance.server.test.ts new file mode 100644 index 00000000000..460119b1740 --- /dev/null +++ b/apps/sim/lib/oauth/salesforce-instance.server.test.ts @@ -0,0 +1,31 @@ +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { fetchSalesforceInstanceUrl } from '@/lib/oauth/salesforce-instance.server' + +const mockFetch = vi.fn() + +describe('fetchSalesforceInstanceUrl', () => { + beforeEach(() => { + vi.stubGlobal('fetch', mockFetch) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + }) + + afterEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) + networkConfigMockFns.mockResolveOutboundRoute.mockResolvedValue({ kind: 'direct' }) + }) + + it('propagates a blocked egress route instead of reporting an unverifiable account', async () => { + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('ROUTE_BLOCKED') + ) + await expect(fetchSalesforceInstanceUrl('salesforce', 'token')).rejects.toBeInstanceOf( + OutboundRoutingError + ) + expect(mockFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/oauth/salesforce-instance.server.ts b/apps/sim/lib/oauth/salesforce-instance.server.ts index 86cb61de311..c2ace3a006b 100644 --- a/apps/sim/lib/oauth/salesforce-instance.server.ts +++ b/apps/sim/lib/oauth/salesforce-instance.server.ts @@ -1,12 +1,18 @@ import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { validateSalesforceInstanceUrl } from '@/lib/core/security/input-validation' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { isSalesforceLoginOrigin, SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' const logger = createLogger('SalesforceAccountDestination') -/** Resolves a token's API origin from Salesforce identity, never from model arguments. */ +/** + * Resolves a token's API origin from Salesforce identity, never from model arguments. An + * unavailable or blocked egress route is Sim's outage, not an unverifiable account, so it + * propagates instead of resolving to `undefined`. + */ export async function fetchSalesforceInstanceUrl( providerId: string, accessToken: string @@ -14,7 +20,7 @@ export async function fetchSalesforceInstanceUrl( const loginHost = SALESFORCE_LOGIN_HOSTS[providerId] if (!loginHost) return undefined try { - const response = await fetch(`https://${loginHost}/services/oauth2/userinfo`, { + const response = await outboundFetch(`https://${loginHost}/services/oauth2/userinfo`, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, redirect: 'error', signal: AbortSignal.timeout(10_000), @@ -35,7 +41,8 @@ export async function fetchSalesforceInstanceUrl( if (origin.isValid && origin.sanitized && !isSalesforceLoginOrigin(origin.sanitized)) return origin.sanitized } - } catch { + } catch (error) { + if (error instanceof OutboundRoutingError) throw error logger.warn('Could not resolve Salesforce account destination', { providerId }) } return undefined diff --git a/apps/sim/lib/oauth/shopify-installation.test.ts b/apps/sim/lib/oauth/shopify-installation.test.ts new file mode 100644 index 00000000000..08ea8753dfa --- /dev/null +++ b/apps/sim/lib/oauth/shopify-installation.test.ts @@ -0,0 +1,46 @@ +import { databaseMock, resetDbChainMock } from '@sim/testing' +import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { redisConfigMockFns } from '@sim/testing/mocks/redis-config.mock' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) + +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { refreshShopifyInstallation } from '@/lib/oauth/shopify-installation' + +const shopDomain = 'fixture.myshopify.com' +const findFirst = vi.fn() +const redis = { get: vi.fn(async () => null), set: vi.fn(async () => 'OK') } + +Object.assign(databaseMock.db.query, { account: { findFirst } }) +setEnv({ SHOPIFY_CLIENT_ID: 'shopify-client', SHOPIFY_CLIENT_SECRET: 'shopify-secret' }) +afterAll(resetEnvMock) + +beforeEach(() => { + resetDbChainMock() + redisConfigMockFns.mockGetRedisClient.mockReturnValue(redis) + findFirst.mockResolvedValue({ + id: 'account-1', + accountId: '123456789012345', + idToken: shopDomain, + accessToken: 'expired-access', + accessTokenExpiresAt: new Date(Date.now() - 60_000), + refreshToken: 'stored-refresh', + refreshTokenExpiresAt: null, + }) + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + networkConfigMockFns.mockResolveOutboundRoute.mockRejectedValue( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) +}) + +describe('refreshShopifyInstallation', () => { + it('surfaces an unavailable egress route instead of a failed refresh, leaving the chain alive', async () => { + await expect( + runWithOutboundOrganization('organization-1', () => refreshShopifyInstallation('account-1')) + ).rejects.toBeInstanceOf(OutboundRoutingError) + expect(redis.set).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/oauth/shopify-installation.ts b/apps/sim/lib/oauth/shopify-installation.ts index b3bada458c2..5f10414c5d5 100644 --- a/apps/sim/lib/oauth/shopify-installation.ts +++ b/apps/sim/lib/oauth/shopify-installation.ts @@ -5,6 +5,8 @@ import { generateId } from '@sim/utils/id' import { toRecord } from '@sim/utils/object' import { and, eq, sql } from 'drizzle-orm' import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -95,7 +97,7 @@ async function exchangeToken( const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout let response: Response try { - response = await fetch(`https://${shopDomain}/admin/oauth/access_token`, { + response = await outboundFetch(`https://${shopDomain}/admin/oauth/access_token`, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ @@ -107,8 +109,9 @@ async function exchangeToken( redirect: 'error', signal: requestSignal, }) - } catch { + } catch (error) { signal?.throwIfAborted() + if (error instanceof OutboundRoutingError) throw error throw new ShopifyOAuthError('Shopify token response unavailable', 'shopify_token_error') } const data = await readShopifyResponse(response, requestSignal, signal) @@ -152,7 +155,7 @@ async function fetchShopAccountId( ): Promise { const timeout = AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS) const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout - const response = await fetch( + const response = await outboundFetch( `https://${shopDomain}/admin/api/${SHOPIFY_API_VERSION}/graphql.json`, { method: 'POST', @@ -266,6 +269,7 @@ export async function refreshShopifyInstallation(accountId: string): Promise { - const accountId = await connectShopifyInstallation(params) + const inDraftScope = await resolveConnectDraftOutboundScope({ + draftId: params.draftId, + userId: params.userId, + providerId: 'shopify', + }) + const accountId = await inDraftScope(() => connectShopifyInstallation(params)) await processCredentialDraft({ draftId: params.draftId, userId: params.userId, diff --git a/apps/sim/lib/oauth/token-resolution.test.ts b/apps/sim/lib/oauth/token-resolution.test.ts index 9c0de2e8eb0..175a14aedff 100644 --- a/apps/sim/lib/oauth/token-resolution.test.ts +++ b/apps/sim/lib/oauth/token-resolution.test.ts @@ -26,7 +26,6 @@ vi.mock('@/lib/credentials/application/managed-oauth-delegation', () => ({ this.name = 'InvalidManagedOAuthDelegationError' } }, - authenticateManagedOAuthDelegation: vi.fn(), })) vi.mock('@/lib/credentials/application/resolve-managed-oauth-token', () => ({ @@ -37,10 +36,11 @@ vi.mock('@/lib/credentials/managed-oauth', () => credentialsManagedOauthMock) vi.unmock('@/lib/oauth/utils') +import { OutboundRoutingError } from '@/lib/core/network/routing' import { OrchestrationError } from '@/lib/core/orchestration/types' import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' -import { resolveCredentialAccessToken, resolveCredentialToken } from '@/lib/oauth/token-resolution' +import { resolveCredentialAccessToken } from '@/lib/oauth/token-resolution' import type { OAuthService } from '@/lib/oauth/types' import { getToolMetadata } from '@/tools/metadata' @@ -54,18 +54,20 @@ const { mockResolveServiceAccountToken, } = authOAuthUtilsMockFns -const INTERNAL_AUTH = { success: true, userId: 'user-1', authType: 'internal_jwt' } as const +describe('resolveCredentialAccessToken for plain credentials', () => { + beforeEach(() => { + mockResolveOAuthAccountId.mockResolvedValue(null) + }) -describe('resolveCredentialToken', () => { it('fails closed when the credential is not authorized', async () => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: false, error: 'You do not have access to this credential.', }) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', }) @@ -95,9 +97,9 @@ describe('resolveCredentialToken', () => { }) mockRefreshTokenIfNeeded.mockResolvedValue({ accessToken: 'fresh', refreshed: true }) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', workflowId: 'wf-1', }) @@ -106,6 +108,10 @@ describe('resolveCredentialToken', () => { ok: true, token: { accessToken: 'fresh', credentialType: 'oauth', idToken: 'id-token' }, }) + expect(mockAuthorizeCredentialUseForAuth).toHaveBeenCalledWith( + { success: true, userId: 'user-1', authType: 'internal_jwt' }, + { credentialId: 'cred-1', workflowId: 'wf-1', callerUserId: undefined } + ) expect(mockGetCredential).toHaveBeenCalledWith('req-1', 'account-1', 'owner-1') expect(auditMockFns.mockRecordAuditRollup).toHaveBeenCalledWith( expect.objectContaining({ @@ -135,9 +141,9 @@ describe('resolveCredentialToken', () => { mockRefreshTokenIfNeeded.mockResolvedValue({ accessToken: 'fresh', refreshed: false }) await expect( - resolveCredentialToken(INTERNAL_AUTH, { + resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', }) ).resolves.toEqual({ @@ -161,9 +167,9 @@ describe('resolveCredentialToken', () => { mockGetCredential.mockResolvedValue({ providerId: 'google' }) mockRefreshTokenIfNeeded.mockRejectedValue(new Error('refresh token revoked')) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', }) @@ -172,20 +178,44 @@ describe('resolveCredentialToken', () => { expect(auditMockFns.mockRecordAuditRollup).not.toHaveBeenCalled() }) + it('reports an unavailable egress route as 503, not a failed credential', async () => { + mockAuthorizeCredentialUseForAuth.mockResolvedValue({ + ok: true, + requesterUserId: 'user-1', + credentialOwnerUserId: 'owner-1', + }) + mockGetCredential.mockResolvedValue({ providerId: 'google' }) + mockRefreshTokenIfNeeded.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + userId: 'user-1', + credentialId: 'cred-1', + }) + + expect(result).toEqual({ + ok: false, + status: 503, + error: "The organization's egress route is unavailable", + }) + }) + it('authorizes service-account credentials before minting a token', async () => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: false, error: 'Unauthorized' }) - const result = await resolveCredentialToken(INTERNAL_AUTH, { + mockResolveOAuthAccountId.mockResolvedValue({ + credentialType: 'service_account', + credentialId: 'sa-1', + providerId: 'google', + workspaceId: 'ws-1', + accountId: '', + usedCredentialTable: true, + }) + + const result = await resolveCredentialAccessToken({ requestId: 'req-1', + userId: 'user-1', credentialId: 'cred-1', - resolvedCredential: { - credentialType: 'service_account', - credentialId: 'sa-1', - providerId: 'google', - workspaceId: 'ws-1', - accountId: '', - usedCredentialTable: true, - }, }) expect(result).toEqual({ ok: false, status: 403, error: 'Unauthorized' }) @@ -193,9 +223,9 @@ describe('resolveCredentialToken', () => { }) it('rejects a malformed impersonation subject before touching the credential', async () => { - const result = await resolveCredentialToken(INTERNAL_AUTH, { + const result = await resolveCredentialAccessToken({ requestId: 'req-1', - resolvedCredential: null, + userId: 'user-1', credentialId: 'cred-1', impersonateEmail: 'not-an-email', }) @@ -222,13 +252,11 @@ const EXECUTOR_PRINCIPAL = { } as never describe('resolveCredentialAccessToken', () => { - const authenticate = vi.fn() const resolveManagedPrincipal = vi.fn() beforeEach(() => { mockExecuteManagedToken.mockReset() mockResolveOAuthAccountId.mockResolvedValue(null) - authenticate.mockResolvedValue(INTERNAL_AUTH) resolveManagedPrincipal.mockResolvedValue(EXECUTOR_PRINCIPAL) mockGetToolMetadata.mockReturnValue({ id: 'gmail_send', @@ -244,7 +272,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - authenticate, + userId: 'user-1', }) expect(result).toEqual({ @@ -253,7 +281,7 @@ describe('resolveCredentialAccessToken', () => { code: 'MANAGED_CREDENTIAL_DELEGATION_REQUIRED', error: 'Managed credentials can only be used by an authenticated workflow execution', }) - expect(authenticate).not.toHaveBeenCalled() + expect(mockAuthorizeCredentialUseForAuth).not.toHaveBeenCalled() }) it('maps an invalid delegation to 401 with its message', async () => { @@ -264,7 +292,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -283,7 +311,7 @@ describe('resolveCredentialAccessToken', () => { const result = await resolveCredentialAccessToken({ requestId: 'req-1', credentialId: 'cred-1', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -303,7 +331,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'http_request', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -337,7 +365,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'slack_add_bookmark', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -362,7 +390,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -377,14 +405,11 @@ describe('resolveCredentialAccessToken', () => { it("scopes a managed credential to the tool's required scopes", async () => { mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) mockExecuteManagedToken.mockResolvedValue({ accessToken: 'managed-token', idToken: 'id-1' }) - const auditRequest = { headers: { get: () => null } } - const result = await resolveCredentialAccessToken({ requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - auditRequest, - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -404,9 +429,8 @@ describe('resolveCredentialAccessToken', () => { requiredScopes: ['scope-a'], toolId: 'gmail_send', }, - request: auditRequest, }) - expect(authenticate).not.toHaveBeenCalled() + expect(mockAuthorizeCredentialUseForAuth).not.toHaveBeenCalled() }) it('projects orchestration failures as managed unauthorized', async () => { @@ -419,7 +443,7 @@ describe('resolveCredentialAccessToken', () => { requestId: 'req-1', credentialId: 'cred-1', toolId: 'gmail_send', - authenticate, + userId: 'user-1', resolveManagedPrincipal, }) @@ -430,10 +454,28 @@ describe('resolveCredentialAccessToken', () => { error: 'Managed credential not found', }) }) + + it('reports a managed refresh blocked by the egress route as a retryable 503, not unauthorized', async () => { + mockResolveOAuthAccountId.mockResolvedValue(MANAGED_RESOLVED) + mockExecuteManagedToken.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + const result = await resolveCredentialAccessToken({ + requestId: 'req-1', + credentialId: 'cred-1', + toolId: 'gmail_send', + userId: 'user-1', + resolveManagedPrincipal, + }) + + expect(result).toEqual({ + ok: false, + status: 503, + error: "The organization's egress route is unavailable", + }) + }) }) describe('ordinary tool credential binding', () => { - const authenticate = async () => INTERNAL_AUTH const serviceAccount = { credentialType: 'service_account', credentialId: 'service-account-1', @@ -459,33 +501,11 @@ describe('ordinary tool credential binding', () => { }) }) - it.each([ - ['a session', { success: true, userId: 'user-1', authType: 'session' }], - ['an API key', { success: true, userId: 'user-1', authType: 'api_key' }], - ] as const)( - 'releases a saved secret to server-side callers only, never %s', - async (_label, auth) => { - const result = await resolveCredentialAccessToken({ - requestId: 'binding-test', - credentialId: 'credential-1', - toolId: 'oracle_fusion_financials_list_payables_invoices', - authenticate: async () => auth, - }) - - expect(result).toMatchObject({ - ok: false, - status: 403, - code: 'SERVICE_ACCOUNT_SESSION_FORBIDDEN', - }) - expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() - } - ) - it('refuses to release a service-account secret without a tool to bind it to', async () => { const result = await resolveCredentialAccessToken({ requestId: 'binding-test', credentialId: 'credential-1', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'SERVICE_ACCOUNT_TOOL_REQUIRED' }) @@ -500,7 +520,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'oracle_fusion_financials_list_payables_invoices', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'CREDENTIAL_PROVIDER_MISMATCH' }) expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() @@ -528,7 +548,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'dynamodb_get', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: true }) expect(mockResolveServiceAccountToken).toHaveBeenCalled() @@ -542,7 +562,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'dynamodb_get', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, @@ -562,7 +582,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'oracle_fusion_financials_list_payables_invoices', - authenticate, + userId: 'user-1', }) ).toEqual({ ok: false, status: 403, error: 'Unauthorized' }) expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() @@ -576,7 +596,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'oracle_fusion_financials_list_payables_invoices', - authenticate, + userId: 'user-1', }) ).toMatchObject({ ok: false, status: 400, code: 'CREDENTIAL_TYPE_MISMATCH' }) expect(mockRefreshTokenIfNeeded).not.toHaveBeenCalled() @@ -603,7 +623,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'fixture-tool', - authenticate, + userId: 'user-1', }) ).toMatchObject({ ok: true, @@ -641,7 +661,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'fixture-tool', - authenticate, + userId: 'user-1', }) ).toMatchObject({ ok: true, @@ -668,7 +688,7 @@ describe('ordinary tool credential binding', () => { requestId: 'binding-test', credentialId: 'credential-1', toolId: 'fixture-tool', - authenticate, + userId: 'user-1', }) expect(result.ok).toBe(allowed) if (!allowed) { @@ -680,7 +700,6 @@ describe('ordinary tool credential binding', () => { }) describe('Microsoft app credential binding', () => { - const authenticate = async () => INTERNAL_AUTH const appCredential = { credentialType: 'service_account', credentialId: 'microsoft-app-1', @@ -724,7 +743,7 @@ describe('Microsoft app credential binding', () => { requestId: 'microsoft-app', credentialId: 'microsoft-app-1', toolId: `${provider}-tool`, - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'CREDENTIAL_PROVIDER_MISMATCH' }) expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() @@ -739,7 +758,7 @@ describe('Microsoft app credential binding', () => { requestId: 'microsoft-app', credentialId: 'microsoft-app-1', toolId: `${provider}-tool`, - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: true, token: { credentialType: 'service_account' } }) expect(mockResolveServiceAccountToken).toHaveBeenCalledWith( @@ -759,7 +778,7 @@ describe('Microsoft app credential binding', () => { requestId: 'microsoft-app', credentialId: 'microsoft-app-1', toolId: resetPasswordTool.id, - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'CREDENTIAL_TYPE_MISMATCH' }) expect(mockResolveServiceAccountToken).not.toHaveBeenCalled() @@ -767,8 +786,6 @@ describe('Microsoft app credential binding', () => { }) describe('keyless Google Cloud credential refusals', () => { - const authenticate = async () => INTERNAL_AUTH - beforeEach(() => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: true, @@ -799,7 +816,7 @@ describe('keyless Google Cloud credential refusals', () => { requestId: 'google-keyless', credentialId: 'google-keyless-1', toolId: 'google-drive-tool', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, @@ -817,15 +834,13 @@ describe('keyless Google Cloud credential refusals', () => { requestId: 'servicenow-host', credentialId: 'google-keyless-1', toolId: 'google-drive-tool', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'host_not_allowed' }) }) }) describe('Microsoft app credential cloud refusals', () => { - const authenticate = async () => INTERNAL_AUTH - beforeEach(() => { mockAuthorizeCredentialUseForAuth.mockResolvedValue({ ok: true, @@ -856,7 +871,7 @@ describe('Microsoft app credential cloud refusals', () => { requestId: 'microsoft-cloud', credentialId: 'microsoft-app-gov', toolId: 'powerbi-tool', - authenticate, + userId: 'user-1', }) expect(result).toMatchObject({ ok: false, status: 400, code: 'cloud_mismatch' }) expect(result.ok ? '' : result.error).toMatch(/Microsoft cloud/) diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index 7776067db83..d75844cae69 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -2,12 +2,9 @@ import { AuditAction, AuditResourceType, recordAudit, recordAuditRollup } from ' import { type DelegatedPrincipal, resolvePrincipalSubject } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { - impersonateEmailSchema, - type OAuthTokenResponse, -} from '@/lib/api/contracts/oauth-connections' +import { impersonateEmailSchema } from '@/lib/api/contracts/oauth-connections' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' -import { type AuthResult, AuthType } from '@/lib/auth/hybrid' +import { AuthType } from '@/lib/auth/hybrid' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' import { resolveManagedOAuthCredentialToken } from '@/lib/credentials/application/resolve-managed-oauth-token' @@ -21,6 +18,7 @@ import { refreshTokenIfNeeded, resolveOAuthAccountId, resolveServiceAccountToken, + type ServiceAccountTokenResult, } from '@/lib/oauth/credential-service' import { extractMicrosoftDataverseEnvironmentUrl, @@ -36,11 +34,29 @@ import { } from '@/lib/oauth/utils' import { captureServerEvent } from '@/lib/posthog/server' import { getToolMetadata } from '@/tools/metadata' +import type { QuickBooksEnvironment } from '@/tools/quickbooks/client' import type { OAuthConfig } from '@/tools/types' import { extractZohoDeskBaseFromScope } from '@/tools/zoho_desk/host-allowlist' const logger = createLogger('OAuthTokenResolution') +/** + * A failure Sim's side can't currently get past (the organization's egress gateway is down or + * refuses the route), or null. Never a sign the credential is bad, so the caller is told to + * retry rather than reconnect. + */ +function unavailableFailure( + error: unknown +): Extract | null { + const classified = asOrchestrationError(error) + if (classified?.code !== 'unavailable') return null + return { + ok: false, + status: statusForOrchestrationError('unavailable'), + error: classified.message, + } +} + /** * Duck type of the inbound request, used only so audit rows record IP and user agent. * In-process callers omit it. @@ -49,17 +65,25 @@ export interface CredentialAuditRequest { headers: { get(name: string): string | null } } -/** Token material a resolved credential yields; taken from the contract so it cannot drift. */ -export type CredentialTokenPayload = OAuthTokenResponse +/** + * Token material a resolved credential yields to the tool that consumes it: everything a + * service-account mint returns, plus the OAuth-only identity fields. + */ +export interface CredentialTokenPayload extends ServiceAccountTokenResult { + credentialType?: 'oauth' | 'managed_oauth' | 'service_account' + idToken?: string + realmId?: string + quickBooksEnvironment?: QuickBooksEnvironment +} -export interface ResolveCredentialTokenInput { +interface ResolveCredentialTokenInput { /** Tool whose canonical provider and credential-kind policy applies before token minting. */ toolId?: string /** * The call is an Assistant turn acting with the person's own connected account, so the * tool's policy is the one the Assistant projects for it (GitHub tools bind the person's * GitHub account, never a workspace token). Set only by the in-process executor for an - * authenticated Assistant turn; the token route never forwards it. + * authenticated Assistant turn. */ assistantConnectedAccount?: boolean /** Correlation id used by the credential service's own logging. */ @@ -72,12 +96,13 @@ export interface ResolveCredentialTokenInput { scopes?: string[] /** Google domain-wide-delegation subject for service-account credentials. */ impersonateEmail?: string + /** Acting user whose credential access is enforced; absent when access is not enforced. */ + callerUserId?: string /** - * Asserted acting user. When the caller authenticated with an internal JWT it - * must equal the token subject, so a forged assertion cannot widen access. + * User the server-side execution runs as, authorizing non-managed credentials. + * Managed OAuth credentials authenticate through delegation instead. */ - callerUserId?: string - auditRequest?: CredentialAuditRequest + userId?: string /** Credential lookup already performed by {@link resolveCredentialAccessToken}'s dispatch. */ resolvedCredential: ResolvedCredential | null } @@ -91,7 +116,7 @@ interface OAuthCredentialContext { accountId?: string | null } -export function validateOAuthCredentialContext( +function validateOAuthCredentialContext( credential: OAuthCredentialContext ): { ok: true } | { ok: false; error: string } { if (credential.providerId !== 'quickbooks') return { ok: true } @@ -228,7 +253,6 @@ export async function completeOAuthCredentialToken(params: { workspaceId: string | null workflowId?: string executionId?: string - auditRequest?: CredentialAuditRequest }): Promise { const { requestId, @@ -238,7 +262,6 @@ export async function completeOAuthCredentialToken(params: { workspaceId, workflowId, executionId, - auditRequest, } = params const contextValidation = validateOAuthCredentialContext(credential) if (!contextValidation.ok) { @@ -257,7 +280,6 @@ export async function completeOAuthCredentialToken(params: { credentialType: 'oauth', workflowId, executionId, - auditRequest, }) } @@ -272,6 +294,11 @@ export async function completeOAuthCredentialToken(params: { }) return { ok: false, status: 401, code: OAUTH_CREDENTIAL_REVOKED, error: error.message } } + const unavailable = unavailableFailure(error) + if (unavailable) { + logger.warn('Egress route unavailable for token refresh', { requestId }) + return unavailable + } logger.error('Failed to refresh access token:', error, { requestId }) return { ok: false, status: 401, error: 'Failed to refresh access token' } } @@ -331,15 +358,11 @@ function validateToolCredentialBinding( } /** - * Resolves a plain OAuth or service-account credential to a token for an - * authenticated caller. Managed OAuth credentials are dispatched one level up by - * {@link resolveCredentialAccessToken}, which every server surface goes through, - * so authorization, refresh, and audit cannot drift between surfaces. - * - * @param auth Result of authenticating the caller (session or internal JWT). + * Resolves a plain OAuth or service-account credential to a token for server-side + * execution. Managed OAuth credentials are dispatched one level up by + * {@link resolveCredentialAccessToken}. */ -export async function resolveCredentialToken( - auth: AuthResult, +async function resolveCredentialToken( input: ResolveCredentialTokenInput ): Promise { const { @@ -350,7 +373,7 @@ export async function resolveCredentialToken( scopes, impersonateEmail, callerUserId, - auditRequest, + userId, } = input try { @@ -365,25 +388,19 @@ export async function resolveCredentialToken( } const resolved = input.resolvedCredential - const authz = await authorizeCredentialUseForAuth(auth, { - credentialId, - workflowId, - callerUserId, - }) + const authz = await authorizeCredentialUseForAuth( + { success: true, userId, authType: AuthType.INTERNAL_JWT }, + { + credentialId, + workflowId, + callerUserId, + } + ) if (resolved?.credentialType === 'service_account' && resolved.credentialId) { if (!authz.ok) { return { ok: false, status: 403, error: authz.error || 'Unauthorized' } } - // A saved secret never leaves the server: only server-side (internal JWT) callers may resolve it. - if (auth.authType !== AuthType.INTERNAL_JWT) { - return { - ok: false, - status: 403, - code: 'SERVICE_ACCOUNT_SESSION_FORBIDDEN', - error: 'Saved credentials are resolved only during server-side execution', - } - } // A saved secret is released only to a tool of its own provider, never on a bare ID. if (!input.toolId) { @@ -425,7 +442,6 @@ export async function resolveCredentialToken( credentialType: 'service_account', workflowId, executionId, - auditRequest, }) } @@ -444,6 +460,11 @@ export async function resolveCredentialToken( }, } } catch (error) { + const unavailable = unavailableFailure(error) + if (unavailable) { + logger.warn('Egress route unavailable for service account token', { requestId }) + return unavailable + } logger.error('Service account token error:', error, { requestId }) if (error instanceof TokenServiceAccountValidationError) { // Classified provider outages are infra failures, not bad credentials. @@ -566,7 +587,6 @@ export async function resolveCredentialToken( workspaceId: authz.workspaceId ?? null, workflowId, executionId, - auditRequest, }) } catch (error) { logger.error('Error getting access token', error, { requestId }) @@ -576,50 +596,39 @@ export async function resolveCredentialToken( export interface ResolveCredentialAccessTokenInput extends Omit { - /** - * Authenticates the caller for non-managed credentials. Invoked only when the - * credential is not managed OAuth, which authenticates through delegation instead. - */ - authenticate: () => AuthResult | Promise /** * Proves a delegation for one managed credential: a workflow execution (the - * route verifies the delegation JWT header; the executor binds its delegation - * origin in-process) or a Chat turn acting as the signed-in user. Absent, - * managed credentials are rejected with `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. + * executor binds its delegation origin in-process) or a Chat turn acting as the + * signed-in user. Absent, managed credentials are rejected with + * `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. * Must throw {@link InvalidManagedOAuthDelegationError} on an invalid delegation. */ resolveManagedPrincipal?: (credentialId: string) => Promise } /** - * Authorized application dispatch behind `POST /api/auth/oauth/token`. Every server - * surface that needs a credential token — the route and the in-process tool - * executor — goes through here, so the managed / service-account / plain-OAuth - * dispatch, authorization, refresh, audit, and analytics cannot drift between them. + * Authorized application dispatch for a credential token during server-side + * execution, so the managed / service-account / plain-OAuth dispatch, + * authorization, refresh, audit, and analytics run in one place. */ export async function resolveCredentialAccessToken( input: ResolveCredentialAccessTokenInput ): Promise { - const { requestId, credentialId, toolId, auditRequest } = input + const { requestId, credentialId, toolId } = input const resolved = credentialId ? await resolveOAuthAccountId(credentialId) : null if (resolved?.credentialType !== 'managed_oauth' || !resolved.credentialId) { - const auth = await input.authenticate() - return resolveCredentialToken(auth, { + return resolveCredentialToken({ requestId, credentialId, workflowId: input.workflowId, executionId: input.executionId, scopes: input.scopes, - /** - * In-process callers forward raw subblock state, where an untouched - * field is '' — treated as absent, matching what the wire contract - * (which rejects '') and the old truthy guards always produced. - */ + /** The executor forwards raw subblock state, where an untouched field is '': treated as absent. */ impersonateEmail: input.impersonateEmail || undefined, callerUserId: input.callerUserId, - auditRequest, + userId: input.userId, resolvedCredential: resolved, toolId, assistantConnectedAccount: input.assistantConnectedAccount, @@ -695,7 +704,6 @@ export async function resolveCredentialAccessToken( requiredScopes, toolId, }, - request: auditRequest, }) const subject = resolvePrincipalSubject(principal) @@ -731,6 +739,8 @@ export async function resolveCredentialAccessToken( return { ok: false, status: error.statusCode, code: error.code, error: error.message } } + const unavailable = unavailableFailure(error) + if (unavailable) return unavailable const orchestrationError = asOrchestrationError(error) if (orchestrationError) { return { diff --git a/apps/sim/lib/provider-models/application/list-models.ts b/apps/sim/lib/provider-models/application/list-models.ts new file mode 100644 index 00000000000..10f6e6d22f2 --- /dev/null +++ b/apps/sim/lib/provider-models/application/list-models.ts @@ -0,0 +1,175 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + basetenUpstreamResponseSchema, + fireworksUpstreamResponseSchema, + ollamaUpstreamResponseSchema, + providerModelsResponseSchema, + togetherUpstreamResponseSchema, +} from '@/lib/api/contracts/providers' +import { getBYOKKey } from '@/lib/api-key/byok' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { env } from '@/lib/core/config/env' +import { isHosted } from '@/lib/core/config/env-flags' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { providerModelOperations } from '@/lib/provider-models/application/operations' +import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/context' +import { filterBlacklistedModels } from '@/providers/utils' + +const logger = createLogger('ProviderModelsList') + +/** Together's catalog includes non-text models; only chat models work with chat completions. */ +const TOGETHER_NON_CHAT_MODEL_TYPES = new Set([ + 'image', + 'video', + 'audio', + 'transcribe', + 'embedding', + 'moderation', + 'rerank', +]) + +interface ModelCatalogSource { + url: string + /** Sim's own key, used when the workspace has none; absent for a BYOK-only provider. */ + platformKey(): string | undefined + /** Upstream body to provider-prefixed model ids. */ + parse(body: unknown): string[] +} + +const MODEL_CATALOGS = { + baseten: { + url: 'https://inference.baseten.co/v1/models', + platformKey: () => env.BASETEN_API_KEY, + parse: (body) => + basetenUpstreamResponseSchema.parse(body).data.map((model) => `baseten/${model.id}`), + }, + fireworks: { + url: 'https://api.fireworks.ai/inference/v1/models', + /** + * On hosted Sim the platform key is the sim-auto pool's inference key, not a workspace + * credential: enumerating the whole Fireworks catalog from it would offer every serverless + * model as selectable when no key can actually run it (`getApiKeyWithBYOK` serves the platform + * key to catalog models only). + */ + platformKey: () => (isHosted ? undefined : env.FIREWORKS_API_KEY), + parse: (body) => + fireworksUpstreamResponseSchema.parse(body).data.map((model) => `fireworks/${model.id}`), + }, + together: { + url: 'https://api.together.ai/v1/models', + platformKey: () => env.TOGETHER_API_KEY, + parse: (body) => + togetherUpstreamResponseSchema + .parse(body) + .filter((model) => !model.type || !TOGETHER_NON_CHAT_MODEL_TYPES.has(model.type)) + .map((model) => `together/${model.id}`), + }, + /** BYOK-only: Sim never supplies a hosted key and never bills usage. */ + 'ollama-cloud': { + url: 'https://ollama.com/api/tags', + platformKey: () => undefined, + parse: (body) => + ollamaUpstreamResponseSchema.parse(body).models.map((model) => `ollama-cloud/${model.name}`), + }, +} as const satisfies Record + +export type ModelCatalogProvider = keyof typeof MODEL_CATALOGS + +/** + * Reads a provider's model catalog with `apiKey`. Every failure is logged and answers an empty + * list, so the model picker degrades instead of erroring. + */ +async function fetchModelCatalog( + provider: ModelCatalogProvider, + apiKey: string | undefined, + fetchImpl: typeof fetch +): Promise { + if (!apiKey) { + logger.info('No provider API key available, returning empty models', { provider }) + return [] + } + const source = MODEL_CATALOGS[provider] + try { + const response = await fetchImpl(source.url, { + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + cache: 'no-store', + }) + if (!response.ok) { + logger.warn('Failed to fetch provider models', { + provider, + status: response.status, + statusText: response.statusText, + }) + return [] + } + const uniqueModels = Array.from(new Set(source.parse(await response.json()))) + const { models } = providerModelsResponseSchema.parse({ + models: filterBlacklistedModels(uniqueModels), + }) + logger.info('Successfully fetched provider models', { + provider, + count: models.length, + filtered: uniqueModels.length - models.length, + }) + return models + } catch (error) { + logger.error('Error fetching provider models', { + provider, + error: getErrorMessage(error, 'Unknown error'), + }) + return [] + } +} + +/** + * Lists a provider's models for a workspace member: the workspace's BYOK key when it has one, + * otherwise Sim's key, sent through the workspace organization's outbound route. + */ +const listWorkspaceProviderModels = defineAuthorizedWorkspaceUseCase({ + operation: providerModelOperations.list, + resolveContext: ({ input }: { input: { provider: ModelCatalogProvider; workspaceId: string } }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: {}, + execute: async ({ input, context }) => { + const byok = await getBYOKKey(context.workspaceId, input.provider) + const apiKey = byok?.apiKey ?? MODEL_CATALOGS[input.provider].platformKey() + return fetchModelCatalog(input.provider, apiKey, scopedOutboundFetch) + }, +}) + +/** A missing workspace or a caller outside it gets the platform catalog, never an error. */ +function isWorkspaceUnavailable(error: unknown): boolean { + return ( + error instanceof OrchestrationError && + (error.code === 'not_found' || error.code === 'forbidden') + ) +} + +/** + * The models a provider offers to the caller. A signed-in workspace member reads them with the + * workspace's key through its egress route; anyone else gets Sim's own catalog, read directly. + */ +export async function listProviderModels( + provider: ModelCatalogProvider, + { principal, workspaceId }: { principal: SessionPrincipal | null; workspaceId?: string } +): Promise { + if (principal && workspaceId) { + try { + return await listWorkspaceProviderModels.execute({ + principal, + input: { provider, workspaceId }, + }) + } catch (error) { + if (!isWorkspaceUnavailable(error)) throw error + } + } + return fetchModelCatalog( + provider, + MODEL_CATALOGS[provider].platformKey(), + // egress-direct: no authorized workspace, so a platform request made with Sim's own key + fetch + ) +} diff --git a/apps/sim/lib/provider-models/application/operations.ts b/apps/sim/lib/provider-models/application/operations.ts new file mode 100644 index 00000000000..5c1170fcb4d --- /dev/null +++ b/apps/sim/lib/provider-models/application/operations.ts @@ -0,0 +1,12 @@ +import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' + +export const providerModelOperations = { + // permission-group-exempt: lists the models a workspace's own provider key can reach; running a model is gated where it executes. + list: defineWorkspaceOperation({ + id: 'provider_models.list', + minimumRole: 'read', + capability: 'none', + workspaceApiKey: 'deny', + principalKinds: ['session'], + }), +} as const diff --git a/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts b/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts index a98da4a940e..ba89b458b69 100644 --- a/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts +++ b/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts @@ -1,5 +1,6 @@ import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' import { describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' const mockResolveOrganizationToken = vi.hoisted(() => vi.fn()) vi.mock('@/lib/credentials/application/organization-credentials', () => ({ @@ -36,6 +37,21 @@ describe('selector credential bundles', () => { expect(protectedValues.contains('prefix-cloud-1-suffix')).toBe(false) }) + it('surfaces an egress outage instead of reporting the connection unavailable', async () => { + const outage = new OutboundRoutingError('GATEWAY_UNAVAILABLE') + mockResolveCredentialAccessToken.mockRejectedValueOnce(outage) + + await expect( + resolveSelectorCredentialBundle({ + credential: { + suppliedId: 'credential-1', + access: { ok: true, credentialOwnerUserId: 'owner-1' }, + }, + protectedValues: createSelectorProtectedValues(), + }) + ).rejects.toBe(outage) + }) + it('preserves a selector abort without canceling the shared resolution', async () => { let resolveShared!: (value: { accessToken: string }) => void const sharedResolution = new Promise<{ accessToken: string }>((resolve) => { diff --git a/apps/sim/lib/selectors/server/providers/credential-bundle.ts b/apps/sim/lib/selectors/server/providers/credential-bundle.ts index 109fd5c6750..cc86a43e3f2 100644 --- a/apps/sim/lib/selectors/server/providers/credential-bundle.ts +++ b/apps/sim/lib/selectors/server/providers/credential-bundle.ts @@ -1,3 +1,4 @@ +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { resolveOrganizationCredentialTokenBundle } from '@/lib/credentials/application/organization-credentials' import { resolveCredentialTokenBundle, @@ -61,7 +62,7 @@ export async function resolveSelectorCredentialBundle(input: { ) credential.signal?.throwIfAborted() } catch (error) { - if (credential.signal?.aborted) throw error + if (credential.signal?.aborted || isUnavailableFailure(error)) throw error throw new SelectorConnectionUnavailableError() } if (!bundle?.accessToken) throw new SelectorConnectionUnavailableError() diff --git a/apps/sim/lib/selectors/server/providers/harmonic.test.ts b/apps/sim/lib/selectors/server/providers/harmonic.test.ts index 6c3a5d409aa..84bc736f29c 100644 --- a/apps/sim/lib/selectors/server/providers/harmonic.test.ts +++ b/apps/sim/lib/selectors/server/providers/harmonic.test.ts @@ -10,6 +10,7 @@ const { mockFetch } = vi.hoisted(() => ({ vi.mock('@/lib/selectors/server/providers/credential-bundle', () => selectorCredentialBundleMock) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values' import { harmonicSelectorAttachments } from '@/lib/selectors/server/providers/harmonic' import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' @@ -40,6 +41,14 @@ describe('Harmonic server selector adapter', () => { afterAll(() => vi.unstubAllGlobals()) + it('surfaces an unavailable egress route instead of unavailable selector options', async () => { + mockFetch.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + harmonicSelectorAttachments['harmonic.savedSearches'].execute(detailArgs('1')) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('hydrates a selected saved search after the list projection cap', async () => { const rows = Array.from({ length: 501 }, (_, index) => ({ id: index + 1, diff --git a/apps/sim/lib/selectors/server/providers/harmonic.ts b/apps/sim/lib/selectors/server/providers/harmonic.ts index b07923a554d..f57bab77c32 100644 --- a/apps/sim/lib/selectors/server/providers/harmonic.ts +++ b/apps/sim/lib/selectors/server/providers/harmonic.ts @@ -1,5 +1,7 @@ import { isPlainRecord } from '@sim/utils/object' import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { ServerSelectorKey } from '@/lib/selectors/manifest' import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' @@ -105,13 +107,13 @@ async function listSavedSearches( let response: Response try { - response = await fetch(HARMONIC_URL, { + response = await outboundFetch(HARMONIC_URL, { headers: { Accept: 'application/json', apikey: accessToken }, redirect: 'error', signal, }) } catch (error) { - if (args.signal?.aborted) throw error + if (args.signal?.aborted || error instanceof OutboundRoutingError) throw error throw new SelectorOptionsUnavailableError() } if (!response.ok) { diff --git a/apps/sim/lib/selectors/server/providers/http.test.ts b/apps/sim/lib/selectors/server/providers/http.test.ts index b32e09c20f9..37883ec8d33 100644 --- a/apps/sim/lib/selectors/server/providers/http.test.ts +++ b/apps/sim/lib/selectors/server/providers/http.test.ts @@ -1,5 +1,6 @@ import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { SelectorConnectionUnavailableError, SelectorOptionsUnavailableError, @@ -65,6 +66,16 @@ describe('provider HTTP selector boundary', () => { expect(cancel).toHaveBeenCalledOnce() }) + it('surfaces an unavailable egress route instead of unavailable selector options', async () => { + mockFetch.mockRejectedValueOnce(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect( + fetchProviderJsonWithStatus('https://provider.example/item', undefined, { + passthroughNetworkErrors: true, + }) + ).rejects.toBeInstanceOf(OutboundRoutingError) + }) + it('can preserve a generic retry signal without exposing the raw network error', async () => { mockFetch.mockRejectedValueOnce(new Error('fetch failed with provider-secret-canary')) diff --git a/apps/sim/lib/selectors/server/providers/http.ts b/apps/sim/lib/selectors/server/providers/http.ts index 98f33b27d2a..727bd858811 100644 --- a/apps/sim/lib/selectors/server/providers/http.ts +++ b/apps/sim/lib/selectors/server/providers/http.ts @@ -1,5 +1,7 @@ import { createLogger } from '@sim/logger' import { parseRetryAfter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { SelectorConnectionUnavailableError, SelectorOptionsUnavailableError, @@ -119,9 +121,9 @@ export async function fetchProviderJsonWithStatus( const timeoutSignal = AbortSignal.timeout(PROVIDER_TIMEOUT_MS) const signal = init?.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal try { - response = await fetch(input, { ...init, signal, redirect: init?.redirect ?? 'error' }) + response = await outboundFetch(input, { ...init, signal, redirect: init?.redirect ?? 'error' }) } catch (error) { - if (init?.signal?.aborted) throw error + if (init?.signal?.aborted || error instanceof OutboundRoutingError) throw error logProviderFailure(input, 'network') if (options.passthroughNetworkErrors) throw new RetryableProviderNetworkError() throw new SelectorOptionsUnavailableError() diff --git a/apps/sim/lib/selectors/server/providers/snowflake.ts b/apps/sim/lib/selectors/server/providers/snowflake.ts index 07e6a96d8b2..845d24579cd 100644 --- a/apps/sim/lib/selectors/server/providers/snowflake.ts +++ b/apps/sim/lib/selectors/server/providers/snowflake.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' import type { ServerSelectorKey } from '@/lib/selectors/manifest' import { @@ -91,7 +92,7 @@ function requireStatementHandle(value: string): string { } async function fetchSnowflakeResponse(url: string, init: RequestInit): Promise { - const response = await fetch(url, { ...init, redirect: 'error' }) + const response = await outboundFetch(url, { ...init, redirect: 'error' }) if (!response.ok) { await response.body?.cancel().catch(() => undefined) throw selectorProviderStatusError(response.status) diff --git a/apps/sim/lib/slack-search/public-install-auth.ts b/apps/sim/lib/slack-search/public-install-auth.ts index 6535c17fd4d..2874bb401b4 100644 --- a/apps/sim/lib/slack-search/public-install-auth.ts +++ b/apps/sim/lib/slack-search/public-install-auth.ts @@ -1,4 +1,5 @@ import { isHosted } from '@/lib/core/config/env-flags' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { exchangeSlackBotAuthorization, @@ -9,16 +10,15 @@ import { getSharedSlackSearchAppConfiguration } from '@/lib/slack-search/shared- /** * Completes installation in Slack without binding it to Sim or retaining tokens. + * No organization owns it yet, so the exchange is a platform request on the direct route. * Organization setup later obtains its own grant through the admin's state-bound OAuth flow. */ export async function authenticateSlackPublicInstallation(code: string) { const app = isHosted ? getSharedSlackSearchAppConfiguration() : null if (!app) throw new OrchestrationError('forbidden', 'The Sim Search app is unavailable') - const grant = await exchangeSlackBotAuthorization({ - clientId: app.clientId, - clientSecret: app.clientSecret, - code, - }) + const grant = await runWithOutboundOrganization(null, () => + exchangeSlackBotAuthorization({ clientId: app.clientId, clientSecret: app.clientSecret, code }) + ) validateSlackBotAuthorization(grant, SLACK_SHARED_SEARCH_BOT_SCOPES) if (grant.app_id !== app.id) throw new OrchestrationError('forbidden', 'Slack returned a different app') diff --git a/apps/sim/lib/tools/falai-pricing.ts b/apps/sim/lib/tools/falai-pricing.ts index 2fdd51f1f2d..a4bb1dc2611 100644 --- a/apps/sim/lib/tools/falai-pricing.ts +++ b/apps/sim/lib/tools/falai-pricing.ts @@ -151,6 +151,8 @@ async function fetchFalAIBillingEvent( let response: Response try { + // egress-direct: meters Sim's own fal billing account; the generation call itself is the + // workspace's request and is routed whichever key it carries response = await fetch(url, { headers: { Authorization: `Key ${apiKey}`, @@ -199,6 +201,8 @@ async function estimateFalAICallCost( ): Promise<{ costDollars?: number; error?: string }> { let response: Response try { + // egress-direct: meters Sim's own fal billing account; the generation call itself is the + // workspace's request and is routed whichever key it carries response = await fetch('https://api.fal.ai/v1/models/pricing/estimate', { method: 'POST', headers: { diff --git a/apps/sim/lib/wand/application/generate.ts b/apps/sim/lib/wand/application/generate.ts new file mode 100644 index 00000000000..ed060a085a3 --- /dev/null +++ b/apps/sim/lib/wand/application/generate.ts @@ -0,0 +1,661 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { workflow } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' +import type OpenAI from 'openai' +import { getBYOKKey } from '@/lib/api-key/byok' +import { + type BillingAttributionSnapshot, + checkAttributedBillingBlocks, + checkAttributedUsageLimits, + resolveBillingAttribution, + toBillingContext, +} from '@/lib/billing/core/attribution' +import { recordUsage } from '@/lib/billing/core/usage-log' +import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { env } from '@/lib/core/config/env' +import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags' +import { scopedOutboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { enrichSandboxCapabilities } from '@/lib/execution/remote-sandbox/wand-enricher' +import { enrichTableSchema } from '@/lib/table/llm/wand' +import { wandOperations } from '@/lib/wand/application/operations' +import { + type ActiveWorkspaceApplicationContext, + resolveActiveWorkspaceApplicationContext, +} from '@/lib/workspaces/application/context' +import { extractResponseText, parseResponsesUsage } from '@/providers/openai/utils' +import { getModelPricing } from '@/providers/utils' + +const logger = createLogger('WandGenerate') + +const azureApiKey = env.AZURE_OPENAI_API_KEY +const azureEndpoint = env.AZURE_OPENAI_ENDPOINT +const azureApiVersion = env.AZURE_OPENAI_API_VERSION +const wandModelName = env.WAND_OPENAI_MODEL_NAME || 'gpt-4o' +const openaiApiKey = env.OPENAI_API_KEY + +const useWandAzure = Boolean(azureApiKey && azureEndpoint && azureApiVersion) + +if (!useWandAzure && !openaiApiKey) { + logger.warn( + 'Neither Azure OpenAI nor OpenAI API key found. Wand generation API will not function.' + ) +} else { + logger.info(`Using ${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} for wand generation`) +} + +const wandApiUrl = useWandAzure + ? `${azureEndpoint?.replace(/\/$/, '')}/openai/v1/responses?api-version=${azureApiVersion}` + : 'https://api.openai.com/v1/responses' + +const requestModel = useWandAzure ? wandModelName : 'gpt-4o' + +/** Which deployment serves wand requests, for the surface's error logging and messages. */ +export const WAND_DEPLOYMENT = { + useWandAzure, + model: requestModel, + endpoint: useWandAzure ? azureEndpoint : 'api.openai.com', + apiVersion: useWandAzure ? azureApiVersion : 'N/A', +} as const + +/** + * Sends a wand completion request. Sim's own Azure OpenAI deployment stays direct; OpenAI calls + * (BYOK or Sim's key) are made for the workspace, so they leave through its egress route. + */ +function postWandRequest(init: RequestInit): Promise { + // egress-direct: Sim's env-configured Azure OpenAI deployment, not a workspace credential + if (useWandAzure) return fetch(wandApiUrl, init) + return scopedOutboundFetch(wandApiUrl, init) +} + +interface WandChatMessage { + role: 'user' | 'assistant' | 'system' + content: string +} + +/** + * Wand enricher function type. + * Enrichers add context to the system prompt based on generationType. + */ +type WandEnricher = ( + workspaceId: string | null, + context: Record +) => Promise + +/** + * Registry of wand enrichers by generationType. + * Each enricher returns additional context to append to the system prompt. + */ +const wandEnrichers: Partial> = { + timestamp: async () => { + const now = new Date() + return `Current date and time context for reference: +- Current UTC timestamp: ${now.toISOString()} +- Current Unix timestamp (seconds): ${Math.floor(now.getTime() / 1000)} +- Current Unix timestamp (milliseconds): ${now.getTime()} +- Current date (UTC): ${now.toISOString().split('T')[0]} +- Current year: ${now.getUTCFullYear()} +- Current month: ${now.getUTCMonth() + 1} +- Current day of month: ${now.getUTCDate()} +- Current day of week: ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][now.getUTCDay()]} + +Use this context to calculate relative dates like "yesterday", "last week", "beginning of this month", etc.` + }, + + 'table-schema': enrichTableSchema, + /** JavaScript, Python, and Shell swap prompt text while retaining sandbox enrichment. */ + 'javascript-function-body': enrichSandboxCapabilities, +} + +interface WandUsage { + prompt_tokens?: number + completion_tokens?: number + total_tokens?: number +} + +async function updateUserStatsForWand( + billingAttribution: BillingAttributionSnapshot, + usage: WandUsage, + requestId: string, + isBYOK = false +): Promise { + if (!isBillingEnabled) { + return + } + + if (!usage.total_tokens || usage.total_tokens <= 0) { + return + } + + try { + const promptTokens = usage.prompt_tokens || 0 + const completionTokens = usage.completion_tokens || 0 + + const modelName = requestModel + let costToStore = 0 + + if (!isBYOK) { + const pricing = getModelPricing(modelName) + const costMultiplier = getCostMultiplier() + let modelCost = 0 + + if (pricing) { + const inputCost = (promptTokens / 1000000) * pricing.input + const outputCost = (completionTokens / 1000000) * pricing.output + modelCost = inputCost + outputCost + } else { + modelCost = (promptTokens / 1000000) * 0.005 + (completionTokens / 1000000) * 0.015 + } + + costToStore = modelCost * costMultiplier + } + + await recordUsage({ + userId: billingAttribution.actorUserId, + workspaceId: billingAttribution.workspaceId ?? undefined, + ...toBillingContext(billingAttribution), + entries: [ + { + category: 'model', + source: 'wand', + description: modelName, + cost: costToStore, + sourceReference: `wand:${requestId}`, + metadata: { inputTokens: promptTokens, outputTokens: completionTokens }, + }, + ], + }) + + await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) + } catch (error) { + logger.error('Failed to update user stats for wand usage', error, { requestId }) + } +} + +/** The workflow a wand request names does not exist. */ +export class WandWorkflowNotFoundError extends OrchestrationError { + constructor() { + super('not_found', 'Workflow not found') + this.name = 'WandWorkflowNotFoundError' + } +} + +/** The workflow a wand request names belongs to no workspace (a deprecated personal workflow). */ +export class WandWorkflowDetachedError extends OrchestrationError { + constructor() { + super( + 'forbidden', + 'This workflow is not attached to a workspace. Personal workflows are deprecated and cannot be accessed.' + ) + this.name = 'WandWorkflowDetachedError' + } +} + +/** Neither Sim's Azure deployment nor an OpenAI key (Sim's or the workspace's) is available. */ +export class WandNotConfiguredError extends Error { + constructor() { + super('Wand generation service is not configured.') + this.name = 'WandNotConfiguredError' + } +} + +/** The billing gate refused the request; `message` is the client-facing explanation. */ +export class WandBillingRefusedError extends Error { + constructor( + message: string, + readonly scope?: 'actor' | 'payer' | 'member' + ) { + super(message) + this.name = 'WandBillingRefusedError' + } +} + +/** The model answered without any text. */ +export class WandEmptyResponseError extends Error { + constructor() { + super('Wand response was empty or invalid') + this.name = 'WandEmptyResponseError' + } +} + +/** The model's endpoint answered a non-streaming request with an error status. */ +class WandUpstreamError extends Error { + constructor( + readonly status: number, + message: string + ) { + super(message) + this.name = 'WandUpstreamError' + } +} + +/** The streaming request could not be opened. */ +export class WandStreamStartError extends Error { + constructor(cause: unknown) { + super('Failed to create wand stream', { cause }) + this.name = 'WandStreamStartError' + } +} + +interface GenerateWandInput { + prompt: string + systemPrompt?: string + stream: boolean + history: WandChatMessage[] + generationType?: string + wandContext: Record + requestId: string +} + +type GenerateWandResult = + | { kind: 'stream'; body: ReadableStream } + | { kind: 'content'; content: string } + +interface ResponsesStreamEvent { + type?: string + delta?: unknown + json?: unknown + text?: unknown + error?: { message?: string } + response?: { usage?: OpenAI.Responses.ResponseUsage } + usage?: OpenAI.Responses.ResponseUsage +} + +interface ResponsesCompletion { + output: OpenAI.Responses.ResponseOutputItem[] + usage?: OpenAI.Responses.ResponseUsage +} + +/** + * Kept apart from `resolveActiveWorkflowApplicationContext`: wand has always accepted archived + * workflows and answers a workflow without a workspace with its own refusal, not a not-found. + */ +async function resolveWorkflowWorkspaceId(workflowId: string): Promise { + const [workflowRecord] = await db + .select({ workspaceId: workflow.workspaceId }) + .from(workflow) + .where(eq(workflow.id, workflowId)) + .limit(1) + if (!workflowRecord) { + logger.warn('Wand workflow not found', { workflowId }) + throw new WandWorkflowNotFoundError() + } + if (!workflowRecord.workspaceId) { + logger.warn('Wand workflow has no workspace; request blocked', { workflowId }) + throw new WandWorkflowDetachedError() + } + return workflowRecord.workspaceId +} + +/** + * Streams the Responses API SSE body as `data: {chunk}` events, closing with `data: {done}` and + * recording usage once the completion reports it. + */ +function relayResponsesStream( + response: Response, + recordUsageOnce: (usage: WandUsage) => Promise +): ReadableStream { + const encoder = new TextEncoder() + const decoder = new TextDecoder() + + return new ReadableStream({ + async start(controller) { + const reader = response.body?.getReader() + if (!reader) { + controller.close() + return + } + + let finalUsage: WandUsage | null = null + let usageRecorded = false + + const flushUsage = async () => { + if (usageRecorded || !finalUsage) { + return + } + + usageRecorded = true + await recordUsageOnce(finalUsage) + } + + try { + let buffer = '' + let chunkCount = 0 + let activeEventType: string | undefined + + while (true) { + const { done, value } = await reader.read() + + if (done) { + logger.info(`Stream completed. Total chunks: ${chunkCount}`) + await flushUsage() + controller.enqueue(encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`)) + controller.close() + break + } + + buffer += decoder.decode(value, { stream: true }) + + const lines = buffer.split('\n') + buffer = lines.pop() || '' + + for (const line of lines) { + const trimmed = line.trim() + if (!trimmed) { + continue + } + + if (trimmed.startsWith('event:')) { + activeEventType = trimmed.slice(6).trim() + continue + } + + if (!trimmed.startsWith('data:')) { + continue + } + + const data = trimmed.slice(5).trim() + if (data === '[DONE]') { + logger.info('Received [DONE] signal') + + await flushUsage() + + controller.enqueue(encoder.encode(`data: ${JSON.stringify({ done: true })}\n\n`)) + controller.close() + return + } + + let parsed: ResponsesStreamEvent | null + try { + parsed = JSON.parse(data) + } catch { + continue + } + + const eventType = parsed?.type ?? activeEventType + + if ( + eventType === 'response.error' || + eventType === 'error' || + eventType === 'response.failed' + ) { + throw new Error(parsed?.error?.message || 'Responses stream error') + } + + if ( + parsed && + (eventType === 'response.output_text.delta' || + eventType === 'response.output_json.delta') + ) { + const content = deltaContent(parsed) + + if (content) { + chunkCount++ + if (chunkCount === 1) { + logger.info('Received first content chunk') + } + + controller.enqueue( + encoder.encode(`data: ${JSON.stringify({ chunk: content })}\n\n`) + ) + } + } + + if (eventType === 'response.completed') { + const usage = parseResponsesUsage(parsed?.response?.usage ?? parsed?.usage) + if (usage) { + finalUsage = { + prompt_tokens: usage.promptTokens, + completion_tokens: usage.completionTokens, + total_tokens: usage.totalTokens, + } + logger.info(`Received usage data: ${JSON.stringify(finalUsage)}`) + } + } + } + } + } catch (streamError) { + logger.error('Streaming error', { + name: streamError instanceof Error ? streamError.name : undefined, + message: (streamError instanceof Error && streamError.message) || 'Unknown error', + stack: streamError instanceof Error ? streamError.stack : undefined, + }) + + try { + await flushUsage() + } catch (usageError) { + logger.warn('Failed to record usage after stream error', usageError) + } + + const errorData = `data: ${JSON.stringify({ error: 'Streaming failed', done: true })}\n\n` + controller.enqueue(encoder.encode(errorData)) + controller.close() + } finally { + reader.releaseLock() + } + }, + cancel() {}, + }) +} + +function deltaContent(event: ResponsesStreamEvent): string { + const { delta } = event + if (typeof delta === 'string') return delta + if (delta && typeof delta === 'object') { + if ('text' in delta && typeof delta.text === 'string') return delta.text + if ('json' in delta && delta.json !== undefined) return JSON.stringify(delta.json) + } + if (event.json !== undefined) return JSON.stringify(event.json) + if (typeof event.text === 'string') return event.text + return '' +} + +/** + * Generates wand content for a workspace member, billed to that workspace. OpenAI requests leave + * through the workspace organization's outbound route. + */ +async function executeWandGeneration({ + principal, + input, + context, +}: { + principal: SessionPrincipal + input: I + context: ActiveWorkspaceApplicationContext +}): Promise { + const { workspaceId } = context + const { prompt, history, generationType, wandContext, requestId } = input + + /** Wand is an interactive action attributed to the authenticated human. */ + const billingAttribution = await resolveBillingAttribution({ + actorUserId: principal.userId, + workspaceId, + }) + + let isBYOK = false + let activeOpenAIKey = openaiApiKey + + if (!useWandAzure) { + const byokResult = await getBYOKKey(workspaceId, 'openai') + if (byokResult) { + isBYOK = true + activeOpenAIKey = byokResult.apiKey + logger.info('Using BYOK OpenAI key for wand generation') + } + } + + if (!useWandAzure && !activeOpenAIKey) { + logger.error('AI client not initialized. Missing API key.') + throw new WandNotConfiguredError() + } + + /** + * BYOK skips metered caps but still checks the actor and exact workspace + * payer. Non-BYOK applies the complete attributed gate. + */ + if (isBYOK) { + const blocked = await checkAttributedBillingBlocks(billingAttribution) + if (blocked.blocked) { + throw new WandBillingRefusedError( + blocked.message || 'Account is not in good standing. Please contact support.' + ) + } + } else { + const usage = await checkAttributedUsageLimits(billingAttribution) + if (usage.isExceeded) { + throw new WandBillingRefusedError( + usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.', + usage.scope + ) + } + } + + let finalSystemPrompt = + input.systemPrompt || + 'You are a helpful AI assistant. Generate content exactly as requested by the user.' + + if (generationType) { + const enricher = wandEnrichers[generationType] + if (enricher) { + const enrichment = await enricher(workspaceId, wandContext) + if (enrichment) { + finalSystemPrompt += `\n\n${enrichment}` + } + } + } + + if (generationType === 'cron-expression') { + finalSystemPrompt += + '\n\nIMPORTANT: Return ONLY the raw cron expression (e.g., "0 9 * * 1-5"). Do NOT wrap it in markdown code blocks, backticks, or quotes. Do NOT include any explanation or text before or after the expression.' + } + + // Both the JavaScript and Python function-body prompts share this type, so + // the reinforcement stays language-neutral. + if (generationType === 'javascript-function-body') { + finalSystemPrompt += + '\n\nIMPORTANT: Return ONLY the raw function body. Do NOT wrap it in markdown code blocks (no ```javascript, no ```python, no ```). Do NOT include any explanation before or after the code.' + } + + if (generationType === 'json-object') { + finalSystemPrompt += + '\n\nIMPORTANT: Return ONLY the raw JSON object. Do NOT wrap it in markdown code blocks (no ```json or ```). Do NOT include any explanation or text before or after the JSON. The response must start with { and end with }.' + } + + // Separate from json-object: that reinforcement demands braces, which would + // fight a field whose contract is an array. + if (generationType === 'json-array') { + finalSystemPrompt += + '\n\nIMPORTANT: Return ONLY the raw JSON array. Do NOT wrap it in markdown code blocks (no ```json or ```). Do NOT include any explanation or text before or after the JSON. The response must start with [ and end with ].' + } + + const messages: WandChatMessage[] = [ + { role: 'system', content: finalSystemPrompt }, + ...history.filter((msg) => msg.role !== 'system'), + { role: 'user', content: prompt }, + ] + + const headers: Record = { + 'Content-Type': 'application/json', + 'OpenAI-Beta': 'responses=v1', + } + + if (useWandAzure && azureApiKey) { + headers['api-key'] = azureApiKey + } else { + headers.Authorization = `Bearer ${activeOpenAIKey}` + } + + const requestBody = { + model: requestModel, + input: messages, + temperature: 0.2, + max_output_tokens: 10000, + } + + const recordWandUsage = (usage: WandUsage) => + updateUserStatsForWand(billingAttribution, usage, requestId, isBYOK) + + if (input.stream) { + let response: Response + try { + logger.info(`About to create stream with model: ${requestModel}`) + + response = await postWandRequest({ + method: 'POST', + headers, + body: JSON.stringify({ ...requestBody, stream: true }), + }) + + if (!response.ok) { + const errorText = await response.text() + logger.error('API request failed', { + status: response.status, + statusText: response.statusText, + error: errorText, + }) + throw new Error(`API request failed: ${response.status} ${response.statusText}`) + } + } catch (error) { + throw new WandStreamStartError(error) + } + + logger.info('Stream response received, starting processing') + return { kind: 'stream', body: relayResponsesStream(response, recordWandUsage) } + } + + const response = await postWandRequest({ + method: 'POST', + headers, + body: JSON.stringify(requestBody), + }) + + if (!response.ok) { + const errorText = await response.text() + throw new WandUpstreamError( + response.status, + `API request failed: ${response.status} ${response.statusText} - ${errorText}` + ) + } + + const completion: ResponsesCompletion = await response.json() + const generatedContent = extractResponseText(completion.output)?.trim() + + if (!generatedContent) { + logger.error(`${useWandAzure ? 'Azure OpenAI' : 'OpenAI'} response was empty or invalid.`) + throw new WandEmptyResponseError() + } + + logger.info('Wand generation successful') + + const usage = parseResponsesUsage(completion.usage) + if (usage) { + await recordWandUsage({ + prompt_tokens: usage.promptTokens, + completion_tokens: usage.completionTokens, + total_tokens: usage.totalTokens, + }) + } + + return { kind: 'content', content: generatedContent } +} + +/** Generates for a workflow's editor, which requires write access to the workflow's workspace. */ +export const generateWandForWorkflow = defineAuthorizedWorkspaceUseCase({ + operation: wandOperations.generateForWorkflow, + resolveContext: async ({ input }: { input: GenerateWandInput & { workflowId: string } }) => + resolveActiveWorkspaceApplicationContext(await resolveWorkflowWorkspaceId(input.workflowId)), + authorizationOptions: {}, + execute: executeWandGeneration, +}) + +/** Generates with no workflow to resolve (a table schema, an unhydrated editor) for a workspace member. */ +export const generateWandInWorkspace = defineAuthorizedWorkspaceUseCase({ + operation: wandOperations.generateInWorkspace, + resolveContext: ({ input }: { input: GenerateWandInput & { workspaceId: string } }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: {}, + execute: executeWandGeneration, +}) diff --git a/apps/sim/lib/wand/application/operations.ts b/apps/sim/lib/wand/application/operations.ts new file mode 100644 index 00000000000..96d11b7daf5 --- /dev/null +++ b/apps/sim/lib/wand/application/operations.ts @@ -0,0 +1,22 @@ +import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' + +export const wandOperations = { + /** Generating for a workflow's editor requires write access to the workflow's workspace. */ + // permission-group-exempt: an editor assist billed to the workspace; it reads no protected resource the caller cannot already edit. + generateForWorkflow: defineWorkspaceOperation({ + id: 'wand.generate_for_workflow', + minimumRole: 'write', + capability: 'none', + workspaceApiKey: 'deny', + principalKinds: ['session'], + }), + /** Generating with no workflow to resolve requires workspace membership. */ + // permission-group-exempt: an editor assist billed to the workspace; it reads no protected resource the caller cannot already see. + generateInWorkspace: defineWorkspaceOperation({ + id: 'wand.generate_in_workspace', + minimumRole: 'read', + capability: 'none', + workspaceApiKey: 'deny', + principalKinds: ['session'], + }), +} as const diff --git a/apps/sim/lib/webhooks/deploy.test.ts b/apps/sim/lib/webhooks/deploy.test.ts index 54a4ae39ff3..d84e1d0051a 100644 --- a/apps/sim/lib/webhooks/deploy.test.ts +++ b/apps/sim/lib/webhooks/deploy.test.ts @@ -67,6 +67,7 @@ vi.mock('@/lib/webhooks/quickbooks-credentials', () => ({ getQuickBooksWebhookClientConfigByCredentialId: mockGetQuickBooksWebhookCredential, })) +import { OutboundRoutingError } from '@/lib/core/network/routing' import { buildProviderConfig, cleanupInactiveDeploymentWebhooks, @@ -780,4 +781,10 @@ describe('prepareStableTriggerWebhooksForDeploy failure status', () => { const result = await prepare(new AggregateError([rejected, new Error('HTTP 503')])) expect(result.error?.status).toBe(500) }) + + it('reports an egress outage as unavailable', async () => { + const outage = new OutboundRoutingError('GATEWAY_UNAVAILABLE') + const result = await prepare(new AggregateError([rejected, outage])) + expect(result.error?.status).toBe(503) + }) }) diff --git a/apps/sim/lib/webhooks/deploy.ts b/apps/sim/lib/webhooks/deploy.ts index 275c353cc8f..abcd22f20b2 100644 --- a/apps/sim/lib/webhooks/deploy.ts +++ b/apps/sim/lib/webhooks/deploy.ts @@ -7,6 +7,7 @@ import { omit, toRecord } from '@sim/utils/object' import { and, asc, eq, inArray, isNull, ne, or } from 'drizzle-orm' import type { NextRequest } from 'next/server' import { isSlackExtendedScopesEnabled } from '@/lib/core/config/env-flags' +import { isUnavailableFailure, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { getProviderIdFromServiceId } from '@/lib/oauth' import { getSlackBotCredential, @@ -772,6 +773,7 @@ async function configurePollingIfNeeded( */ function webhookConfigFailureStatus(error: unknown): number { const failures = error instanceof AggregateError ? error.errors : [error] + if (failures.some(isUnavailableFailure)) return statusForOrchestrationError('unavailable') const permanent = failures.length > 0 && failures.every((failure) => failure instanceof WebhookDeploymentConfigurationError) @@ -1012,7 +1014,9 @@ export async function saveTriggerWebhooksForDeploy({ success: false, error: { message: getErrorMessage(error, 'Failed to activate webhook subscription'), - status: 500, + status: isUnavailableFailure(error) + ? statusForOrchestrationError('unavailable') + : 500, }, } } @@ -1302,7 +1306,7 @@ export async function saveTriggerWebhooksForDeploy({ (cleanupFailure as Error)?.message || (error as Error)?.message || 'Failed to save webhook records', - status: 500, + status: isUnavailableFailure(error) ? statusForOrchestrationError('unavailable') : 500, }, } } diff --git a/apps/sim/lib/webhooks/polling/gmail.test.ts b/apps/sim/lib/webhooks/polling/gmail.test.ts index c1394a9dc82..3dc62d2ad73 100644 --- a/apps/sim/lib/webhooks/polling/gmail.test.ts +++ b/apps/sim/lib/webhooks/polling/gmail.test.ts @@ -10,6 +10,7 @@ import { webhooksProcessorMockFns, } from '@sim/testing/mocks/webhooks-processor.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' vi.mock('@/lib/core/idempotency/service', () => idempotencyServiceMock) @@ -24,8 +25,11 @@ import type { WorkflowRecord, } from '@/lib/webhooks/polling/types' -const { mockUpdateWebhookProviderConfig: mockUpdateConfig, mockResolveOAuthCredential } = - webhooksPollingUtilsMockFns +const { + mockUpdateWebhookProviderConfig: mockUpdateConfig, + mockResolveOAuthCredential, + mockMarkWebhookFailed, +} = webhooksPollingUtilsMockFns const mockProcessEvent = webhooksProcessorMockFns.mockProcessPolledWebhookEvent const EXPIRED_HISTORY_ID = '30000' @@ -191,3 +195,27 @@ describe('gmailPollingHandler history cursor', () => { expect(updates).not.toHaveProperty('historyId') }) }) + +describe('gmailPollingHandler egress outage', () => { + it('skips the poll without counting a failure when the egress route is down', async () => { + mockResolveOAuthCredential.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(gmailPollingHandler.pollWebhook(context({}))).resolves.toBe('skipped') + + expect(mockMarkWebhookFailed).not.toHaveBeenCalled() + }) + + it('leaves the mail for a later poll without counting a failure when the route drops mid-batch', async () => { + mockResolveOAuthCredential.mockResolvedValue('access-token') + mailboxWithExpiredHistory({ + profileHistoryId: '40100', + inboxMessages: [{ id: 'm-new', historyId: '40050' }], + }) + mockProcessEvent.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + + await expect(gmailPollingHandler.pollWebhook(context({}))).resolves.toBe('skipped') + + expect(mockMarkWebhookFailed).not.toHaveBeenCalled() + expect(mockUpdateConfig).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/webhooks/polling/gmail.ts b/apps/sim/lib/webhooks/polling/gmail.ts index 58b40ba8d4b..52dc437e596 100644 --- a/apps/sim/lib/webhooks/polling/gmail.ts +++ b/apps/sim/lib/webhooks/polling/gmail.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getProviderConfig, type PollingProviderHandler, @@ -8,11 +9,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -123,9 +124,8 @@ export const gmailPollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing Gmail webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -149,7 +149,7 @@ async function fetchNewEmails(accessToken: string, config: GmailWebhookConfig, l historyUrl += `&pageToken=${pageToken}` } - const historyResponse = await fetch(historyUrl, { + const historyResponse = await outboundFetch(historyUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -303,7 +303,7 @@ async function searchEmails(accessToken: string, config: GmailWebhookConfig, log const query = `${baseQuery}${timeConstraint}` const searchUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages?q=${encodeURIComponent(query)}&maxResults=${config.maxEmailsPerPoll || 25}` - const searchResponse = await fetch(searchUrl, { + const searchResponse = await outboundFetch(searchUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -386,7 +386,7 @@ async function getGmailProfileHistoryId( logger: Logger ): Promise { try { - const response = await fetch('https://gmail.googleapis.com/gmail/v1/users/me/profile', { + const response = await outboundFetch('https://gmail.googleapis.com/gmail/v1/users/me/profile', { headers: { Authorization: `Bearer ${accessToken}` }, }) if (!response.ok) { @@ -404,7 +404,7 @@ async function getGmailProfileHistoryId( async function getEmailDetails(accessToken: string, messageId: string): Promise { const messageUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}?format=full` - const messageResponse = await fetch(messageUrl, { + const messageResponse = await outboundFetch(messageUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -563,7 +563,7 @@ async function processEmails( logger.info(`Successfully processed email ${email.id} for webhook ${webhookData.id}`) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error const errorMessage = getErrorMessage(error, 'Unknown error') logger.error(`Error processing email ${email.id}:`, errorMessage) failedCount++ @@ -577,7 +577,7 @@ async function markEmailAsRead(accessToken: string, messageId: string, logger: L const modifyUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}/modify` try { - const response = await fetch(modifyUrl, { + const response = await outboundFetch(modifyUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/polling/google-calendar.ts b/apps/sim/lib/webhooks/polling/google-calendar.ts index 4a4a75d05f3..38c04d4c58c 100644 --- a/apps/sim/lib/webhooks/polling/google-calendar.ts +++ b/apps/sim/lib/webhooks/polling/google-calendar.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -9,11 +10,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -185,9 +186,8 @@ export const googleCalendarPollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing Google Calendar webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -225,7 +225,7 @@ async function fetchChangedEvents( const encodedCalendarId = encodeURIComponent(calendarId) const url = `${CALENDAR_API_BASE}/calendars/${encodedCalendarId}/events?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -370,7 +370,7 @@ async function processEvents( logger.info(`Successfully processed event ${event.id} for webhook ${webhookData.id}`) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error const errorMessage = getErrorMessage(error, 'Unknown error') logger.error(`Error processing event ${event.id}:`, errorMessage) failedCount++ diff --git a/apps/sim/lib/webhooks/polling/google-drive.ts b/apps/sim/lib/webhooks/polling/google-drive.ts index cd98379d2f7..c463105a2e0 100644 --- a/apps/sim/lib/webhooks/polling/google-drive.ts +++ b/apps/sim/lib/webhooks/polling/google-drive.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -9,11 +10,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -167,9 +168,8 @@ export const googleDrivePollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped if (error instanceof Error && error.name === 'DrivePageTokenInvalidError') { await updateWebhookProviderConfig(webhookId, { pageToken: undefined }, logger) await markWebhookSuccess(webhookId, logger) @@ -210,7 +210,7 @@ async function getStartPageToken( } const url = `${DRIVE_API_BASE}/changes/startPageToken?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -257,7 +257,7 @@ async function fetchChanges( } const url = `${DRIVE_API_BASE}/changes?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -413,7 +413,7 @@ async function processChanges( ) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error const errorMessage = getErrorMessage(error, 'Unknown error') logger.error(`Error processing change for file ${change.fileId}:`, errorMessage) failedCount++ diff --git a/apps/sim/lib/webhooks/polling/google-sheets.ts b/apps/sim/lib/webhooks/polling/google-sheets.ts index 111858e41a8..bf223aca59d 100644 --- a/apps/sim/lib/webhooks/polling/google-sheets.ts +++ b/apps/sim/lib/webhooks/polling/google-sheets.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -9,11 +10,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -226,9 +227,8 @@ export const googleSheetsPollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing Google Sheets webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -259,7 +259,7 @@ async function getDriveFileModifiedTime( fileId: string ): Promise { try { - const response = await fetch( + const response = await outboundFetch( `https://www.googleapis.com/drive/v3/files/${fileId}?fields=modifiedTime`, { headers: { Authorization: `Bearer ${accessToken}` } } ) @@ -296,7 +296,7 @@ async function fetchSheetState( }) const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!A:Z?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -348,7 +348,7 @@ async function fetchRowRange( }) const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!${startRow}:${endRow}?${params.toString()}` - const response = await fetch(url, { + const response = await outboundFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -443,7 +443,7 @@ async function processRows( logger.info(`Successfully processed row ${rowNumber} for webhook ${webhookData.id}`) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error const errorMessage = getErrorMessage(error, 'Unknown error') logger.error(`Error processing row ${rowNumber}:`, errorMessage) failedCount++ diff --git a/apps/sim/lib/webhooks/polling/hubspot.ts b/apps/sim/lib/webhooks/polling/hubspot.ts index 4eb2c5fe303..fc742467ee0 100644 --- a/apps/sim/lib/webhooks/polling/hubspot.ts +++ b/apps/sim/lib/webhooks/polling/hubspot.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getProviderConfig, type PollingProviderHandler, @@ -8,11 +9,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -209,9 +210,8 @@ export const hubspotPollingHandler: PollingProviderHandler = { } return await pollSearchBased(ctx, config, accessToken) } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing HubSpot webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -459,7 +459,7 @@ async function pollListMembership( ) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error failedCount++ logger.error( `Error processing HubSpot list membership ${member.recordId}:`, @@ -676,7 +676,7 @@ async function fetchHubSpotChanges(args: FetchArgs): Promise '') diff --git a/apps/sim/lib/webhooks/polling/imap.ts b/apps/sim/lib/webhooks/polling/imap.ts index 993fb9db798..fde8a4a6b63 100644 --- a/apps/sim/lib/webhooks/polling/imap.ts +++ b/apps/sim/lib/webhooks/polling/imap.ts @@ -17,10 +17,10 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -203,9 +203,8 @@ export const imapPollingHandler: PollingProviderHandler = { throw innerError } } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing IMAP webhook ${webhookId}`) await markWebhookFailed(webhookId, logger) return 'failure' @@ -608,7 +607,7 @@ async function processEmails( ) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error logger.error(`Error processing email ${email.uid}`) failedCount++ } diff --git a/apps/sim/lib/webhooks/polling/outlook.ts b/apps/sim/lib/webhooks/polling/outlook.ts index 8655ecc53a9..8b48cc22ec0 100644 --- a/apps/sim/lib/webhooks/polling/outlook.ts +++ b/apps/sim/lib/webhooks/polling/outlook.ts @@ -11,11 +11,11 @@ import { type PollWebhookContext, } from '@/lib/webhooks/polling/types' import { + isUncountedPollFailure, markWebhookFailed, markWebhookSuccess, - PollAdmissionRefusedError, resolveOAuthCredential, - skipAdmissionRefusedPoll, + skipUncountedPollFailure, throwIfAdmissionRefused, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -195,9 +195,8 @@ export const outlookPollingHandler: PollingProviderHandler = { ) return 'success' } catch (error) { - if (error instanceof PollAdmissionRefusedError) { - return skipAdmissionRefusedPoll(logger, webhookId) - } + const skipped = skipUncountedPollFailure(error, logger, webhookId) + if (skipped) return skipped logger.error(`Error processing Outlook webhook ${webhookId}:`, error) await markWebhookFailed(webhookId, logger) return 'failure' @@ -515,7 +514,7 @@ async function processOutlookEmails( logger.info(`Successfully processed email ${email.id} for webhook ${webhookData.id}`) processedCount++ } catch (error) { - if (error instanceof PollAdmissionRefusedError && processedCount === 0) throw error + if (isUncountedPollFailure(error) && processedCount === 0) throw error logger.error(`Error processing email ${email.id}:`, error) failedCount++ } diff --git a/apps/sim/lib/webhooks/polling/utils.test.ts b/apps/sim/lib/webhooks/polling/utils.test.ts index 4049ab7d75d..56f084605fb 100644 --- a/apps/sim/lib/webhooks/polling/utils.test.ts +++ b/apps/sim/lib/webhooks/polling/utils.test.ts @@ -6,11 +6,13 @@ vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) vi.mock('@/triggers/constants', () => ({ MAX_CONSECUTIVE_FAILURES: 5 })) import { sql } from 'drizzle-orm' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { isPollBackedOff, PollFetchError, readPollRetryAfterMs, recordPollSourceFailure, + skipUncountedPollFailure, updateWebhookProviderConfig, } from '@/lib/webhooks/polling/utils' @@ -113,6 +115,26 @@ describe('poll source backoff', () => { expect(Date.parse(String(stored.pollBackoffUntil))).toBe(Date.now() + minutes(10)) }) + /** Whether any write in this test incremented the webhook's consecutive-failure count. */ + function countedAFailure(): boolean { + return dbChainMockFns.set.mock.calls.some(([values]) => 'failedCount' in Object(values)) + } + + it('counts an ordinary source failure toward the auto-disable', async () => { + await failOnce(0) + expect(countedAFailure()).toBe(true) + }) + + it('backs off an egress outage without counting it against the webhook', async () => { + const outage = new Error('feed fetch failed', { + cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + }) + const stored = await failOnce(0, outage) + + expect(stored.pollSourceFailures).toBe(1) + expect(countedAFailure()).toBe(false) + }) + it('ignores a missing or malformed window', () => { for (const config of [{}, { pollBackoffUntil: 'not-a-date' }, { pollBackoffUntil: 42 }, null]) { expect(isPollBackedOff(config, pollStartedAt)).toBe(false) @@ -131,3 +153,16 @@ describe('readPollRetryAfterMs', () => { expect(readPollRetryAfterMs(header, body)).toBe(expected) }) }) + +describe('skipUncountedPollFailure', () => { + it('skips a poll stopped by an egress outage anywhere in the cause chain', () => { + const outage = new Error('token refresh failed', { + cause: new OutboundRoutingError('ROUTE_BLOCKED'), + }) + expect(skipUncountedPollFailure(outage, logger, 'wh-1')).toBe('skipped') + }) + + it('leaves any other failure to be counted', () => { + expect(skipUncountedPollFailure(new Error('invalid_grant'), logger, 'wh-1')).toBeNull() + }) +}) diff --git a/apps/sim/lib/webhooks/polling/utils.ts b/apps/sim/lib/webhooks/polling/utils.ts index 46ceec0ff9f..18025ad4401 100644 --- a/apps/sim/lib/webhooks/polling/utils.ts +++ b/apps/sim/lib/webhooks/polling/utils.ts @@ -6,6 +6,7 @@ import { toRecord } from '@sim/utils/object' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { and, eq, isNull, ne, or, sql } from 'drizzle-orm' import { getDeterministicAdmissionRejectionCode } from '@/lib/core/admission/rejection' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { getOAuthToken, refreshAccessTokenIfNeeded, @@ -62,12 +63,37 @@ export function throwIfAdmissionRefused(result: PolledWebhookEventResult): void if (getDeterministicAdmissionRejectionCode(result)) throw new PollAdmissionRefusedError(result) } -/** Logs a poll stopped by {@link PollAdmissionRefusedError} and reports it as skipped. */ -export function skipAdmissionRefusedPoll(logger: Logger, webhookId: string): 'skipped' { - logger.info('Execution admission refused the webhook; left its items for a later poll', { - webhookId, - }) - return 'skipped' +/** + * Whether a poll stopped by `error` is stopped by something that is not the webhook's fault: an + * execution admission refusal ({@link PollAdmissionRefusedError}) or an outage of Sim's own egress + * route. A poller rethrows it out of its batch while no item has completed, as it does a refusal. + */ +export function isUncountedPollFailure(error: unknown): boolean { + return error instanceof PollAdmissionRefusedError || isUnavailableFailure(error) +} + +/** + * Reports a poll stopped by a failure that is not the webhook's, so it never counts toward + * `MAX_CONSECUTIVE_FAILURES`: an execution admission refusal ({@link PollAdmissionRefusedError}) + * or an outage of Sim's own egress route. Returns `null` for any other error, which the poller + * records as a failure. + */ +export function skipUncountedPollFailure( + error: unknown, + logger: Logger, + webhookId: string +): 'skipped' | null { + if (error instanceof PollAdmissionRefusedError) { + logger.info('Execution admission refused the webhook; left its items for a later poll', { + webhookId, + }) + return 'skipped' + } + if (isUnavailableFailure(error)) { + logger.warn('Egress route unavailable; left the webhook for a later poll', { webhookId }) + return 'skipped' + } + return null } /** @@ -109,7 +135,8 @@ export function clearPollBackoff(providerConfig: unknown): Record, @@ -144,7 +171,7 @@ export async function recordPollSourceFailure( }, logger ) - await markWebhookFailed(webhookData.id, logger) + if (!isUnavailableFailure(error)) await markWebhookFailed(webhookData.id, logger) } /** diff --git a/apps/sim/lib/webhooks/provider-subscriptions.test.ts b/apps/sim/lib/webhooks/provider-subscriptions.test.ts index decb331a631..c61cece9c6c 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.test.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.test.ts @@ -5,6 +5,8 @@ import { } from '@sim/testing/mocks/billing-attribution.mock' import type { NextRequest } from 'next/server' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' const { mockGetEffectiveDecryptedEnv, mockGetExecutionEnvironment } = environmentUtilsMockFns @@ -57,6 +59,25 @@ describe('createExternalWebhookSubscription', () => { } ) + it('keeps an egress outage classified as unavailable', async () => { + mockGetProviderHandler.mockReturnValue({ + createSubscription: async () => { + throw new Error('Provider call failed', { + cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + }) + }, + }) + const failure = await createExternalWebhookSubscription( + {} as NextRequest, + { provider: 'ashby', providerConfig: {} }, + { workspaceId: 'ws-1' }, + 'user-1', + 'req-1' + ).catch((error: unknown) => error) + + expect(isUnavailableFailure(failure)).toBe(true) + }) + it('resolves {{ENV_VAR}} references in providerConfig before calling the provider', async () => { const createSubscription = vi.fn().mockResolvedValue({ providerConfigUpdates: { externalId: 'ext-1' }, diff --git a/apps/sim/lib/webhooks/provider-subscriptions.ts b/apps/sim/lib/webhooks/provider-subscriptions.ts index 61f172c6f18..d764c28a9df 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { omit, toRecord } from '@sim/utils/object' import type { NextRequest } from 'next/server' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { isSensitiveKey } from '@/lib/core/security/redaction' import { resolveBackgroundWebhookEnv, @@ -19,12 +20,17 @@ import { OPAQUE_RESOLVED_SECRET_REPLACEMENT } from '@/executor/utils/resolved-se const logger = createLogger('WebhookProviderSubscriptions') -/** Resolving credentials must not make a provider's exception a durable plaintext export. */ +/** + * Resolving credentials must not make a provider's exception a durable plaintext export. An egress + * outage keeps its `unavailable` classification (with fixed text) so callers can report it as such. + */ function projectProviderFailure( error: unknown, secrets: ReadonlyMap, providerConfig: Record ): Error { + const classified = asOrchestrationError(error) + if (classified?.code === 'unavailable') return classified let message = 'Webhook provider request failed' try { const matches = [...secrets].map(([name, plaintext]) => ({ diff --git a/apps/sim/lib/webhooks/providers/airtable.ts b/apps/sim/lib/webhooks/providers/airtable.ts index 6d7b88985dc..3fb29a65c9d 100644 --- a/apps/sim/lib/webhooks/providers/airtable.ts +++ b/apps/sim/lib/webhooks/providers/airtable.ts @@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger' import { toStringOrNull } from '@sim/utils/coerce' import { toRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAirtableId } from '@/lib/core/security/input-validation' import { getOAuthToken, @@ -198,7 +199,7 @@ async function fetchAndProcessAirtablePayloads( const fullUrl = `${apiUrl}?${queryParams.toString()}` try { - const response = await fetch(fullUrl, { + const response = await outboundFetch(fullUrl, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -509,7 +510,7 @@ export const airtableHandler: WebhookProviderHandler = { specification: specification, } - const airtableResponse = await fetch(airtableApiUrl, { + const airtableResponse = await outboundFetch(airtableApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -622,7 +623,7 @@ export const airtableHandler: WebhookProviderHandler = { const expectedNotificationUrl = getNotificationUrl(webhookRecord) const listUrl = `https://api.airtable.com/v0/bases/${baseId}/webhooks` - const listResp = await fetch(listUrl, { + const listResp = await outboundFetch(listUrl, { headers: { Authorization: `Bearer ${accessToken}`, }, @@ -694,7 +695,7 @@ export const airtableHandler: WebhookProviderHandler = { } const airtableDeleteUrl = `https://api.airtable.com/v0/bases/${baseId}/webhooks/${resolvedExternalId}` - const airtableResponse = await fetch(airtableDeleteUrl, { + const airtableResponse = await outboundFetch(airtableDeleteUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/ashby.ts b/apps/sim/lib/webhooks/providers/ashby.ts index 10e3a6bb066..a2fa371c2fd 100644 --- a/apps/sim/lib/webhooks/providers/ashby.ts +++ b/apps/sim/lib/webhooks/providers/ashby.ts @@ -4,6 +4,7 @@ import { hmacSha256Hex } from '@sim/security/hmac' import { generateId } from '@sim/utils/id' import { isRecordLike, omit, toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -271,7 +272,7 @@ export const ashbyHandler: WebhookProviderHandler = { secretToken, } - const ashbyResponse = await fetch('https://api.ashbyhq.com/webhook.create', { + const ashbyResponse = await outboundFetch('https://api.ashbyhq.com/webhook.create', { method: 'POST', headers: { Authorization: `Basic ${authString}`, @@ -362,7 +363,7 @@ export const ashbyHandler: WebhookProviderHandler = { const authString = Buffer.from(`${apiKey}:`).toString('base64') - const ashbyResponse = await fetch('https://api.ashbyhq.com/webhook.delete', { + const ashbyResponse = await outboundFetch('https://api.ashbyhq.com/webhook.delete', { method: 'POST', headers: { Authorization: `Basic ${authString}`, diff --git a/apps/sim/lib/webhooks/providers/attio.ts b/apps/sim/lib/webhooks/providers/attio.ts index 83f67aa0b54..d6dbd4b6421 100644 --- a/apps/sim/lib/webhooks/providers/attio.ts +++ b/apps/sim/lib/webhooks/providers/attio.ts @@ -4,6 +4,7 @@ import { hmacSha256Hex } from '@sim/security/hmac' import { toError } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getCredentialAccessToken, getNotificationUrl, @@ -169,7 +170,7 @@ export const attioHandler: WebhookProviderHandler = { }, } - const attioResponse = await fetch('https://api.attio.com/v2/webhooks', { + const attioResponse = await outboundFetch('https://api.attio.com/v2/webhooks', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -273,7 +274,7 @@ export const attioHandler: WebhookProviderHandler = { return } - const attioResponse = await fetch(`https://api.attio.com/v2/webhooks/${externalId}`, { + const attioResponse = await outboundFetch(`https://api.attio.com/v2/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/bitbucket.ts b/apps/sim/lib/webhooks/providers/bitbucket.ts index ae9f8077155..7d063a48b26 100644 --- a/apps/sim/lib/webhooks/providers/bitbucket.ts +++ b/apps/sim/lib/webhooks/providers/bitbucket.ts @@ -6,6 +6,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { toRecord, toRecordOrNull } from '@sim/utils/object' import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getCredentialAccessToken, getNotificationUrl, @@ -71,7 +72,7 @@ function bitbucketHooksUrl(workspaceSlug: string, repoSlug: string): string { } function fetchBitbucketManagement(url: string, init: RequestInit = {}): Promise { - return fetch(url, { + return outboundFetch(url, { ...init, signal: AbortSignal.timeout(BITBUCKET_MANAGEMENT_REQUEST_TIMEOUT_MS), }) diff --git a/apps/sim/lib/webhooks/providers/calendly.ts b/apps/sim/lib/webhooks/providers/calendly.ts index 9bf401df6b9..bde8e0d0cce 100644 --- a/apps/sim/lib/webhooks/providers/calendly.ts +++ b/apps/sim/lib/webhooks/providers/calendly.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -88,7 +89,7 @@ export const calendlyHandler: WebhookProviderHandler = { scope: 'organization', } - const calendlyResponse = await fetch(calendlyApiUrl, { + const calendlyResponse = await outboundFetch(calendlyApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -191,7 +192,7 @@ export const calendlyHandler: WebhookProviderHandler = { const calendlyApiUrl = `https://api.calendly.com/webhook_subscriptions/${externalId}` - const calendlyResponse = await fetch(calendlyApiUrl, { + const calendlyResponse = await outboundFetch(calendlyApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/checkr.ts b/apps/sim/lib/webhooks/providers/checkr.ts index a7618573f5c..5e01656c63f 100644 --- a/apps/sim/lib/webhooks/providers/checkr.ts +++ b/apps/sim/lib/webhooks/providers/checkr.ts @@ -6,6 +6,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toArray, toRecord } from '@sim/utils/object' import { and, eq, isNull, ne } from 'drizzle-orm' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -77,7 +78,7 @@ function isValidCheckrSignature(apiKey: string, signature: string, rawBody: stri * adopts the live registration instead of creating another. */ async function findCheckrWebhookForUrl(apiKey: string, url: string): Promise { - const response = await fetch(CHECKR_WEBHOOKS_URL, { headers: checkrAuthHeaders(apiKey) }) + const response = await outboundFetch(CHECKR_WEBHOOKS_URL, { headers: checkrAuthHeaders(apiKey) }) if (!response.ok) return null const body = await readCheckrResponse(response, 'Checkr webhook list response') const match = toArray(body.data) @@ -219,7 +220,7 @@ export const checkrHandler: WebhookProviderHandler = { triggerId: providerConfig.triggerId, }) - const response = await fetch(CHECKR_WEBHOOKS_URL, { + const response = await outboundFetch(CHECKR_WEBHOOKS_URL, { method: 'POST', headers: checkrAuthHeaders(apiKey), body: JSON.stringify({ @@ -279,10 +280,13 @@ export const checkrHandler: WebhookProviderHandler = { return } - const response = await fetch(`${CHECKR_WEBHOOKS_URL}/${encodeURIComponent(externalId)}`, { - method: 'DELETE', - headers: checkrAuthHeaders(apiKey), - }) + const response = await outboundFetch( + `${CHECKR_WEBHOOKS_URL}/${encodeURIComponent(externalId)}`, + { + method: 'DELETE', + headers: checkrAuthHeaders(apiKey), + } + ) if (response.ok) { logger.info(`Deleted Checkr webhook ${externalId}`, { requestId: ctx.requestId }) diff --git a/apps/sim/lib/webhooks/providers/clickup.ts b/apps/sim/lib/webhooks/providers/clickup.ts index 44593b59d24..6187a7da723 100644 --- a/apps/sim/lib/webhooks/providers/clickup.ts +++ b/apps/sim/lib/webhooks/providers/clickup.ts @@ -4,6 +4,7 @@ import { hmacSha256Hex } from '@sim/security/hmac' import { toError } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getCredentialAccessToken, getNotificationUrl, @@ -87,7 +88,7 @@ function parseOptionalStringId(value: unknown): string | undefined { } async function deleteClickUpWebhook(accessToken: string, externalId: string): Promise { - return fetch(`${CLICKUP_API_BASE_URL}/webhook/${externalId}`, { + return outboundFetch(`${CLICKUP_API_BASE_URL}/webhook/${externalId}`, { method: 'DELETE', headers: { Authorization: clickupAuthorizationHeader(accessToken) }, }) @@ -184,7 +185,7 @@ export const clickupHandler: WebhookProviderHandler = { if (listId !== undefined) requestBody.list_id = listId if (taskId !== undefined) requestBody.task_id = taskId - const clickupResponse = await fetch( + const clickupResponse = await outboundFetch( `${CLICKUP_API_BASE_URL}/team/${encodeURIComponent(workspaceId)}/webhook`, { method: 'POST', diff --git a/apps/sim/lib/webhooks/providers/email-polling-outage.test.ts b/apps/sim/lib/webhooks/providers/email-polling-outage.test.ts new file mode 100644 index 00000000000..1685f6e8635 --- /dev/null +++ b/apps/sim/lib/webhooks/providers/email-polling-outage.test.ts @@ -0,0 +1,33 @@ +import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' +import { queueTableRows } from '@sim/testing/mocks/database.mock' +import { schemaMock } from '@sim/testing/mocks/schema.mock' +import { describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { gmailHandler } from '@/lib/webhooks/providers/gmail' +import { outlookHandler } from '@/lib/webhooks/providers/outlook' + +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) + +describe.each([ + ['Gmail', gmailHandler], + ['Outlook', outlookHandler], +])('%s polling setup during an egress outage', (_name, handler) => { + it('reports the outage instead of a permissions failure', async () => { + const outage = new OutboundRoutingError('GATEWAY_UNAVAILABLE') + authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce({ accountId: 'acct-1' }) + queueTableRows(schemaMock.account, [{ userId: 'owner-1' }]) + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockRejectedValueOnce(outage) + + const configurePolling = handler.configurePolling + if (!configurePolling) throw new Error('Handler has no configurePolling') + await expect( + configurePolling({ + webhook: { id: 'webhook-1', providerConfig: { credentialId: 'cred-1' } }, + requestId: 'req-1', + userId: 'user-1', + workspaceId: 'ws-1', + deploymentVersionId: null, + }) + ).rejects.toBe(outage) + }) +}) diff --git a/apps/sim/lib/webhooks/providers/fathom.ts b/apps/sim/lib/webhooks/providers/fathom.ts index 27d5f6f7534..e927b4d8fc2 100644 --- a/apps/sim/lib/webhooks/providers/fathom.ts +++ b/apps/sim/lib/webhooks/providers/fathom.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -59,7 +60,7 @@ export const fathomHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const fathomResponse = await fetch('https://api.fathom.ai/external/v1/webhooks', { + const fathomResponse = await outboundFetch('https://api.fathom.ai/external/v1/webhooks', { method: 'POST', headers: { 'X-Api-Key': apiKey, @@ -157,7 +158,7 @@ export const fathomHandler: WebhookProviderHandler = { const fathomApiUrl = `https://api.fathom.ai/external/v1/webhooks/${externalId}` - const fathomResponse = await fetch(fathomApiUrl, { + const fathomResponse = await outboundFetch(fathomApiUrl, { method: 'DELETE', headers: { 'X-Api-Key': apiKey, diff --git a/apps/sim/lib/webhooks/providers/fivetran.ts b/apps/sim/lib/webhooks/providers/fivetran.ts index c471257bb47..f786423f0b4 100644 --- a/apps/sim/lib/webhooks/providers/fivetran.ts +++ b/apps/sim/lib/webhooks/providers/fivetran.ts @@ -6,6 +6,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { toRecord, toRecordOrNull } from '@sim/utils/object' import { generateRandomHex } from '@sim/utils/random' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -57,7 +58,7 @@ async function fivetranRequest( method: string, body?: Record ): Promise { - return fetch(url, { + return outboundFetch(url, { method, headers: fivetranAuthHeaders(credentials), body: body === undefined ? undefined : JSON.stringify(body), diff --git a/apps/sim/lib/webhooks/providers/gmail.ts b/apps/sim/lib/webhooks/providers/gmail.ts index bc70d25b556..c5e6a452324 100644 --- a/apps/sim/lib/webhooks/providers/gmail.ts +++ b/apps/sim/lib/webhooks/providers/gmail.ts @@ -3,6 +3,7 @@ import { account, webhook } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/lib/oauth/credential-service' import type { FormatInputContext, @@ -112,6 +113,7 @@ export const gmailHandler: WebhookProviderHandler = { }) return true } catch (error: unknown) { + if (isUnavailableFailure(error)) throw error const err = error as Error logger.error('Failed to configure Gmail polling', { requestId, diff --git a/apps/sim/lib/webhooks/providers/grain.ts b/apps/sim/lib/webhooks/providers/grain.ts index 3741e49a191..19af8940ee0 100644 --- a/apps/sim/lib/webhooks/providers/grain.ts +++ b/apps/sim/lib/webhooks/providers/grain.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -63,7 +64,7 @@ async function createGrainV2Hooks(params: { try { for (const hookType of hookTypes) { - const response = await fetch(`${GRAIN_V2_HOOKS_BASE}/create`, { + const response = await outboundFetch(`${GRAIN_V2_HOOKS_BASE}/create`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -107,7 +108,7 @@ async function deleteGrainV2Hook(params: { hookId: string requestId: string }): Promise { - const response = await fetch(`${GRAIN_V2_HOOKS_BASE}/${params.hookId}`, { + const response = await outboundFetch(`${GRAIN_V2_HOOKS_BASE}/${params.hookId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${params.apiKey}`, @@ -193,7 +194,7 @@ async function createLegacyV1Subscription(params: { requestBody.actions = actions } - const grainResponse = await fetch('https://api.grain.com/_/public-api/hooks', { + const grainResponse = await outboundFetch('https://api.grain.com/_/public-api/hooks', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -238,13 +239,16 @@ async function deleteLegacyV1Hook(params: { hookId: string requestId: string }): Promise { - const response = await fetch(`https://api.grain.com/_/public-api/hooks/${params.hookId}`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${params.apiKey}`, - 'Content-Type': 'application/json', - }, - }) + const response = await outboundFetch( + `https://api.grain.com/_/public-api/hooks/${params.hookId}`, + { + method: 'DELETE', + headers: { + Authorization: `Bearer ${params.apiKey}`, + 'Content-Type': 'application/json', + }, + } + ) if (!response.ok && response.status !== 404 && response.status !== 410) { throw new Error(`Failed to delete Grain webhook ${params.hookId}: ${response.status}`) } diff --git a/apps/sim/lib/webhooks/providers/granola.ts b/apps/sim/lib/webhooks/providers/granola.ts index d44d0273b69..30917a3c633 100644 --- a/apps/sim/lib/webhooks/providers/granola.ts +++ b/apps/sim/lib/webhooks/providers/granola.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toRecordOrNull } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -72,7 +73,7 @@ function granolaUserFacingError(status: number, body: string): string { /** Delete one Granola webhook endpoint. Treats an already-deleted endpoint as success. */ async function deleteGranolaEndpoint(apiKey: string, endpointId: string): Promise { - const response = await fetch( + const response = await outboundFetch( `${GRANOLA_WEBHOOK_ENDPOINTS_URL}/${encodeURIComponent(endpointId)}`, { method: 'DELETE', @@ -244,7 +245,7 @@ export const granolaHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(GRANOLA_WEBHOOK_ENDPOINTS_URL, { + const response = await outboundFetch(GRANOLA_WEBHOOK_ENDPOINTS_URL, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/helpscout.ts b/apps/sim/lib/webhooks/providers/helpscout.ts index 89fcccd4ca2..05b33b0c6c2 100644 --- a/apps/sim/lib/webhooks/providers/helpscout.ts +++ b/apps/sim/lib/webhooks/providers/helpscout.ts @@ -5,6 +5,7 @@ import { toNumberOrNull, toStringOrNull } from '@sim/utils/coerce' import { toRecord } from '@sim/utils/object' import { generateRandomHex } from '@sim/utils/random' import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -131,7 +132,7 @@ export const helpScoutHandler: WebhookProviderHandler = { const accessToken = await requireAccessToken(config, requestId) const webhookSecret = generateRandomHex(WEBHOOK_SECRET_LENGTH) - const response = await fetch(`${HELPSCOUT_API_BASE}/webhooks`, { + const response = await outboundFetch(`${HELPSCOUT_API_BASE}/webhooks`, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -191,7 +192,7 @@ export const helpScoutHandler: WebhookProviderHandler = { if (strict) throw new Error('Missing Help Scout token for webhook deletion') return } - const response = await fetch(`${HELPSCOUT_API_BASE}/webhooks/${externalId}`, { + const response = await outboundFetch(`${HELPSCOUT_API_BASE}/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}` }, redirect: 'error', diff --git a/apps/sim/lib/webhooks/providers/instantly.ts b/apps/sim/lib/webhooks/providers/instantly.ts index 142096bc963..e62c21bb671 100644 --- a/apps/sim/lib/webhooks/providers/instantly.ts +++ b/apps/sim/lib/webhooks/providers/instantly.ts @@ -4,6 +4,7 @@ import { toError } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' import { isRecordLike, toRecord, toRecordOrNull } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -170,7 +171,7 @@ export const instantlyHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(instantlyUrl('/api/v2/webhooks'), { + const response = await outboundFetch(instantlyUrl('/api/v2/webhooks'), { method: 'POST', headers: { Authorization: `Bearer ${apiKey.trim()}`, @@ -235,7 +236,7 @@ export const instantlyHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await outboundFetch( instantlyUrl(`/api/v2/webhooks/${encodeURIComponent(externalId.trim())}`), { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/jotform.ts b/apps/sim/lib/webhooks/providers/jotform.ts index 4cb0011cbf3..3e0fb62dace 100644 --- a/apps/sim/lib/webhooks/providers/jotform.ts +++ b/apps/sim/lib/webhooks/providers/jotform.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' import { and, eq, isNull, ne } from 'drizzle-orm' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -68,7 +69,7 @@ async function listWebhookIdForUrl( credentials: JotformSubscriptionCredentials, notificationUrl: string ): Promise { - const response = await fetch( + const response = await outboundFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks` @@ -186,7 +187,7 @@ export const jotformHandler: WebhookProviderHandler = { return {} } - const response = await fetch( + const response = await outboundFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks` @@ -260,7 +261,7 @@ export const jotformHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await outboundFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks/${encodeURIComponent(webhookId)}` diff --git a/apps/sim/lib/webhooks/providers/lemlist.ts b/apps/sim/lib/webhooks/providers/lemlist.ts index 7c2befa3c85..f34cb57cc6b 100644 --- a/apps/sim/lib/webhooks/providers/lemlist.ts +++ b/apps/sim/lib/webhooks/providers/lemlist.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -68,7 +69,7 @@ export const lemlistHandler: WebhookProviderHandler = { requestBody.campaignId = campaignId } - const lemlistResponse = await fetch(lemlistApiUrl, { + const lemlistResponse = await outboundFetch(lemlistApiUrl, { method: 'POST', headers: { Authorization: `Basic ${authString}`, @@ -149,7 +150,7 @@ export const lemlistHandler: WebhookProviderHandler = { } const lemlistApiUrl = `https://api.lemlist.com/api/hooks/${id}` - const lemlistResponse = await fetch(lemlistApiUrl, { + const lemlistResponse = await outboundFetch(lemlistApiUrl, { method: 'DELETE', headers: { Authorization: `Basic ${authString}`, @@ -184,7 +185,7 @@ export const lemlistHandler: WebhookProviderHandler = { } const notificationUrl = getNotificationUrl(webhook) - const listResponse = await fetch('https://api.lemlist.com/api/hooks', { + const listResponse = await outboundFetch('https://api.lemlist.com/api/hooks', { method: 'GET', headers: { Authorization: `Basic ${authString}`, diff --git a/apps/sim/lib/webhooks/providers/linear.ts b/apps/sim/lib/webhooks/providers/linear.ts index c47e649a2ee..93858bc8b44 100644 --- a/apps/sim/lib/webhooks/providers/linear.ts +++ b/apps/sim/lib/webhooks/providers/linear.ts @@ -5,6 +5,7 @@ import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { isRecordLike, toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -217,7 +218,7 @@ export const linearHandler: WebhookProviderHandler = { } try { - const response = await fetch('https://api.linear.app/graphql', { + const response = await outboundFetch('https://api.linear.app/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -298,7 +299,7 @@ export const linearHandler: WebhookProviderHandler = { } try { - const response = await fetch('https://api.linear.app/graphql', { + const response = await outboundFetch('https://api.linear.app/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/webhooks/providers/linq.ts b/apps/sim/lib/webhooks/providers/linq.ts index 6f05fd820d2..783af98a8a2 100644 --- a/apps/sim/lib/webhooks/providers/linq.ts +++ b/apps/sim/lib/webhooks/providers/linq.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -134,7 +135,7 @@ export const linqHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(`${LINQ_API_BASE}/webhook-subscriptions`, { + const response = await outboundFetch(`${LINQ_API_BASE}/webhook-subscriptions`, { method: 'POST', headers: linqHeaders(apiKey), body: JSON.stringify(requestBody), @@ -196,7 +197,7 @@ export const linqHandler: WebhookProviderHandler = { return } - const response = await fetch(`${LINQ_API_BASE}/webhook-subscriptions/${externalId}`, { + const response = await outboundFetch(`${LINQ_API_BASE}/webhook-subscriptions/${externalId}`, { method: 'DELETE', headers: linqHeaders(apiKey), }) diff --git a/apps/sim/lib/webhooks/providers/microsoft-teams.ts b/apps/sim/lib/webhooks/providers/microsoft-teams.ts index 7bc6ab3a3b7..128c3cb9598 100644 --- a/apps/sim/lib/webhooks/providers/microsoft-teams.ts +++ b/apps/sim/lib/webhooks/providers/microsoft-teams.ts @@ -7,6 +7,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike, toRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isMicrosoftContentUrl } from '@/lib/core/security/input-validation' import { type SecureFetchResponse, @@ -272,7 +273,9 @@ async function formatTeamsGraphNotification( if (accessToken) { const msgUrl = `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(resolvedChatId)}/messages/${encodeURIComponent(resolvedMessageId)}` - const res = await fetch(msgUrl, { headers: { Authorization: `Bearer ${accessToken}` } }) + const res = await outboundFetch(msgUrl, { + headers: { Authorization: `Bearer ${accessToken}` }, + }) if (res.ok) { message = await readResponseJsonWithLimit>(res, { maxBytes: MAX_TEAMS_MESSAGE_BYTES, @@ -336,7 +339,7 @@ async function formatTeamsGraphNotification( .replace(/=+$/, '') const graphUrl = `https://graph.microsoft.com/v1.0/shares/u!${encodedUrl}/driveItem/content` - const graphRes = await fetch(graphUrl, { + const graphRes = await outboundFetch(graphUrl, { headers: { Authorization: `Bearer ${accessToken}` }, redirect: 'follow', }) @@ -386,7 +389,7 @@ async function formatTeamsGraphNotification( } const metadataUrl = `https://graph.microsoft.com/v1.0/shares/${shareToken}/driveItem` - const metadataRes = await fetch(metadataUrl, { + const metadataRes = await outboundFetch(metadataUrl, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -395,7 +398,7 @@ async function formatTeamsGraphNotification( if (!metadataRes.ok) { const directUrl = `https://graph.microsoft.com/v1.0/shares/${shareToken}/driveItem/content` - const directRes = await fetch(directUrl, { + const directRes = await outboundFetch(directUrl, { headers: { Authorization: `Bearer ${accessToken}` }, redirect: 'follow', }) @@ -714,7 +717,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { const existingSubscriptionId = config.externalSubscriptionId as string | undefined if (existingSubscriptionId) { try { - const checkRes = await fetch( + const checkRes = await outboundFetch( `https://graph.microsoft.com/v1.0/subscriptions/${existingSubscriptionId}`, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` } } ) @@ -753,7 +756,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { } try { - const res = await fetch('https://graph.microsoft.com/v1.0/subscriptions', { + const res = await outboundFetch('https://graph.microsoft.com/v1.0/subscriptions', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -846,7 +849,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { return } - const res = await fetch( + const res = await outboundFetch( `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/monday.ts b/apps/sim/lib/webhooks/providers/monday.ts index a70174d3253..d613d4471d9 100644 --- a/apps/sim/lib/webhooks/providers/monday.ts +++ b/apps/sim/lib/webhooks/providers/monday.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateMondayNumericId } from '@/lib/core/security/input-validation' import { getOAuthToken } from '@/lib/oauth/credential-service' import { @@ -106,7 +107,7 @@ export const mondayHandler: WebhookProviderHandler = { const notificationUrl = getNotificationUrl(ctx.webhook) try { - const response = await fetch(MONDAY_API_URL, { + const response = await outboundFetch(MONDAY_API_URL, { method: 'POST', headers: mondayHeaders(accessToken), body: JSON.stringify({ @@ -216,7 +217,7 @@ export const mondayHandler: WebhookProviderHandler = { } try { - const response = await fetch(MONDAY_API_URL, { + const response = await outboundFetch(MONDAY_API_URL, { method: 'POST', headers: mondayHeaders(accessToken), body: JSON.stringify({ diff --git a/apps/sim/lib/webhooks/providers/outlook.ts b/apps/sim/lib/webhooks/providers/outlook.ts index 58c6374faa9..9d564a31447 100644 --- a/apps/sim/lib/webhooks/providers/outlook.ts +++ b/apps/sim/lib/webhooks/providers/outlook.ts @@ -3,6 +3,7 @@ import { account, webhook } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/lib/oauth/credential-service' import type { FormatInputContext, @@ -108,6 +109,7 @@ export const outlookHandler: WebhookProviderHandler = { }) return true } catch (error: unknown) { + if (isUnavailableFailure(error)) throw error const err = error as Error logger.error('Failed to configure Outlook polling', { requestId, diff --git a/apps/sim/lib/webhooks/providers/pagerduty.ts b/apps/sim/lib/webhooks/providers/pagerduty.ts index abb9017a387..de2f82838dd 100644 --- a/apps/sim/lib/webhooks/providers/pagerduty.ts +++ b/apps/sim/lib/webhooks/providers/pagerduty.ts @@ -2,6 +2,7 @@ import crypto from 'crypto' import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -56,7 +57,7 @@ async function cleanupPagerDutySubscription( ): Promise { let id = subscriptionId if (!id) { - const listRes = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { + const listRes = await outboundFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { headers: pagerdutyHeaders(apiKey), }).catch(() => null) if (!listRes || !listRes.ok) return @@ -66,7 +67,7 @@ async function cleanupPagerDutySubscription( id = body?.webhook_subscriptions?.find((sub) => sub.delivery_method?.url === url)?.id } if (!id) return - await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${id}`, { + await outboundFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${id}`, { method: 'DELETE', headers: pagerdutyHeaders(apiKey), }).catch(() => null) @@ -153,7 +154,7 @@ export const pagerdutyHandler: WebhookProviderHandler = { } const { getPagerDutyEvents } = await import('@/triggers/pagerduty/utils') - const res = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { + const res = await outboundFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { method: 'POST', headers: pagerdutyHeaders(apiKey), body: JSON.stringify({ @@ -213,7 +214,7 @@ export const pagerdutyHandler: WebhookProviderHandler = { return } - const res = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${externalId}`, { + const res = await outboundFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${externalId}`, { method: 'DELETE', headers: pagerdutyHeaders(apiKey), }) diff --git a/apps/sim/lib/webhooks/providers/resend.ts b/apps/sim/lib/webhooks/providers/resend.ts index 935911c6604..b200cd1b853 100644 --- a/apps/sim/lib/webhooks/providers/resend.ts +++ b/apps/sim/lib/webhooks/providers/resend.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -145,7 +146,7 @@ export const resendHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const resendResponse = await fetch('https://api.resend.com/webhooks', { + const resendResponse = await outboundFetch('https://api.resend.com/webhooks', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -232,7 +233,7 @@ export const resendHandler: WebhookProviderHandler = { return } - const resendResponse = await fetch(`https://api.resend.com/webhooks/${externalId}`, { + const resendResponse = await outboundFetch(`https://api.resend.com/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/revenuecat.ts b/apps/sim/lib/webhooks/providers/revenuecat.ts index ffbca9b15ec..2f0e424aba4 100644 --- a/apps/sim/lib/webhooks/providers/revenuecat.ts +++ b/apps/sim/lib/webhooks/providers/revenuecat.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -109,7 +110,7 @@ export const revenueCatHandler: WebhookProviderHandler = { requestBody.environment = environment } - const response = await fetch( + const response = await outboundFetch( `${REVENUECAT_API_BASE}/projects/${encodeURIComponent(projectId)}/integrations/webhooks`, { method: 'POST', @@ -187,7 +188,7 @@ export const revenueCatHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await outboundFetch( `${REVENUECAT_API_BASE}/projects/${encodeURIComponent(projectId)}/integrations/webhooks/${encodeURIComponent(externalId)}`, { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/rootly.ts b/apps/sim/lib/webhooks/providers/rootly.ts index cd1ea420d80..2f5e0059355 100644 --- a/apps/sim/lib/webhooks/providers/rootly.ts +++ b/apps/sim/lib/webhooks/providers/rootly.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -177,7 +178,7 @@ export const rootlyHandler: WebhookProviderHandler = { }, } - const response = await fetch('https://api.rootly.com/v1/webhooks/endpoints', { + const response = await outboundFetch('https://api.rootly.com/v1/webhooks/endpoints', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -260,13 +261,16 @@ export const rootlyHandler: WebhookProviderHandler = { return } - const response = await fetch(`https://api.rootly.com/v1/webhooks/endpoints/${externalId}`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${apiKey}`, - Accept: 'application/vnd.api+json', - }, - }) + const response = await outboundFetch( + `https://api.rootly.com/v1/webhooks/endpoints/${externalId}`, + { + method: 'DELETE', + headers: { + Authorization: `Bearer ${apiKey}`, + Accept: 'application/vnd.api+json', + }, + } + ) if (response.ok || response.status === 404) { await response.body?.cancel() diff --git a/apps/sim/lib/webhooks/providers/slack.test.ts b/apps/sim/lib/webhooks/providers/slack.test.ts index f38a5a0387c..5f7f7047ac0 100644 --- a/apps/sim/lib/webhooks/providers/slack.test.ts +++ b/apps/sim/lib/webhooks/providers/slack.test.ts @@ -1,5 +1,9 @@ import { createHmac } from 'node:crypto' -import { describe, expect, it } from 'vitest' +import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' +import { queueTableRows } from '@sim/testing/mocks/database.mock' +import { schemaMock } from '@sim/testing/mocks/schema.mock' +import { describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' import { handleSlackChallenge, resolveSlackEventKey, @@ -7,6 +11,8 @@ import { slackHandler, } from '@/lib/webhooks/providers/slack' +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) + const ctx = (body: unknown) => ({ webhook: {}, workflow: { id: 'wf', userId: 'u' }, @@ -146,6 +152,51 @@ describe('slackHandler formatInput - Events API', () => { }) }) +describe('slackHandler formatInput - native app bot token', () => { + async function formatNativeAppEvent(event: Record) { + const formatInput = slackHandler.formatInput + if (!formatInput) throw new Error('Slack handler has no formatInput') + return formatInput({ + ...ctx({ event }), + webhook: { providerConfig: { credentialId: 'cred-1' } }, + query: {}, + method: 'POST', + }) + } + + function resolveCredentialOwner() { + authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce({ accountId: 'acct-1' }) + queueTableRows(schemaMock.account, [{ userId: 'owner-1' }]) + } + + it('still runs the event without a bot token when the egress route is down', async () => { + resolveCredentialOwner() + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockRejectedValueOnce( + new OutboundRoutingError('GATEWAY_UNAVAILABLE') + ) + + const { input } = await formatNativeAppEvent({ + type: 'message', + channel: 'C1', + user: 'U1', + text: 'hi', + }) + + expect(eventOf(input)).toMatchObject({ event_type: 'message', channel: 'C1', text: 'hi' }) + }) + + it('fails the event on a token error that is not an outage', async () => { + resolveCredentialOwner() + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockRejectedValueOnce( + new Error('decrypt failed') + ) + + await expect(formatNativeAppEvent({ type: 'message', channel: 'C1' })).rejects.toThrow( + 'decrypt failed' + ) + }) +}) + describe('slackHandler formatInput - interactivity (block_actions)', () => { it('carries the full view (state.values + private_metadata) through for a view_submission', async () => { const { input } = await slackHandler.formatInput!( diff --git a/apps/sim/lib/webhooks/providers/slack.ts b/apps/sim/lib/webhooks/providers/slack.ts index b103879edc0..2f0cc3b884f 100644 --- a/apps/sim/lib/webhooks/providers/slack.ts +++ b/apps/sim/lib/webhooks/providers/slack.ts @@ -9,6 +9,8 @@ import { isRecordLike, toRecord, toRecordOrNull } from '@sim/utils/object' import { eq } from 'drizzle-orm' import { LRUCache } from 'lru-cache' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { isUnavailableFailure } from '@/lib/core/orchestration/types' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -287,7 +289,7 @@ async function resolveSlackFileInfo( botToken: string ): Promise<{ url_private?: string; name?: string; mimetype?: string; size?: number } | null> { try { - const response = await fetch( + const response = await outboundFetch( `https://slack.com/api/files.info?file=${encodeURIComponent(fileId)}`, { headers: { Authorization: `Bearer ${botToken}` } } ) @@ -432,7 +434,7 @@ async function fetchSlackMessageText( ): Promise { try { const params = new URLSearchParams({ channel, timestamp: messageTs }) - const response = await fetch(`https://slack.com/api/reactions.get?${params}`, { + const response = await outboundFetch(`https://slack.com/api/reactions.get?${params}`, { headers: { Authorization: `Bearer ${botToken}` }, }) const data = (await response.json()) as { @@ -486,7 +488,7 @@ export async function fetchSlackTeamId(botToken: string): Promise<{ userId: string | undefined teamName: string | undefined }> { - const response = await fetch('https://slack.com/api/auth.test', { + const response = await outboundFetch('https://slack.com/api/auth.test', { headers: { Authorization: `Bearer ${botToken}` }, }) const data = (await response.json()) as { @@ -962,8 +964,18 @@ export const slackHandler: WebhookProviderHandler = { .where(eq(account.id, resolved.accountId)) .limit(1) if (owner?.userId) { - botToken = - (await refreshAccessTokenIfNeeded(credentialId, owner.userId, requestId)) ?? undefined + try { + botToken = + (await refreshAccessTokenIfNeeded(credentialId, owner.userId, requestId)) ?? undefined + } catch (error) { + // An egress outage degrades the event (no reaction text or file downloads) rather + // than failing it, matching a missing token. + if (!isUnavailableFailure(error)) throw error + logger.warn('Slack bot token unavailable during an egress outage', { + credentialId, + requestId, + }) + } } } } diff --git a/apps/sim/lib/webhooks/providers/telegram.ts b/apps/sim/lib/webhooks/providers/telegram.ts index 778ec28330f..ab5242698f8 100644 --- a/apps/sim/lib/webhooks/providers/telegram.ts +++ b/apps/sim/lib/webhooks/providers/telegram.ts @@ -4,6 +4,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' import { and, eq, isNull, ne } from 'drizzle-orm' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getEffectiveDecryptedEnv } from '@/lib/environment/utils' import { resolveBackgroundWebhookEnv } from '@/lib/webhooks/env-resolver' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' @@ -160,7 +161,7 @@ export const telegramHandler: WebhookProviderHandler = { const secretToken = await resolveSubscriptionSecretToken(ctx, config, botToken) try { - const telegramResponse = await fetch(telegramApiUrl, { + const telegramResponse = await outboundFetch(telegramApiUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -236,7 +237,7 @@ export const telegramHandler: WebhookProviderHandler = { } const telegramApiUrl = `https://api.telegram.org/bot${botToken}/deleteWebhook` - const telegramResponse = await fetch(telegramApiUrl, { + const telegramResponse = await outboundFetch(telegramApiUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, }) diff --git a/apps/sim/lib/webhooks/providers/typeform.ts b/apps/sim/lib/webhooks/providers/typeform.ts index 3db80586d01..d1aa10a3df7 100644 --- a/apps/sim/lib/webhooks/providers/typeform.ts +++ b/apps/sim/lib/webhooks/providers/typeform.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Base64 } from '@sim/security/hmac' import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -110,7 +111,7 @@ export const typeformHandler: WebhookProviderHandler = { requestBody.secret = secret } - const typeformResponse = await fetch(typeformApiUrl, { + const typeformResponse = await outboundFetch(typeformApiUrl, { method: 'PUT', headers: { Authorization: `Bearer ${apiKey}`, @@ -199,7 +200,7 @@ export const typeformHandler: WebhookProviderHandler = { const tag = webhookTag || `sim-${(ctx.webhook.id as string).substring(0, 8)}` const typeformApiUrl = `https://api.typeform.com/forms/${formId}/webhooks/${tag}` - const typeformResponse = await fetch(typeformApiUrl, { + const typeformResponse = await outboundFetch(typeformApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/types.ts b/apps/sim/lib/webhooks/providers/types.ts index 4507c82dabc..dd357a7c893 100644 --- a/apps/sim/lib/webhooks/providers/types.ts +++ b/apps/sim/lib/webhooks/providers/types.ts @@ -222,7 +222,10 @@ export interface WebhookProviderHandler { /** Delete an external webhook subscription during cleanup. Strict outbox cleanup should throw. */ deleteSubscription?(ctx: DeleteSubscriptionContext): Promise - /** Configure polling after webhook creation (gmail, outlook, rss, imap). */ + /** + * Configure polling after webhook creation (gmail, outlook, rss, imap). Resolves false only for a + * verdict on the account or configuration; an unavailable failure (egress outage) must throw. + */ configurePolling?(ctx: PollingConfigContext): Promise /** Handle verification challenges before webhook lookup (Slack url_verification, WhatsApp hub.verify_token, Teams validationToken). */ diff --git a/apps/sim/lib/webhooks/providers/vercel.ts b/apps/sim/lib/webhooks/providers/vercel.ts index 96961a17bea..c45998b1115 100644 --- a/apps/sim/lib/webhooks/providers/vercel.ts +++ b/apps/sim/lib/webhooks/providers/vercel.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { isRecordLike } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -146,7 +147,7 @@ export const vercelHandler: WebhookProviderHandler = { ? `https://api.vercel.com/v1/webhooks?teamId=${encodeURIComponent(teamId)}` : 'https://api.vercel.com/v1/webhooks' - const vercelResponse = await fetch(apiUrl, { + const vercelResponse = await outboundFetch(apiUrl, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -234,7 +235,7 @@ export const vercelHandler: WebhookProviderHandler = { ? `https://api.vercel.com/v1/webhooks/${encodeURIComponent(externalId)}?teamId=${encodeURIComponent(teamId)}` : `https://api.vercel.com/v1/webhooks/${encodeURIComponent(externalId)}` - const response = await fetch(apiUrl, { + const response = await outboundFetch(apiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/webflow.ts b/apps/sim/lib/webhooks/providers/webflow.ts index 693afd5cbc5..8979b83c29c 100644 --- a/apps/sim/lib/webhooks/providers/webflow.ts +++ b/apps/sim/lib/webhooks/providers/webflow.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { getOAuthToken } from '@/lib/oauth/credential-service' import { @@ -104,7 +105,7 @@ export const webflowHandler: WebhookProviderHandler = { } } - const webflowResponse = await fetch(webflowApiUrl, { + const webflowResponse = await outboundFetch(webflowApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -213,7 +214,7 @@ export const webflowHandler: WebhookProviderHandler = { const webflowApiUrl = `https://api.webflow.com/v2/sites/${siteId}/webhooks/${externalId}` - const webflowResponse = await fetch(webflowApiUrl, { + const webflowResponse = await outboundFetch(webflowApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/zendesk.ts b/apps/sim/lib/webhooks/providers/zendesk.ts index aecba884097..977052f01a9 100644 --- a/apps/sim/lib/webhooks/providers/zendesk.ts +++ b/apps/sim/lib/webhooks/providers/zendesk.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' import type { @@ -46,7 +47,7 @@ async function deleteZendeskWebhookQuietly( authHeader: string, webhookId: string ): Promise { - await fetch(`${apiBase}/webhooks/${webhookId}`, { + await outboundFetch(`${apiBase}/webhooks/${webhookId}`, { method: 'DELETE', headers: { Authorization: authHeader }, }).catch(() => {}) @@ -198,7 +199,7 @@ export const zendeskHandler: WebhookProviderHandler = { const apiBase = zendeskApiBase(subdomain) const authHeader = zendeskAuthHeader(email, apiToken) - const createRes = await fetch(`${apiBase}/webhooks`, { + const createRes = await outboundFetch(`${apiBase}/webhooks`, { method: 'POST', headers: { Authorization: authHeader, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -231,7 +232,7 @@ export const zendeskHandler: WebhookProviderHandler = { const externalId = toRecord(created.webhook).id as string | undefined if (!externalId) throw new Error('Zendesk webhook created but no webhook ID was returned.') - const secretRes = await fetch(`${apiBase}/webhooks/${externalId}/signing_secret`, { + const secretRes = await outboundFetch(`${apiBase}/webhooks/${externalId}/signing_secret`, { headers: { Authorization: authHeader }, }) if (!secretRes.ok) { @@ -278,7 +279,7 @@ export const zendeskHandler: WebhookProviderHandler = { return } - const res = await fetch(`${zendeskApiBase(subdomain)}/webhooks/${externalId}`, { + const res = await outboundFetch(`${zendeskApiBase(subdomain)}/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: zendeskAuthHeader(email, apiToken) }, }) diff --git a/apps/sim/lib/webhooks/providers/zoho-desk.ts b/apps/sim/lib/webhooks/providers/zoho-desk.ts index 3c94d389c90..294e0f323ff 100644 --- a/apps/sim/lib/webhooks/providers/zoho-desk.ts +++ b/apps/sim/lib/webhooks/providers/zoho-desk.ts @@ -7,6 +7,7 @@ import { eq } from 'drizzle-orm' import * as jose from 'jose' import { LRUCache } from 'lru-cache' import { NextResponse } from 'next/server' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getNotificationUrl } from '@/lib/webhooks/provider-subscription-utils' import { WebhookDeploymentConfigurationError } from '@/lib/webhooks/providers/errors' @@ -271,7 +272,7 @@ export const zohoDeskHandler: WebhookProviderHandler = { if (direction === 'in' || direction === 'out') filter.direction = direction } - const response = await fetch(`${apiDomain}/api/v1/webhooks`, { + const response = await outboundFetch(`${apiDomain}/api/v1/webhooks`, { method: 'POST', headers: { Authorization: `Zoho-oauthtoken ${accessToken}`, @@ -364,7 +365,7 @@ export const zohoDeskHandler: WebhookProviderHandler = { : await resolveZohoDeskApiDomain(owner.accountId) const webhookPath = safeUrlPathSegment(externalId, 'externalId') - const response = await fetch(`${apiDomain}/api/v1/webhooks/${webhookPath}`, { + const response = await outboundFetch(`${apiDomain}/api/v1/webhooks/${webhookPath}`, { method: 'DELETE', headers: { Authorization: `Zoho-oauthtoken ${accessToken}`, diff --git a/apps/sim/lib/webhooks/slack-agent-api.ts b/apps/sim/lib/webhooks/slack-agent-api.ts index 7beaf370ce5..6259a4be51a 100644 --- a/apps/sim/lib/webhooks/slack-agent-api.ts +++ b/apps/sim/lib/webhooks/slack-agent-api.ts @@ -1,5 +1,6 @@ import { toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { SlackDeliveryError } from '@/lib/webhooks/slack-delivery-error' @@ -43,7 +44,7 @@ async function callSlackAgentApi( ): Promise { let response: Response try { - response = await fetch(`https://slack.com/api/${method}`, { + response = await outboundFetch(`https://slack.com/api/${method}`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, diff --git a/apps/sim/lib/webhooks/slack-custom-ingress.test.ts b/apps/sim/lib/webhooks/slack-custom-ingress.test.ts index 44d98649ee7..d1ecd342c8c 100644 --- a/apps/sim/lib/webhooks/slack-custom-ingress.test.ts +++ b/apps/sim/lib/webhooks/slack-custom-ingress.test.ts @@ -1,4 +1,10 @@ import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' +import { networkConfigMock, networkConfigMockFns } from '@sim/testing/mocks/network-config.mock' +import { + createMockWorkspaceApplicationContext, + workspaceContextMock, + workspaceContextMockFns, +} from '@sim/testing/mocks/workspace-context.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -12,6 +18,8 @@ vi.mock('@/lib/execution/cancel-workflow-execution', () => ({ cancelWorkflowExecution: mocks.cancel, })) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) +vi.mock('@/lib/core/network/config.server', () => networkConfigMock) +vi.mock('@/lib/workspaces/application/context', () => workspaceContextMock) vi.mock('@/lib/webhooks/slack-agent-api', () => ({ setSlackAgentSessionStatus: mocks.setStatus, })) @@ -22,6 +30,7 @@ vi.mock('@/lib/webhooks/slack-stream-sessions', () => ({ unregisterSlackStreamSession: mocks.unregister, })) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { getLegacySlackCustomBotCredentialId, handleSlackAgentSessionStopped, @@ -51,6 +60,7 @@ describe('getLegacySlackCustomBotCredentialId', () => { describe('handleSlackAgentSessionStopped', () => { beforeEach(() => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(false) mocks.listSessions.mockResolvedValue([]) authOAuthUtilsMockFns.mockGetSlackBotCredential.mockResolvedValue({ botToken: 'xoxb-test' }) }) @@ -111,4 +121,34 @@ describe('handleSlackAgentSessionStopped', () => { expect(mocks.unregister).not.toHaveBeenCalled() expect(mocks.setStatus).not.toHaveBeenCalled() }) + + it("returns the session to active through the bot credential's organization egress route", async () => { + networkConfigMockFns.mockIsOutboundRoutingEnabled.mockReturnValue(true) + mocks.listSessions.mockResolvedValue([ + { + executionId: 'execution-1', + workflowId: 'workflow-1', + userId: 'user-1', + workspaceId: 'workspace-1', + }, + ]) + mocks.cancel.mockResolvedValue(undefined) + authOAuthUtilsMockFns.mockGetSlackBotCredential.mockResolvedValue({ + botToken: 'xoxb-test', + workspaceId: 'workspace-1', + }) + workspaceContextMockFns.mockLoadWorkspaceApplicationContext.mockResolvedValue( + createMockWorkspaceApplicationContext({ + workspaceId: 'workspace-1', + workspaceOrganizationId: 'org-1', + }) + ) + mocks.setStatus.mockImplementation(async () => { + await resolveCurrentOutboundRoute() + }) + + await handleSlackAgentSessionStopped('credential-1', { kind: 'stop' }) + + expect(networkConfigMockFns.mockResolveOutboundRoute).toHaveBeenCalledWith('org-1') + }) }) diff --git a/apps/sim/lib/webhooks/slack-custom-ingress.ts b/apps/sim/lib/webhooks/slack-custom-ingress.ts index 84dab841f46..2c0b2dff41c 100644 --- a/apps/sim/lib/webhooks/slack-custom-ingress.ts +++ b/apps/sim/lib/webhooks/slack-custom-ingress.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toRecord } from '@sim/utils/object' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { cancelWorkflowExecution } from '@/lib/execution/cancel-workflow-execution' import { getSlackBotCredential } from '@/lib/oauth/credential-service' import { findWebhooksByRoutingKey, type WebhookDispatchResult } from '@/lib/webhooks/processor' @@ -163,5 +164,7 @@ export async function handleSlackAgentSessionStopped( if (!credential) { throw new Error('Slack agent session stop credential is unavailable') } - await setSlackAgentSessionStatus(credential.botToken, target, 'active') + const { botToken, workspaceId } = credential + const restoreStatus = () => setSlackAgentSessionStatus(botToken, target, 'active') + await (workspaceId ? withResourceOutboundScope({ workspaceId }, restoreStatus) : restoreStatus()) } diff --git a/apps/sim/providers/anthropic/index.ts b/apps/sim/providers/anthropic/index.ts index 7beb53c79e5..e554e5f92aa 100644 --- a/apps/sim/providers/anthropic/index.ts +++ b/apps/sim/providers/anthropic/index.ts @@ -1,5 +1,6 @@ import Anthropic from '@anthropic-ai/sdk' import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { executeAnthropicProviderRequest } from '@/providers/anthropic/core' import { getCachedProviderClient } from '@/providers/client-cache' @@ -27,7 +28,7 @@ export const anthropicProvider: ProviderConfig = { const cacheKey = `anthropic::${apiKey}` return getCachedProviderClient( cacheKey, - () => new Anthropic({ apiKey, maxRetries: PROVIDER_MAX_RETRIES }) + () => new Anthropic({ apiKey, maxRetries: PROVIDER_MAX_RETRIES, fetch: outboundFetch }) ) }, logger, diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index d68dbc3a97d..252adef6e3e 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -121,14 +121,14 @@ describe('azureOpenAIProvider — SSRF pinning', () => { expect(responsesConfig().fetch).toBe(sentinelFetch) }) - it('passes no custom fetch when the endpoint comes from trusted server env', async () => { + it("sends Sim's trusted env endpoint direct, not through the organization router", async () => { setEnv({ AZURE_OPENAI_ENDPOINT: 'https://trusted.openai.azure.com' }) await azureOpenAIProvider.executeRequest(request({ azureEndpoint: undefined })) expect(mockValidate).not.toHaveBeenCalled() expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(responsesConfig().fetch).toBeUndefined() + expect(responsesConfig().fetch).toBe(globalThis.fetch) }) it.each([false, true])( diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index 37db6e20761..307accdd16c 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -767,6 +767,8 @@ export const azureOpenAIProvider: ProviderConfig = { } pinnedFetch = createPinnedFetch(validation.resolvedIP, { profile: 'configuredEndpoint' }) } + // egress-direct: without a user endpoint this is Sim's own Azure deployment + const responsesFetch = pinnedFetch ?? fetch const apiKey = request.apiKey if (!apiKey) { @@ -826,7 +828,7 @@ export const azureOpenAIProvider: ProviderConfig = { 'api-key': apiKey, }, logger, - fetch: pinnedFetch, + fetch: responsesFetch, }) } @@ -848,7 +850,7 @@ export const azureOpenAIProvider: ProviderConfig = { 'api-key': apiKey, }, logger, - fetch: pinnedFetch, + fetch: responsesFetch, }) }, } diff --git a/apps/sim/providers/baseten/index.ts b/apps/sim/providers/baseten/index.ts index 3d6ffacfe78..de30620d661 100644 --- a/apps/sim/providers/baseten/index.ts +++ b/apps/sim/providers/baseten/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { @@ -44,6 +45,7 @@ export const basetenProvider: ProviderConfig = { const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://inference.baseten.co/v1', }) diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index 1cf2f8931b2..ec67c109c64 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -17,6 +17,7 @@ import { sha256Hex } from '@sim/security/hash' import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { getAwsCredentials } from '@/lib/core/config/aws' +import { organizationAwsRequestHandler } from '@/lib/core/network/organization-aws-handler.server' import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' import { validateAwsRegion } from '@/lib/core/security/input-validation' import type { IterationToolCall, NormalizedBlockOutput, StreamingExecution } from '@/executor/types' @@ -70,6 +71,9 @@ import { const logger = createLogger('BedrockProvider') +/** Bedrock Runtime defaults to HTTP/2, which a customer's direct requests keep. */ +const customerRequestHandler = () => organizationAwsRequestHandler({ http2: true }) + function enrichLastModelSegmentFromBedrockResponse( timeSegments: TimeSegment[], response: ConverseResponse, @@ -137,12 +141,17 @@ function enrichLastModelSegmentFromBedrockResponse( function resolveBedrockClientCredentials(request: ProviderRequest): { credentials: BedrockRuntimeClientConfig['credentials'] credentialKey: string | null + createRequestHandler: () => BedrockRuntimeClientConfig['requestHandler'] } { const session = request.bedrockSession if (session) { const { accessKeyId, secretAccessKey, sessionToken } = session if (sessionToken) { - return { credentials: { accessKeyId, secretAccessKey, sessionToken }, credentialKey: null } + return { + credentials: { accessKeyId, secretAccessKey, sessionToken }, + credentialKey: null, + createRequestHandler: customerRequestHandler, + } } return { credentials: { accessKeyId, secretAccessKey }, @@ -151,6 +160,7 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { session.credentialId, sha256Hex([accessKeyId, secretAccessKey].join('\0')), ].join(':'), + createRequestHandler: customerRequestHandler, } } @@ -172,9 +182,15 @@ function resolveBedrockClientCredentials(request: ProviderRequest): { secretAccessKey: request.bedrockSecretKey, }, credentialKey: `${request.bedrockAccessKeyId}:${request.bedrockSecretKey}`, + createRequestHandler: customerRequestHandler, } } - return { credentials: getAwsCredentials(), credentialKey: 'default-chain' } + // The deployment's own credentials stay on Sim's network, never an organization's route. + return { + credentials: getAwsCredentials(), + credentialKey: 'default-chain', + createRequestHandler: awsRequestHandler, + } } export const bedrockProvider: ProviderConfig = { @@ -206,9 +222,10 @@ export const bedrockProvider: ProviderConfig = { region, }) - const { credentials, credentialKey } = resolveBedrockClientCredentials(request) + const { credentials, credentialKey, createRequestHandler } = + resolveBedrockClientCredentials(request) const buildClient = () => - new BedrockRuntimeClient({ region, credentials, requestHandler: awsRequestHandler() }) + new BedrockRuntimeClient({ region, credentials, requestHandler: createRequestHandler() }) const client = credentialKey === null ? buildClient() diff --git a/apps/sim/providers/cerebras/index.ts b/apps/sim/providers/cerebras/index.ts index 1a6d94f0db8..efb04cb21c1 100644 --- a/apps/sim/providers/cerebras/index.ts +++ b/apps/sim/providers/cerebras/index.ts @@ -67,6 +67,9 @@ export const cerebrasProvider: ProviderConfig = { const providerStartTimeISO = new Date(providerStartTime).toISOString() try { + // Not routed through the organization gateway: this SDK's default transport is node-fetch + // on a keep-alive agent it stretches to the 600s header budget, and any custom `fetch` + // replaces both with the global fetch, cutting that budget for every organization. const client = new Cerebras({ apiKey: request.apiKey, ...openAICompatTransport(), diff --git a/apps/sim/providers/deepseek/index.ts b/apps/sim/providers/deepseek/index.ts index b2750a8090c..b47f791d83e 100644 --- a/apps/sim/providers/deepseek/index.ts +++ b/apps/sim/providers/deepseek/index.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -65,6 +66,7 @@ export const deepseekProvider: ProviderConfig = { try { const deepseek = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.deepseek.com', }) diff --git a/apps/sim/providers/file-attachments.server.ts b/apps/sim/providers/file-attachments.server.ts index 9940801c1a5..a08209d2770 100644 --- a/apps/sim/providers/file-attachments.server.ts +++ b/apps/sim/providers/file-attachments.server.ts @@ -2,6 +2,7 @@ import { FileState, GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { assertUserFileContentAccess } from '@/lib/execution/payloads/materialization.server' import { resolveExecutorFileMaterializationContext } from '@/lib/internal/file/materialization-context' import { StorageService } from '@/lib/uploads' @@ -148,7 +149,10 @@ export async function uploadLargeFilesToProvider( if (groups.length === 0) return const maxBytes = getProviderAttachmentMaxBytes(providerId) - const ai = providerId === 'google' ? new GoogleGenAI({ apiKey: request.apiKey }) : null + const ai = + providerId === 'google' + ? new GoogleGenAI({ apiKey: request.apiKey, httpOptions: { fetch: outboundFetch } }) + : null for (const group of groups) { const [representative] = group @@ -250,7 +254,7 @@ async function uploadOpenAIFile( form.append('expires_after[seconds]', String(OPENAI_FILE_EXPIRY_SECONDS)) form.append('file', blob, file.name) - const response = await fetch(OPENAI_FILES_ENDPOINT, { + const response = await outboundFetch(OPENAI_FILES_ENDPOINT, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}` }, body: form, diff --git a/apps/sim/providers/fireworks/index.ts b/apps/sim/providers/fireworks/index.ts index 53c340ce485..1be08ed305f 100644 --- a/apps/sim/providers/fireworks/index.ts +++ b/apps/sim/providers/fireworks/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { resolveFireworksWireModel } from '@/providers/fireworks/utils' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' @@ -45,6 +46,7 @@ export const fireworksProvider: ProviderConfig = { const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.fireworks.ai/inference/v1', }) diff --git a/apps/sim/providers/google/index.ts b/apps/sim/providers/google/index.ts index e381ef9da79..ca279cd9766 100644 --- a/apps/sim/providers/google/index.ts +++ b/apps/sim/providers/google/index.ts @@ -1,5 +1,6 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { executeGeminiRequest } from '@/providers/gemini/core' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' @@ -30,7 +31,7 @@ export const googleProvider: ProviderConfig = { logger.info('Creating Google Gemini client', { model: request.model }) - const ai = new GoogleGenAI({ apiKey: request.apiKey }) + const ai = new GoogleGenAI({ apiKey: request.apiKey, httpOptions: { fetch: outboundFetch } }) return executeGeminiRequest({ ai, diff --git a/apps/sim/providers/groq/index.ts b/apps/sim/providers/groq/index.ts index 1357a763d18..3b28a975717 100644 --- a/apps/sim/providers/groq/index.ts +++ b/apps/sim/providers/groq/index.ts @@ -7,6 +7,7 @@ import type { ChatCompletionChunk, ChatCompletionMessageParam, } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -66,7 +67,11 @@ export const groqProvider: ProviderConfig = { throw new Error('API key is required for Groq') } - const groq = new Groq({ apiKey: request.apiKey, ...openAICompatTransport() }) + const groq = new Groq({ + apiKey: request.apiKey, + ...openAICompatTransport(), + fetch: outboundFetch, + }) const allMessages = [] diff --git a/apps/sim/providers/kie/index.ts b/apps/sim/providers/kie/index.ts index b3440ac2809..0a0b8521560 100644 --- a/apps/sim/providers/kie/index.ts +++ b/apps/sim/providers/kie/index.ts @@ -1,5 +1,6 @@ import Anthropic from '@anthropic-ai/sdk' import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { executeAnthropicProviderRequest } from '@/providers/anthropic/core' import { getCachedProviderClient } from '@/providers/client-cache' @@ -16,7 +17,7 @@ import { PROVIDER_MAX_RETRIES } from '@/providers/transport' import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types' const logger = createLogger('KieProvider') -const kieFetch = createKieFetch() +const kieFetch = createKieFetch(outboundFetch) export const kieProvider: ProviderConfig = { id: 'kie', diff --git a/apps/sim/providers/kie/transport.ts b/apps/sim/providers/kie/transport.ts index 4281b6e96f6..fa5c83e8f9d 100644 --- a/apps/sim/providers/kie/transport.ts +++ b/apps/sim/providers/kie/transport.ts @@ -52,9 +52,9 @@ function withKieThinkingFlag( * are labeled as JSON. Streams are never buffered: they are typed * `text/event-stream`, and a failed streaming request comes back as JSON. */ -export function createKieFetch(baseFetch?: typeof fetch): typeof fetch { +export function createKieFetch(baseFetch: typeof fetch): typeof fetch { return async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const response = await (baseFetch ?? fetch)(input, withKieThinkingFlag(input, init)) + const response = await baseFetch(input, withKieThinkingFlag(input, init)) const contentType = response.headers.get('content-type')?.toLowerCase() if (response.status !== 200 || (contentType && !contentType.includes('application/json'))) { return response diff --git a/apps/sim/providers/kimi/index.ts b/apps/sim/providers/kimi/index.ts index 0434dc047d5..37d9e8f86dd 100644 --- a/apps/sim/providers/kimi/index.ts +++ b/apps/sim/providers/kimi/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -125,6 +126,7 @@ export const kimiProvider: ProviderConfig = { try { const kimi = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: KIMI_BASE_URL, }) diff --git a/apps/sim/providers/litellm/index.ts b/apps/sim/providers/litellm/index.ts index 913816b5e60..8cca38eff5e 100644 --- a/apps/sim/providers/litellm/index.ts +++ b/apps/sim/providers/litellm/index.ts @@ -81,6 +81,7 @@ export const litellmProvider: ProviderConfig = { headers.Authorization = `Bearer ${env.LITELLM_API_KEY}` } + // egress-direct: operator's own LiteLLM proxy named by the deployment const response = await fetch(`${baseUrl}/v1/models`, { headers }) if (!response.ok) { await response.text().catch(() => {}) diff --git a/apps/sim/providers/meta/index.ts b/apps/sim/providers/meta/index.ts index 25e33098d4f..d29213658f3 100644 --- a/apps/sim/providers/meta/index.ts +++ b/apps/sim/providers/meta/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -69,6 +70,7 @@ export const metaProvider: ProviderConfig = { try { const meta = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: META_BASE_URL, }) diff --git a/apps/sim/providers/mistral/index.ts b/apps/sim/providers/mistral/index.ts index 03d225be391..a9657bb262c 100644 --- a/apps/sim/providers/mistral/index.ts +++ b/apps/sim/providers/mistral/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -78,6 +79,7 @@ export const mistralProvider: ProviderConfig = { const mistral = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.mistral.ai/v1', }) diff --git a/apps/sim/providers/nebius/index.ts b/apps/sim/providers/nebius/index.ts index 193b756bc09..10f6e167ef6 100644 --- a/apps/sim/providers/nebius/index.ts +++ b/apps/sim/providers/nebius/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getModelCapabilities, @@ -52,6 +53,7 @@ export const nebiusProvider: ProviderConfig = { if (!request.apiKey) throw new Error('API key is required for Nebius') const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.tokenfactory.nebius.com/v1', }) diff --git a/apps/sim/providers/nvidia/index.ts b/apps/sim/providers/nvidia/index.ts index 956433308b9..8865e8c6b5e 100644 --- a/apps/sim/providers/nvidia/index.ts +++ b/apps/sim/providers/nvidia/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -81,6 +82,7 @@ export const nvidiaProvider: ProviderConfig = { try { const nvidia = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: NVIDIA_BASE_URL, }) diff --git a/apps/sim/providers/ollama-cloud/index.ts b/apps/sim/providers/ollama-cloud/index.ts index 350769082aa..20d11e5e8a7 100644 --- a/apps/sim/providers/ollama-cloud/index.ts +++ b/apps/sim/providers/ollama-cloud/index.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { inheritConversationGenerationContext } from '@/providers/conversation-generation' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' @@ -38,6 +39,7 @@ export const ollamaCloudProvider: ProviderConfig = { createClient: () => new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey, baseURL: OLLAMA_CLOUD_BASE_URL, }), diff --git a/apps/sim/providers/ollama/index.ts b/apps/sim/providers/ollama/index.ts index c15f79870a7..36eb784975e 100644 --- a/apps/sim/providers/ollama/index.ts +++ b/apps/sim/providers/ollama/index.ts @@ -28,6 +28,7 @@ export const ollamaProvider: ProviderConfig = { } try { + // egress-direct: operator-local Ollama server named by the deployment const response = await fetch(`${OLLAMA_HOST}/api/tags`) if (!response.ok) { await response.text().catch(() => {}) diff --git a/apps/sim/providers/openai/core.ts b/apps/sim/providers/openai/core.ts index 85d53ec8efe..2e3df6a39ea 100644 --- a/apps/sim/providers/openai/core.ts +++ b/apps/sim/providers/openai/core.ts @@ -141,11 +141,11 @@ export interface ResponsesProviderConfig { headers: Record logger: Logger /** - * Optional fetch implementation. Used to pin the connection to a pre-validated - * IP (DNS-rebinding/SSRF protection) when the endpoint is user-supplied. - * Defaults to the global fetch. + * Fetch implementation for the request, chosen by the caller: `outboundFetch` for a vendor + * called on the workspace's behalf, a pinned fetch for a user-supplied endpoint + * (DNS-rebinding/SSRF protection), or the global `fetch` for Sim's own deployment. */ - fetch?: typeof fetch + fetch: typeof fetch } /** @@ -156,7 +156,7 @@ export async function executeResponsesProviderRequest( config: ResponsesProviderConfig ): Promise { const { logger } = config - const fetchImpl = config.fetch ?? fetch + const fetchImpl = config.fetch logger.info(`Preparing ${config.providerLabel} request`, { model: request.model, diff --git a/apps/sim/providers/openai/index.ts b/apps/sim/providers/openai/index.ts index db95598e8c7..b85560709db 100644 --- a/apps/sim/providers/openai/index.ts +++ b/apps/sim/providers/openai/index.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types' @@ -33,6 +34,7 @@ export const openaiProvider: ProviderConfig = { 'OpenAI-Beta': 'responses=v1', }, logger, + fetch: outboundFetch, }) }, } diff --git a/apps/sim/providers/openrouter/index.ts b/apps/sim/providers/openrouter/index.ts index 75fdaa11f69..0d0fa9e609f 100644 --- a/apps/sim/providers/openrouter/index.ts +++ b/apps/sim/providers/openrouter/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { bindConversationGenerationContextWindow } from '@/providers/conversation-generation' import { getConversationRequestContext } from '@/providers/conversation-history' @@ -69,6 +70,7 @@ export const openRouterProvider: ProviderConfig = { const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://openrouter.ai/api/v1', }) diff --git a/apps/sim/providers/openrouter/utils.ts b/apps/sim/providers/openrouter/utils.ts index fd65eff65ef..1dd6bc7b47d 100644 --- a/apps/sim/providers/openrouter/utils.ts +++ b/apps/sim/providers/openrouter/utils.ts @@ -23,6 +23,7 @@ const MODEL_CAPABILITIES_TIMEOUT_MS = 5000 async function fetchModelCapabilities(): Promise | undefined> { try { + // egress-direct: public model catalog, read without a workspace const response = await fetch('https://openrouter.ai/api/v1/models', { headers: { 'Content-Type': 'application/json' }, signal: AbortSignal.timeout(MODEL_CAPABILITIES_TIMEOUT_MS), diff --git a/apps/sim/providers/sakana/index.ts b/apps/sim/providers/sakana/index.ts index 5ab637f4fde..91d8155e891 100644 --- a/apps/sim/providers/sakana/index.ts +++ b/apps/sim/providers/sakana/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -71,6 +72,7 @@ export const sakanaProvider: ProviderConfig = { try { const sakana = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: SAKANA_BASE_URL, }) diff --git a/apps/sim/providers/thinkingmachines/index.ts b/apps/sim/providers/thinkingmachines/index.ts index 305ce26f07d..8eb07ba1aba 100644 --- a/apps/sim/providers/thinkingmachines/index.ts +++ b/apps/sim/providers/thinkingmachines/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, @@ -65,6 +66,7 @@ export const thinkingMachinesProvider: ProviderConfig = { if (!request.apiKey) throw new Error('API key is required for Thinking Machines') const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: TINKER_BASE_URL, }) diff --git a/apps/sim/providers/together/index.ts b/apps/sim/providers/together/index.ts index a3746e57a19..8d00508d79e 100644 --- a/apps/sim/providers/together/index.ts +++ b/apps/sim/providers/together/index.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { @@ -44,6 +45,7 @@ export const togetherProvider: ProviderConfig = { const client = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.together.ai/v1', }) diff --git a/apps/sim/providers/transport.ts b/apps/sim/providers/transport.ts index d882cfe9936..fec962eaee4 100644 --- a/apps/sim/providers/transport.ts +++ b/apps/sim/providers/transport.ts @@ -54,9 +54,11 @@ export interface OpenAICompatTransport { * what makes them safe to set in a constructor that {@link getCachedProviderClient} * memoises — anything varying per request must be passed per call instead. * - * Deliberately stamps no `fetch`: a wrapper would turn the SDK's header-only timer - * into a total deadline that truncates a live stream, and would drop the caller's - * abort signal. + * Stamps no `fetch`. A wrapper that adds its own deadline would turn the SDK's + * header-only timer into a total one that truncates a live stream, and could drop the + * caller's abort signal. A pass-through router such as `outboundFetch` does neither and + * is fine, but providers add it per constructor rather than here: operator-local + * providers (Ollama, vLLM) share this transport and must stay direct. */ export const openAICompatTransport = (): OpenAICompatTransport => ({ timeout: PROVIDER_HEADERS_TIMEOUT_MS, diff --git a/apps/sim/providers/typesafe/transport.ts b/apps/sim/providers/typesafe/transport.ts index 658267367f0..b2a17363fd7 100644 --- a/apps/sim/providers/typesafe/transport.ts +++ b/apps/sim/providers/typesafe/transport.ts @@ -1,5 +1,6 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { stringifyBoundedJson } from '@/lib/core/utils/bounded-json' import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { @@ -39,7 +40,7 @@ export async function requestJevEvaluation( let response: Response | undefined try { signal.throwIfAborted() - response = await fetch('https://api.typesafe.ai/v1/systemone', { + response = await outboundFetch('https://api.typesafe.ai/v1/systemone', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, body: payload, diff --git a/apps/sim/providers/vertex/index.ts b/apps/sim/providers/vertex/index.ts index ea5edba6328..83c0d255d17 100644 --- a/apps/sim/providers/vertex/index.ts +++ b/apps/sim/providers/vertex/index.ts @@ -2,6 +2,7 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { OAuth2Client } from 'google-auth-library' import { env } from '@/lib/core/config/env' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { validateGoogleCloudLocation, validateGoogleCloudProject, @@ -91,6 +92,7 @@ export const vertexProvider: ProviderConfig = { googleAuthOptions: { authClient, }, + httpOptions: { fetch: outboundFetch }, }) return executeGeminiRequest({ diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index 0fac82e9eaa..c98fdef1dcd 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -86,6 +86,7 @@ export const vllmProvider: ProviderConfig = { headers.Authorization = `Bearer ${env.VLLM_API_KEY}` } + // egress-direct: operator's own vLLM server named by the deployment const response = await fetch(`${apiBaseUrl}/models`, { headers }) if (!response.ok) { await response.text().catch(() => {}) diff --git a/apps/sim/providers/xai/index.ts b/apps/sim/providers/xai/index.ts index bf708f25058..0ca496b8d40 100644 --- a/apps/sim/providers/xai/index.ts +++ b/apps/sim/providers/xai/index.ts @@ -4,6 +4,7 @@ import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' import type { CompletionUsage } from 'openai/resources/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -91,6 +92,7 @@ export const xAIProvider: ProviderConfig = { const xai = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: 'https://api.x.ai/v1', }) diff --git a/apps/sim/providers/zai/index.ts b/apps/sim/providers/zai/index.ts index 10ff0aec053..b461150344f 100644 --- a/apps/sim/providers/zai/index.ts +++ b/apps/sim/providers/zai/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -98,6 +99,7 @@ export const zaiProvider: ProviderConfig = { try { const zai = new OpenAI({ ...openAICompatTransport(), + fetch: outboundFetch, apiKey: request.apiKey, baseURL: ZAI_BASE_URL, }) diff --git a/apps/sim/tools/apify/run_actor_async.ts b/apps/sim/tools/apify/run_actor_async.ts index 1cc3c33c449..2e0e548124e 100644 --- a/apps/sim/tools/apify/run_actor_async.ts +++ b/apps/sim/tools/apify/run_actor_async.ts @@ -1,5 +1,6 @@ import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { ApifyRun, RunActorParams, RunActorResult } from '@/tools/apify/types' import { APIFY_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' @@ -148,7 +149,7 @@ export const apifyRunActorAsyncTool: ToolConfig await sleep(POLL_INTERVAL_MS) elapsedTime += POLL_INTERVAL_MS - const statusResponse = await fetch(`https://api.apify.com/v2/actor-runs/${runId}`, { + const statusResponse = await outboundFetch(`https://api.apify.com/v2/actor-runs/${runId}`, { headers: { Authorization: `Bearer ${params.apiKey}`, }, @@ -173,7 +174,7 @@ export const apifyRunActorAsyncTool: ToolConfig ) { if (run.status === 'SUCCEEDED' && run.defaultDatasetId) { const limit = Math.max(1, Math.min(params.itemLimit || 100, 250000)) - const itemsResponse = await fetch( + const itemsResponse = await outboundFetch( `https://api.apify.com/v2/datasets/${run.defaultDatasetId}/items?limit=${limit}`, { headers: { diff --git a/apps/sim/tools/azure_devops/get_work_items_batch.ts b/apps/sim/tools/azure_devops/get_work_items_batch.ts index 03c2c4229b8..07e8ff1f74b 100644 --- a/apps/sim/tools/azure_devops/get_work_items_batch.ts +++ b/apps/sim/tools/azure_devops/get_work_items_batch.ts @@ -6,12 +6,11 @@ import type { import type { AzureDevOpsRawWorkItem } from '@/tools/azure_devops/utils' import { azureDevOpsProjectUrl, - fetchWorkItemsChunk, formatWorkItem, mapWorkItem, WORK_ITEM_LIST_DETAIL_OUTPUT_PROPERTIES, - WORK_ITEMS_BATCH_SIZE, } from '@/tools/azure_devops/utils' +import { fetchWorkItemsChunk, WORK_ITEMS_BATCH_SIZE } from '@/tools/azure_devops/work-items.server' import { AZURE_DEVOPS_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' diff --git a/apps/sim/tools/azure_devops/query_work_items.ts b/apps/sim/tools/azure_devops/query_work_items.ts index 6c0d796e22a..4a10a005c19 100644 --- a/apps/sim/tools/azure_devops/query_work_items.ts +++ b/apps/sim/tools/azure_devops/query_work_items.ts @@ -6,11 +6,10 @@ import type { } from '@/tools/azure_devops/types' import { azureDevOpsProjectUrl, - fetchWorkItemsChunk, formatWorkItem, WORK_ITEM_LIST_DETAIL_OUTPUT_PROPERTIES, - WORK_ITEMS_BATCH_SIZE, } from '@/tools/azure_devops/utils' +import { fetchWorkItemsChunk, WORK_ITEMS_BATCH_SIZE } from '@/tools/azure_devops/work-items.server' import { AZURE_DEVOPS_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' diff --git a/apps/sim/tools/azure_devops/utils.ts b/apps/sim/tools/azure_devops/utils.ts index 568ea5a9bd7..aba8ada8713 100644 --- a/apps/sim/tools/azure_devops/utils.ts +++ b/apps/sim/tools/azure_devops/utils.ts @@ -1,11 +1,5 @@ import { toStringOrNull } from '@sim/utils/coerce' import { toRecordOrNull } from '@sim/utils/object' -import { - DEFAULT_MAX_ERROR_BODY_BYTES, - readResponseJsonWithLimit, - readResponseTextWithLimit, -} from '@/lib/core/utils/stream-limits' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import type { AzureDevOpsComment, AzureDevOpsWorkItem } from '@/tools/azure_devops/types' import type { OutputProperty } from '@/tools/types' import { safeUrlPathSegment } from '@/tools/url-path' @@ -29,52 +23,6 @@ function pathSegment(value: string, paramName: string): string { return safeUrlPathSegment(decoded, paramName) } -/** Work Items - List accepts at most 200 ids per request. */ -export const WORK_ITEMS_BATCH_SIZE = 200 - -/** - * Fetches one Work Items - List chunk with `$expand=all`, bounded to the buffered - * transfer cap (100 MB) and cancelled with the tool call. - */ -export async function fetchWorkItemsChunk( - params: { organization: string; project: string; accessToken: string }, - ids: Array, - errorPrefix: string, - signal?: AbortSignal -): Promise { - const url = new URL(`${azureDevOpsProjectUrl(params)}/_apis/wit/workitems`) - url.searchParams.set('ids', ids.join(',')) - url.searchParams.set('$expand', 'all') - url.searchParams.set('api-version', '7.2-preview.3') - - const response = await fetch(url.toString(), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: `Basic ${btoa(`:${params.accessToken}`)}`, - }, - signal, - }) - - if (!response.ok) { - const errorBody = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Azure DevOps work items error body', - signal, - }).catch(() => '') - throw new Error(`${errorPrefix} (${response.status}): ${errorBody || response.statusText}`) - } - - const data = await readResponseJsonWithLimit<{ value?: AzureDevOpsRawWorkItem[] }>(response, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - label: 'Azure DevOps work items response', - signal, - }) - return (data.value ?? []).map((raw: AzureDevOpsRawWorkItem) => - mapWorkItem(raw, { includeRawDetails: false }) - ) -} - /** States for Azure DevOps Basic process work items (Issue, Task, Epic). */ export const AZURE_DEVOPS_BASIC_WORK_ITEM_STATES = ['To Do', 'Doing', 'Done'] as const diff --git a/apps/sim/tools/azure_devops/work-items.server.ts b/apps/sim/tools/azure_devops/work-items.server.ts new file mode 100644 index 00000000000..47ba15c00a0 --- /dev/null +++ b/apps/sim/tools/azure_devops/work-items.server.ts @@ -0,0 +1,59 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { AzureDevOpsWorkItem } from '@/tools/azure_devops/types' +import { + type AzureDevOpsRawWorkItem, + azureDevOpsProjectUrl, + mapWorkItem, +} from '@/tools/azure_devops/utils' + +/** Work Items - List accepts at most 200 ids per request. */ +export const WORK_ITEMS_BATCH_SIZE = 200 + +/** + * Fetches one Work Items - List chunk with `$expand=all`, bounded to the buffered + * transfer cap (100 MB) and cancelled with the tool call. + */ +export async function fetchWorkItemsChunk( + params: { organization: string; project: string; accessToken: string }, + ids: Array, + errorPrefix: string, + signal?: AbortSignal +): Promise { + const url = new URL(`${azureDevOpsProjectUrl(params)}/_apis/wit/workitems`) + url.searchParams.set('ids', ids.join(',')) + url.searchParams.set('$expand', 'all') + url.searchParams.set('api-version', '7.2-preview.3') + + const response = await outboundFetch(url.toString(), { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${btoa(`:${params.accessToken}`)}`, + }, + signal, + }) + + if (!response.ok) { + const errorBody = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Azure DevOps work items error body', + signal, + }).catch(() => '') + throw new Error(`${errorPrefix} (${response.status}): ${errorBody || response.statusText}`) + } + + const data = await readResponseJsonWithLimit<{ value?: AzureDevOpsRawWorkItem[] }>(response, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Azure DevOps work items response', + signal, + }) + return (data.value ?? []).map((raw: AzureDevOpsRawWorkItem) => + mapWorkItem(raw, { includeRawDetails: false }) + ) +} diff --git a/apps/sim/tools/brightdata/discover.ts b/apps/sim/tools/brightdata/discover.ts index f0cebce36cd..8e5295939ac 100644 --- a/apps/sim/tools/brightdata/discover.ts +++ b/apps/sim/tools/brightdata/discover.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { BrightDataDiscoverParams, BrightDataDiscoverResponse } from '@/tools/brightdata/types' import { BRIGHTDATA_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { ToolConfig } from '@/tools/types' @@ -147,7 +148,7 @@ export const brightDataDiscoverTool: ToolConfig< while (elapsedTime < MAX_POLL_TIME_MS) { try { - const pollResponse = await fetch( + const pollResponse = await outboundFetch( `https://api.brightdata.com/discover?task_id=${encodeURIComponent(taskId)}`, { method: 'GET', diff --git a/apps/sim/tools/cbinsights/utils.ts b/apps/sim/tools/cbinsights/utils.ts index fc0c85d323a..c2d6e8c77fc 100644 --- a/apps/sim/tools/cbinsights/utils.ts +++ b/apps/sim/tools/cbinsights/utils.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { toArray } from '@sim/utils/object' import { LRUCache } from 'lru-cache' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -125,7 +126,7 @@ async function authorize( clientSecret: string, signal?: AbortSignal ): Promise { - const response = await fetch(`${CBINSIGHTS_API_BASE}/v2/authorize`, { + const response = await outboundFetch(`${CBINSIGHTS_API_BASE}/v2/authorize`, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify({ clientId, clientSecret }), @@ -203,7 +204,7 @@ export async function cbInsightsRequest( const effectiveSignal = signal ? AbortSignal.any([signal, timeout]) : timeout const send = async (token: string) => - fetch(`${CBINSIGHTS_API_BASE}${spec.path}`, { + outboundFetch(`${CBINSIGHTS_API_BASE}${spec.path}`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, diff --git a/apps/sim/tools/dropbox/download.ts b/apps/sim/tools/dropbox/download.ts index dbd57e8894c..69614bcff8a 100644 --- a/apps/sim/tools/dropbox/download.ts +++ b/apps/sim/tools/dropbox/download.ts @@ -1,4 +1,5 @@ import { omit } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { httpHeaderSafeJson } from '@/lib/core/utils/validation' import type { DropboxDownloadParams, @@ -28,14 +29,17 @@ async function transformDownloadResponse(response: Response, params?: DropboxDow let temporaryLink: string | undefined if (params?.accessToken) { try { - const linkResponse = await fetch('https://api.dropboxapi.com/2/files/get_temporary_link', { - method: 'POST', - headers: { - Authorization: `Bearer ${params.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ path: params.path.trim() }), - }) + const linkResponse = await outboundFetch( + 'https://api.dropboxapi.com/2/files/get_temporary_link', + { + method: 'POST', + headers: { + Authorization: `Bearer ${params.accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ path: params.path.trim() }), + } + ) if (linkResponse.ok) { const linkData = await linkResponse.json() temporaryLink = linkData.link diff --git a/apps/sim/tools/dropcontact/enrich_contact.ts b/apps/sim/tools/dropcontact/enrich_contact.ts index f3682d196fc..da123efe99b 100644 --- a/apps/sim/tools/dropcontact/enrich_contact.ts +++ b/apps/sim/tools/dropcontact/enrich_contact.ts @@ -1,4 +1,5 @@ import { sleep } from '@sim/utils/helpers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { dropcontactHosting } from '@/tools/dropcontact/hosting' import type { DropcontactEmailEntry, @@ -254,7 +255,7 @@ export const dropcontactEnrichContactTool: ToolConfig< // Poll endpoint: GET https://api.dropcontact.com/v1/enrich/all/{request_id} // Source: https://developer.dropcontact.com (retrieved 2026-05) - const pollResponse = await fetch( + const pollResponse = await outboundFetch( `https://api.dropcontact.com/v1/enrich/all/${encodeURIComponent(requestId)}`, { headers: { diff --git a/apps/sim/tools/enrow/find_email.ts b/apps/sim/tools/enrow/find_email.ts index bcead35e5fa..3cc5cea05a6 100644 --- a/apps/sim/tools/enrow/find_email.ts +++ b/apps/sim/tools/enrow/find_email.ts @@ -1,4 +1,5 @@ import { sleep } from '@sim/utils/helpers' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { enrowHosting } from '@/tools/enrow/hosting' import type { EnrowFindEmailParams, @@ -142,7 +143,7 @@ export const enrowFindEmailTool: ToolConfig = { elapsedTime += POLL_INTERVAL_MS try { - const statusResponse = await fetch(`https://api.exa.ai/agent/runs/${runId}`, { + const statusResponse = await outboundFetch(`https://api.exa.ai/agent/runs/${runId}`, { method: 'GET', headers: { 'x-api-key': params.apiKey, diff --git a/apps/sim/tools/firecrawl/agent.ts b/apps/sim/tools/firecrawl/agent.ts index 13c77ffda28..964ed100450 100644 --- a/apps/sim/tools/firecrawl/agent.ts +++ b/apps/sim/tools/firecrawl/agent.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { AgentParams, AgentResponse } from '@/tools/firecrawl/types' import type { ToolConfig } from '@/tools/types' @@ -119,7 +120,7 @@ export const agentTool: ToolConfig = { while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, { + const statusResponse = await outboundFetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, { method: 'GET', headers: { Authorization: `Bearer ${params.apiKey}`, diff --git a/apps/sim/tools/firecrawl/batch-scrape.ts b/apps/sim/tools/firecrawl/batch-scrape.ts index f2d23d82044..26f4d655bdd 100644 --- a/apps/sim/tools/firecrawl/batch-scrape.ts +++ b/apps/sim/tools/firecrawl/batch-scrape.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlFormatModelInput, @@ -196,13 +197,16 @@ export const batchScrapeTool: ToolConfig while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/crawl/${jobId}`, { + const statusResponse = await outboundFetch(`https://api.firecrawl.dev/v2/crawl/${jobId}`, { method: 'GET', headers: { Authorization: `Bearer ${params.apiKey}`, diff --git a/apps/sim/tools/firecrawl/extract.ts b/apps/sim/tools/firecrawl/extract.ts index b6c06b6000d..2d2d2a31020 100644 --- a/apps/sim/tools/firecrawl/extract.ts +++ b/apps/sim/tools/firecrawl/extract.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlScrapeOptionsModelInput, @@ -168,13 +169,16 @@ export const extractTool: ToolConfig = { while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/extract/${jobId}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${params.apiKey}`, - 'Content-Type': 'application/json', - }, - }) + const statusResponse = await outboundFetch( + `https://api.firecrawl.dev/v2/extract/${jobId}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${params.apiKey}`, + 'Content-Type': 'application/json', + }, + } + ) if (!statusResponse.ok) { throw new Error(`Failed to get extract status: ${statusResponse.statusText}`) diff --git a/apps/sim/tools/github/pr.ts b/apps/sim/tools/github/pr.ts index 8c2525bfe87..1a2bfca5bda 100644 --- a/apps/sim/tools/github/pr.ts +++ b/apps/sim/tools/github/pr.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { nullableBoolean, nullableString, @@ -150,7 +151,7 @@ async function fetchPullRequestFiles( const maxPages = MAX_PULL_REQUEST_FILES / PULL_REQUEST_FILES_PER_PAGE for (let page = 1; page <= maxPages; page += 1) { - const response = await fetch( + const response = await outboundFetch( `https://api.github.com/repos/${params.owner}/${params.repo}/pulls/${pullNumber}/files?per_page=${PULL_REQUEST_FILES_PER_PAGE}&page=${page}`, { headers: { diff --git a/apps/sim/tools/gmail/read.ts b/apps/sim/tools/gmail/read.ts index 860dfa2d2d2..a75329696fe 100644 --- a/apps/sim/tools/gmail/read.ts +++ b/apps/sim/tools/gmail/read.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -163,7 +164,7 @@ export const gmailReadTool: ToolConfig = { try { // Get the first message details const messageId = data.messages[0].id - const messageResponse = await fetch( + const messageResponse = await outboundFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}?format=full`, { headers: { @@ -210,7 +211,7 @@ export const gmailReadTool: ToolConfig = { const messages: GmailMessage[] = [] for (const msg of data.messages.slice(0, maxResults)) { context?.signal?.throwIfAborted() - const messageResponse = await fetch( + const messageResponse = await outboundFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(msg.id)}?format=full`, { headers: { Authorization: `Bearer ${params?.accessToken || ''}` }, diff --git a/apps/sim/tools/gmail/search.ts b/apps/sim/tools/gmail/search.ts index 6aef77d9876..acfdc0ae042 100644 --- a/apps/sim/tools/gmail/search.ts +++ b/apps/sim/tools/gmail/search.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { GmailMessage, GmailSearchParams, GmailToolResponse } from '@/tools/gmail/types' import { createMessagesSummary, @@ -97,7 +98,7 @@ export const gmailSearchTool: ToolConfig = context?.signal?.throwIfAborted() const batch = await Promise.all( listed.slice(i, i + MESSAGE_FETCH_CONCURRENCY).map(async (msg) => { - const messageResponse = await fetch( + const messageResponse = await outboundFetch( `${GMAIL_API_BASE}/messages/${safeUrlPathSegment(msg.id, 'messageId')}?format=full`, { headers: { diff --git a/apps/sim/tools/gmail/utils.ts b/apps/sim/tools/gmail/utils.ts index f4002bbcbdd..dcb828fbca6 100644 --- a/apps/sim/tools/gmail/utils.ts +++ b/apps/sim/tools/gmail/utils.ts @@ -1,5 +1,6 @@ import { generateRandomString } from '@sim/utils/random' import { convert } from 'html-to-text' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -32,7 +33,7 @@ export async function fetchThreadingHeaders( subject?: string }> { try { - const messageResponse = await fetch( + const messageResponse = await outboundFetch( `${GMAIL_API_BASE}/messages/${safeUrlPathSegment(messageId, 'messageId')}?format=metadata&metadataHeaders=Message-ID&metadataHeaders=References&metadataHeaders=Subject`, { headers: { @@ -276,7 +277,7 @@ export async function downloadAttachments( for (const attachment of attachmentInfo) { try { budget.assertSize(attachment.size, 'Gmail attachments') - const response = await fetch( + const response = await outboundFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}/attachments/${encodeURIComponent(attachment.attachmentId)}`, { headers: { Authorization: `Bearer ${accessToken}` }, diff --git a/apps/sim/tools/google_calendar/invite.ts b/apps/sim/tools/google_calendar/invite.ts index 32f7a3e3d97..6d44e4e9793 100644 --- a/apps/sim/tools/google_calendar/invite.ts +++ b/apps/sim/tools/google_calendar/invite.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { CALENDAR_API_BASE, type CalendarAttendee, @@ -80,7 +81,7 @@ async function inviteAttendees( queryParams.append('sendUpdates', params?.sendUpdates ?? 'all') const putUrl = `${CALENDAR_API_BASE}/calendars/${safeUrlPathSegment(calendarId, 'calendarId')}/events/${safeUrlPathSegment(params?.eventId ?? '', 'eventId')}?${queryParams.toString()}` - const putResponse = await fetch(putUrl, { + const putResponse = await outboundFetch(putUrl, { method: 'PUT', headers: { Authorization: `Bearer ${params?.accessToken}`, diff --git a/apps/sim/tools/google_calendar/quick_add.ts b/apps/sim/tools/google_calendar/quick_add.ts index cb430425700..1d51a4e7861 100644 --- a/apps/sim/tools/google_calendar/quick_add.ts +++ b/apps/sim/tools/google_calendar/quick_add.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { CALENDAR_API_BASE, type GoogleCalendarApiEventResponse, @@ -113,7 +114,7 @@ export const quickAddTool: ToolConfig< const updateUrl = `${CALENDAR_API_BASE}/calendars/${safeUrlPathSegment(calendarId, 'calendarId')}/events/${safeUrlPathSegment(eventId, 'eventId')}${updateQueryParams.toString() ? `?${updateQueryParams.toString()}` : ''}` - const updateResponse = await fetch(updateUrl, { + const updateResponse = await outboundFetch(updateUrl, { method: 'PATCH', headers: { Authorization: `Bearer ${params.accessToken}`, @@ -244,7 +245,7 @@ export const quickAddV2Tool: ToolConfig< const updateUrl = `${CALENDAR_API_BASE}/calendars/${safeUrlPathSegment(calendarId, 'calendarId')}/events/${safeUrlPathSegment(eventId, 'eventId')}${updateQueryParams.toString() ? `?${updateQueryParams.toString()}` : ''}` - const updateResponse = await fetch(updateUrl, { + const updateResponse = await outboundFetch(updateUrl, { method: 'PATCH', headers: { Authorization: `Bearer ${params.accessToken}`, diff --git a/apps/sim/tools/google_calendar/respond.ts b/apps/sim/tools/google_calendar/respond.ts index 64918e92502..c95f3baebf9 100644 --- a/apps/sim/tools/google_calendar/respond.ts +++ b/apps/sim/tools/google_calendar/respond.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { CALENDAR_API_BASE, type CalendarAttendee, @@ -67,7 +68,7 @@ async function respondToEvent( const patchUrl = `${CALENDAR_API_BASE}/calendars/${safeUrlPathSegment(calendarId, 'calendarId')}/events/${safeUrlPathSegment(params?.eventId ?? '', 'eventId')}${queryString ? `?${queryString}` : ''}` signal?.throwIfAborted() - const patchResponse = await fetch(patchUrl, { + const patchResponse = await outboundFetch(patchUrl, { method: 'PATCH', signal, headers: { diff --git a/apps/sim/tools/google_drive/create_folder.ts b/apps/sim/tools/google_drive/create_folder.ts index 67c512a0864..e0dda457231 100644 --- a/apps/sim/tools/google_drive/create_folder.ts +++ b/apps/sim/tools/google_drive/create_folder.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { GoogleDriveFile, @@ -96,7 +97,7 @@ export const createFolderTool: ToolConfig { executorDelegationOrigin, }) ) - expect( - fetchMock.mock.calls.some(([url]) => String(url).includes('/api/auth/oauth/token')) - ).toBe(false) }) it('fails before transport when managed credential delegation lacks current workflow authority', async () => { diff --git a/apps/sim/tools/jira/add_comment.ts b/apps/sim/tools/jira/add_comment.ts index c3ea57bd92c..fc4eec2fc67 100644 --- a/apps/sim/tools/jira/add_comment.ts +++ b/apps/sim/tools/jira/add_comment.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraAddCommentParams, JiraAddCommentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { extractAdfText, getJiraCloudId, toAdf, transformUser } from '@/tools/jira/utils' @@ -110,7 +111,7 @@ export const jiraAddCommentTool: ToolConfig { const commentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey ?? '', 'issueKey')}/comment` - const commentResponse = await fetch(commentUrl, { + const commentResponse = await outboundFetch(commentUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/add_watcher.ts b/apps/sim/tools/jira/add_watcher.ts index 2699df81494..23d2f739a1c 100644 --- a/apps/sim/tools/jira/add_watcher.ts +++ b/apps/sim/tools/jira/add_watcher.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraAddWatcherParams, JiraAddWatcherResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -80,7 +81,7 @@ export const jiraAddWatcherTool: ToolConfig { const worklogUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey ?? '', 'issueKey')}/worklog` - const worklogResponse = await fetch(worklogUrl, { + const worklogResponse = await outboundFetch(worklogUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/assign_issue.ts b/apps/sim/tools/jira/assign_issue.ts index 5b39127afc1..f6dd197d0ba 100644 --- a/apps/sim/tools/jira/assign_issue.ts +++ b/apps/sim/tools/jira/assign_issue.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraAssignIssueParams, JiraAssignIssueResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -89,7 +90,7 @@ export const jiraAssignIssueTool: ToolConfig { const cloudId = params?.cloudId || (await getJiraCloudId(params!.domain, params!.accessToken)) - const typesResp = await fetch( + const typesResp = await outboundFetch( `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issueLinkType`, { method: 'GET', @@ -115,7 +116,7 @@ export const jiraCreateIssueLinkTool: ToolConfig< } const linkUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issueLink` - const linkResponse = await fetch(linkUrl, { + const linkResponse = await outboundFetch(linkUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/delete_attachment.ts b/apps/sim/tools/jira/delete_attachment.ts index 0dbd4846545..f341393aa05 100644 --- a/apps/sim/tools/jira/delete_attachment.ts +++ b/apps/sim/tools/jira/delete_attachment.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraDeleteAttachmentParams, JiraDeleteAttachmentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -67,7 +68,7 @@ export const jiraDeleteAttachmentTool: ToolConfig< const cloudId = await getJiraCloudId(params!.domain, params!.accessToken) // Make the actual request with the resolved cloudId const attachmentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/attachment/${safeUrlPathSegment(params?.attachmentId ?? '', 'attachmentId')}` - const attachmentResponse = await fetch(attachmentUrl, { + const attachmentResponse = await outboundFetch(attachmentUrl, { method: 'DELETE', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/delete_comment.ts b/apps/sim/tools/jira/delete_comment.ts index a8f61ea1a60..19b019c6d7c 100644 --- a/apps/sim/tools/jira/delete_comment.ts +++ b/apps/sim/tools/jira/delete_comment.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraDeleteCommentParams, JiraDeleteCommentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -71,7 +72,7 @@ export const jiraDeleteCommentTool: ToolConfig { const attachmentsUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey ?? '', 'issueKey')}?fields=attachment` - const attachmentsResponse = await fetch(attachmentsUrl, { + const attachmentsResponse = await outboundFetch(attachmentsUrl, { method: 'GET', signal: context?.signal, headers: { diff --git a/apps/sim/tools/jira/get_comments.ts b/apps/sim/tools/jira/get_comments.ts index a0c41c7f569..346b6a8b40b 100644 --- a/apps/sim/tools/jira/get_comments.ts +++ b/apps/sim/tools/jira/get_comments.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraGetCommentsParams, JiraGetCommentsResponse } from '@/tools/jira/types' import { COMMENT_ITEM_PROPERTIES, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { extractAdfText, getJiraCloudId, transformUser } from '@/tools/jira/utils' @@ -106,7 +107,7 @@ export const jiraGetCommentsTool: ToolConfig { const fetchFields = async (cloudId: string) => { - const fieldsResponse = await fetch(buildFieldsUrl(cloudId), { + const fieldsResponse = await outboundFetch(buildFieldsUrl(cloudId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/get_project.ts b/apps/sim/tools/jira/get_project.ts index b7020c6453b..5cb773c37ff 100644 --- a/apps/sim/tools/jira/get_project.ts +++ b/apps/sim/tools/jira/get_project.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraGetProjectParams, JiraGetProjectResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -65,7 +66,7 @@ export const jiraGetProjectTool: ToolConfig { const fetchProject = async (cloudId: string) => { - const projectResponse = await fetch(buildProjectUrl(cloudId, params!.projectId), { + const projectResponse = await outboundFetch(buildProjectUrl(cloudId, params!.projectId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/get_transitions.ts b/apps/sim/tools/jira/get_transitions.ts index 51daaf83b14..dd102f4f20d 100644 --- a/apps/sim/tools/jira/get_transitions.ts +++ b/apps/sim/tools/jira/get_transitions.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraGetTransitionsParams, JiraGetTransitionsResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -67,13 +68,16 @@ export const jiraGetTransitionsTool: ToolConfig< transformResponse: async (response: Response, params?: JiraGetTransitionsParams) => { const fetchTransitions = async (cloudId: string) => { - const transitionsResponse = await fetch(buildTransitionsUrl(cloudId, params!.issueKey), { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${params!.accessToken}`, - }, - }) + const transitionsResponse = await outboundFetch( + buildTransitionsUrl(cloudId, params!.issueKey), + { + method: 'GET', + headers: { + Accept: 'application/json', + Authorization: `Bearer ${params!.accessToken}`, + }, + } + ) if (!transitionsResponse.ok) { const errorText = await transitionsResponse.text() diff --git a/apps/sim/tools/jira/get_users.ts b/apps/sim/tools/jira/get_users.ts index 4fa1219c0d7..204fb1de06d 100644 --- a/apps/sim/tools/jira/get_users.ts +++ b/apps/sim/tools/jira/get_users.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraGetUsersParams, JiraGetUsersResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -109,7 +110,7 @@ export const jiraGetUsersTool: ToolConfig { const fetchIssueTypes = async (cloudId: string) => { - const issueTypesResponse = await fetch(buildIssueTypesUrl(cloudId), { + const issueTypesResponse = await outboundFetch(buildIssueTypesUrl(cloudId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/list_projects.ts b/apps/sim/tools/jira/list_projects.ts index 7e65df9dec1..eedbd862aeb 100644 --- a/apps/sim/tools/jira/list_projects.ts +++ b/apps/sim/tools/jira/list_projects.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraListProjectsParams, JiraListProjectsResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -101,7 +102,7 @@ export const jiraListProjectsTool: ToolConfig { const fetchProjects = async (cloudId: string) => { - const projectsResponse = await fetch(buildSearchUrl(cloudId, params!), { + const projectsResponse = await outboundFetch(buildSearchUrl(cloudId, params!), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/remove_watcher.ts b/apps/sim/tools/jira/remove_watcher.ts index 4b8424cc98c..a3e64232980 100644 --- a/apps/sim/tools/jira/remove_watcher.ts +++ b/apps/sim/tools/jira/remove_watcher.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraRemoveWatcherParams, JiraRemoveWatcherResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' @@ -70,7 +71,7 @@ export const jiraRemoveWatcherTool: ToolConfig { const issueUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params.issueKey, 'issueKey')}?expand=renderedFields,names,schema,transitions,operations,editmeta,changelog,versionedRepresentations` - const issueResponse = await fetch(issueUrl, { + const issueResponse = await outboundFetch(issueUrl, { method: 'GET', signal: context?.signal, headers: { @@ -120,15 +121,15 @@ export const jiraRetrieveTool: ToolConfig { const base = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params.issueKey, 'issueKey')}` const [commentsResp, worklogResp, watchersResp] = await Promise.all([ - fetch(`${base}/comment?maxResults=100&orderBy=-created`, { + outboundFetch(`${base}/comment?maxResults=100&orderBy=-created`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), - fetch(`${base}/worklog?maxResults=100`, { + outboundFetch(`${base}/worklog?maxResults=100`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), - fetch(`${base}/watchers`, { + outboundFetch(`${base}/watchers`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), diff --git a/apps/sim/tools/jira/search_issues.ts b/apps/sim/tools/jira/search_issues.ts index 25d9017166c..803a4b533e3 100644 --- a/apps/sim/tools/jira/search_issues.ts +++ b/apps/sim/tools/jira/search_issues.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraSearchIssuesParams, JiraSearchIssuesResponse } from '@/tools/jira/types' import { SEARCH_ISSUE_ITEM_PROPERTIES, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { @@ -212,7 +213,7 @@ export const jiraSearchIssuesTool: ToolConfig { // First, fetch available transitions to get the name and target status const transitionsUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey, 'issueKey')}/transitions` - const transitionsResp = await fetch(transitionsUrl, { + const transitionsResp = await outboundFetch(transitionsUrl, { method: 'GET', headers: { Accept: 'application/json', @@ -122,7 +123,7 @@ export const jiraTransitionIssueTool: ToolConfig< } // Perform the transition - const transitionResponse = await fetch(transitionsUrl, { + const transitionResponse = await outboundFetch(transitionsUrl, { method: 'POST', headers: { Accept: 'application/json', @@ -167,7 +168,7 @@ export const jiraTransitionIssueTool: ToolConfig< // Fetch transition metadata for the response try { const transitionsUrl = `https://api.atlassian.com/ex/jira/${params.cloudId}/rest/api/3/issue/${safeUrlPathSegment(params.issueKey, 'issueKey')}/transitions` - const transitionsResp = await fetch(transitionsUrl, { + const transitionsResp = await outboundFetch(transitionsUrl, { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/update_comment.ts b/apps/sim/tools/jira/update_comment.ts index e2daeac5549..b1b1d8a0e74 100644 --- a/apps/sim/tools/jira/update_comment.ts +++ b/apps/sim/tools/jira/update_comment.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraUpdateCommentParams, JiraUpdateCommentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { extractAdfText, getJiraCloudId, toAdf, transformUser } from '@/tools/jira/utils' @@ -108,7 +109,7 @@ export const jiraUpdateCommentTool: ToolConfig { const commentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${safeUrlPathSegment(params!.issueKey ?? '', 'issueKey')}/comment/${safeUrlPathSegment(params!.commentId ?? '', 'commentId')}` - const commentResponse = await fetch(commentUrl, { + const commentResponse = await outboundFetch(commentUrl, { method: 'PUT', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/update_worklog.ts b/apps/sim/tools/jira/update_worklog.ts index 130e2715296..2347e397c4e 100644 --- a/apps/sim/tools/jira/update_worklog.ts +++ b/apps/sim/tools/jira/update_worklog.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { JiraUpdateWorklogParams, JiraUpdateWorklogResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { @@ -145,7 +146,7 @@ export const jiraUpdateWorklogTool: ToolConfig { const basePath = getItemBasePath(spreadsheetId, driveId) try { - const response = await fetch(`${basePath}?$select=id,webUrl`, { + const response = await outboundFetch(`${basePath}?$select=id,webUrl`, { signal, headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/tools/microsoft_teams/utils.ts b/apps/sim/tools/microsoft_teams/utils.ts index 3e1dfc13e97..1a7c6daecb7 100644 --- a/apps/sim/tools/microsoft_teams/utils.ts +++ b/apps/sim/tools/microsoft_teams/utils.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toStringOrNull } from '@sim/utils/coerce' import { isRecordLike, toRecord, toRecordOrNull } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -220,7 +221,7 @@ async function fetchHostedContents( const results: ToolFileData[] = [] try { budget.signal?.throwIfAborted() - const response = await fetch(path, { + const response = await outboundFetch(path, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -237,7 +238,7 @@ async function fetchHostedContents( for (const item of data.value ?? []) { if (!item.id) continue budget.signal?.throwIfAborted() - const content = await fetch(`${path}/${encodeURIComponent(item.id)}/$value`, { + const content = await outboundFetch(`${path}/${encodeURIComponent(item.id)}/$value`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -298,7 +299,7 @@ async function downloadReferenceAttachment( budget.signal?.throwIfAborted() const shareId = `u!${Buffer.from(attachment.contentUrl).toString('base64url')}` const path = `https://graph.microsoft.com/v1.0/shares/${shareId}/driveItem` - const metadataResponse = await fetch(`${path}?$select=name,size,file`, { + const metadataResponse = await outboundFetch(`${path}?$select=name,size,file`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -313,7 +314,7 @@ async function downloadReferenceAttachment( file?: { mimeType?: string } }>(metadataResponse, 'Teams attachment metadata', budget.signal) if (item.size !== undefined) budget.assertSize(item.size, 'Teams attachments') - const content = await fetch(`${path}/content`, { + const content = await outboundFetch(`${path}/content`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -371,7 +372,7 @@ function parseMentions(content: string): ParsedMention[] { } async function fetchChatMembers(chatId: string, accessToken: string): Promise { - const response = await fetch( + const response = await outboundFetch( `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(chatId)}/members`, { headers: { @@ -398,7 +399,7 @@ async function fetchChannelMembers( channelId: string, accessToken: string ): Promise { - const response = await fetch( + const response = await outboundFetch( `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/members`, { headers: { diff --git a/apps/sim/tools/netsuite/utils.ts b/apps/sim/tools/netsuite/utils.ts index c52c0c435bb..0f405a6bb35 100644 --- a/apps/sim/tools/netsuite/utils.ts +++ b/apps/sim/tools/netsuite/utils.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { truncate } from '@sim/utils/string' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -447,7 +448,7 @@ async function sendSuiteTalkRequest( headers['X-NetSuite-PropertyNameValidation'] = 'error' } } - return fetch(url, { + return outboundFetch(url, { method: request.method, headers, ...(serializedBody !== undefined ? { body: serializedBody } : {}), diff --git a/apps/sim/tools/notion/read.ts b/apps/sim/tools/notion/read.ts index 47eab0a5fd1..534cf98e009 100644 --- a/apps/sim/tools/notion/read.ts +++ b/apps/sim/tools/notion/read.ts @@ -1,6 +1,7 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { parseRetryAfter } from '@sim/utils/retry' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { NotionReadParams, NotionResponse } from '@/tools/notion/types' import { NOTION_READ_MAX_BLOCK_PAGES, @@ -41,7 +42,7 @@ async function fetchPageBlocks( if (cursor) url.searchParams.set('start_cursor', cursor) const request = () => - fetch(url.toString(), { + outboundFetch(url.toString(), { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/tools/outlook/read.ts b/apps/sim/tools/outlook/read.ts index b7e8a8397d1..2c71084b46c 100644 --- a/apps/sim/tools/outlook/read.ts +++ b/apps/sim/tools/outlook/read.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { AttachmentDownloadBudget, @@ -45,7 +46,8 @@ export async function downloadAttachments( budget = new AttachmentDownloadBudget(), options: OutlookAttachmentDownloadOptions = {} ): Promise { - const fetchImpl = options.fetchImpl ?? ((url: string, init: RequestInit) => fetch(url, init)) + const fetchImpl = + options.fetchImpl ?? ((url: string, init: RequestInit) => outboundFetch(url, init)) const attachments: OutlookAttachment[] = [] const path = `https://graph.microsoft.com/v1.0/me/messages/${encodeURIComponent(messageId)}/attachments` const handleError = (error: unknown, attachment?: OutlookDownloadAttachmentMetadata) => { diff --git a/apps/sim/tools/parallel/deep_research.ts b/apps/sim/tools/parallel/deep_research.ts index cbc0f086820..bbe92fda299 100644 --- a/apps/sim/tools/parallel/deep_research.ts +++ b/apps/sim/tools/parallel/deep_research.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { PlatformEvents } from '@/lib/core/telemetry' import { toList } from '@/tools/parallel/search' import type { ParallelDeepResearchParams } from '@/tools/parallel/types' @@ -199,7 +200,7 @@ export const deepResearchTool: ToolConfig { diff --git a/apps/sim/tools/powerbi/listing.ts b/apps/sim/tools/powerbi/listing.ts index 8f6d8d28af7..400e5a451dc 100644 --- a/apps/sim/tools/powerbi/listing.ts +++ b/apps/sim/tools/powerbi/listing.ts @@ -1,4 +1,5 @@ import { z } from 'zod' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { SelectorContextUnavailableError, SelectorOptionsUnavailableError, @@ -70,7 +71,7 @@ export async function listPowerBIOptions(input: { let response: Response | undefined try { // boundary-raw-fetch: Power BI's fixed external origin uses the shared bounded provider reader. - response = await fetch(url, { headers, signal: providerSignal, redirect: 'error' }) + response = await outboundFetch(url, { headers, signal: providerSignal, redirect: 'error' }) providerSignal.throwIfAborted() } catch { await response?.body?.cancel().catch(() => undefined) diff --git a/apps/sim/tools/quickbooks/client.ts b/apps/sim/tools/quickbooks/client.ts index 9888ec5392d..a3cd2c9943c 100644 --- a/apps/sim/tools/quickbooks/client.ts +++ b/apps/sim/tools/quickbooks/client.ts @@ -1,3 +1,4 @@ +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -125,7 +126,7 @@ export async function fetchValidatedQuickBooksCompanyInfo( environment: QuickBooksEnvironment ): Promise { const normalizedRealmId = normalizeQuickBooksRealmId(realmId) - const response = await fetch( + const response = await outboundFetch( buildQuickBooksCompanyUrl( normalizedRealmId, `companyinfo/${encodeURIComponent(normalizedRealmId)}`, diff --git a/apps/sim/tools/quickbooks/utils.ts b/apps/sim/tools/quickbooks/utils.ts index 647c05c40cb..5ef2ddcd2f2 100644 --- a/apps/sim/tools/quickbooks/utils.ts +++ b/apps/sim/tools/quickbooks/utils.ts @@ -1,4 +1,5 @@ import { omit } from '@sim/utils/object' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ErrorExtractorId, extractErrorMessage } from '@/tools/error-extractors' import { @@ -484,7 +485,7 @@ export async function executeQuickBooksFullUpdate< const recordId = requiredQuickBooksString(options.recordId, 'recordId') const syncToken = requiredQuickBooksString(options.syncToken, 'syncToken') const patch = options.buildPatch(options.params) - const readResponse = await fetch( + const readResponse = await outboundFetch( buildQuickBooksEntityUrl(options.params, options.resource, recordId), { method: 'GET', @@ -525,7 +526,7 @@ export async function executeQuickBooksFullUpdate< * references the item. */ if (options.entity === 'Item') updateUrl.searchParams.set('include', 'donotupdateaccountontxns') - const updateResponse = await fetch(updateUrl, { + const updateResponse = await outboundFetch(updateUrl, { method: 'POST', headers: getQuickBooksToolHeaders(options.params.accessToken, 'application/json'), body: JSON.stringify(fullBody), diff --git a/apps/sim/tools/sharepoint/read_page.ts b/apps/sim/tools/sharepoint/read_page.ts index cd95ae96bcd..8eaa7acb28a 100644 --- a/apps/sim/tools/sharepoint/read_page.ts +++ b/apps/sim/tools/sharepoint/read_page.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { outboundFetch } from '@/lib/core/network/outbound-fetch.server' import type { GraphApiPageFields, GraphApiResponse, @@ -246,7 +247,7 @@ export const readPageTool: ToolConfig = { max_results: '100', // Max allowed }) - const repliesResponse = await fetch( + const repliesResponse = await outboundFetch( `https://api.twitter.com/2/tweets/search/recent?${searchParams.toString()}`, { method: 'GET', diff --git a/package.json b/package.json index 9bf33d89b40..5bfee8991b3 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "lint:all": "bun run lint && bun run lint:helm", "check": "bun run format:check", "check:egress-boundary": "bun run scripts/check-egress-boundary.ts", + "check:organization-egress": "bun run scripts/check-organization-egress.ts", "check:boundaries": "bun run scripts/check-monorepo-boundaries.ts", "check:api-validation": "bun run scripts/check-api-validation-contracts.ts --check", "check:api-contract-routes": "bun run scripts/check-api-contract-routes.ts", diff --git a/packages/testing/src/mocks/webhooks-polling-utils.mock.ts b/packages/testing/src/mocks/webhooks-polling-utils.mock.ts index 3e77f839fec..7b3184f53c9 100644 --- a/packages/testing/src/mocks/webhooks-polling-utils.mock.ts +++ b/packages/testing/src/mocks/webhooks-polling-utils.mock.ts @@ -14,6 +14,23 @@ class PollAdmissionRefusedError extends Error { } } +/** + * Faithful copy of the production egress-outage test (`asOrchestrationError(error)?.code === + * 'unavailable'`): an `OutboundRoutingError`, or an `OrchestrationError` coded `unavailable`, + * anywhere in the cause chain. + */ +function isSimOutage(error: unknown): boolean { + let current: unknown = error + while (current instanceof Error) { + if (current.name === 'OutboundRoutingError') return true + if (current.name === 'OrchestrationError') { + return (current as Error & { code?: string }).code === 'unavailable' + } + current = current.cause + } + return false +} + /** Faithful copy of the production `PollFetchError`. */ class PollFetchError extends Error { readonly status: number @@ -32,8 +49,8 @@ class PollFetchError extends Error { * * State writes (`markWebhookFailed`, `markWebhookSuccess`, `updateWebhookProviderConfig`, * `recordPollSourceFailure`) resolve `undefined`; `fetchActiveWebhooks` resolves `[]`; - * `resolveOAuthCredential` is a bare `vi.fn()`. `throwIfAdmissionRefused` and - * `skipAdmissionRefusedPoll` keep production behavior so a poller's refusal path runs as it + * `resolveOAuthCredential` is a bare `vi.fn()`. `throwIfAdmissionRefused`, + * `isUncountedPollFailure`, and `skipUncountedPollFailure` keep production behavior so a poller's refusal path runs as it * does in production. Defaults are `vi.fn(impl)`, so `mockReset()` restores them. * * @example @@ -52,7 +69,12 @@ export const webhooksPollingUtilsMockFns = { } } ), - mockSkipAdmissionRefusedPoll: vi.fn((..._args: unknown[]): 'skipped' => 'skipped'), + mockIsUncountedPollFailure: vi.fn( + (error: unknown): boolean => error instanceof PollAdmissionRefusedError || isSimOutage(error) + ), + mockSkipUncountedPollFailure: vi.fn((error: unknown, ..._args: unknown[]): 'skipped' | null => + error instanceof PollAdmissionRefusedError || isSimOutage(error) ? 'skipped' : null + ), mockReadPollRetryAfterMs: vi.fn((_header: string | null, _body: string): number | null => null), mockClearPollBackoff: vi.fn((_providerConfig: unknown): Record => ({})), mockRecordPollSourceFailure: vi.fn(async (..._args: unknown[]): Promise => {}), @@ -79,7 +101,8 @@ export const webhooksPollingUtilsMock = { PollFetchError, isPollBackedOff: webhooksPollingUtilsMockFns.mockIsPollBackedOff, throwIfAdmissionRefused: webhooksPollingUtilsMockFns.mockThrowIfAdmissionRefused, - skipAdmissionRefusedPoll: webhooksPollingUtilsMockFns.mockSkipAdmissionRefusedPoll, + isUncountedPollFailure: webhooksPollingUtilsMockFns.mockIsUncountedPollFailure, + skipUncountedPollFailure: webhooksPollingUtilsMockFns.mockSkipUncountedPollFailure, readPollRetryAfterMs: webhooksPollingUtilsMockFns.mockReadPollRetryAfterMs, clearPollBackoff: webhooksPollingUtilsMockFns.mockClearPollBackoff, recordPollSourceFailure: webhooksPollingUtilsMockFns.mockRecordPollSourceFailure, diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 438f2e1f027..6f49965e3a1 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -21,7 +21,6 @@ "apps/sim/app/api/schedules/route.test.ts": 1, "apps/sim/app/api/v1/logs/[id]/route.ts": 2, "apps/sim/app/api/v1/logs/route.ts": 3, - "apps/sim/app/api/wand/route.ts": 8, "apps/sim/app/api/webhooks/[id]/route.ts": 1, "apps/sim/app/api/webhooks/route.ts": 7, "apps/sim/app/api/workflows/[id]/execute/route.ts": 11, @@ -1132,7 +1131,6 @@ "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/pull/route.ts": 1, "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/push/route.ts": 1, "apps/sim/app/api/v2/workspaces/[workspaceId]/fork/route.ts": 1, - "apps/sim/app/api/wand/route.ts": 2, "apps/sim/app/api/workflows/[id]/execute/route.test.ts": 5, "apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.test.ts": 1, "apps/sim/app/credential-groups/enroll/[token]/page.test.tsx": 1, diff --git a/scripts/check-organization-egress.test.ts b/scripts/check-organization-egress.test.ts new file mode 100644 index 00000000000..2a54a952936 --- /dev/null +++ b/scripts/check-organization-egress.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest' +import { findDirectFetches } from './check-organization-egress' + +const lines = (source: string) => findDirectFetches('file.ts', source).map((hit) => hit.line) + +describe('organization egress scan', () => { + it('reports the global fetch and globalThis.fetch', () => { + expect( + lines(`const a = await fetch(url) +const b = await globalThis.fetch(url)`) + ).toEqual([1, 2]) + }) + + it('reports an alias, a fallback, a shorthand property and another global object', () => { + expect( + lines(`const f = fetch +const g = custom ?? fetch +new OpenAI({ fetch }) +function run(transport = fetch) {} +await window.fetch(url) +await self['fetch'](url)`) + ).toEqual([1, 2, 3, 4, 5, 6]) + }) + + it('ignores fetch in type positions and as a property key', () => { + expect( + lines(`type Fetch = typeof fetch +const transport: typeof globalThis.fetch = outboundFetch +new Anthropic({ fetch: outboundFetch })`) + ).toEqual([]) + }) + + it('ignores a client method or a local binding named fetch', () => { + expect( + lines(`class Client { async fetch(path: string) { return this.fetch(path) } } +await client.fetch(url) +async function run(fetch: typeof globalThis.fetch) { return fetch(url) }`) + ).toEqual([]) + }) + + it('reports a fetch imported from anywhere but the outbound router, at its import', () => { + expect( + lines(`import { fetch } from 'undici' +await fetch(url) +import nodeFetch, { fetch as undiciFetch } from 'undici' +import fetch from 'node-fetch'`) + ).toEqual([1, 3, 4]) + }) + + it('accepts outboundFetch imported under the name fetch', () => { + expect( + lines(`import { outboundFetch as fetch } from '@/lib/core/network/outbound-fetch.server' +await fetch(url)`) + ).toEqual([]) + }) + + it('accepts an egress-direct annotation within two lines above the call', () => { + expect( + lines(`// egress-direct: operator-local Ollama server +const a = await fetch(url, { +}) +// egress-direct: public catalog + +const b = await fetch(url) + +const c = 1 +const d = await fetch(url)`) + ).toEqual([9]) + }) + + it('ignores fetch named in strings and comments', () => { + expect(lines(`const description = 'fetch() is available'\n// call fetch(url) here`)).toEqual([]) + }) +}) diff --git a/scripts/check-organization-egress.ts b/scripts/check-organization-egress.ts new file mode 100644 index 00000000000..8b543076bc4 --- /dev/null +++ b/scripts/check-organization-egress.ts @@ -0,0 +1,273 @@ +#!/usr/bin/env bun +/** + * Keeps requests made on a workspace's behalf on the organization's egress route. + * + * An organization with a dedicated egress gateway gets static IPs that its vendors and its own + * tenants allowlist. That only holds if Sim's request reaches the network through + * `outboundFetch` (`apps/sim/lib/core/network/outbound-fetch.server.ts`), or one of the + * SSRF-guarded helpers, which pick the gateway from the current outbound scope. The global + * `fetch` never does: a token mint, OAuth refresh or tool call written with it leaves from Sim's + * shared IPs, and a customer who locked their endpoint to their own IPs sees a 403 that no test + * catches, because the code works everywhere else. + * + * So inside the directories whose code acts for a workspace (credentials, OAuth, integration + * clients, tools, webhooks, providers and connectors), a call to the global `fetch` fails this + * check. A call that rightly goes direct (an operator-local server such as Ollama, or a public + * catalog read with no workspace behind it) takes `// egress-direct: ` on one of the two + * lines above it. A method named `fetch` on a client class is not the global and is ignored. A + * `fetch` imported from another module (undici, node-fetch) bypasses the gateway just the same and + * is reported at its import; only `outboundFetch` may be imported under that name. + * + * Parsed with the TypeScript AST, so `fetch` inside strings, comments and tool descriptions is + * never reported. The tree was swept when this landed, so there is no baseline. + * + * Usage: bun run check:organization-egress + */ +import type { Dirent } from 'node:fs' +import { readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import ts from '@typescript/typescript6' + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') + +/** Code under these directories makes requests for a workspace or organization. */ +const SCAN_DIRS = [ + 'apps/sim/connectors', + 'apps/sim/providers', + 'apps/sim/tools', + 'apps/sim/lib/auth/connectors', + 'apps/sim/lib/credential-groups', + 'apps/sim/lib/credentials', + 'apps/sim/lib/data-drains', + 'apps/sim/lib/embeddings', + 'apps/sim/lib/internal', + 'apps/sim/lib/knowledge', + 'apps/sim/lib/managed-agents', + 'apps/sim/lib/media', + 'apps/sim/lib/oauth', + 'apps/sim/lib/provider-models', + 'apps/sim/lib/selectors', + 'apps/sim/lib/wand', + 'apps/sim/lib/webhooks', +] + +const SKIP_DIRS = new Set([ + 'node_modules', + 'dist', + '.next', + '.turbo', + 'coverage', + 'client', + '__fixtures__', +]) +const ANNOTATION = '// egress-direct:' +const ANNOTATION_REACH = 2 +/** The only module a binding of `fetch` may be imported from: it is the router itself. */ +const ROUTED_FETCH_MODULE = '@/lib/core/network/outbound-fetch.server' + +export interface DirectFetch { + line: number + text: string +} + +function walk(dir: string, out: string[] = []): string[] { + let entries: Dirent[] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + throw new Error(`check-organization-egress: SCAN_DIRS entry "${dir}" does not exist`) + } + for (const entry of entries) { + if (SKIP_DIRS.has(entry.name)) continue + const full = path.join(dir, entry.name) + if (entry.isDirectory()) walk(full, out) + else if (/\.ts$/.test(entry.name) && !/\.(test|integration|live\.test)\.ts$/.test(entry.name)) + out.push(full) + } + return out +} + +function bindsName(name: ts.BindingName, target: string): boolean { + if (ts.isIdentifier(name)) return name.text === target + return name.elements.some( + (element) => !ts.isOmittedExpression(element) && bindsName(element.name, target) + ) +} + +/** Whether this scope node itself declares a binding called `target`. */ +function declares(scope: ts.Node, target: string): boolean { + if (ts.isFunctionLike(scope)) { + if (scope.parameters.some((parameter) => bindsName(parameter.name, target))) return true + } + const statements = + ts.isSourceFile(scope) || ts.isBlock(scope) || ts.isModuleBlock(scope) ? scope.statements : [] + return statements.some((statement) => { + if (ts.isVariableStatement(statement)) { + return statement.declarationList.declarations.some((d) => bindsName(d.name, target)) + } + if (ts.isFunctionDeclaration(statement) || ts.isClassDeclaration(statement)) { + return statement.name?.text === target + } + if (ts.isImportDeclaration(statement)) { + const clause = statement.importClause + if (!clause) return false + if (clause.name?.text === target) return true + const bindings = clause.namedBindings + if (!bindings) return false + if (ts.isNamespaceImport(bindings)) return bindings.name.text === target + return bindings.elements.some((element) => element.name.text === target) + } + return false + }) +} + +/** Whether `name` here refers to a local declaration rather than the global. */ +function shadowsGlobal(node: ts.Node, name: string): boolean { + for (let scope: ts.Node | undefined = node.parent; scope; scope = scope.parent) { + if (declares(scope, name)) return true + } + return false +} + +const GLOBAL_OBJECTS = new Set(['globalThis', 'window', 'self', 'global']) + +/** + * Whether this `fetch` identifier is a value reference to the global: a call, but also an alias + * (`const f = fetch`), a fallback (`custom ?? fetch`), a shorthand property (`{ fetch }`) or a + * default parameter. Names in declarations, property keys, member names and type positions are + * not references. + */ +function refersToGlobalFetch(node: ts.Identifier): boolean { + const parent = node.parent + if (ts.isPropertyAccessExpression(parent) && parent.name === node) return false + if ( + (ts.isPropertyAssignment(parent) || + ts.isMethodDeclaration(parent) || + ts.isPropertyDeclaration(parent) || + ts.isPropertySignature(parent) || + ts.isMethodSignature(parent) || + ts.isGetAccessorDeclaration(parent) || + ts.isSetAccessorDeclaration(parent) || + ts.isParameter(parent) || + ts.isVariableDeclaration(parent) || + ts.isBindingElement(parent) || + ts.isFunctionDeclaration(parent) || + ts.isImportSpecifier(parent) || + ts.isExportSpecifier(parent) || + ts.isImportClause(parent)) && + parent.name === node + ) { + return false + } + if (ts.isQualifiedName(parent) && parent.right === node) return false + if (ts.isBindingElement(parent) && parent.propertyName === node) return false + if (ts.isImportSpecifier(parent) || ts.isExportSpecifier(parent)) return false + if (ts.isTypeQueryNode(parent) || ts.isTypeReferenceNode(parent)) return false + return !shadowsGlobal(node, 'fetch') +} + +/** + * Import bindings of a `fetch` from anywhere but {@link ROUTED_FETCH_MODULE}, such as undici's + * or node-fetch's: each is the global's bypass under another name, whatever it is called locally. + * Uses of the binding are then local and are not reported again. + */ +function importedFetchBindings(declaration: ts.ImportDeclaration): ts.Node[] { + const clause = declaration.importClause + if (!clause || clause.isTypeOnly) return [] + if ( + ts.isStringLiteral(declaration.moduleSpecifier) && + declaration.moduleSpecifier.text === ROUTED_FETCH_MODULE + ) { + return [] + } + const bindings: ts.Node[] = [] + if (clause.name?.text === 'fetch') bindings.push(clause.name) + const named = clause.namedBindings + if (named && ts.isNamespaceImport(named) && named.name.text === 'fetch') bindings.push(named) + if (named && ts.isNamedImports(named)) { + for (const element of named.elements) { + if (element.isTypeOnly) continue + const imported = element.propertyName ?? element.name + if (imported.text === 'fetch' || element.name.text === 'fetch') bindings.push(element) + } + } + return bindings +} + +/** + * Every reference to the global `fetch` (a call, an alias, `globalThis.fetch`, `window['fetch']`) + * without an `egress-direct` annotation on one of the two lines above it. + */ +export function findDirectFetches(file: string, source: string): DirectFetch[] { + if (!source.includes('fetch')) return [] + const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true) + const lines = source.split('\n') + const found: DirectFetch[] = [] + + const report = (node: ts.Node) => { + const line = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line + const annotated = lines + .slice(Math.max(0, line - ANNOTATION_REACH), line) + .some((text) => text.trim().startsWith(ANNOTATION)) + if (!annotated) found.push({ line: line + 1, text: lines[line].trim() }) + } + + const visit = (node: ts.Node): void => { + if (ts.isImportDeclaration(node)) { + for (const binding of importedFetchBindings(node)) report(binding) + } else if (ts.isIdentifier(node) && node.text === 'fetch' && refersToGlobalFetch(node)) { + report(node) + } else if ( + (ts.isPropertyAccessExpression(node) && node.name.text === 'fetch') || + (ts.isElementAccessExpression(node) && + ts.isStringLiteralLike(node.argumentExpression) && + node.argumentExpression.text === 'fetch') + ) { + if (ts.isIdentifier(node.expression) && GLOBAL_OBJECTS.has(node.expression.text)) { + report(node) + } + } + ts.forEachChild(node, visit) + } + + visit(sourceFile) + return found +} + +function main() { + const violations: Array = [] + let scanned = 0 + for (const scanDir of SCAN_DIRS) { + for (const file of walk(path.join(ROOT, scanDir))) { + scanned++ + const rel = path.relative(ROOT, file).split(path.sep).join('/') + for (const hit of findDirectFetches(rel, readFileSync(file, 'utf8'))) { + violations.push({ file: rel, ...hit }) + } + } + } + + if (violations.length === 0) { + console.log( + `✓ check-organization-egress: ${scanned} files, every request is organization-routed` + ) + process.exit(0) + } + + console.error('✗ check-organization-egress: global fetch in code that acts for a workspace\n') + for (const violation of violations) { + console.error(` ${violation.file}:${violation.line}`) + console.error(` ${violation.text}`) + } + console.error( + "\n The global fetch always leaves from Sim's shared IPs, so an organization with a\n" + + ' dedicated egress gateway loses its static IPs for this request. Use outboundFetch from\n' + + ' @/lib/core/network/outbound-fetch.server (same signature as fetch). If the destination\n' + + " is genuinely not on the organization's behalf (an operator-local server, a public\n" + + ' catalog), mark the call with `// egress-direct: ` on the line above it.\n' + ) + process.exit(1) +} + +if (import.meta.main) main() diff --git a/scripts/check-source-conventions.baseline.json b/scripts/check-source-conventions.baseline.json index 138033a0538..0129b4138ed 100644 --- a/scripts/check-source-conventions.baseline.json +++ b/scripts/check-source-conventions.baseline.json @@ -646,7 +646,6 @@ "apps/sim/app/api/v1/workflows/[id]/route.ts": 2, "apps/sim/app/api/v1/workflows/route.ts": 2, "apps/sim/app/api/v2/workflows/[workflowId]/execute/route.ts": 2, - "apps/sim/app/api/wand/route.ts": 2, "apps/sim/app/api/webhooks/[id]/route.ts": 2, "apps/sim/app/api/webhooks/agentmail/route.ts": 1, "apps/sim/app/api/webhooks/route.ts": 2, diff --git a/scripts/check-test-types.baseline.json b/scripts/check-test-types.baseline.json index ad34b315e79..36e295e7ee2 100644 --- a/scripts/check-test-types.baseline.json +++ b/scripts/check-test-types.baseline.json @@ -19,12 +19,6 @@ "TS2345": 2, "TS2554": 5 }, - "apps/sim/app/api/auth/oauth/token/route.test.ts": { - "TS2322": 1, - "TS2345": 1, - "TS2554": 11, - "TS2739": 2 - }, "apps/sim/app/api/auth/oauth2/authorize/route.test.ts": { "TS2345": 2 }, @@ -1172,9 +1166,6 @@ "apps/sim/lib/credential-groups/slack-managed-users.test.ts": { "TS2339": 1 }, - "apps/sim/lib/credential-groups/slack-provider.test.ts": { - "TS2741": 2 - }, "apps/sim/lib/credential-groups/standard-oauth-provider.test.ts": { "TS2739": 1, "TS2741": 1 diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index 1199b592d45..1d35047b0bf 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -1680,10 +1680,6 @@ "apps/sim/lib/api/contracts/oauth-connections.ts#disconnectOAuthBodySchema", "apps/sim/lib/api/contracts/oauth-connections.ts#oauthAccountSummarySchema", "apps/sim/lib/api/contracts/oauth-connections.ts#oauthConnectionSchema", - "apps/sim/lib/api/contracts/oauth-connections.ts#oauthTokenGetQuerySchema", - "apps/sim/lib/api/contracts/oauth-connections.ts#oauthTokenPostHeadersSchema", - "apps/sim/lib/api/contracts/oauth-connections.ts#oauthTokenPostQuerySchema", - "apps/sim/lib/api/contracts/oauth-connections.ts#oauthTokenRequestBodySchema", "apps/sim/lib/api/contracts/oauth-connections.ts#quickBooksCallbackQuerySchema", "apps/sim/lib/api/contracts/oauth-connections.ts#trelloTokenBodySchema", "apps/sim/lib/api/contracts/oauth-provider.ts#RegisterOAuthClientBody",