improvement(egress): keep org egress outages from disabling connectors or failing degradable work

- slack native formatInput: an outage resolving the bot token degrades the event (no token) instead of failing it
- connector content and member sync: an outage retries at the first backoff rung without touching the auto-disable breaker
- selector credential bundle: rethrow an outage instead of reporting the connection unavailable
- internal LLM Vertex credential: an outage returns 503 with the fixed egress message
- webhook provider subscriptions: projected failures keep the unavailable classification; the webhook save route and deploy report it as 503
- gmail/outlook configurePolling: rethrow an outage instead of the permissions rollback message

Author: waleedlatif1

apps/sim/app/api/webhooks/route.test.ts

@@ -50,6 +50,8 @@ vi.mock('@/lib/webhooks/utils.server', () => ({
   findConflictingWebhookPathOwner: mocks.findConflictingWebhookPathOwner,
 }))
 
+import { OutboundRoutingError } from '@/lib/core/network/routing'
+import { EGRESS_UNAVAILABLE_MESSAGE, OrchestrationError } from '@/lib/core/orchestration/types'
 import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields'
 import { POST } from '@/app/api/webhooks/route'
 
@@ -619,3 +621,34 @@ describe('POST /api/webhooks subscription replacement recovery', () => {
     }
   )
 })
+
+describe('POST /api/webhooks egress outage', () => {
+  beforeEach(setupUpsertMocks)
+
+  it('reports an outage while creating the external subscription as unavailable', async () => {
+    queueCreatePathRows()
+    mocks.createExternalWebhookSubscription.mockRejectedValue(
+      new OrchestrationError('unavailable', EGRESS_UNAVAILABLE_MESSAGE)
+    )
+
+    const response = await POST(upsertRequest(), {})
+
+    expect(response.status).toBe(503)
+    expect(await response.json()).toEqual({ error: EGRESS_UNAVAILABLE_MESSAGE })
+  })
+
+  it('reports an outage while configuring polling as unavailable', async () => {
+    queueCreatePathRows()
+    dbChainMockFns.returning.mockImplementationOnce(async () => [
+      { id: 'webhook-1', workflowId: 'workflow-1', path: 'inbound-orders', isActive: true },
+    ])
+    mocks.activateExternalWebhookSubscription.mockResolvedValue(undefined)
+    mocks.configurePolling.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE'))
+    mocks.getProviderHandler.mockReturnValue({ configurePolling: mocks.configurePolling })
+
+    const response = await POST(upsertRequest(), {})
+
+    expect(response.status).toBe(503)
+    expect(await response.json()).toEqual({ error: EGRESS_UNAVAILABLE_MESSAGE })
+  })
+})

apps/sim/app/api/webhooks/route.ts

@@ -17,6 +17,11 @@ import { parseRequest } from '@/lib/api/server'
 import { getSession } from '@/lib/auth'
 import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access'
 import { AuthType } from '@/lib/auth/hybrid'
+import {
+  EGRESS_UNAVAILABLE_MESSAGE,
+  isUnavailableFailure,
+  statusForOrchestrationError,
+} from '@/lib/core/orchestration/types'
 import { PlatformEvents } from '@/lib/core/telemetry'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
@@ -569,6 +574,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
           }
         }
       } catch (err) {
+        if (isUnavailableFailure(err)) {
+          logger.warn('External webhook subscription unavailable during an egress outage')
+          return NextResponse.json(
+            { error: EGRESS_UNAVAILABLE_MESSAGE },
+            { status: statusForOrchestrationError('unavailable') }
+          )
+        }
         const isConfigurationError = err instanceof WebhookDeploymentConfigurationError
         if (isConfigurationError) {
           logger.warn('Rejected webhook trigger configuration', {
@@ -746,6 +758,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
             err
           )
           await revertSavedWebhook(savedWebhook, existingWebhook, requestId)
+          if (isUnavailableFailure(err)) {
+            return NextResponse.json(
+              { error: EGRESS_UNAVAILABLE_MESSAGE },
+              { status: statusForOrchestrationError('unavailable') }
+            )
+          }
           return NextResponse.json(
             {
               error: `Failed to configure ${provider} webhook`,

apps/sim/lib/internal/llm/operations.test.ts

@@ -14,6 +14,8 @@ import { permissionCheckMock } from '@sim/testing/mocks/permission-check.mock'
 import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock'
 import { providersMock, providersMockFns } from '@sim/testing/mocks/providers.mock'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { OutboundRoutingError } from '@/lib/core/network/routing'
+import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types'
 
 const mocks = vi.hoisted(() => ({
   authorizeCredential: vi.fn(),
@@ -132,6 +134,26 @@ describe('executeLlmProviderOperation', () => {
     )
   })
 
+  it('reports a Vertex credential the egress outage kept from resolving as unavailable', async () => {
+    mockExecuteProviderRequest.mockClear()
+    mocks.resolveVertexAccessToken.mockRejectedValueOnce(
+      new OutboundRoutingError('GATEWAY_UNAVAILABLE')
+    )
+
+    await expect(
+      executeLlmProviderOperation(
+        {
+          provider: 'vertex',
+          model: 'vertex/gemini-2.5-pro',
+          vertexCredential: 'credential-1',
+          workspaceId: 'workspace-1',
+        },
+        { actorUserId: 'user-1', headers: new Headers(), requestId: 'request-1' }
+      )
+    ).rejects.toMatchObject({ status: 503, body: { error: EGRESS_UNAVAILABLE_MESSAGE } })
+    expect(mockExecuteProviderRequest).not.toHaveBeenCalled()
+  })
+
   it("signs a Bedrock call with the selected AWS credential's session for the caller", async () => {
     mocks.authorizeCredential.mockResolvedValueOnce(AUTHORIZED_AWS_CREDENTIAL)
     mockResolveAuthorizedAwsCredential.mockResolvedValueOnce({

apps/sim/lib/internal/llm/operations.ts

@@ -8,6 +8,11 @@ import {
   type BillingAttributionSnapshot,
   requireBillingAttributionHeader,
 } from '@/lib/billing/core/attribution'
+import {
+  EGRESS_UNAVAILABLE_MESSAGE,
+  isUnavailableFailure,
+  statusForOrchestrationError,
+} from '@/lib/core/orchestration/types'
 import {
   inspectModelInputProjectionState,
   inspectModelInputProvenanceRequest,
@@ -158,6 +163,13 @@ async function prepareProviderRequest(
     if (error instanceof BedrockCredentialError && error.reason === 'unavailable') {
       fail(503, error.message)
     }
+    if (isUnavailableFailure(error)) {
+      logger.warn('Provider credential unavailable during an egress outage', {
+        requestId: context.requestId,
+        provider: input.provider,
+      })
+      fail(statusForOrchestrationError('unavailable'), EGRESS_UNAVAILABLE_MESSAGE)
+    }
     logger.error('Failed to resolve provider credential', {
       requestId: context.requestId,
       provider: input.provider,

apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts

@@ -4,6 +4,8 @@ import { knowledgeDocumentsServiceMock } from '@sim/testing/mocks/knowledge-docu
 import { knowledgeMemberAccessMock } from '@sim/testing/mocks/knowledge-member-access.mock'
 import { DrizzleQueryError } from 'drizzle-orm/errors'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { OutboundRoutingError } from '@/lib/core/network/routing'
+import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types'
 
 vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} }))
 vi.mock('@/lib/knowledge/documents/service', () => knowledgeDocumentsServiceMock)
@@ -273,6 +275,19 @@ describe('member sync engine decisions', () => {
       })
       expect(update.nextMemberSyncAt).not.toBeNull()
     })
+
+    it('does not disable a connector one failure from the breaker over an egress outage', async () => {
+      const update = await resolveMemberSyncFailureUpdate(
+        new Error('listing failed', { cause: new OutboundRoutingError('GATEWAY_UNAVAILABLE') }),
+        failure
+      )
+      expect(update).toMatchObject({
+        memberSyncStatus: 'error',
+        lastMemberSyncError: EGRESS_UNAVAILABLE_MESSAGE,
+        memberSyncConsecutiveFailures: MAX_CONSECUTIVE_FAILURES - 1,
+      })
+      expect(update.nextMemberSyncAt).not.toBeNull()
+    })
   })
 
   describe('nextMemberSyncTime', () => {

apps/sim/lib/knowledge/connectors/member-sync-engine.ts

@@ -28,6 +28,7 @@ import {
   type BillingAttributionSnapshot,
 } from '@/lib/billing/core/attribution'
 import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server'
+import { EGRESS_UNAVAILABLE_MESSAGE, isUnavailableFailure } from '@/lib/core/orchestration/types'
 import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope'
 import { resourceScopeCondition } from '@/lib/core/resource-scope.server'
 import {
@@ -382,8 +383,8 @@ export function memberRunMadeProgress(result: MemberSyncResult): boolean {
 
 /**
  * The connector row a failed members-mode run writes. A deterministic capacity rejection waits for
- * an operator, a transient database failure retries without touching the breaker, and anything
- * else climbs the ladder toward auto-disable.
+ * an operator, a transient database failure or an organization egress outage retries without
+ * touching the breaker, and anything else climbs the ladder toward auto-disable.
  */
 export async function resolveMemberSyncFailureUpdate(
   error: unknown,
@@ -423,6 +424,14 @@ export async function resolveMemberSyncFailureUpdate(
       })
     )
   }
+  if (isUnavailableFailure(error)) {
+    return buildMemberSyncDatabaseRetryUpdate(
+      now,
+      failure.previousFailures,
+      EGRESS_UNAVAILABLE_MESSAGE,
+      connectorFailureBackoffMinutes(1) * 60 * 1000
+    )
+  }
   return buildMemberSyncFailureUpdate(
     now,
     failure.previousFailures,

apps/sim/lib/knowledge/connectors/sync-engine.test.ts

@@ -25,6 +25,8 @@ import {
 import { generateShortId } from '@sim/utils/id'
 import { DrizzleQueryError } from 'drizzle-orm/errors'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { OutboundRoutingError } from '@/lib/core/network/routing'
+import { EGRESS_UNAVAILABLE_MESSAGE } from '@/lib/core/orchestration/types'
 import * as connectorTokens from '@/lib/knowledge/connectors/access-token'
 import {
   routeWindowTallies,
@@ -820,6 +822,19 @@ describe('executeSync database failures', () => {
       expect.objectContaining({ status: 'failed', databaseFailureClass: 'capacity' })
     )
   })
+
+  it('does not disable a connector one failure from the breaker over an egress outage', async () => {
+    const { terminal, MAX_CONSECUTIVE_FAILURES } = await failSyncWith(
+      new OutboundRoutingError('GATEWAY_UNAVAILABLE')
+    )
+
+    expect(terminal).toMatchObject({
+      status: 'error',
+      lastSyncError: EGRESS_UNAVAILABLE_MESSAGE,
+      consecutiveFailures: MAX_CONSECUTIVE_FAILURES - 1,
+    })
+    expect((terminal?.nextSyncAt as Date).getTime()).toBeGreaterThan(NOW.getTime())
+  })
 })
 
 describe('previous complete listing evidence', () => {

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -20,6 +20,7 @@ import {
   type BillingAttributionSnapshot,
 } from '@/lib/billing/core/attribution'
 import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server'
+import { EGRESS_UNAVAILABLE_MESSAGE, isUnavailableFailure } from '@/lib/core/orchestration/types'
 import {
   type ResourceScope,
   resourceScopeFields,
@@ -1645,19 +1646,26 @@ export async function executeSync(
                     madeProgress: result.docsAdded + result.docsUpdated + result.docsDeleted > 0,
                   })
                 )
-              : rateLimited
-                ? buildSyncRateLimitUpdate(
+              : isUnavailableFailure(error)
+                ? buildSyncDatabaseRetryUpdate(
                     new Date(),
                     connector.consecutiveFailures,
-                    errorMessage,
-                    retryAfterMs
-                  )
-                : buildSyncFailureUpdate(
-                    new Date(),
-                    connector.consecutiveFailures,
-                    errorMessage,
-                    retryAfterMs
+                    EGRESS_UNAVAILABLE_MESSAGE,
+                    connectorFailureBackoffMinutes(1) * 60 * 1000
                   )
+                : rateLimited
+                  ? buildSyncRateLimitUpdate(
+                      new Date(),
+                      connector.consecutiveFailures,
+                      errorMessage,
+                      retryAfterMs
+                    )
+                  : buildSyncFailureUpdate(
+                      new Date(),
+                      connector.consecutiveFailures,
+                      errorMessage,
+                      retryAfterMs
+                    )
 
         if (failureUpdate.status === 'disabled') {
           logger.warn('Connector disabled after repeated failures', {

apps/sim/lib/selectors/server/providers/credential-bundle.test.ts

@@ -1,5 +1,6 @@
 import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock'
 import { describe, expect, it, vi } from 'vitest'
+import { OutboundRoutingError } from '@/lib/core/network/routing'
 
 const mockResolveOrganizationToken = vi.hoisted(() => vi.fn())
 vi.mock('@/lib/credentials/application/organization-credentials', () => ({
@@ -36,6 +37,21 @@ describe('selector credential bundles', () => {
     expect(protectedValues.contains('prefix-cloud-1-suffix')).toBe(false)
   })
 
+  it('surfaces an egress outage instead of reporting the connection unavailable', async () => {
+    const outage = new OutboundRoutingError('GATEWAY_UNAVAILABLE')
+    mockResolveCredentialAccessToken.mockRejectedValueOnce(outage)
+
+    await expect(
+      resolveSelectorCredentialBundle({
+        credential: {
+          suppliedId: 'credential-1',
+          access: { ok: true, credentialOwnerUserId: 'owner-1' },
+        },
+        protectedValues: createSelectorProtectedValues(),
+      })
+    ).rejects.toBe(outage)
+  })
+
   it('preserves a selector abort without canceling the shared resolution', async () => {
     let resolveShared!: (value: { accessToken: string }) => void
     const sharedResolution = new Promise<{ accessToken: string }>((resolve) => {

apps/sim/lib/selectors/server/providers/credential-bundle.ts

@@ -1,3 +1,4 @@
+import { isUnavailableFailure } from '@/lib/core/orchestration/types'
 import { resolveOrganizationCredentialTokenBundle } from '@/lib/credentials/application/organization-credentials'
 import {
   resolveCredentialTokenBundle,
@@ -61,7 +62,7 @@ export async function resolveSelectorCredentialBundle(input: {
     )
     credential.signal?.throwIfAborted()
   } catch (error) {
-    if (credential.signal?.aborted) throw error
+    if (credential.signal?.aborted || isUnavailableFailure(error)) throw error
     throw new SelectorConnectionUnavailableError()
   }
   if (!bundle?.accessToken) throw new SelectorConnectionUnavailableError()