diff --git a/apps/docs/content/docs/cli/knowledge.mdx b/apps/docs/content/docs/cli/knowledge.mdx index b08a06d5396..1a03bb030e4 100644 --- a/apps/docs/content/docs/cli/knowledge.mdx +++ b/apps/docs/content/docs/cli/knowledge.mdx @@ -1040,7 +1040,7 @@ sim knowledge connectors create [options] | Option | Required | Description | | --- | --- | --- | | `--connector-type ` | Yes | Registered connector type. | -| `--credential-id ` | No | OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types); omit it for `apiKey` connectors. | +| `--credential-id ` | No | OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types), or, when this deployment enables it, a saved credential for an `apiKey` connector type that declares `auth.savedCredential` (such as Amazon S3 with a saved AWS credential); otherwise omit it for `apiKey` connectors. | | `--api-key ` | No | Write-only API key for connector types whose `auth.mode` is `apiKey` (see connector types), or a personal access token for an OAuth connector that also accepts one, such as GitHub. Send it instead of `credentialId`. Pass a raw key, or a secret reference written as the whole value `{{SECRET_NAME}}`, which the server resolves; `$SECRET_NAME` is not a reference. | | `--source-config ` | Yes | Connector-specific source selection and filtering configuration. (JSON, or @path / @- to read a file or stdin). | | `--sync-interval-minutes ` | No | Scheduled synchronization interval in minutes; zero disables scheduling. Defaults to 1440. | diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 0bacce6d88f..579ddc3c433 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -2645,7 +2645,7 @@ sim knowledge connectors create [options] | Option | Required | Description | | --- | --- | --- | | `--connector-type ` | Yes | Registered connector type. | -| `--credential-id ` | No | OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types); omit it for `apiKey` connectors. | +| `--credential-id ` | No | OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types), or, when this deployment enables it, a saved credential for an `apiKey` connector type that declares `auth.savedCredential` (such as Amazon S3 with a saved AWS credential); otherwise omit it for `apiKey` connectors. | | `--api-key ` | No | Write-only API key for connector types whose `auth.mode` is `apiKey` (see connector types), or a personal access token for an OAuth connector that also accepts one, such as GitHub. Send it instead of `credentialId`. Pass a raw key, or a secret reference written as the whole value `{{SECRET_NAME}}`, which the server resolves; `$SECRET_NAME` is not a reference. | | `--source-config ` | Yes | Connector-specific source selection and filtering configuration. (JSON, or @path / @- to read a file or stdin). | | `--sync-interval-minutes ` | No | Scheduled synchronization interval in minutes; zero disables scheduling. Defaults to 1440. | diff --git a/apps/docs/content/docs/knowledgebase/connectors.mdx b/apps/docs/content/docs/knowledgebase/connectors.mdx index 35fcecf501c..8c98e6ced2e 100644 --- a/apps/docs/content/docs/knowledgebase/connectors.mdx +++ b/apps/docs/content/docs/knowledgebase/connectors.mdx @@ -46,7 +46,7 @@ API-key and personal-access-token connections use the credential shown in their | Azure DevOps | PAT with Project and Team Read, Wiki Read, Work Items Read, and Code Read. | | GitLab | PAT with `read_api` and access to the selected project; permission-aware methods have additional requirements. | | YouTube | YouTube Data API key from Google Cloud Console. | -| Amazon S3 | Secret Access Key; enter Access Key ID, region, and bucket as configuration fields. | +| Amazon S3 | Secret Access Key; enter Access Key ID, region, and bucket as configuration fields. Where enabled, **Authentication → Saved AWS credential** uses a workspace AWS credential instead (access keys, an assumed IAM role, or a keyless role trusted through Sim OIDC); it signs only for Amazon S3 in a commercial or GovCloud AWS region (not China or isolated regions), so it has no Access Key ID or custom endpoint. Either way, the identity needs `s3:ListBucket` on the bucket and `s3:GetObject` on the synced objects. | | Sentry | Auth token with `project:read` and `event:read`. | | PagerDuty | Create a [REST API key](https://support.pagerduty.com/main/docs/api-access-keys) under **Integrations → Developer Tools → API Access Keys**. | | SFTP | Password or unencrypted private key, plus a required SHA-256 host-key fingerprint; enter host, port, username, and root path as configuration fields. | diff --git a/apps/docs/openapi-v2-knowledge.json b/apps/docs/openapi-v2-knowledge.json index ae3f904261c..c98a9c09f81 100644 --- a/apps/docs/openapi-v2-knowledge.json +++ b/apps/docs/openapi-v2-knowledge.json @@ -6933,7 +6933,7 @@ "type": "null" } ], - "description": "OAuth credential identifier, or null for API-key and unauthenticated sources." + "description": "OAuth credential identifier, or the saved credential an `apiKey` source syncs with (see `auth.savedCredential`); null for API-key and unauthenticated sources." }, "sourceConfig": { "type": "object", @@ -7155,7 +7155,7 @@ "description": "Registered connector type." }, "credentialId": { - "description": "OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types); omit it for `apiKey` connectors.", + "description": "OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types), or, when this deployment enables it, a saved credential for an `apiKey` connector type that declares `auth.savedCredential` (such as Amazon S3 with a saved AWS credential); otherwise omit it for `apiKey` connectors.", "type": "string", "minLength": 1, "maxLength": 255 @@ -7331,7 +7331,7 @@ "type": "null" } ], - "description": "OAuth credential identifier, or null for API-key and unauthenticated sources." + "description": "OAuth credential identifier, or the saved credential an `apiKey` source syncs with (see `auth.savedCredential`); null for API-key and unauthenticated sources." }, "sourceConfig": { "type": "object", diff --git a/apps/docs/openapi-v2-resources.json b/apps/docs/openapi-v2-resources.json index d1970caf12a..e9f650b62d7 100644 --- a/apps/docs/openapi-v2-resources.json +++ b/apps/docs/openapi-v2-resources.json @@ -21713,7 +21713,7 @@ "mode": { "type": "string", "const": "apiKey", - "description": "Authenticates with a stored API key; pass `apiKey`, and no `credentialId`, when creating the connector." + "description": "Authenticates with a stored API key; pass `apiKey` when creating the connector, or a saved `credentialId` instead when `savedCredential` is present." }, "label": { "description": "Label shown above the key field.", @@ -21726,13 +21726,29 @@ "optional": { "type": "boolean", "description": "Whether the key may be left blank, for a source reachable without authentication." + }, + "savedCredential": { + "description": "When present, a saved credential of this service may be passed as `credentialId` instead of `apiKey`, once saved-credential setup is enabled on this deployment (until then it is refused with a validation error). Fields marked `apiKeyOnly` are then omitted.", + "type": "object", + "properties": { + "provider": { + "type": "string", + "description": "Service the saved credential must authenticate with, e.g. `aws`." + }, + "label": { + "type": "string", + "description": "Label for the saved-credential option." + } + }, + "required": ["provider", "label"], + "additionalProperties": false } }, "required": ["mode", "optional"], "additionalProperties": false } ], - "description": "How the connector authenticates against its source: `oauth` connectors take `credentialId` (GitHub also accepts a personal access token as `apiKey`), `apiKey` connectors take `apiKey`." + "description": "How the connector authenticates against its source: `oauth` connectors take `credentialId` (GitHub also accepts a personal access token as `apiKey`), `apiKey` connectors take `apiKey` (or, when they declare `savedCredential`, a saved `credentialId`)." }, "configFields": { "type": "array", @@ -21882,6 +21898,10 @@ "multi": { "description": "When true the stored `sourceConfig` value is a `string[]`, not a `string`: a `selector` renders a multi-select picker and a `short-input` accepts a comma-separated list.", "type": "boolean" + }, + "apiKeyOnly": { + "description": "When true the field describes the connector’s API key (such as its key ID or a custom host the key signs for). It is required as stated only with `apiKey`, and omitted when a saved `credentialId` is sent.", + "type": "boolean" } }, "required": ["id", "title", "type"], @@ -21910,7 +21930,7 @@ "mode": { "type": "string", "enum": ["oauth", "apiKey"], - "description": "How the connector authenticates against its source: `oauth` connectors take `credentialId` (GitHub also accepts a personal access token as `apiKey`), `apiKey` connectors take `apiKey`." + "description": "How the connector authenticates against its source: `oauth` connectors take `credentialId` (GitHub also accepts a personal access token as `apiKey`), `apiKey` connectors take `apiKey` (or, when they declare `savedCredential`, a saved `credentialId`)." } }, "required": ["mode"], diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index b14b5b9aac3..5e57cb57234 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -106,6 +106,7 @@ import { kbDocumentSortParams, } from '@/app/workspace/[workspaceId]/knowledge/[id]/search-params' import { canDeleteKnowledgeBase } from '@/app/workspace/[workspaceId]/knowledge/permissions' +import { useFeatureFlag } from '@/app/workspace/[workspaceId]/providers/feature-flags-provider' import { useRegisterGlobalCommands } from '@/app/workspace/[workspaceId]/providers/global-commands-provider' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { BrandIcon } from '@/blocks/brand-icon' @@ -420,6 +421,7 @@ export function KnowledgeBase({ const [showDocumentTagsModal, setShowDocumentTagsModal] = useState(false) const [documentForTagsId, setDocumentForTagsId] = useState(null) const showAddConnectorModal = addConnectorType != null + const savedCredentialSetupEnabled = useFeatureFlag('knowledge-connector-saved-credentials') const updateAddConnectorParam = useCallback( (value: string | null) => { void setAddConnectorType(value, { history: 'replace', scroll: false }) @@ -1472,6 +1474,7 @@ export function KnowledgeBase({ knowledgeBaseId={id} initialConnectorType={addConnectorType || undefined} isSearchIndex={knowledgeBase?.isSearchIndex} + savedCredentialSetupEnabled={savedCredentialSetupEnabled} /> )} @@ -1513,6 +1516,7 @@ export function KnowledgeBase({ isLoading={isLoadingConnectors} canEdit={userPermissions.canEdit} className='mt-0' + savedCredentialSetupEnabled={savedCredentialSetupEnabled} /> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index 9cc992c2cf4..2950343d65a 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -63,7 +63,11 @@ import { } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' import { withBrandIcon } from '@/blocks/brand-icon' -import { getConnectorApiKeyConfig, isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { + getConnectorApiKeyConfig, + isConnectorCredentialTypeAllowed, + isConnectorFieldForAuth, +} from '@/connectors/auth' import { GitHubInstallationConnectionField } from '@/connectors/github/installation-connection-field' import { GitLabPermissionTabs, @@ -109,6 +113,8 @@ interface AddConnectorModalProps { initialSyncIntervalMinutes?: number onCreated?: (connectorType: string, connector: ConnectorData) => void setupDraftKey?: string + /** Offers a key connector's saved-credential alternative, such as a saved AWS credential for S3. */ + savedCredentialSetupEnabled?: boolean } type Step = 'select-type' | 'configure' @@ -125,6 +131,7 @@ export function AddConnectorModal({ lockedAccessMode, initialSyncIntervalMinutes = 1440, onCreated, + savedCredentialSetupEnabled = false, setupDraftKey, scope: explicitScope, }: AddConnectorModalProps) { @@ -213,8 +220,14 @@ export function AddConnectorModal({ const showGitLabPermissions = selectedType === 'gitlab' && access.accessMode === 'admin' const isMembersMode = access.accessMode === 'members' const apiKeyConfig = connectorConfig ? getConnectorApiKeyConfig(connectorConfig.auth) : undefined + const savedCredentialConfig = + savedCredentialSetupEnabled && connectorConfig?.auth.mode === 'apiKey' + ? connectorConfig.auth.savedCredential + : undefined const isApiKeyMode = - connectorConfig?.auth.mode === 'apiKey' || Boolean(apiKeyConfig && !isMembersMode && useApiKey) + connectorConfig?.auth.mode === 'apiKey' + ? !savedCredentialConfig || useApiKey + : Boolean(apiKeyConfig && !isMembersMode && useApiKey) const { integrationAvailability, oauthServiceAvailability, @@ -249,10 +262,13 @@ export function AddConnectorModal({ /** True when the connector declares its key optional (public sources need none). */ const isApiKeyOptional = connectorConfig?.auth.mode === 'apiKey' && connectorConfig.auth.optional === true - const connectorProviderId = + const credentialServiceId = connectorConfig?.auth.mode === 'oauth' - ? (getProviderIdFromServiceId(connectorConfig.auth.provider) as OAuthProvider) - : null + ? connectorConfig.auth.provider + : savedCredentialConfig?.provider + const connectorProviderId = credentialServiceId + ? (getProviderIdFromServiceId(credentialServiceId) as OAuthProvider) + : null const serviceAccountProviderId = connectorProviderId ? getServiceAccountProviderForProviderId(connectorProviderId) @@ -263,7 +279,7 @@ export function AddConnectorModal({ !isConnectorCredentialTypeAllowed(connectorConfig.auth, access.accessMode, 'oauth') const serviceAccountTarget = useServiceAccountConnectTarget({ serviceAccountProviderId: - isSearchIndex || requiresServiceAccount + isSearchIndex || requiresServiceAccount || savedCredentialConfig ? asServiceAccountProviderId(serviceAccountProviderId) : undefined, serviceName: connectorConfig?.name, @@ -303,7 +319,7 @@ export function AddConnectorModal({ isConnectorCredentialTypeAllowed(connectorConfig.auth, access.accessMode, credential.type) ) const canConnectOAuth = - connectorConfig && + connectorConfig?.auth.mode === 'oauth' && isConnectorCredentialTypeAllowed(connectorConfig.auth, access.accessMode, 'oauth') const effectiveCredentialId = selectedCredentialId && credentials.some((credential) => credential.id === selectedCredentialId) @@ -367,6 +383,7 @@ export function AddConnectorModal({ ) const isFieldVisible = (field: ConnectorConfigField) => isConfigFieldVisible(field) && + isConnectorFieldForAuth(field, isApiKeyMode) && (connectorConfig?.auth.mode !== 'oauth' || connectorConfig.auth.serviceAccountSubjectFieldId !== field.id || indexingCredential?.type === 'service_account') @@ -541,6 +558,13 @@ export function AddConnectorModal({ ) ) + /** Key-only settings typed before switching to a saved credential are never sent with it. */ + const keyOnlyFieldIds = new Set( + (connectorConfig?.configFields ?? []) + .filter((field) => !isConnectorFieldForAuth(field, isApiKeyMode)) + .map((field) => field.canonicalParamId ?? field.id) + ) + const handleSubmit = () => { if (!selectedType || !canSubmit) return @@ -548,7 +572,7 @@ export function AddConnectorModal({ const resolvedConfig: Record = {} for (const [key, value] of Object.entries(resolveSourceConfig())) { - if (hiddenCapFieldIds.has(key)) continue + if (hiddenCapFieldIds.has(key) || keyOnlyFieldIds.has(key)) continue if (Array.isArray(value)) { if (value.length > 0) resolvedConfig[key] = value } else if (typeof value === 'string') { @@ -714,23 +738,28 @@ export function AddConnectorModal({ {!slackSetupRequired && ( <> - {connectorConfig.auth.mode === 'oauth' && apiKeyConfig && !isMembersMode && ( - - { - setUseApiKey(value === 'apiKey') - setApiKeyValue('') - setSelectedCredentialId(null) - }} - /> - - )} + {apiKeyConfig && + (savedCredentialConfig || + (connectorConfig.auth.mode === 'oauth' && !isMembersMode)) && ( + + { + setUseApiKey(value === 'apiKey') + setApiKeyValue('') + setSelectedCredentialId(null) + }} + /> + + )} {isApiKeyMode ? ( {credentialsError && rawCredentials.length === 0 ? ( @@ -801,7 +830,9 @@ export function AddConnectorModal({ placeholder={ canConnectOAuth ? `Select ${connectorConfig.name} account` - : 'Select a service account' + : savedCredentialConfig + ? 'Select a credential' + : 'Select a service account' } isLoading={credentialsLoading || isIntegrationAvailabilityLoading} disabled={isCreating || !isIntegrationAvailabilityReady} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx index 5eec846a5e9..b659418ae9e 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx @@ -31,6 +31,7 @@ interface ConnectorsSectionProps { isLoading: boolean canEdit: boolean className?: string + savedCredentialSetupEnabled?: boolean } const STATUS_CONFIG = { @@ -51,6 +52,7 @@ export function ConnectorsSection({ isLoading, canEdit, className, + savedCredentialSetupEnabled = false, }: ConnectorsSectionProps) { const scope = explicitScope ?? resourceScopeFromOwner({ workspaceId }) const [editingConnector, setEditingConnector] = useState(null) @@ -84,6 +86,7 @@ export function ConnectorsSection({ knowledgeBaseId={knowledgeBaseId} isSearchIndex={isSearchIndex} connector={editingConnector} + savedCredentialSetupEnabled={savedCredentialSetupEnabled} /> )} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx index 3ea7a10d590..ea9b30290ca 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx @@ -44,7 +44,7 @@ import type { ConfigFieldValue, } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { SettingsQueryErrorState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' -import { isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { getConnectorCredentialService, isConnectorCredentialTypeAllowed } from '@/connectors/auth' import { GitHubInstallationConnectionField } from '@/connectors/github/installation-connection-field' import { GitLabPermissionTabs, @@ -162,10 +162,14 @@ export function ConnectorSettingsFields({ const connectorConfig = liveSearchSourceMeta(originalConfig, Boolean(liveSearch), { githubInstallation: usesGitHubInstallation && scope.kind === 'organization', }) - const providerId = - connectorConfig?.auth.mode === 'oauth' - ? (getProviderIdFromServiceId(connectorConfig.auth.provider) as OAuthProvider) - : null + const credentialServiceId = connectorConfig + ? getConnectorCredentialService(connectorConfig.auth) + : undefined + const providerId = credentialServiceId + ? (getProviderIdFromServiceId(credentialServiceId) as OAuthProvider) + : null + const savedCredentialConfig = + connectorConfig?.auth.mode === 'apiKey' ? connectorConfig.auth.savedCredential : undefined const syncsPerMember = access.accessMode === 'members' const isGitHubInstallationSource = usesGitHubInstallation && @@ -182,7 +186,7 @@ export function ConnectorSettingsFields({ : undefined const serviceAccountTarget = useServiceAccountConnectTarget({ serviceAccountProviderId: - isSearchIndex || requiresServiceAccount + isSearchIndex || requiresServiceAccount || savedCredentialConfig ? asServiceAccountProviderId(serviceAccountProviderId) : undefined, serviceName: connectorConfig?.name, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index 4c6b9471546..045814e9628 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -26,6 +26,7 @@ interface EditConnectorModalProps { knowledgeBaseId: string isSearchIndex?: boolean connector: ConnectorData + savedCredentialSetupEnabled?: boolean } export function EditConnectorModal({ @@ -35,6 +36,7 @@ export function EditConnectorModal({ isSearchIndex = false, connector, scope, + savedCredentialSetupEnabled = false, }: EditConnectorModalProps) { const [activeTab, setActiveTab] = useState('settings') const [documentFilter, setDocumentFilter] = useState('active') @@ -44,6 +46,7 @@ export function EditConnectorModal({ isSearchIndex, connector, onSaved: () => onOpenChange(false), + savedCredentialSetupEnabled, }) return ( diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts index 300366d0e7a..8cad515af54 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts @@ -6,6 +6,7 @@ import { isEqual } from 'es-toolkit' import type { UpdateConnectorBody } from '@/lib/api/contracts/knowledge/connectors' import type { ResourceScope } from '@/lib/core/resource-scope' import { isContentEngineAccessMode } from '@/lib/knowledge/connectors/access-modes' +import { CREDENTIAL_REMOVED_SYNC_ERROR } from '@/lib/knowledge/connectors/sync-limits' import { getConnectorAccessAvailability } from '@/lib/sim-search/connectors' import { liveSearchSourceMeta } from '@/lib/sim-search/live/source-settings' import { readSourceSelectionLabels, SOURCE_LABELS_KEY } from '@/lib/sim-search/source-identity' @@ -20,9 +21,10 @@ import { useConnectorConfigFields, } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { useConnectorScope } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-scope' -import { isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { isConnectorCredentialTypeAllowed, isConnectorFieldForAuth } from '@/connectors/auth' import { useGitLabPermissionForm } from '@/connectors/gitlab/permission-config/use-permission-form' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import type { ConnectorConfigField } from '@/connectors/types' import { type ConnectorData, isConnectorSyncingOrPending, @@ -125,6 +127,8 @@ interface UseConnectorSettingsFormOptions { connector: ConnectorData onSaved: (connector: ConnectorData) => void syncing?: boolean + /** Whether a key connector's saved credential may be replaced; the server refuses it while setup is off. */ + savedCredentialSetupEnabled?: boolean } /** Editable connector settings shared by the source page and knowledge-base modal. */ @@ -135,6 +139,7 @@ export function useConnectorSettingsForm({ connector, onSaved, syncing = isConnectorSyncingOrPending(connector), + savedCredentialSetupEnabled = false, }: UseConnectorSettingsFormOptions) { const liveSearch = isSearchIndex const connectorConfig = liveSearchSourceMeta( @@ -232,6 +237,18 @@ export function useConnectorSettingsForm({ initialSelectionLabels, }) + /** + * A key connector that syncs with its saved credential, or did until the credential was + * removed: its credential stays replaceable here while saved-credential setup is enabled, + * and its key-only settings do not apply. + */ + const usesSavedCredential = + connectorConfig?.auth.mode === 'apiKey' && + Boolean(connectorConfig.auth.savedCredential) && + (Boolean(connector.credentialId) || connector.lastSyncError === CREDENTIAL_REMOVED_SYNC_ERROR) + const isAuthFieldVisible = (field: ConnectorConfigField) => + isFieldVisible(field) && isConnectorFieldForAuth(field, !usesSavedCredential) + const { scope, canAdmin, memberAccessAvailable, mirroredAccessAvailable, hasMaxAccess } = useConnectorScope(explicitScope) const { mutate: updateConnector, isPending: isSavingSettings } = useUpdateConnector() @@ -280,10 +297,11 @@ export function useConnectorSettingsForm({ accessDirty && !accessModeChanged && isContentEngineAccessMode(access.accessMode) /** Exposes credential selection for mode changes and administrator credential recovery. */ const needsWorkspaceCredential = - connectorConfig?.auth.mode === 'oauth' && - isContentEngineAccessMode(access.accessMode) && - (persistedAccess.accessMode === 'members' || - !isConnectorCredentialTypeAllowed(connectorConfig.auth, access.accessMode, 'oauth')) + (usesSavedCredential && savedCredentialSetupEnabled) || + (connectorConfig?.auth.mode === 'oauth' && + isContentEngineAccessMode(access.accessMode) && + (persistedAccess.accessMode === 'members' || + !isConnectorCredentialTypeAllowed(connectorConfig.auth, access.accessMode, 'oauth'))) const missingAdminField = accessDirty && access.accessMode === 'admin' ? connectorConfig?.configFields.find((field) => { @@ -316,7 +334,7 @@ export function useConnectorSettingsForm({ const settingsComplete = connectorConfig?.configFields.every( (field) => !isConnectorFieldRequired(field, connectorConfig, persistedAccess.accessMode) || - !isFieldVisible(field) || + !isAuthFieldVisible(field) || hiddenCapFieldIds.has(field.id) || isFieldPopulated(field) ) @@ -526,7 +544,7 @@ export function useConnectorSettingsForm({ canonicalModes, onToggleCanonicalMode: toggleCanonicalMode, onFieldChange: handleFieldChange, - isFieldVisible: (field) => isFieldVisible(field) && !hiddenCapFieldIds.has(field.id), + isFieldVisible: (field) => isAuthFieldVisible(field) && !hiddenCapFieldIds.has(field.id), syncInterval, setSyncInterval, hasMaxAccess, diff --git a/apps/sim/app/workspace/[workspaceId]/layout.tsx b/apps/sim/app/workspace/[workspaceId]/layout.tsx index fb9ac90703f..d3227f10e41 100644 --- a/apps/sim/app/workspace/[workspaceId]/layout.tsx +++ b/apps/sim/app/workspace/[workspaceId]/layout.tsx @@ -13,6 +13,7 @@ import { hasDesktopBackgroundExecutor, isDesktopBackgroundExecutorAvailable, } from '@/lib/desktop/executor/availability' +import { isConnectorSavedCredentialSetupEnabled } from '@/lib/knowledge/connectors/saved-credential-rollout' import { isMothershipModelSelectorEnabled, isPlanModeEnabled } from '@/lib/mothership/feature-flags' import { resolveOrganizationEntryPath } from '@/lib/navigation/resolve-app-entry' import { isProjectsEnabled } from '@/lib/projects/rollout.server' @@ -137,6 +138,7 @@ async function WorkspaceContent({ computerUseEnabled, projectsEnabled, simLinkOrganizationId, + connectorSavedCredentialsEnabled, ] = await Promise.all([ cookies(), orgSettings, @@ -157,6 +159,7 @@ async function WorkspaceContent({ isComputerUseAvailable(hostContext.hostOrganizationId), isProjectsEnabled(), resolveSimLinkOrganization(principal), + isConnectorSavedCredentialSetupEnabled(), prefetchWorkspaceAccess(queryClient, workspaceId, principal), prefetchWorkspaceForkAvailability(queryClient, workspaceId, principal, hostContext), ]) @@ -173,6 +176,7 @@ async function WorkspaceContent({ 'mothership-computer-use': computerUseEnabled, 'mothership-plan-mode': planModeEnabled, projects: projectsEnabled, + 'knowledge-connector-saved-credentials': connectorSavedCredentialsEnabled, }} > diff --git a/apps/sim/app/workspace/[workspaceId]/providers/feature-flags-provider.tsx b/apps/sim/app/workspace/[workspaceId]/providers/feature-flags-provider.tsx index f33fd8f230f..44d40828ebc 100644 --- a/apps/sim/app/workspace/[workspaceId]/providers/feature-flags-provider.tsx +++ b/apps/sim/app/workspace/[workspaceId]/providers/feature-flags-provider.tsx @@ -11,6 +11,7 @@ export interface WorkspaceFeatureFlags { 'mothership-computer-use'?: boolean 'mothership-memory-spaces'?: boolean projects?: boolean + 'knowledge-connector-saved-credentials'?: boolean } const FeatureFlagsContext = createContext(null) diff --git a/apps/sim/connectors/auth.test.ts b/apps/sim/connectors/auth.test.ts index ad6d02739d4..3349e3b56b7 100644 --- a/apps/sim/connectors/auth.test.ts +++ b/apps/sim/connectors/auth.test.ts @@ -1,7 +1,15 @@ import { describe, expect, it } from 'vitest' -import { isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { + connectorHasAuthSource, + connectorUsesCredential, + getConnectorCredentialService, + isConnectorCredentialTypeAllowed, +} from '@/connectors/auth' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' +import { databricksConnectorMeta } from '@/connectors/databricks/meta' +import { githubConnectorMeta } from '@/connectors/github/meta' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' +import { s3ConnectorMeta } from '@/connectors/s3/meta' describe('connector credential eligibility', () => { it.each([confluenceConnectorMeta, googleDriveConnectorMeta])( @@ -24,3 +32,47 @@ describe('connector credential eligibility', () => { } ) }) + +describe('connector auth source', () => { + const s3Auth = s3ConnectorMeta.auth + const keyOnlyAuth = databricksConnectorMeta.auth + + it('keeps a stored-key S3 connector on its key', () => { + const row = { credentialId: null, encryptedApiKey: 'encrypted-key' } + expect(connectorUsesCredential(s3Auth, row)).toBe(false) + expect(connectorHasAuthSource(s3Auth, row)).toBe(true) + }) + + it('authenticates an S3 connector that names a saved credential with that credential', () => { + const row = { credentialId: 'credential-1', encryptedApiKey: null } + expect(connectorUsesCredential(s3Auth, row)).toBe(true) + expect(connectorHasAuthSource(s3Auth, row)).toBe(true) + }) + + it('treats an S3 connector whose credential was removed as needing reconnection', () => { + expect(connectorHasAuthSource(s3Auth, { credentialId: null, encryptedApiKey: null })).toBe( + false + ) + }) + + it('never routes a key-only connector to a credential it does not accept', () => { + const row = { credentialId: 'credential-1', encryptedApiKey: 'encrypted-key' } + expect(connectorUsesCredential(keyOnlyAuth, row)).toBe(false) + expect(getConnectorCredentialService(keyOnlyAuth)).toBeUndefined() + }) + + it('keeps an OAuth connector with a token alternative on the token it stored', () => { + expect( + connectorUsesCredential(githubConnectorMeta.auth, { + credentialId: null, + encryptedApiKey: 'encrypted-token', + }) + ).toBe(false) + expect( + connectorUsesCredential(githubConnectorMeta.auth, { + credentialId: 'credential-1', + encryptedApiKey: null, + }) + ).toBe(true) + }) +}) diff --git a/apps/sim/connectors/auth.ts b/apps/sim/connectors/auth.ts index adf4dc9e4aa..8d74d5a8b59 100644 --- a/apps/sim/connectors/auth.ts +++ b/apps/sim/connectors/auth.ts @@ -1,4 +1,5 @@ -import type { ConnectorAuthConfig } from '@/connectors/types' +import type { OAuthService } from '@/lib/oauth/types' +import type { ConnectorAuthConfig, ConnectorConfigField } from '@/connectors/types' /** The same selected-source requirements drive credential filtering, validation, and token minting. */ export function getConnectorRequiredScopes( @@ -25,6 +26,36 @@ export function isConnectorCredentialTypeAllowed( ) } +/** The service whose saved credentials can authenticate this connector, if any. */ +export function getConnectorCredentialService(auth: ConnectorAuthConfig): OAuthService | undefined { + return auth.mode === 'oauth' ? auth.provider : auth.savedCredential?.provider +} + +/** + * Whether a connector row authenticates with its saved credential rather than a stored key. + * + * An OAuth connector uses its credential unless it also takes a token and the row stores + * only that. A key connector uses a credential only when it declares a `savedCredential` + * alternative and the row names one; creation and access updates never bind a credential to + * a row that stores a key, so such a row keeps using its key. + */ +export function connectorUsesCredential( + auth: ConnectorAuthConfig, + connector: { credentialId: string | null; encryptedApiKey: string | null } +): boolean { + if (auth.mode === 'apiKey') return Boolean(auth.savedCredential && connector.credentialId) + return !(auth.apiKey && !connector.credentialId && connector.encryptedApiKey) +} + +/** How a connector row authenticates, as recorded in audit metadata. */ +export function connectorAuthModeLabel( + authMode: ConnectorAuthConfig['mode'], + usesCredential: boolean +): 'apiKey' | 'oauth' | 'savedCredential' { + if (!usesCredential) return 'apiKey' + return authMode === 'apiKey' ? 'savedCredential' : 'oauth' +} + /** * Whether a workspace-mode connector row still carries something to authenticate with. A * connector whose credential was removed keeps its documents but has no token source, so a @@ -34,9 +65,16 @@ export function connectorHasAuthSource( auth: ConnectorAuthConfig, connector: { credentialId: string | null; encryptedApiKey: string | null } ): boolean { - const apiKeyConfig = getConnectorApiKeyConfig(auth) - if (apiKeyConfig && connector.encryptedApiKey) return true - return auth.mode === 'apiKey' ? apiKeyConfig?.optional === true : Boolean(connector.credentialId) + if (connectorUsesCredential(auth, connector)) return Boolean(connector.credentialId) + return Boolean(connector.encryptedApiKey) || getConnectorApiKeyConfig(auth)?.optional === true +} + +/** Whether a config field applies to how the connector authenticates: key-only fields drop out with a saved credential. */ +export function isConnectorFieldForAuth( + field: Pick, + authenticatesWithKey: boolean +): boolean { + return authenticatesWithKey || !field.apiKeyOnly } /** Workspace token input supported by a connector, independent of its member OAuth method. */ diff --git a/apps/sim/connectors/aws-session.ts b/apps/sim/connectors/aws-session.ts new file mode 100644 index 00000000000..e0289d5b84f --- /dev/null +++ b/apps/sim/connectors/aws-session.ts @@ -0,0 +1,44 @@ +/** + * The AWS keys a saved AWS credential resolved to for one connector run. + * + * A saved AWS credential has no bearer token: the server resolves it to a key + * pair (and, for a role, an STS session token) for the knowledge base's own + * scope. The resolver seeds them into the run's `syncContext` under one slot, + * the same way it seeds a credential's bound site, so a connector reads them + * from one place and never from tool or user input. Server-only state: the + * slot is never persisted or logged. + */ +export interface ConnectorAwsSession { + accessKeyId: string + secretAccessKey: string + /** Present for STS sessions (assumed and keyless roles); absent for saved long-lived keys. */ + sessionToken?: string +} + +const AWS_SESSION_CONTEXT_KEY = 'credentialAwsSession' + +/** The `syncContext` entries that carry a resolved AWS session to a connector. */ +export function awsSessionSyncContext(session: ConnectorAwsSession): Record { + return { + [AWS_SESSION_CONTEXT_KEY]: { + accessKeyId: session.accessKeyId, + secretAccessKey: session.secretAccessKey, + ...(session.sessionToken ? { sessionToken: session.sessionToken } : {}), + }, + } +} + +/** The AWS session seeded into this run, or undefined when the connector signs with inline keys. */ +export function readAwsSession( + syncContext: Record | undefined +): ConnectorAwsSession | undefined { + const value = syncContext?.[AWS_SESSION_CONTEXT_KEY] + if (typeof value !== 'object' || value === null) return undefined + const { accessKeyId, secretAccessKey, sessionToken } = value as Record + if (typeof accessKeyId !== 'string' || !accessKeyId) return undefined + if (typeof secretAccessKey !== 'string' || !secretAccessKey) return undefined + if (sessionToken !== undefined && (typeof sessionToken !== 'string' || !sessionToken)) { + return undefined + } + return { accessKeyId, secretAccessKey, ...(sessionToken ? { sessionToken } : {}) } +} diff --git a/apps/sim/connectors/s3/meta.ts b/apps/sim/connectors/s3/meta.ts index 49367964ba4..de40b65b44b 100644 --- a/apps/sim/connectors/s3/meta.ts +++ b/apps/sim/connectors/s3/meta.ts @@ -13,6 +13,7 @@ export const s3ConnectorMeta: ConnectorMeta = { mode: 'apiKey', label: 'Secret Access Key', placeholder: 'Enter your AWS Secret Access Key', + savedCredential: { provider: 'aws', label: 'Saved AWS credential' }, }, configFields: [ @@ -22,6 +23,7 @@ export const s3ConnectorMeta: ConnectorMeta = { type: 'short-input', placeholder: 'e.g. AKIAIOSFODNN7EXAMPLE', required: true, + apiKeyOnly: true, }, { id: 'region', @@ -30,7 +32,7 @@ export const s3ConnectorMeta: ConnectorMeta = { placeholder: 'e.g. us-east-1 (use auto for Cloudflare R2)', required: true, description: - 'AWS region for the bucket. For Cloudflare R2 use "auto"; for MinIO use the region the server is configured with (commonly us-east-1).', + 'AWS region for the bucket. For Cloudflare R2 use "auto"; for MinIO use the region the server is configured with (commonly us-east-1). A saved AWS credential needs a commercial or GovCloud AWS region.', }, { id: 'bucket', @@ -47,6 +49,7 @@ export const s3ConnectorMeta: ConnectorMeta = { required: false, description: 'S3-compatible endpoint for Cloudflare R2, MinIO, etc. Leave empty for AWS S3. Uses path-style addressing. Plain http:// is only allowed for localhost.', + apiKeyOnly: true, }, { id: 'prefix', diff --git a/apps/sim/connectors/s3/s3.test.ts b/apps/sim/connectors/s3/s3.test.ts new file mode 100644 index 00000000000..6dc93080fb6 --- /dev/null +++ b/apps/sim/connectors/s3/s3.test.ts @@ -0,0 +1,167 @@ +/** + * Pins the S3 connector's two signing paths. Inline keys sign exactly as they + * always have; a saved AWS credential (seeded into `syncContext` by the token + * resolver) signs with its STS session token and only ever for Amazon S3 — a + * custom S3-compatible endpoint would otherwise receive the customer's AWS + * session. + */ +import { knowledgeDocumentsUtilsMock } from '@sim/testing/mocks/knowledge-documents-utils.mock' +import { + knowledgeSecureFetchMock, + knowledgeSecureFetchMockFns, +} from '@sim/testing/mocks/knowledge-secure-fetch.mock' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/knowledge/documents/utils', () => knowledgeDocumentsUtilsMock) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => knowledgeSecureFetchMock) + +import { awsSessionSyncContext } from '@/connectors/aws-session' +import { s3Connector } from '@/connectors/s3/s3' + +const mockSecureFetchWithRetry = knowledgeSecureFetchMockFns.mockSecureFetchWithRetry + +const SESSION = { + accessKeyId: 'ASIAIOSFODNN7EXAMPLE', + secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + sessionToken: 'IQoJb3JpZ2luX2VjEXAMPLE+token/with=chars', +} + +const EMPTY_LISTING = + 'false' + +function recordRequests() { + const requests: { url: string; headers: Record }[] = [] + mockSecureFetchWithRetry.mockImplementation( + async (url: string, init: { headers: Record }) => { + requests.push({ url, headers: init.headers }) + return new Response(EMPTY_LISTING, { status: 200 }) + } + ) + return requests +} + +describe('s3Connector signing', () => { + beforeEach(() => { + mockSecureFetchWithRetry.mockReset() + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(new Date('2026-11-06T12:00:00Z')) + }) + + it('signs a saved credential session token into the request, as SigV4 requires', async () => { + const requests = recordRequests() + + await s3Connector.listDocuments( + 'credential-handle', + { region: 'us-east-1', bucket: 'docs-bucket', prefix: 'kb/' }, + undefined, + { ...awsSessionSyncContext(SESSION) } + ) + + expect(requests).toHaveLength(1) + const [request] = requests + expect(request.url).toBe( + 'https://docs-bucket.s3.us-east-1.amazonaws.com/?encoding-type=url&list-type=2&max-keys=1000&prefix=kb%2F' + ) + expect(request.headers['X-Amz-Security-Token']).toBe(SESSION.sessionToken) + // Reference value from @smithy/signature-v4 (the AWS SDK signer) for the same request. + expect(request.headers.Authorization).toBe( + 'AWS4-HMAC-SHA256 Credential=ASIAIOSFODNN7EXAMPLE/20261106/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=dfe6577a8c0c32916977121ba18620aeca7758bf39d0230a02d29183ff49677a' + ) + }) + + it('never sends a saved AWS credential to a custom endpoint', async () => { + recordRequests() + + const validation = await s3Connector.validateConfig( + 'credential-handle', + { region: 'us-east-1', bucket: 'docs', endpoint: 'https://attacker.example.com' }, + { ...awsSessionSyncContext(SESSION) } + ) + + expect(validation.valid).toBe(false) + expect(validation.error).toMatch(/custom endpoint/i) + expect(mockSecureFetchWithRetry).not.toHaveBeenCalled() + }) + + it('refuses a non-AWS region with a saved AWS credential', async () => { + recordRequests() + + const validation = await s3Connector.validateConfig( + 'credential-handle', + { accessKeyId: 'AKIAIOSFODNN7EXAMPLE', region: 'auto', bucket: 'docs' }, + { ...awsSessionSyncContext(SESSION) } + ) + + expect(validation.valid).toBe(false) + expect(validation.error).toMatch(/AWS region/i) + expect(mockSecureFetchWithRetry).not.toHaveBeenCalled() + }) + + it.each(['cn-north-1', 'us-iso-east-1', 'eu-isoe-west-1'])( + 'refuses %s, whose S3 host is not under amazonaws.com, with a saved AWS credential', + async (region) => { + recordRequests() + + const validation = await s3Connector.validateConfig( + 'credential-handle', + { region, bucket: 'docs' }, + { ...awsSessionSyncContext(SESSION) } + ) + + expect(validation.valid).toBe(false) + expect(validation.error).toMatch(/commercial or GovCloud/i) + expect(mockSecureFetchWithRetry).not.toHaveBeenCalled() + } + ) + + it('signs a saved AWS credential for a GovCloud bucket', async () => { + const requests = recordRequests() + + await s3Connector.listDocuments( + '', + { region: 'us-gov-west-1', bucket: 'docs-bucket' }, + undefined, + { ...awsSessionSyncContext(SESSION) } + ) + + expect(requests[0]?.url).toMatch(/^https:\/\/docs-bucket\.s3\.us-gov-west-1\.amazonaws\.com\//) + }) + + it('keeps signing inline keys without a session token', async () => { + const requests = recordRequests() + + await s3Connector.listDocuments( + SESSION.secretAccessKey, + { + accessKeyId: SESSION.accessKeyId, + region: 'us-east-1', + bucket: 'docs-bucket', + prefix: 'kb/', + }, + undefined, + {} + ) + + const [request] = requests + expect(request.headers['X-Amz-Security-Token']).toBeUndefined() + // Reference value from @smithy/signature-v4 for the same request without a session token. + expect(request.headers.Authorization).toBe( + 'AWS4-HMAC-SHA256 Credential=ASIAIOSFODNN7EXAMPLE/20261106/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=bceba1b42847dc164dfe175c63bc088449c244e6e91cce4ecd6d9b06f6508b0d' + ) + }) + + it('ignores an inline access key ID once a saved credential signs', async () => { + const requests = recordRequests() + + await s3Connector.getDocument( + 'credential-handle', + { accessKeyId: 'AKIAIOSFODNN7EXAMPLE', region: 'us-east-1', bucket: 'docs-bucket' }, + 'notes/readme.md', + { ...awsSessionSyncContext(SESSION) } + ) + + expect(requests[0].headers.Authorization).toContain( + `Credential=${SESSION.accessKeyId}/20261106/us-east-1/s3/aws4_request` + ) + }) +}) diff --git a/apps/sim/connectors/s3/s3.ts b/apps/sim/connectors/s3/s3.ts index 5e07d171339..d9686d10353 100644 --- a/apps/sim/connectors/s3/s3.ts +++ b/apps/sim/connectors/s3/s3.ts @@ -7,6 +7,7 @@ import { isHosted } from '@/lib/core/config/env-flags' import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { readAwsSession } from '@/connectors/aws-session' import { s3ConnectorMeta } from '@/connectors/s3/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { @@ -52,6 +53,13 @@ const BUCKET_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{1,62}$/ /** Accepted region shape — AWS region codes plus Cloudflare R2's `auto`. */ const REGION_PATTERN = /^[A-Za-z0-9-]{1,64}$/ +/** + * A commercial or GovCloud AWS region code (`us-east-1`, `us-gov-west-1`): the only regions a + * saved AWS credential signs for, because their S3 hosts are all `s3..amazonaws.com`. + * China (`cn-*`, under `amazonaws.com.cn`) and the isolated partitions use other domains. + */ +const AWS_REGION_PATTERN = /^(?!cn-)[a-z]{2}(-gov)?-[a-z]+-\d+$/ + /** * Default set of file extensions considered safely text-extractable. Objects * with any other extension (or no extension) are skipped, since their content @@ -115,6 +123,8 @@ interface S3Endpoint { interface S3Context { accessKeyId: string secretAccessKey: string + /** The STS session token of a saved AWS credential's role session; signed into every request. */ + sessionToken?: string region: string bucket: string endpoint?: S3Endpoint @@ -206,20 +216,29 @@ function parseEndpoint(raw: string): S3Endpoint { } /** - * Resolves AWS credentials and the target bucket from the connector's - * sourceConfig and the encrypted secret (delivered as accessToken). When an - * `endpoint` is configured it is parsed/validated into an {@link S3Endpoint} so - * the connector targets an S3-compatible store via path-style addressing. + * Resolves AWS credentials and the target bucket. + * + * With a saved AWS credential, the server seeds its resolved keys into + * `syncContext` and those sign every request; the inline `accessKeyId` and the + * access token are not read. A saved credential signs only for Amazon S3 in an + * AWS region: a custom endpoint would hand the customer's AWS session to + * whatever host it names, so it is refused rather than ignored. + * + * Otherwise the inline key pair comes from `sourceConfig.accessKeyId` and the + * encrypted secret (delivered as accessToken), and an `endpoint` is parsed into + * an {@link S3Endpoint} so the connector targets an S3-compatible store via + * path-style addressing. */ -function resolveContext(accessToken: string, sourceConfig: Record): S3Context { - const accessKeyId = ((sourceConfig.accessKeyId as string) ?? '').trim() +function resolveContext( + accessToken: string, + sourceConfig: Record, + syncContext: Record | undefined +): S3Context { const region = ((sourceConfig.region as string) ?? '').trim() const bucket = ((sourceConfig.bucket as string) ?? '').trim() - const secretAccessKey = (accessToken ?? '').trim() const rawEndpoint = ((sourceConfig.endpoint as string) ?? '').trim() + const savedSession = readAwsSession(syncContext) - if (!accessKeyId) throw new Error('Missing AWS Access Key ID') - if (!secretAccessKey) throw new Error('Missing AWS Secret Access Key') if (!region) throw new Error('Missing AWS region') if (!bucket) throw new Error('Missing S3 bucket name') if (!REGION_PATTERN.test(region)) { @@ -231,6 +250,25 @@ function resolveContext(accessToken: string, sourceConfig: Record ): Promise => { - const ctx = resolveContext(accessToken, sourceConfig) + const ctx = resolveContext(accessToken, sourceConfig, syncContext) const prefix = ((sourceConfig.prefix as string) ?? '').trim() const allowedExtensions = resolveExtensions(sourceConfig.extensions) @@ -602,9 +649,10 @@ export const s3Connector: ConnectorConfig = { getDocument: async ( accessToken: string, sourceConfig: Record, - externalId: string + externalId: string, + syncContext?: Record ): Promise => { - const ctx = resolveContext(accessToken, sourceConfig) + const ctx = resolveContext(accessToken, sourceConfig, syncContext) const key = externalId if (hasDotSegment(key)) { @@ -680,11 +728,12 @@ export const s3Connector: ConnectorConfig = { validateConfig: async ( accessToken: string, - sourceConfig: Record + sourceConfig: Record, + syncContext?: Record ): Promise<{ valid: boolean; error?: string }> => { let ctx: S3Context try { - ctx = resolveContext(accessToken, sourceConfig) + ctx = resolveContext(accessToken, sourceConfig, syncContext) } catch (error) { return { valid: false, error: getErrorMessage(error, 'Invalid configuration') } } diff --git a/apps/sim/connectors/types.ts b/apps/sim/connectors/types.ts index 92dc23b7e44..22cb3c18dc1 100644 --- a/apps/sim/connectors/types.ts +++ b/apps/sim/connectors/types.ts @@ -60,6 +60,16 @@ export type ConnectorAuthConfig = * connector receives an empty access token. */ optional?: boolean + /** + * A saved credential that may authenticate a workspace crawl instead of + * the key, such as a saved AWS credential for Amazon S3. Additive: a + * connector row with a stored key keeps using it, and one that names a + * credential resolves it for the knowledge base's own scope. Fields + * marked {@link ConnectorConfigField.apiKeyOnly} describe the key and + * do not apply to a credential. Only AWS resolves to signing keys today; + * another provider needs its own resolution in the connector access token. + */ + savedCredential?: { provider: 'aws'; label: string } } /** A group in an external directory, named as the connector's ACLs name it. */ @@ -335,6 +345,12 @@ export interface ConnectorConfigField { hideInAdminMode?: true /** Only applies to a central crawl that mirrors source permissions. */ showInAdminModeOnly?: true + /** + * Describes the connector's own API key (an access key ID, a custom host the + * key signs for), so it is hidden, not required, and not sent when the + * connector authenticates with its `savedCredential` instead. + */ + apiKeyOnly?: true options?: { label: string; id: string }[] /** Selector key from the selector registry (used when type is 'selector') */ diff --git a/apps/sim/lib/api/contracts/v2/catalog.ts b/apps/sim/lib/api/contracts/v2/catalog.ts index 59167a71c39..67942cb26d6 100644 --- a/apps/sim/lib/api/contracts/v2/catalog.ts +++ b/apps/sim/lib/api/contracts/v2/catalog.ts @@ -696,6 +696,12 @@ export const v2ConnectorConfigFieldSchema = z .describe( 'When true the stored `sourceConfig` value is a `string[]`, not a `string`: a `selector` renders a multi-select picker and a `short-input` accepts a comma-separated list.' ), + apiKeyOnly: z + .boolean() + .optional() + .describe( + 'When true the field describes the connector’s API key (such as its key ID or a custom host the key signs for). It is required as stated only with `apiKey`, and omitted when a saved `credentialId` is sent.' + ), }) .meta({ id: 'V2ConnectorConfigField', @@ -731,7 +737,7 @@ export const v2ConnectorTypeSchema = z mode: z .literal('apiKey') .describe( - 'Authenticates with a stored API key; pass `apiKey`, and no `credentialId`, when creating the connector.' + 'Authenticates with a stored API key; pass `apiKey` when creating the connector, or a saved `credentialId` instead when `savedCredential` is present.' ), label: z.string().optional().describe('Label shown above the key field.'), placeholder: z.string().optional().describe('Placeholder shown in the key field.'), @@ -740,10 +746,21 @@ export const v2ConnectorTypeSchema = z .describe( 'Whether the key may be left blank, for a source reachable without authentication.' ), + savedCredential: z + .object({ + provider: z + .string() + .describe('Service the saved credential must authenticate with, e.g. `aws`.'), + label: z.string().describe('Label for the saved-credential option.'), + }) + .optional() + .describe( + 'When present, a saved credential of this service may be passed as `credentialId` instead of `apiKey`, once saved-credential setup is enabled on this deployment (until then it is refused with a validation error). Fields marked `apiKeyOnly` are then omitted.' + ), }), ]) .describe( - 'How the connector authenticates against its source: `oauth` connectors take `credentialId` (GitHub also accepts a personal access token as `apiKey`), `apiKey` connectors take `apiKey`.' + 'How the connector authenticates against its source: `oauth` connectors take `credentialId` (GitHub also accepts a personal access token as `apiKey`), `apiKey` connectors take `apiKey` (or, when they declare `savedCredential`, a saved `credentialId`).' ), configFields: z .array(v2ConnectorConfigFieldSchema) @@ -787,7 +804,7 @@ export const v2ConnectorTypeSummarySchema = z mode: z .enum(['oauth', 'apiKey']) .describe( - 'How the connector authenticates against its source: `oauth` connectors take `credentialId` (GitHub also accepts a personal access token as `apiKey`), `apiKey` connectors take `apiKey`.' + 'How the connector authenticates against its source: `oauth` connectors take `credentialId` (GitHub also accepts a personal access token as `apiKey`), `apiKey` connectors take `apiKey` (or, when they declare `savedCredential`, a saved `credentialId`).' ), }) .describe( diff --git a/apps/sim/lib/api/contracts/v2/knowledge.ts b/apps/sim/lib/api/contracts/v2/knowledge.ts index 2aab9f6129c..a3bc5cb8108 100644 --- a/apps/sim/lib/api/contracts/v2/knowledge.ts +++ b/apps/sim/lib/api/contracts/v2/knowledge.ts @@ -1777,7 +1777,9 @@ export const v2KnowledgeConnectorSchema = z credentialId: z .string() .nullable() - .describe('OAuth credential identifier, or null for API-key and unauthenticated sources.'), + .describe( + 'OAuth credential identifier, or the saved credential an `apiKey` source syncs with (see `auth.savedCredential`); null for API-key and unauthenticated sources.' + ), sourceConfig: z .record(z.string(), z.unknown().describe('Connector-specific source configuration value.')) .describe('Connector-specific source selection and filtering configuration.'), @@ -1932,7 +1934,7 @@ export const v2CreateKnowledgeConnectorBodySchema = z .max(255) .optional() .describe( - 'OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types); omit it for `apiKey` connectors.' + 'OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types), or, when this deployment enables it, a saved credential for an `apiKey` connector type that declares `auth.savedCredential` (such as Amazon S3 with a saved AWS credential); otherwise omit it for `apiKey` connectors.' ), apiKey: z .string() diff --git a/apps/sim/lib/catalog/projection/connector-type.ts b/apps/sim/lib/catalog/projection/connector-type.ts index 4e0c7ee6237..1f65fc62c60 100644 --- a/apps/sim/lib/catalog/projection/connector-type.ts +++ b/apps/sim/lib/catalog/projection/connector-type.ts @@ -16,7 +16,14 @@ import type { /** How a connector authenticates against its source. */ export type CatalogConnectorAuth = | { mode: 'oauth'; provider: string; requiredScopes?: string[] } - | { mode: 'apiKey'; label?: string; placeholder?: string; optional: boolean } + | { + mode: 'apiKey' + label?: string + placeholder?: string + optional: boolean + /** A saved credential of this service may be sent as `credentialId` instead of the key. */ + savedCredential?: { provider: string; label: string } + } /** * One field of a connector's `sourceConfig`. @@ -48,6 +55,8 @@ export interface CatalogConnectorConfigField { mode?: 'basic' | 'advanced' canonicalParamId?: string multi?: boolean + /** Applies only with the connector's API key; omit it when sending a saved `credentialId`. */ + apiKeyOnly?: boolean } /** A tag slot a connector populates on the documents it syncs. */ @@ -83,6 +92,14 @@ function projectAuth(auth: ConnectorMeta['auth']): CatalogConnectorAuth { optional: auth.optional === true, ...(auth.label !== undefined ? { label: auth.label } : {}), ...(auth.placeholder !== undefined ? { placeholder: auth.placeholder } : {}), + ...(auth.savedCredential + ? { + savedCredential: { + provider: auth.savedCredential.provider, + label: auth.savedCredential.label, + }, + } + : {}), } } @@ -120,6 +137,7 @@ function projectConfigField(field: ConnectorConfigField): CatalogConnectorConfig if (field.mode !== undefined) projected.mode = field.mode if (field.canonicalParamId !== undefined) projected.canonicalParamId = field.canonicalParamId if (field.multi !== undefined) projected.multi = field.multi + if (field.apiKeyOnly) projected.apiKeyOnly = true return projected } diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 1775d3f6f02..5aae00c5a09 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -667,6 +667,7 @@ export const env = createEnv({ AGENT_MEMORY_HISTORY: z.boolean().optional(), CREDENTIAL_GROUPS: z.boolean().optional(), // Enable Credential Groups on self-hosted (bypasses hosted requirements) KNOWLEDGE_MEMBER_ACCESS: z.boolean().optional(), // Enable per-member knowledge connectors and hybrid-by-default retrieval globally + KNOWLEDGE_CONNECTOR_SAVED_CREDENTIALS: z.boolean().optional(), // Fallback for the `knowledge-connector-saved-credentials` feature flag off AppConfig // Organizations - for self-hosted deployments ORGANIZATIONS_ENABLED: z.boolean().optional(), // Enable organizations on self-hosted (bypasses plan requirements) diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index 5394541ae6e..5ebab319615 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -114,6 +114,17 @@ const FEATURE_FLAGS = { 'Off-AppConfig falls back to PROJECTS.', fallback: 'PROJECTS', }, + 'knowledge-connector-saved-credentials': { + description: + 'Let a key-based knowledge connector that declares a saved-credential alternative (Amazon S3 ' + + 'with a saved AWS credential) be set up with that credential. Global on/off only; gates new ' + + 'setup and credential changes, never syncing of connectors already using a credential. Turn ' + + 'on only after the release that adds it is live on both the app and Trigger.dev workers. ' + + 'Once any connector uses a saved credential, rolling back to an earlier release is unsafe ' + + 'even with the flag off: older workers unschedule those connectors as credential-removed. ' + + 'Off-AppConfig falls back to KNOWLEDGE_CONNECTOR_SAVED_CREDENTIALS.', + fallback: 'KNOWLEDGE_CONNECTOR_SAVED_CREDENTIALS', + }, 'knowledge-member-access': { description: 'Organization Search (live) and the permission-aware workspace connector modes: members ' + diff --git a/apps/sim/lib/internal/aws/credentials.ts b/apps/sim/lib/internal/aws/credentials.ts index 94cd67e2533..1d24f170217 100644 --- a/apps/sim/lib/internal/aws/credentials.ts +++ b/apps/sim/lib/internal/aws/credentials.ts @@ -1,7 +1,5 @@ import { AsyncLocalStorage } from 'node:async_hooks' -import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' -import { isRetryableNetworkError } from '@/lib/core/errors/retryable-infrastructure' import { awsRequestHandler } from '@/lib/core/network/proxy-agent.server' import { type AwsSessionCredentials, @@ -9,6 +7,7 @@ import { getAwsCredentialConfig, resolveAwsSessionCredentials, } from '@/lib/credentials/aws-service-account.server' +import { awsCredentialFailureMessage, isTransientAwsFailure } from '@/lib/internal/aws/failure' import type { InternalToolOperationCall, InternalToolOperationHandler, @@ -112,27 +111,6 @@ export async function resolveAuthorizedAwsCredential( }) } -const THROTTLING_ERROR_NAMES = new Set(['Throttling', 'ThrottlingException']) - -/** - * Whether resolving a credential failed because AWS was briefly unavailable rather - * than because it denied the credential: an error STS models as retryable - * (IDPCommunicationError, when STS could not reach Sim's OIDC issuer), throttling, - * an STS 5xx, or a network failure. The SDK has already retried these. - */ -export function isTransientAwsFailure(error: unknown): boolean { - if (!isRecordLike(error)) return false - const status = isRecordLike(error.$metadata) ? error.$metadata.httpStatusCode : undefined - return ( - error.$retryable !== undefined || - THROTTLING_ERROR_NAMES.has(String(error.name)) || - status === 429 || - (typeof status === 'number' && status >= 500) || - error.name === 'TimeoutError' || - isRetryableNetworkError(error) - ) -} - /** Wraps a family's operation handler so it also accepts a saved AWS credential. */ export function withAwsCredential( handler: InternalToolOperationHandler @@ -152,7 +130,7 @@ export function withAwsCredential( } catch (error) { request.signal?.throwIfAborted() return Response.json( - { error: `Could not use the AWS credential: ${getErrorMessage(error, 'Unknown error')}` }, + { error: awsCredentialFailureMessage(error) }, { status: isTransientAwsFailure(error) ? 503 : 403 } ) } diff --git a/apps/sim/lib/internal/aws/failure.ts b/apps/sim/lib/internal/aws/failure.ts new file mode 100644 index 00000000000..ce5188ac2fc --- /dev/null +++ b/apps/sim/lib/internal/aws/failure.ts @@ -0,0 +1,29 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { isRetryableNetworkError } from '@/lib/core/errors/retryable-infrastructure' + +const THROTTLING_ERROR_NAMES = new Set(['Throttling', 'ThrottlingException']) + +/** + * Whether resolving a saved AWS credential failed because AWS was briefly unavailable rather + * than because it denied the credential: an error STS models as retryable + * (IDPCommunicationError, when STS could not reach Sim's OIDC issuer), throttling, an STS 5xx, + * or a network failure. The SDK has already retried these. + */ +export function isTransientAwsFailure(error: unknown): boolean { + if (!isRecordLike(error)) return false + const status = isRecordLike(error.$metadata) ? error.$metadata.httpStatusCode : undefined + return ( + error.$retryable !== undefined || + THROTTLING_ERROR_NAMES.has(String(error.name)) || + status === 429 || + (typeof status === 'number' && status >= 500) || + error.name === 'TimeoutError' || + isRetryableNetworkError(error) + ) +} + +/** The caller-facing reason a saved AWS credential could not be used: the AWS message, never a key. */ +export function awsCredentialFailureMessage(error: unknown): string { + return `Could not use the AWS credential: ${getErrorMessage(error, 'Unknown error')}` +} diff --git a/apps/sim/lib/internal/llm/bedrock-credential.ts b/apps/sim/lib/internal/llm/bedrock-credential.ts index 493f18e1532..dd98e4d9ebb 100644 --- a/apps/sim/lib/internal/llm/bedrock-credential.ts +++ b/apps/sim/lib/internal/llm/bedrock-credential.ts @@ -1,10 +1,7 @@ -import { getErrorMessage } from '@sim/utils/errors' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' import { AuthType } from '@/lib/auth/hybrid' -import { - isTransientAwsFailure, - resolveAuthorizedAwsCredential, -} from '@/lib/internal/aws/credentials' +import { resolveAuthorizedAwsCredential } from '@/lib/internal/aws/credentials' +import { awsCredentialFailureMessage, isTransientAwsFailure } from '@/lib/internal/aws/failure' import { recordCredentialAccess } from '@/lib/oauth/token-resolution' import { AWS_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' import type { BedrockCredentialSession } from '@/providers/types' @@ -95,10 +92,10 @@ export async function resolveBedrockCredentialSession({ ) } catch (error) { signal?.throwIfAborted() - throw new BedrockCredentialError( - `Could not use the AWS credential: ${getErrorMessage(error, 'Unknown error')}`, - { reason: isTransientAwsFailure(error) ? 'unavailable' : 'unusable', cause: error } - ) + throw new BedrockCredentialError(awsCredentialFailureMessage(error), { + reason: isTransientAwsFailure(error) ? 'unavailable' : 'unusable', + cause: error, + }) } recordCredentialAccess({ diff --git a/apps/sim/lib/internal/llm/operations.test.ts b/apps/sim/lib/internal/llm/operations.test.ts index 0869b5c1a43..93209b0370a 100644 --- a/apps/sim/lib/internal/llm/operations.test.ts +++ b/apps/sim/lib/internal/llm/operations.test.ts @@ -2,6 +2,10 @@ import { billingAttributionMock, billingAttributionMockFns, } from '@sim/testing/mocks/billing-attribution.mock' +import { + internalAwsCredentialsMock, + internalAwsCredentialsMockFns, +} from '@sim/testing/mocks/internal-aws-credentials.mock' import { mothershipEnvironmentContextMock, mothershipEnvironmentContextMockFns, @@ -16,7 +20,6 @@ const mocks = vi.hoisted(() => ({ importProvenance: vi.fn(), isComplete: vi.fn(), resolveVertexAccessToken: vi.fn(), - resolveAuthorizedAwsCredential: vi.fn(), })) vi.mock('@/providers', () => providersMock) @@ -28,11 +31,7 @@ vi.mock('@/lib/mothership/environment-context', () => mothershipEnvironmentConte vi.mock('@/lib/internal/llm/credentials', () => ({ resolveVertexAccessToken: mocks.resolveVertexAccessToken, })) -vi.mock('@/lib/internal/aws/credentials', async (importOriginal) => ({ - isTransientAwsFailure: (await importOriginal()) - .isTransientAwsFailure, - resolveAuthorizedAwsCredential: mocks.resolveAuthorizedAwsCredential, -})) +vi.mock('@/lib/internal/aws/credentials', () => internalAwsCredentialsMock) vi.mock('@/lib/oauth/token-resolution', () => ({ recordCredentialAccess: vi.fn() })) vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({ @@ -41,6 +40,7 @@ vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({ vi.mock('@/ee/access-control/utils/permission-check', () => permissionCheckMock) const { mockRequireBillingAttributionHeader } = billingAttributionMockFns +const { mockResolveAuthorizedAwsCredential } = internalAwsCredentialsMockFns import type { LlmOperationError } from '@/lib/internal/llm/errors' import { executeLlmProviderOperation } from '@/lib/internal/llm/operations' @@ -134,7 +134,7 @@ describe('executeLlmProviderOperation', () => { it("signs a Bedrock call with the selected AWS credential's session for the caller", async () => { mocks.authorizeCredential.mockResolvedValueOnce(AUTHORIZED_AWS_CREDENTIAL) - mocks.resolveAuthorizedAwsCredential.mockResolvedValueOnce({ + mockResolveAuthorizedAwsCredential.mockResolvedValueOnce({ accessKeyId: 'ASIASESSIONKEY000001', secretAccessKey: 'session-secret', sessionToken: 'session-token', @@ -150,7 +150,7 @@ describe('executeLlmProviderOperation', () => { } ) - expect(mocks.resolveAuthorizedAwsCredential).toHaveBeenCalledWith( + expect(mockResolveAuthorizedAwsCredential).toHaveBeenCalledWith( 'aws-cred', { workspaceId: 'workspace-1', executionId: 'execution-1' }, undefined @@ -201,7 +201,7 @@ describe('executeLlmProviderOperation', () => { it('fails a Bedrock call whose AWS credential cannot be resolved before provider work', async () => { mockExecuteProviderRequest.mockClear() mocks.authorizeCredential.mockResolvedValueOnce(AUTHORIZED_AWS_CREDENTIAL) - mocks.resolveAuthorizedAwsCredential.mockRejectedValueOnce(new Error('AccessDenied')) + mockResolveAuthorizedAwsCredential.mockRejectedValueOnce(new Error('AccessDenied')) await expect( executeLlmProviderOperation(BEDROCK_CREDENTIAL_INPUT, { @@ -219,7 +219,7 @@ describe('executeLlmProviderOperation', () => { it('reports an AWS credential STS could not resolve for a moment as unavailable, not invalid', async () => { mockExecuteProviderRequest.mockClear() mocks.authorizeCredential.mockResolvedValueOnce(AUTHORIZED_AWS_CREDENTIAL) - mocks.resolveAuthorizedAwsCredential.mockRejectedValueOnce( + mockResolveAuthorizedAwsCredential.mockRejectedValueOnce( Object.assign(new Error('Rate exceeded'), { name: 'ThrottlingException', $metadata: { httpStatusCode: 400 }, diff --git a/apps/sim/lib/knowledge/application/connector-access.test.ts b/apps/sim/lib/knowledge/application/connector-access.test.ts index 11f031f2a64..1ea4ac82c75 100644 --- a/apps/sim/lib/knowledge/application/connector-access.test.ts +++ b/apps/sim/lib/knowledge/application/connector-access.test.ts @@ -13,6 +13,7 @@ import { credentialGroupsSelfEnrollmentMockFns, } from '@sim/testing/mocks/credential-groups-self-enrollment.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' import { githubInstallationMock, githubInstallationMockFns, @@ -61,6 +62,7 @@ vi.mock('@/lib/knowledge/application/connector-credential', () => ({ requireConnectorCredential: hoisted.credential, })) vi.mock('@/lib/core/security/encryption', () => encryptionMock) +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) vi.mock('@/lib/oauth/github-installation', () => githubInstallationMock) vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) vi.mock('@/connectors/registry', () => ({ getConnectorMeta: hoisted.meta })) @@ -132,6 +134,10 @@ const row = { encryptedApiKey: 'encrypted-fixture', sourceConfig: { host: 'gitlab.example.test', project: 'one' }, } +const savedAwsAuth = { + mode: 'apiKey', + savedCredential: { provider: 'aws', label: 'Saved AWS credential' }, +} as const beforeEach(() => { knowledgeContextsMockFns.mockResolveActiveKnowledgeConnectorContext.mockResolvedValue({ @@ -145,6 +151,7 @@ beforeEach(() => { }) workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') mocks.connector.mockResolvedValue(row) + featureFlagsMockFns.mockIsFeatureEnabled.mockResolvedValue(false) mocks.meta.mockReturnValue({ name: 'GitLab', auth: { mode: 'apiKey' }, mirrorsSourceAcls: true }) mocks.validate.mockResolvedValue(null) mocks.token.mockResolvedValue({ accessToken: 'provider-fixture' }) @@ -357,6 +364,37 @@ describe('connector access application boundary', () => { expect(mocks.validate).not.toHaveBeenCalled() }) + it('refuses moving a key connector that stores its key onto a saved credential', async () => { + featureFlagsMockFns.mockIsFeatureEnabled.mockResolvedValue(true) + mocks.meta.mockReturnValue({ name: 'Amazon S3', auth: savedAwsAuth }) + await expect( + updateKnowledgeConnectorAccess.execute({ + principal, + input: { ...input, accessMode: 'workspace', credentialId: 'aws-credential' }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mocks.update).not.toHaveBeenCalled() + }) + + it('lets a connector already on a saved credential move to another one', async () => { + featureFlagsMockFns.mockIsFeatureEnabled.mockResolvedValue(true) + mocks.meta.mockReturnValue({ name: 'Amazon S3', auth: savedAwsAuth }) + mocks.connector.mockResolvedValue({ + ...row, + credentialId: 'aws-credential', + encryptedApiKey: null, + }) + await updateKnowledgeConnectorAccess.execute({ + principal, + input: { ...input, accessMode: 'workspace', credentialId: 'other-aws-credential' }, + }) + expect(mocks.update).toHaveBeenCalledWith( + expect.objectContaining({ + target: expect.objectContaining({ credentialId: 'other-aws-credential' }), + }) + ) + }) + it('rejects separate content credentials for providers without the capability', async () => { mocks.meta.mockReturnValue({ name: 'Confluence', diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts index 7bcd21e0718..f0f37152b7c 100644 --- a/apps/sim/lib/knowledge/application/connector-access.ts +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -35,12 +35,14 @@ import { mirrorsSourceAcls, supportsConnectorAccessMode, } from '@/lib/knowledge/connectors/access-modes' +import { requireConnectorCredentialService } from '@/lib/knowledge/connectors/credential-service' import { validateKnowledgeConnectorMembersBinding } from '@/lib/knowledge/connectors/member-access' import { provisionKnowledgeConnectorMembersBinding, sourceIdentityBinding, } from '@/lib/knowledge/connectors/member-provisioning' import { assertConnectorMirrorsSourceAcls } from '@/lib/knowledge/connectors/mirrored-access' +import { requireConnectorSavedCredentialSetup } from '@/lib/knowledge/connectors/saved-credential-rollout' import { type ConnectorAccessTarget, performUpdateKnowledgeConnectorAccess, @@ -49,7 +51,7 @@ import { import { getKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' import type { KnowledgeOperationSource } from '@/lib/knowledge/orchestration/shared' import { requireOrganizationSearchApproval } from '@/lib/knowledge/search/integration-policy' -import { getServiceConfigByProviderId, getServiceConfigByServiceId } from '@/lib/oauth' +import { getConnectorCredentialService } from '@/connectors/auth' import { getConnectorMeta } from '@/connectors/registry' import type { ConnectorMeta } from '@/connectors/types' @@ -282,6 +284,7 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ await requireUsableCredential({ principal, credentialId, + storesApiKey: Boolean(connector.encryptedApiKey), connectorMeta, sourceConfig, ...owner, @@ -312,6 +315,7 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ credentialId: await requireUsableCredential({ principal, credentialId: input.credentialId, + storesApiKey: Boolean(connector.encryptedApiKey), connectorMeta, sourceConfig, ...owner, @@ -391,10 +395,16 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ * provider, and one that yields a token, since the connector syncs as it from * then on. API-key connectors keep their canonical encrypted key; the shared * source validation verifies it against the target mode before any mutation. + * + * A key connector that stores its key never moves onto its saved-credential + * alternative: the key would stay on the row and silently take over again once + * the credential is deleted, so switching means creating the connector anew. */ async function requireUsableCredential(input: { principal: Principal credentialId: string | null | undefined + /** Whether the connector row stores an encrypted API key. */ + storesApiKey: boolean connectorMeta: Pick sourceConfig: Record workspaceId?: string @@ -404,26 +414,28 @@ async function requireUsableCredential(input: { accessMode: ConnectorAccessMode }): Promise { const { auth } = input.connectorMeta - if (auth.mode !== 'oauth') { + const credentialService = getConnectorCredentialService(auth) + if (!credentialService) { if (input.credentialId) { throw new OrchestrationError('validation', 'API-key connectors use their configured API key') } return null } if (!input.credentialId) { + if (auth.mode === 'apiKey') return null throw new OrchestrationError( 'validation', 'credentialId is required for a mode that syncs with one credential' ) } - const service = - getServiceConfigByServiceId(auth.provider) ?? getServiceConfigByProviderId(auth.provider) - if (!service) { + if (auth.mode === 'apiKey' && input.storesApiKey) { throw new OrchestrationError( 'validation', - `${input.connectorMeta.name} has no OAuth service to validate the credential against` + `This ${input.connectorMeta.name} connector signs with its stored key. Create a new connector to use a saved credential.` ) } + await requireConnectorSavedCredentialSetup(auth, input.credentialId) + const service = requireConnectorCredentialService(input.connectorMeta) const token = await resolveConnectorCredentialAccessToken({ principal: input.principal, credentialId: input.credentialId, diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index 2156453d5a2..c7dbf6c5df3 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -8,6 +8,11 @@ import { credentialsAccessMockFns, } from '@sim/testing/mocks/credentials-access.mock' import { environmentUtilsMockFns } from '@sim/testing/mocks/environment-utils.mock' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { + internalAwsCredentialsMock, + internalAwsCredentialsMockFns, +} from '@sim/testing/mocks/internal-aws-credentials.mock' import { knowledgeAccessScopeMock, knowledgeAccessScopeMockFns, @@ -71,6 +76,8 @@ vi.mock('@/lib/credentials/application/organization-credentials', () => ({ })) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) +vi.mock('@/lib/internal/aws/credentials', () => internalAwsCredentialsMock) +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) vi.mock('@/lib/api-key/crypto', () => ({ decryptApiKey: hoisted.decryptApiKey })) vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) @@ -89,6 +96,15 @@ vi.mock('@/connectors/registry.server', () => ({ auth: { mode: 'oauth', requiredScopes: ['read:confluence-content.all'] }, validateConfig: hoisted.validateConnectorConfig, }, + s3: { + name: 'Amazon S3', + auth: { + mode: 'apiKey', + label: 'Secret Access Key', + savedCredential: { provider: 'aws', label: 'Saved AWS credential' }, + }, + validateConfig: hoisted.validateConnectorConfig, + }, google_drive: { name: 'Google Drive', auth: { @@ -130,6 +146,7 @@ import { confluenceConnectorMeta } from '@/connectors/confluence/meta' import { gmailConnectorMeta } from '@/connectors/gmail/meta' import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' +import { s3ConnectorMeta } from '@/connectors/s3/meta' const mocks = { ...hoisted, @@ -341,6 +358,43 @@ describe('knowledge connector application use cases', () => { expect(auditMockFns.mockRecordAudit).not.toHaveBeenCalled() }) + it('refuses another service’s credential for an S3 connector before resolving it', async () => { + featureFlagsMockFns.mockIsFeatureEnabled.mockResolvedValueOnce(true) + mocks.getCredentialActorContext.mockResolvedValueOnce({ + credential: { + id: 'credential-1', + workspaceId: 'workspace-b', + providerId: 'google-service-account', + }, + member: { role: 'admin' }, + hasWorkspaceAccess: true, + canWriteWorkspace: true, + isAdmin: true, + }) + mocks.createConnector.mockImplementationOnce( + async (input: { resolveAccessToken: (credentialId: string) => Promise }) => { + await input.resolveAccessToken('credential-1') + throw new Error('An ineligible credential reached connector persistence') + } + ) + await expect( + createKnowledgeConnector.execute({ + principal: patPrincipal, + input: { + knowledgeBaseId: 'knowledge-b', + connectorType: 's3', + credentialId: 'credential-1', + sourceConfig: { region: 'us-east-1', bucket: 'docs' }, + syncIntervalMinutes: 1440, + }, + }) + ).rejects.toMatchObject({ + code: 'validation', + message: expect.stringContaining('belongs to another service'), + }) + expect(internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential).not.toHaveBeenCalled() + }) + it('rejects a personal Confluence OAuth credential before central token use', async () => { await expect( resolveConnectorCredentialAccessToken({ @@ -1022,6 +1076,75 @@ describe('organization connector credential authorization', () => { } ) + it.each([ + ['refused by AWS', { name: 'AccessDenied' }, 400], + ['throttled by AWS', { name: 'ThrottlingException', $metadata: { httpStatusCode: 400 } }, 500], + ])( + 'projects a saved AWS credential %s with the matching setup status', + async (_case, failure, status) => { + mocks.authorizeOrganizationCredentialUse.mockResolvedValueOnce({ + credential: { ...credential, providerId: 'aws' }, + userId: principal.userId, + }) + internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential.mockRejectedValueOnce( + Object.assign(new Error('AWS said no'), failure) + ) + const error = await resolveConnectorCredentialAccessToken({ + ...input, + auth: s3ConnectorMeta.auth, + accessMode: 'workspace', + sourceConfig: { region: 'us-east-1', bucket: 'docs' }, + }).catch((error: unknown) => error) + expect(internalOrchestrationErrorPolicy.project(error)?.status ?? 500).toBe(status) + } + ) + + it.each(['result', 'exception'] as const)( + 'redacts a saved AWS session from settings validation %s', + async (failure) => { + const awsSession = { + accessKeyId: 'ASIAEXAMPLE', + secretAccessKey: 'session/secret', + sessionToken: 'session-token', + } + mocks.authorizeOrganizationCredentialUse.mockResolvedValueOnce({ + credential: { ...credential, providerId: 'aws' }, + userId: principal.userId, + }) + internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential.mockResolvedValueOnce( + awsSession + ) + const message = `Denied ${awsSession.secretAccessKey} ${awsSession.sessionToken}` + if (failure === 'exception') { + mocks.validateConnectorConfig.mockRejectedValueOnce( + new OrchestrationError('validation', message) + ) + } else { + mocks.validateConnectorConfig.mockResolvedValueOnce({ valid: false, error: message }) + } + const result = validateConnectorSourceConfig({ + principal, + requestId: 'request', + organizationId: 'org', + actingUserId: principal.userId, + sourceConfig: { region: 'us-east-1', bucket: 'docs' }, + connector: { + id: 'connector-s3', + connectorType: 's3', + credentialId: credential.id, + encryptedApiKey: null, + accessMode: 'workspace', + } as Parameters[0]['connector'], + }) + const redacted = 'Denied [REDACTED] [REDACTED]' + if (failure === 'exception') { + await expect(result).rejects.toMatchObject({ code: 'validation', message: redacted }) + } else { + await expect(result).resolves.toEqual({ errorCode: 'validation', message: redacted }) + } + } + ) + it('does not mint a token after the credential creator leaves the organization', async () => { mocks.resolveTokenIdentity.mockResolvedValueOnce(null) await expect(resolveConnectorCredentialAccessToken(input)).resolves.toBeNull() diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 33a2b3ab683..735932b5178 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -58,9 +58,12 @@ import { } from '@/lib/knowledge/connectors/access-modes' import { type ConnectorAccessToken, + ConnectorAwsCredentialError, + connectorTokenSecrets, resolveConnectorAccessToken, syncContextForToken, } from '@/lib/knowledge/connectors/access-token' +import { requireConnectorCredentialService } from '@/lib/knowledge/connectors/credential-service' import { requiresConnectorIndexing } from '@/lib/knowledge/connectors/indexing-policy' import { provisionKnowledgeConnectorMembersBinding, @@ -78,6 +81,7 @@ import { readConnectorPermissionSummaries, readConnectorPermissionSummary, } from '@/lib/knowledge/connectors/permission-config.server' +import { requireConnectorSavedCredentialSetup } from '@/lib/knowledge/connectors/saved-credential-rollout' import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' import { connectorIsLive } from '@/lib/knowledge/connectors/sync-lock' import { @@ -118,7 +122,12 @@ import { ServiceAccountTokenError } from '@/lib/oauth/credential-service' import { CAPABILITY_RULES, refuseCapability } from '@/lib/permission-groups/capabilities' import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' import { getUserPermissionConfigForOrganization } from '@/lib/permission-groups/resolve.server' -import { getConnectorApiKeyConfig, isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { + connectorAuthModeLabel, + connectorUsesCredential, + getConnectorApiKeyConfig, + isConnectorCredentialTypeAllowed, +} from '@/connectors/auth' import { getConnectorMeta } from '@/connectors/registry' import type { ConnectorAuthConfig } from '@/connectors/types' import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' @@ -322,12 +331,16 @@ export async function resolveConnectorCredentialAccessToken(input: { connector: { credentialId: input.credentialId, encryptedApiKey: null }, userId: identity.kind === 'oauth' ? identity.userId : input.actingUserId, requestId: input.requestId, + scope: resourceScopeFromOwner(input), sourceConfig: input.sourceConfig, }) } /** Exposes actionable credential refusals without returning raw provider payloads. */ function rethrowConnectorCredentialError(error: unknown): never { + if (error instanceof ConnectorAwsCredentialError && !error.transient) { + throw new OrchestrationError('validation', error.message) + } if (error instanceof ServiceAccountTokenError && [400, 401, 403].includes(error.statusCode)) { let message: string switch (error.errorCode) { @@ -420,10 +433,7 @@ export async function validateConnectorSourceConfig(input: { */ let tokenUserId = input.actingUserId const apiKeyConfig = getConnectorApiKeyConfig(connectorConfig.auth) - if ( - connectorConfig.auth.mode === 'apiKey' || - (apiKeyConfig && !input.connector.credentialId && input.connector.encryptedApiKey) - ) { + if (!connectorUsesCredential(connectorConfig.auth, input.connector)) { if (!input.connector.encryptedApiKey && !apiKeyConfig?.optional) { return { message: 'API key not found. Please reconfigure the connector.', @@ -464,6 +474,7 @@ export async function validateConnectorSourceConfig(input: { connector: input.connector, userId: tokenUserId, requestId: input.requestId, + scope: resourceScopeFromOwner(input), sourceConfig: input.sourceConfig, }) if (!resolved) { @@ -488,19 +499,21 @@ export async function validateConnectorSourceConfig(input: { ) } } + const secrets = connectorTokenSecrets(resolved) const validation = await connectorConfig .validateConfig(resolved.accessToken, input.sourceConfig, validationContext) .catch((error: unknown) => { const sanitized = toError(error) - sanitized.message = redactKnownSensitiveValues(sanitized.message, [resolved.accessToken]) + sanitized.message = redactKnownSensitiveValues(sanitized.message, secrets) throw sanitized }) return validation.valid ? null : { - message: redactKnownSensitiveValues(validation.error || 'Invalid source configuration', [ - resolved.accessToken, - ]), + message: redactKnownSensitiveValues( + validation.error || 'Invalid source configuration', + secrets + ), errorCode: 'validation', } } @@ -741,6 +754,12 @@ async function executeCreateKnowledgeConnector({ if (input.apiKey && input.credentialId) { throw new OrchestrationError('validation', 'Choose either an OAuth account or an API key') } + await requireConnectorSavedCredentialSetup(connectorMeta.auth, input.credentialId) + /** A key connector's saved credential must be of its declared service, as on an access update. */ + const savedCredentialService = + connectorMeta.auth.mode === 'apiKey' && connectorMeta.auth.savedCredential && input.credentialId + ? requireConnectorCredentialService(connectorMeta) + : undefined if ( context.knowledgeBase.isSearchIndex && (!connectorMeta.search || !input.accessMode || input.accessMode === 'workspace') @@ -840,6 +859,7 @@ async function executeCreateKnowledgeConnector({ ...owner, actingUserId, requestId, + ...(savedCredentialService ? { service: savedCredentialService } : {}), auth: connectorMeta.auth, accessMode: input.accessMode ?? 'workspace', sourceConfig, @@ -890,7 +910,10 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ knowledgeBaseName: context.knowledgeBase.name, connectorType: result.connector.connectorType, syncIntervalMinutes: result.connector.syncIntervalMinutes, - authMode: result.connector.credentialId ? 'oauth' : 'apiKey', + authMode: connectorAuthModeLabel( + getConnectorMeta(result.connector.connectorType)?.auth.mode ?? 'oauth', + Boolean(result.connector.credentialId) + ), accessMode: result.connector.accessMode, }, }, diff --git a/apps/sim/lib/knowledge/connectors/access-token.test.ts b/apps/sim/lib/knowledge/connectors/access-token.test.ts index 0674996d174..afdd14a92e6 100644 --- a/apps/sim/lib/knowledge/connectors/access-token.test.ts +++ b/apps/sim/lib/knowledge/connectors/access-token.test.ts @@ -1,21 +1,30 @@ import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' +import { + internalAwsCredentialsMock, + internalAwsCredentialsMockFns, +} from '@sim/testing/mocks/internal-aws-credentials.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockDecryptApiKey } = vi.hoisted(() => ({ mockDecryptApiKey: vi.fn() })) vi.mock('@/lib/api-key/crypto', () => ({ decryptApiKey: mockDecryptApiKey })) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) +vi.mock('@/lib/internal/aws/credentials', () => internalAwsCredentialsMock) import type { ConnectorAccessMode } from '@/lib/knowledge/connectors/access-modes' import { + ConnectorAwsCredentialError, connectorServiceAccountScopes, connectorServiceAccountSubject, + createConnectorRunToken, resolveConnectorAccessToken, syncContextForToken, } from '@/lib/knowledge/connectors/access-token' import { isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { readAwsSession } from '@/connectors/aws-session' import { gmailConnectorMeta } from '@/connectors/gmail/meta' import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' +import { s3ConnectorMeta } from '@/connectors/s3/meta' import type { ConnectorAuthConfig } from '@/connectors/types' const { @@ -32,6 +41,8 @@ const OAUTH_AUTH: ConnectorAuthConfig = { const NO_CREDENTIAL = { credentialId: null, encryptedApiKey: null } +const WORKSPACE_SCOPE = { kind: 'workspace', workspaceId: 'workspace-1' } as const + function credentialConnector(credentialId: string) { return { credentialId, encryptedApiKey: null } } @@ -62,6 +73,7 @@ describe('resolveConnectorAccessToken', () => { connector: credentialConnector('docusign-credential'), userId: 'user-1', requestId: 'req-1', + scope: WORKSPACE_SCOPE, sourceConfig: {}, }) @@ -83,6 +95,7 @@ describe('resolveConnectorAccessToken', () => { connector: { credentialId: null, encryptedApiKey: 'cipher' }, userId: 'user-1', requestId: 'req-1', + scope: WORKSPACE_SCOPE, sourceConfig: {}, }) ).rejects.toThrow('missing credential ID') @@ -96,6 +109,7 @@ describe('resolveConnectorAccessToken', () => { connector: NO_CREDENTIAL, userId: 'user-1', requestId: 'req-1', + scope: WORKSPACE_SCOPE, sourceConfig: {}, }) ).rejects.toThrow('missing encrypted API key') @@ -108,6 +122,7 @@ describe('resolveConnectorAccessToken', () => { connector: NO_CREDENTIAL, userId: 'user-1', requestId: 'req-1', + scope: WORKSPACE_SCOPE, sourceConfig: {}, }) ).rejects.toThrow('missing credential ID') @@ -124,12 +139,110 @@ describe('resolveConnectorAccessToken', () => { connector: credentialConnector('credential-1'), userId: 'credential-owner', requestId: 'req-1', + scope: WORKSPACE_SCOPE, sourceConfig: {}, }) ).resolves.toBeNull() }) }) +describe('saved AWS credential on a key connector', () => { + const session = { + accessKeyId: 'ASIAIOSFODNN7EXAMPLE', + secretAccessKey: 'session-secret', + sessionToken: 'session-token', + expiration: new Date('2026-11-06T13:00:00Z'), + } + + beforeEach(() => { + mockDecryptApiKey.mockResolvedValue({ decrypted: 'inline-secret-key' }) + internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential.mockReset() + }) + + it('asks for the credential in the knowledge base scope and seeds its session, not a bearer token', async () => { + internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential.mockResolvedValueOnce(session) + const organizationScope = { kind: 'organization', organizationId: 'org-1' } as const + + const token = await resolveConnectorAccessToken({ + auth: s3ConnectorMeta.auth, + connector: credentialConnector('aws-credential'), + userId: 'user-1', + requestId: 'sync-run', + scope: organizationScope, + sourceConfig: {}, + }) + if (!token) throw new Error('Expected an AWS connector token') + + const [credentialId, context] = + internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential.mock.calls[0] + expect(credentialId).toBe('aws-credential') + expect(context).toMatchObject({ organizationId: 'org-1' }) + expect(context.workspaceId).toBeUndefined() + expect(token.expiresAt).toEqual(session.expiration) + expect(token.accessToken).toBe('') + expect(readAwsSession(syncContextForToken(token))).toEqual({ + accessKeyId: session.accessKeyId, + secretAccessKey: session.secretAccessKey, + sessionToken: session.sessionToken, + }) + expect(mockResolveTokenBundle).not.toHaveBeenCalled() + }) + + it('keeps a stored-key S3 connector on its key without touching AWS', async () => { + const token = await resolveConnectorAccessToken({ + auth: s3ConnectorMeta.auth, + connector: { credentialId: null, encryptedApiKey: 'encrypted' }, + userId: 'user-1', + requestId: 'sync-run', + scope: WORKSPACE_SCOPE, + sourceConfig: {}, + }) + + expect(token).toEqual({ accessToken: 'inline-secret-key' }) + expect(internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential).not.toHaveBeenCalled() + }) + + it('reports an unusable AWS credential without leaking its keys', async () => { + internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential.mockRejectedValueOnce( + Object.assign(new Error('Not authorized to perform sts:AssumeRoleWithWebIdentity'), { + name: 'AccessDenied', + }) + ) + + await expect( + resolveConnectorAccessToken({ + auth: s3ConnectorMeta.auth, + connector: credentialConnector('aws-credential'), + userId: 'user-1', + requestId: 'sync-run', + scope: WORKSPACE_SCOPE, + sourceConfig: {}, + }) + ).rejects.toMatchObject({ name: 'ConnectorAwsCredentialError', transient: false }) + }) + + it('marks an AWS outage as transient rather than a refused credential', async () => { + internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential.mockRejectedValueOnce( + Object.assign(new Error('Rate exceeded'), { + name: 'ThrottlingException', + $metadata: { httpStatusCode: 400 }, + }) + ) + + const failure = await resolveConnectorAccessToken({ + auth: s3ConnectorMeta.auth, + connector: credentialConnector('aws-credential'), + userId: 'user-1', + requestId: 'sync-run', + scope: WORKSPACE_SCOPE, + sourceConfig: {}, + }).catch((error: unknown) => error) + + expect(failure).toBeInstanceOf(ConnectorAwsCredentialError) + expect(failure).toMatchObject({ transient: true }) + }) +}) + describe('connectorServiceAccountSubject', () => { const withSubject: ConnectorAuthConfig = { ...OAUTH_AUTH, @@ -166,6 +279,7 @@ describe('delegated connector access', () => { connector: credentialConnector('service-credential'), userId: 'actor', requestId: 'request', + scope: WORKSPACE_SCOPE, sourceConfig: { adminEmail: 'admin@example.com' }, }) } @@ -273,6 +387,7 @@ describe.each([ connector: credentialConnector('google-service'), userId: 'actor', requestId: 'request', + scope: WORKSPACE_SCOPE, sourceConfig: { adminEmail: ' Directory.Admin@Example.com ', scopes: ['untrusted.write'], @@ -349,6 +464,7 @@ describe('impersonation on the connector path', () => { connector: credentialConnector('credential-1'), userId: 'credential-owner', requestId: 'req-1', + scope: WORKSPACE_SCOPE, sourceConfig: { adminEmail: 'admin@corp.com' }, }) ).resolves.toEqual({ accessToken: 'access-token' }) @@ -362,3 +478,92 @@ describe('impersonation on the connector path', () => { ) }) }) + +describe('createConnectorRunToken', () => { + const NOW = new Date('2026-11-06T12:00:00Z') + const session = (accessKeyId: string, expiresAt: Date) => ({ + accessToken: 'aws-credential', + awsSession: { accessKeyId, secretAccessKey: `${accessKeyId}-secret`, sessionToken: 'token' }, + expiresAt, + }) + + beforeEach(() => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(NOW) + }) + + it('keeps an STS session that has time left instead of exchanging it on every page', async () => { + const resolve = vi.fn() + const syncContext: Record = {} + const runToken = createConnectorRunToken({ + token: session('ASIA1', new Date(NOW.getTime() + 30 * 60_000)), + syncContext, + resolve, + }) + + await runToken.refresh() + + expect(resolve).not.toHaveBeenCalled() + expect(runToken.current().awsSession?.accessKeyId).toBe('ASIA1') + }) + + it('replaces a session close to expiry and signs later calls with the new one', async () => { + const renewed = session('ASIA2', new Date(NOW.getTime() + 60 * 60_000)) + const syncContext: Record = {} + const runToken = createConnectorRunToken({ + token: session('ASIA1', new Date(NOW.getTime() + 60_000)), + syncContext, + resolve: async () => renewed, + }) + + await runToken.refresh() + + expect(runToken.current()).toBe(renewed) + expect(readAwsSession(syncContext)?.accessKeyId).toBe('ASIA2') + }) + + it('re-resolves a token that states no expiry on every refresh', async () => { + const resolve = vi.fn(async () => ({ accessToken: 'fresh-oauth-token' })) + const runToken = createConnectorRunToken({ + token: { accessToken: 'oauth-token' }, + syncContext: {}, + resolve, + }) + + await runToken.refresh() + + expect(resolve).toHaveBeenCalledTimes(1) + expect(runToken.current().accessToken).toBe('fresh-oauth-token') + }) + + it('keeps long-lived saved AWS keys for the whole run instead of reloading them on every page', async () => { + const resolve = vi.fn() + const runToken = createConnectorRunToken({ + token: { + accessToken: '', + awsSession: { accessKeyId: 'AKIA1', secretAccessKey: 'AKIA1-secret' }, + }, + syncContext: {}, + resolve, + }) + + await runToken.refresh() + + expect(resolve).not.toHaveBeenCalled() + expect(runToken.current().awsSession?.accessKeyId).toBe('AKIA1') + }) + + it('never re-resolves a stored key', async () => { + const resolve = vi.fn() + const runToken = createConnectorRunToken({ + token: { accessToken: 'inline-key' }, + syncContext: {}, + resolve: undefined, + }) + + await runToken.refresh() + + expect(resolve).not.toHaveBeenCalled() + expect(runToken.current().accessToken).toBe('inline-key') + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/access-token.ts b/apps/sim/lib/knowledge/connectors/access-token.ts index 0464a520749..2c0027906fd 100644 --- a/apps/sim/lib/knowledge/connectors/access-token.ts +++ b/apps/sim/lib/knowledge/connectors/access-token.ts @@ -1,7 +1,12 @@ import { isValidEmailSyntax, normalizeEmail } from '@sim/utils/string' import { decryptApiKey } from '@/lib/api-key/crypto' -import { resourceScopeFromOwner } from '@/lib/core/resource-scope' +import { + type ResourceScope, + resourceScopeFields, + resourceScopeFromOwner, +} from '@/lib/core/resource-scope' import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' +import { awsCredentialFailureMessage, isTransientAwsFailure } from '@/lib/internal/aws/failure' import type { ConnectorAccessMode } from '@/lib/knowledge/connectors/access-modes' import { getServiceAccountToken, @@ -9,7 +14,12 @@ import { resolveOAuthAccountId, } from '@/lib/oauth/credential-service' import { GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' -import { getConnectorApiKeyConfig } from '@/connectors/auth' +import { + connectorUsesCredential, + getConnectorApiKeyConfig, + getConnectorCredentialService, +} from '@/connectors/auth' +import { awsSessionSyncContext, type ConnectorAwsSession } from '@/connectors/aws-session' import type { ConnectorAuthConfig } from '@/connectors/types' /** @@ -36,6 +46,83 @@ export interface ConnectorAccessToken { instanceUrl?: string /** Server-only capability bound to this credential and the connector's declared scopes. */ getDelegatedAccessToken?: (subject: string, signal?: AbortSignal) => Promise + /** + * AWS only — the keys a saved AWS credential resolved to for the knowledge + * base's scope. A saved AWS credential has no bearer token, so `accessToken` + * is empty and the connector signs with these instead. + */ + awsSession?: ConnectorAwsSession + /** When the credential stops working, if it says; a holder may reuse it until shortly before. */ + expiresAt?: Date +} + +/** + * Every secret a resolved token carries, for redacting provider text that may + * echo one back. A saved AWS credential's secrets sit in `awsSession`, not in + * the empty bearer, so callers redact this list rather than `accessToken` alone. + */ +export function connectorTokenSecrets(token: ConnectorAccessToken): string[] { + return [ + token.accessToken, + ...(token.awsSession + ? [token.awsSession.secretAccessKey, token.awsSession.sessionToken ?? ''] + : []), + ] +} + +/** The connector-facing service that a saved AWS credential authenticates. */ +const AWS_CREDENTIAL_SERVICE = 'aws' + +/** + * A saved AWS credential that could not be used: unavailable in this scope, + * refused by AWS STS, or not resolvable while AWS was briefly unavailable + * (`transient`). The message is the AWS reason, never a secret. + */ +export class ConnectorAwsCredentialError extends Error { + readonly transient: boolean + + constructor(message: string, options: { cause: unknown; transient: boolean }) { + super(message, { cause: options.cause }) + this.name = 'ConnectorAwsCredentialError' + this.transient = options.transient + } +} + +/** + * Resolves a saved AWS credential to signing keys for the knowledge base's own + * scope. The owner check and the STS exchange are the same ones AWS tools use, + * so a credential from another workspace or organization never resolves here. + */ +async function resolveConnectorAwsToken( + credentialId: string, + scope: ResourceScope, + requestId: string, + signal: AbortSignal | undefined +): Promise { + const { resolveAuthorizedAwsCredential } = await import('@/lib/internal/aws/credentials') + let session: Awaited> + try { + session = await resolveAuthorizedAwsCredential( + credentialId, + { ...resourceScopeFields(scope), executionId: requestId }, + signal + ) + } catch (error) { + signal?.throwIfAborted() + throw new ConnectorAwsCredentialError(awsCredentialFailureMessage(error), { + cause: error, + transient: isTransientAwsFailure(error), + }) + } + return { + accessToken: '', + awsSession: { + accessKeyId: session.accessKeyId, + secretAccessKey: session.secretAccessKey, + ...(session.sessionToken ? { sessionToken: session.sessionToken } : {}), + }, + ...(session.expiration ? { expiresAt: session.expiration } : {}), + } } /** @@ -95,6 +182,11 @@ export async function resolveConnectorAccessToken(params: { connector: { credentialId: string | null; encryptedApiKey: string | null } userId: string requestId: string + /** + * The knowledge base's owner. A credential that resolves per scope (a saved + * AWS credential) is loaded only from this workspace or organization. + */ + scope: ResourceScope /** * Where a service-account credential's impersonation subject lives. Always * passed on a path that has it. Without a subject, a Drive service account @@ -108,10 +200,7 @@ export async function resolveConnectorAccessToken(params: { params.signal?.throwIfAborted() const apiKeyConfig = getConnectorApiKeyConfig(auth) - if ( - auth.mode === 'apiKey' || - (apiKeyConfig && !connector.credentialId && connector.encryptedApiKey) - ) { + if (!connectorUsesCredential(auth, connector)) { if (!connector.encryptedApiKey) { if (apiKeyConfig?.optional) return { accessToken: '' } throw new Error('API key connector is missing encrypted API key') @@ -124,6 +213,13 @@ export async function resolveConnectorAccessToken(params: { throw new Error('OAuth connector is missing credential ID') } + if (getConnectorCredentialService(auth) === AWS_CREDENTIAL_SERVICE) { + return resolveConnectorAwsToken(connector.credentialId, params.scope, requestId, params.signal) + } + if (auth.mode !== 'oauth') { + throw new Error('This connector cannot authenticate with a saved credential') + } + const subject = connectorServiceAccountSubject(auth, params.sourceConfig) const githubRepositoryScope = auth.mode === 'oauth' && auth.provider === 'github-repositories' @@ -239,5 +335,43 @@ export function syncContextForToken(token: ConnectorAccessToken): Record +} + +/** + * Holds the token one connector run syncs with. Each refresh re-resolves it, so a long run + * outlives a short-lived token, and re-seeds what the token binds into the run's + * `syncContext`, so a connector reading its credential's site or AWS session from there signs + * with the current one. A token that states its expiry (an AWS STS session) is kept until + * shortly before it rather than exchanged again on every page. Long-lived saved AWS keys (an + * AWS session with no session token) are kept for the whole run, like a stored key, for which + * `resolve` is absent. + */ +export function createConnectorRunToken(input: { + token: ConnectorAccessToken + syncContext: Record + resolve: (() => Promise) | undefined +}): ConnectorRunToken { + let token = input.token + return { + current: () => token, + async refresh() { + if (!input.resolve) return + if (token.awsSession && !token.awsSession.sessionToken) return + if (token.expiresAt && token.expiresAt.getTime() - Date.now() > TOKEN_REUSE_MARGIN_MS) return + token = await input.resolve() + Object.assign(input.syncContext, syncContextForToken(token)) + }, } } diff --git a/apps/sim/lib/knowledge/connectors/credential-service.ts b/apps/sim/lib/knowledge/connectors/credential-service.ts new file mode 100644 index 00000000000..ecc19537b64 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/credential-service.ts @@ -0,0 +1,33 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + getServiceConfigByProviderId, + getServiceConfigByServiceId, + type ServiceProviderIdentity, +} from '@/lib/oauth' +import { getConnectorCredentialService } from '@/connectors/auth' +import type { ConnectorMeta } from '@/connectors/types' + +/** + * The OAuth service a connector's credential must belong to. Setting up a key + * connector on its saved credential and every access update resolve it here, so + * a connector whose declared service has no config is refused instead of + * skipping the provider check. Callers reach it only for a connector that takes + * a credential, so a missing service is a connector-definition fault, not one + * the caller can fix. + */ +export function requireConnectorCredentialService( + connectorMeta: Pick +): ServiceProviderIdentity { + const credentialService = getConnectorCredentialService(connectorMeta.auth) + const service = credentialService + ? (getServiceConfigByServiceId(credentialService) ?? + getServiceConfigByProviderId(credentialService)) + : null + if (!service) { + throw new OrchestrationError( + 'internal', + `${connectorMeta.name} has no OAuth service to validate the credential against` + ) + } + return service +} diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.ts index 13d94d92396..91b77e52d90 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.ts @@ -629,6 +629,7 @@ export async function refreshConnectorDirectory( connector, userId: credentialUserId, requestId, + scope: resourceScopeFromOwner(connector), sourceConfig, }) if (!token) return 'unusable' diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 2d8318add4f..3e5ffe7088b 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -1657,6 +1657,7 @@ async function syncDedicatedMemberContent(input: { connector, userId, requestId: run.runId, + scope: resourceScopeFromOwner(run), sourceConfig, }) if (!token) throw new Error('The content credential needs to be reconnected') diff --git a/apps/sim/lib/knowledge/connectors/saved-credential-rollout.test.ts b/apps/sim/lib/knowledge/connectors/saved-credential-rollout.test.ts new file mode 100644 index 00000000000..d18540824c6 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/saved-credential-rollout.test.ts @@ -0,0 +1,44 @@ +/** + * Pins the setup gate for a key connector's saved-credential alternative. Workers from an + * earlier release treat a key connector without a key as one whose credential was removed and + * unschedule it, so no such connector may be set up until the switch is turned on. + */ +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +import { requireConnectorSavedCredentialSetup } from '@/lib/knowledge/connectors/saved-credential-rollout' +import { databricksConnectorMeta } from '@/connectors/databricks/meta' +import { s3ConnectorMeta } from '@/connectors/s3/meta' + +const mockIsFeatureEnabled = featureFlagsMockFns.mockIsFeatureEnabled + +describe('requireConnectorSavedCredentialSetup', () => { + beforeEach(() => { + mockIsFeatureEnabled.mockReset() + }) + + it('refuses to bind a saved credential to S3 while setup is off', async () => { + mockIsFeatureEnabled.mockResolvedValueOnce(false) + + await expect( + requireConnectorSavedCredentialSetup(s3ConnectorMeta.auth, 'aws-credential') + ).rejects.toThrow(/cannot use a saved credential yet/) + }) + + it('allows a saved credential once setup is on', async () => { + mockIsFeatureEnabled.mockResolvedValueOnce(true) + + await expect( + requireConnectorSavedCredentialSetup(s3ConnectorMeta.auth, 'aws-credential') + ).resolves.toBeUndefined() + }) + + it('leaves key setup, and key-only connectors, untouched by the switch', async () => { + await requireConnectorSavedCredentialSetup(s3ConnectorMeta.auth, undefined) + await requireConnectorSavedCredentialSetup(databricksConnectorMeta.auth, 'stray-credential') + + expect(mockIsFeatureEnabled).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/saved-credential-rollout.ts b/apps/sim/lib/knowledge/connectors/saved-credential-rollout.ts new file mode 100644 index 00000000000..3f10b42fa70 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/saved-credential-rollout.ts @@ -0,0 +1,31 @@ +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { ConnectorAuthConfig } from '@/connectors/types' + +/** + * Whether a key-based connector may be set up with its saved-credential alternative. + * + * A global switch that gates setup only. Workers from an older release read a key + * connector without a key as one whose credential was removed and unschedule it, so + * the switch stays off until every app and worker runs a release that reads these rows. + * Turning it off later does not protect rows already set up: once any exist, a release + * rollback unschedules them as credential-removed, and they stay idle until edited or + * synced manually. + */ +export function isConnectorSavedCredentialSetupEnabled(): Promise { + return isFeatureEnabled('knowledge-connector-saved-credentials') +} + +/** Refuses to bind a saved credential to a key-based connector while setup is not enabled. */ +export async function requireConnectorSavedCredentialSetup( + auth: ConnectorAuthConfig, + credentialId: string | null | undefined +): Promise { + if (auth.mode !== 'apiKey' || !auth.savedCredential || !credentialId) return + if (!(await isConnectorSavedCredentialSetupEnabled())) { + throw new OrchestrationError( + 'validation', + 'This source cannot use a saved credential yet. Use its API key instead.' + ) + } +} diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 6fcf9c9f905..2265263d505 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -20,7 +20,11 @@ import { type BillingAttributionSnapshot, } from '@/lib/billing/core/attribution' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' -import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' +import { + type ResourceScope, + resourceScopeFields, + resourceScopeFromOwner, +} from '@/lib/core/resource-scope' import { EMPTY_ACL } from '@/lib/knowledge/access/tokens' import type { MirroredDocumentAcl } from '@/lib/knowledge/access/types' import { @@ -32,6 +36,7 @@ import { } from '@/lib/knowledge/connectors/access-modes' import { type ConnectorAccessToken, + createConnectorRunToken, resolveConnectorAccessToken, resolveConnectorTokenUserId, syncContextForToken, @@ -99,7 +104,7 @@ import { hardDeleteDocuments } from '@/lib/knowledge/documents/service' import { getRetryAfterMs, isRateLimitError } from '@/lib/knowledge/documents/utils' import { getCredentialTerminalRefreshError } from '@/lib/oauth/credential-service' import { isCredentialRevocationError } from '@/lib/oauth/refresh-error-codes' -import { connectorHasAuthSource } from '@/connectors/auth' +import { connectorHasAuthSource, connectorUsesCredential } from '@/connectors/auth' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' import type { ConnectorAuthConfig, @@ -799,7 +804,8 @@ async function resolveAccessToken( connectorConfig: { auth: ConnectorAuthConfig }, userId: string, sourceConfig: Record, - accessMode: ContentEngineAccessMode + accessMode: ContentEngineAccessMode, + scope: ResourceScope ): Promise { const requestId = `sync-${connector.credentialId}` const resolved = await resolveConnectorAccessToken({ @@ -808,6 +814,7 @@ async function resolveAccessToken( connector, userId, requestId, + scope, sourceConfig, }) @@ -1108,25 +1115,15 @@ export async function executeSync( ) } - let credentialToken = await resolveAccessToken( + const kbScope = resourceScopeFromOwner(kbOwner) + const initialToken = await resolveAccessToken( connector, connectorConfig, credentialUserId, sourceConfig, - accessMode + accessMode, + kbScope ) - /** Re-resolves the token for every OAuth call after the first, so a long run outlives a short-lived token. */ - const refreshOAuthToken = async (): Promise => { - if (connectorConfig.auth.mode === 'oauth') { - credentialToken = await resolveAccessToken( - connector, - connectorConfig, - credentialUserId, - sourceConfig, - accessMode - ) - } - } /** * A credential that already knows its cloud id seeds the same `syncContext` @@ -1137,13 +1134,31 @@ export async function executeSync( */ const syncContext: Record = { syncRunId: generateId(), - ...syncContextForToken(credentialToken), + ...syncContextForToken(initialToken), /** Tells a connector to carry permissions with its listing; without it, none are read. */ ...(mirrored ? { mirrorsSourceAcls: true } : {}), } if (mirrored) await connectorConfig.permissionConfig?.populateSyncContext(connectorId, syncContext) + /** Re-resolved before every call after the first, so a long run outlives a short-lived token. */ + const runToken = createConnectorRunToken({ + token: initialToken, + syncContext, + resolve: connectorUsesCredential(connectorConfig.auth, connector) + ? () => + resolveAccessToken( + connector, + connectorConfig, + credentialUserId, + sourceConfig, + accessMode, + kbScope + ) + : undefined, + }) + const refreshRunToken = () => runToken.refresh() + // Shared cutoff for both the tombstone-retry bound below and the stuck-document // retry near the end of this sync — same RETRY_WINDOW_DAYS window, one computation. const retryCutoff = new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000) @@ -1276,7 +1291,7 @@ export async function executeSync( connectorConfig, sourceConfig, syncContext, - accessToken: credentialToken.accessToken, + accessToken: runToken.current().accessToken, force: Boolean(options.fullSync) || connector.consecutiveFailures > 0 || @@ -1296,15 +1311,15 @@ export async function executeSync( result, forceRehydrate, getAccessToken: async (pageNum) => { - if (pageNum > 0) await refreshOAuthToken() - return credentialToken.accessToken + if (pageNum > 0) await refreshRunToken() + return runToken.current().accessToken }, hydration: { concurrency: connectorConfig.contentConcurrency, - beforeHydration: refreshOAuthToken, + beforeHydration: refreshRunToken, getDocument: (externalId) => connectorConfig.getDocument( - credentialToken.accessToken, + runToken.current().accessToken, sourceConfig, externalId, syncContext @@ -1332,7 +1347,7 @@ export async function executeSync( connectorConfig, sourceConfig, syncContext, - accessToken: credentialToken.accessToken, + accessToken: runToken.current().accessToken, externalDocs, generationStartedAt, ownedExternalIds: [], diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index 05efa5d5d21..cd36618dd2b 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -34,12 +34,14 @@ const { mockEncryptApiKey, mockValidateGitHub, + mockValidateS3, mockEnqueueConnectorDeletion, mockEnqueueConnectorDetachment, } = vi.hoisted(() => ({ mockDispatchSync: vi.fn(), mockEncryptApiKey: vi.fn(), mockValidateGitHub: vi.fn(), + mockValidateS3: vi.fn(), mockEnqueueConnectorDeletion: vi.fn(), mockEnqueueConnectorDetachment: vi.fn(), })) @@ -76,6 +78,16 @@ vi.mock('@/connectors/registry.server', () => ({ configFields: [], validateConfig: mockValidateGitHub, }, + s3: { + name: 'Amazon S3', + auth: { + mode: 'apiKey', + label: 'Secret Access Key', + savedCredential: { provider: 'aws', label: 'Saved AWS credential' }, + }, + configFields: [], + validateConfig: mockValidateS3, + }, notion: { auth: { mode: 'apiKey', optional: true }, configFields: [], @@ -188,6 +200,35 @@ describe('performCreateKnowledgeConnector', () => { } ) + it.each(['result', 'exception'])( + 'redacts a saved AWS session from provider validation %s errors', + async (failure) => { + const awsSession = { + accessKeyId: 'ASIAEXAMPLE', + secretAccessKey: 'session/secret', + sessionToken: 'session-token', + } + const message = `Denied ${awsSession.secretAccessKey} ${awsSession.sessionToken}` + if (failure === 'result') + mockValidateS3.mockResolvedValueOnce({ valid: false, error: message }) + else mockValidateS3.mockRejectedValueOnce(new OrchestrationError('validation', message)) + const request = performCreateKnowledgeConnector({ + ...createParams, + connectorType: 's3', + credentialId: 'aws-credential', + resolveAccessToken: vi.fn().mockResolvedValue({ accessToken: '', awsSession }), + }) + const expected = { errorCode: 'validation', error: 'Denied [REDACTED] [REDACTED]' } + if (failure === 'result') await expect(request).resolves.toMatchObject(expected) + else + await expect(request).rejects.toMatchObject({ + code: expected.errorCode, + message: expected.error, + }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + } + ) + it('validates and encrypts a GitHub PAT without resolving an OAuth account or returning the secret', async () => { dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'kb-1' }]) dbChainMockFns.returning.mockResolvedValueOnce([ diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index 058439ab50f..a8c0e49e30b 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -43,6 +43,7 @@ import { } from '@/lib/knowledge/connectors/access-modes' import { type ConnectorAccessToken, + connectorTokenSecrets, syncContextForToken, } from '@/lib/knowledge/connectors/access-token' import { enqueueConnectorDeletion } from '@/lib/knowledge/connectors/deletion' @@ -72,7 +73,11 @@ import { lockOrganizationSearchApproval } from '@/lib/knowledge/search/integrati import { createTagDefinition } from '@/lib/knowledge/tags/service' import { captureServerEvent } from '@/lib/posthog/server' import { searchSourceIdentity } from '@/lib/sim-search/source-identity' -import { getConnectorApiKeyConfig } from '@/connectors/auth' +import { + connectorAuthModeLabel, + connectorUsesCredential, + getConnectorApiKeyConfig, +} from '@/connectors/auth' import { getConnectorMeta } from '@/connectors/registry' import { findSourceSelectionError } from '@/connectors/selection' import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' @@ -296,6 +301,7 @@ export async function performCreateKnowledgeConnector( let resolvedCredentialId: string | null = null let resolvedEncryptedApiKey: string | null = null let accessToken: string | null = null + let tokenSecrets: string[] = [] let tokenContext: Record = {} const apiKeyConfig = getConnectorApiKeyConfig(connectorConfig.auth) if (apiKey && (!apiKeyConfig || accessMode === 'members')) { @@ -304,7 +310,10 @@ export async function performCreateKnowledgeConnector( if (apiKey && credentialId) { return fail('Choose either an OAuth account or an API key', 'validation') } - const usesApiKey = connectorConfig.auth.mode === 'apiKey' || Boolean(apiKeyConfig && apiKey) + const usesApiKey = !connectorUsesCredential(connectorConfig.auth, { + credentialId: credentialId ?? null, + encryptedApiKey: apiKey ?? null, + }) if (membersBinding) { /** @@ -331,6 +340,7 @@ export async function performCreateKnowledgeConnector( return fail('API key is required', 'validation') } accessToken = apiKey ?? '' + tokenSecrets = [accessToken] } else if (!membersBinding || credentialId) { if (!credentialId) { return fail('Credential is required', 'validation') @@ -345,6 +355,7 @@ export async function performCreateKnowledgeConnector( return fail('Credential has no access token. Please reconnect your account.', 'validation') } accessToken = token.accessToken + tokenSecrets = connectorTokenSecrets(token) tokenContext = syncContextForToken(token) resolvedCredentialId = credentialId } @@ -360,13 +371,13 @@ export async function performCreateKnowledgeConnector( .validateConfig(accessToken, sourceConfig, validationContext) .catch((error: unknown) => { const sanitized = toError(error) - sanitized.message = redactKnownSensitiveValues(sanitized.message, [accessToken]) + sanitized.message = redactKnownSensitiveValues(sanitized.message, tokenSecrets) throw sanitized }) if (!configValidation.valid) { return fail( (configValidation.error && - redactKnownSensitiveValues(configValidation.error, [accessToken])) || + redactKnownSensitiveValues(configValidation.error, tokenSecrets)) || `The ${connectorType} connector rejected sourceConfig without a reason — re-check its required fields with \`sim connector-types list --detail full\` (GET /api/v2/connector-types?detail=full) before retrying; the same config will fail again.`, 'validation' ) @@ -642,7 +653,7 @@ export async function performCreateKnowledgeConnector( knowledgeBaseName: kb.name, connectorType, syncIntervalMinutes, - authMode: usesApiKey ? 'apiKey' : 'oauth', + authMode: connectorAuthModeLabel(connectorConfig.auth.mode, !usesApiKey), }, ...(request ? { request } : {}), }) diff --git a/apps/sim/lib/sim-search/live/service-session.ts b/apps/sim/lib/sim-search/live/service-session.ts index e39c54c9ad3..86d335fdbaa 100644 --- a/apps/sim/lib/sim-search/live/service-session.ts +++ b/apps/sim/lib/sim-search/live/service-session.ts @@ -81,6 +81,7 @@ export async function createLiveServiceSession(input: { connector: source, userId: credentialUserId, requestId: 'live-source-search', + scope: { kind: 'organization', organizationId: source.organizationId }, sourceConfig: source.config, signal, }) diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index 81b168d403d..08720bd756c 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -13,6 +13,10 @@ import { executorPrincipalMock, executorPrincipalMockFns, } from '@sim/testing/mocks/executor-principal.mock' +import { + internalAwsCredentialsMock, + internalAwsCredentialsMockFns, +} from '@sim/testing/mocks/internal-aws-credentials.mock' import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { permissionCheckMock, @@ -204,18 +208,12 @@ vi.mock( () => workspaceFileSecretProvenanceMock ) -const { mockResolveExecutorCredentialToken, mockResolveAuthorizedAwsCredential } = vi.hoisted( - () => ({ - mockResolveExecutorCredentialToken: vi.fn(), - mockResolveAuthorizedAwsCredential: vi.fn(), - }) -) - -vi.mock('@/lib/internal/aws/credentials', () => ({ - resolveAuthorizedAwsCredential: (...args: unknown[]) => - mockResolveAuthorizedAwsCredential(...args), +const { mockResolveExecutorCredentialToken } = vi.hoisted(() => ({ + mockResolveExecutorCredentialToken: vi.fn(), })) +vi.mock('@/lib/internal/aws/credentials', () => internalAwsCredentialsMock) + vi.mock('@/executor/utils/credential-token', () => ({ resolveExecutorCredentialToken: (...args: unknown[]) => mockResolveExecutorCredentialToken(...args), @@ -498,6 +496,8 @@ import { createToolConfig, getTool } from '@/tools/utils' import { getToolAsync } from '@/tools/utils.server' const mockGetBYOKKey = apiKeyByokMockFns.mockGetBYOKKey +const mockResolveAuthorizedAwsCredential = + internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential const mockResolveWorkspaceBillingPayer = billingAttributionMockFns.mockResolveWorkspaceBillingPayer const mockGenerateInternalToken = authInternalMockFns.mockGenerateInternalToken const mockCreateExecutorPrincipalFromExecutionContext = diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 9447cb1ce48..cc633376b8f 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -8736,6 +8736,10 @@ type ListConnectorTypesResponseRef0 = { label?: string placeholder?: string optional: boolean + savedCredential?: { + provider: string + label: string + } } configFields: Array supportsIncrementalSync: boolean @@ -8768,6 +8772,7 @@ type ListConnectorTypesResponseRef1 = { mode?: 'basic' | 'advanced' canonicalParamId?: string multi?: boolean + apiKeyOnly?: boolean } type ListConnectorTypesResponseRef2 = { @@ -17347,7 +17352,7 @@ export const V2_OPERATIONS = { credentialId: { kind: 'string', describe: - 'OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types); omit it for `apiKey` connectors.', + 'OAuth credential identifier for connector types whose `auth.mode` is `oauth` (see connector types), or, when this deployment enables it, a saved credential for an `apiKey` connector type that declares `auth.savedCredential` (such as Amazon S3 with a saved AWS credential); otherwise omit it for `apiKey` connectors.', }, apiKey: { kind: 'string', diff --git a/packages/testing/src/mocks/index.ts b/packages/testing/src/mocks/index.ts index 51a3aab2233..8bb5f081e68 100644 --- a/packages/testing/src/mocks/index.ts +++ b/packages/testing/src/mocks/index.ts @@ -388,6 +388,10 @@ export { integrationsAvailabilityMock, integrationsAvailabilityMockFns, } from './integrations-availability.mock' +export { + internalAwsCredentialsMock, + internalAwsCredentialsMockFns, +} from './internal-aws-credentials.mock' export { invitationsCoreMock, invitationsCoreMockFns, diff --git a/packages/testing/src/mocks/internal-aws-credentials.mock.ts b/packages/testing/src/mocks/internal-aws-credentials.mock.ts new file mode 100644 index 00000000000..b408e7bd114 --- /dev/null +++ b/packages/testing/src/mocks/internal-aws-credentials.mock.ts @@ -0,0 +1,69 @@ +import { vi } from 'vitest' + +interface MockAwsSession { + accessKeyId: string + secretAccessKey: string + sessionToken?: string + durationSeconds?: number + expiration?: Date +} + +/** + * Controllable mock functions for `@/lib/internal/aws/credentials`. + * + * Defaults: `mockResolveAuthorizedAwsCredential` rejects (no saved credential + * is configured), the client helpers return the inline keys unchanged, and the + * wrappers run their operation as-is. + * + * @example + * ```ts + * import { internalAwsCredentialsMockFns } from '@sim/testing/mocks/internal-aws-credentials.mock' + * + * internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential.mockResolvedValueOnce({ + * accessKeyId: 'ASIA...', + * secretAccessKey: 'secret', + * sessionToken: 'token', + * }) + * ``` + */ +export const internalAwsCredentialsMockFns = { + mockResolveAuthorizedAwsCredential: vi.fn( + async ( + _credentialId: string, + _context: { workspaceId?: string; organizationId?: string; executionId?: string }, + _signal?: AbortSignal + ): Promise => { + throw new Error('AWS credential is unavailable') + } + ), + mockAwsClientCredentials: vi.fn((accessKeyId: string, secretAccessKey: string) => ({ + accessKeyId, + secretAccessKey, + })), + mockAwsSessionLifetime: vi.fn((_accessKeyId: string) => undefined), + mockAwsClientConfig: vi.fn((region: string, accessKeyId: string, secretAccessKey: string) => ({ + region, + credentials: { accessKeyId, secretAccessKey }, + })), + mockRunWithAwsSession: vi.fn( + (_session: MockAwsSession, operation: () => Promise) => operation() + ), + mockWithAwsCredential: vi.fn((handler: Handler) => handler), +} + +/** + * Static mock module for `@/lib/internal/aws/credentials`. + * + * @example + * ```ts + * vi.mock('@/lib/internal/aws/credentials', () => internalAwsCredentialsMock) + * ``` + */ +export const internalAwsCredentialsMock = { + resolveAuthorizedAwsCredential: internalAwsCredentialsMockFns.mockResolveAuthorizedAwsCredential, + awsClientCredentials: internalAwsCredentialsMockFns.mockAwsClientCredentials, + awsSessionLifetime: internalAwsCredentialsMockFns.mockAwsSessionLifetime, + awsClientConfig: internalAwsCredentialsMockFns.mockAwsClientConfig, + runWithAwsSession: internalAwsCredentialsMockFns.mockRunWithAwsSession, + withAwsCredential: internalAwsCredentialsMockFns.mockWithAwsCredential, +}