feat(fivetran): add connector setup, schema config, group, and destination operations

apps/sim/lib/webhooks/providers/fivetran.ts

@@ -182,8 +182,13 @@ export const fivetranHandler: WebhookProviderHandler = {
       }
     }
 
+    // Transformation events describe a dbt job and carry no connection, so the filter skips them.
     const connectionId = readString(providerConfig, 'triggerConnectionId')
-    if (connectionId && payload.connector_id !== connectionId) {
+    if (
+      connectionId &&
+      !event.startsWith('transformation_') &&
+      payload.connector_id !== connectionId
+    ) {
       logger.debug(`[${requestId}] Fivetran event is for another connection. Skipping execution.`, {
         webhookId: webhook.id,
       })

apps/sim/tools/fivetran/cancel_transformation.ts

@@ -1,4 +1,8 @@
-import type { FivetranMessageResponse, FivetranTransformationParams } from '@/tools/fivetran/types'
+import {
+  type FivetranMessageResponse,
+  type FivetranTransformationParams,
+  MESSAGE_OUTPUTS,
+} from '@/tools/fivetran/types'
 import {
   FIVETRAN_AUTH_PARAMS,
   FIVETRAN_ERROR_EXTRACTOR,
@@ -7,7 +11,6 @@ import {
   fivetranHeaders,
   fivetranId,
   fivetranUrl,
-  MESSAGE_OUTPUTS,
   transformMessageResponse,
 } from '@/tools/fivetran/utils'
 import type { ToolConfig } from '@/tools/types'

apps/sim/tools/fivetran/create_connection.ts

@@ -1,9 +1,9 @@
-import type {
-  FivetranConnectionResponse,
-  FivetranCreateConnectionParams,
-} from '@/tools/fivetran/types'
 import {
   CONNECTION_PROPERTIES,
+  type FivetranConnectionResponse,
+  type FivetranCreateConnectionParams,
+} from '@/tools/fivetran/types'
+import {
   compactBody,
   envelopeData,
   FIVETRAN_AUTH_PARAMS,

apps/sim/tools/fivetran/create_destination.ts

@@ -0,0 +1,114 @@
+import {
+  DESTINATION_PROPERTIES,
+  type FivetranCreateDestinationParams,
+  type FivetranDestinationResponse,
+} from '@/tools/fivetran/types'
+import {
+  compactBody,
+  envelopeData,
+  FIVETRAN_AUTH_PARAMS,
+  FIVETRAN_ERROR_EXTRACTOR,
+  FIVETRAN_OAUTH,
+  FIVETRAN_TRUST_PARAMS,
+  fivetranHeaders,
+  fivetranUrl,
+  mapDestination,
+  optionalBoolean,
+  parseJsonObject,
+  trimmed,
+} from '@/tools/fivetran/utils'
+import type { ToolConfig } from '@/tools/types'
+
+export const fivetranCreateDestinationTool: ToolConfig<
+  FivetranCreateDestinationParams,
+  FivetranDestinationResponse
+> = {
+  id: 'fivetran_create_destination',
+  name: 'Fivetran Create Destination',
+  description:
+    'Create the destination for a Fivetran group, such as a Databricks workspace. The config object holds the destination-specific connection settings.',
+  version: '1.0.0',
+  errorExtractor: FIVETRAN_ERROR_EXTRACTOR,
+  oauth: FIVETRAN_OAUTH,
+
+  params: {
+    ...FIVETRAN_AUTH_PARAMS,
+    groupId: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'ID of the group the destination belongs to (each group has one destination)',
+    },
+    service: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'Destination type, e.g. databricks, snowflake, or big_query',
+    },
+    timeZoneOffset: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'UTC offset used for sync scheduling, from -11 to +12, e.g. -5 or 0',
+    },
+    region: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Data processing region, e.g. AWS_US_EAST_1 or GCP_US_EAST4',
+    },
+    config: {
+      type: 'json',
+      required: false,
+      visibility: 'user-or-llm',
+      description:
+        'Destination-specific settings, e.g. for Databricks {"server_host_name": "...", "http_path": "...", "catalog": "...", "auth_type": "PERSONAL_ACCESS_TOKEN", "personal_access_token": "..."}',
+    },
+    daylightSavingTimeEnabled: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Shift the UTC offset with daylight saving time (US only)',
+    },
+    runSetupTests: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Run the setup tests after creating the destination',
+    },
+    ...FIVETRAN_TRUST_PARAMS,
+  },
+
+  request: {
+    url: () => fivetranUrl('/destinations'),
+    method: 'POST',
+    headers: (params) => fivetranHeaders(params),
+    body: (params) =>
+      compactBody({
+        group_id: params.groupId?.trim(),
+        service: params.service?.trim(),
+        time_zone_offset: trimmed(params.timeZoneOffset),
+        region: trimmed(params.region),
+        config: parseJsonObject(params.config, 'config'),
+        daylight_saving_time_enabled: optionalBoolean(params.daylightSavingTimeEnabled),
+        run_setup_tests: optionalBoolean(params.runSetupTests),
+        trust_certificates: optionalBoolean(params.trustCertificates),
+        trust_fingerprints: optionalBoolean(params.trustFingerprints),
+      }),
+  },
+
+  transformResponse: async (response: Response) => {
+    return {
+      success: true,
+      output: { destination: mapDestination(envelopeData(await response.json())) },
+    }
+  },
+
+  outputs: {
+    destination: {
+      type: 'object',
+      description: 'The created destination, with setup test results when they ran',
+      properties: DESTINATION_PROPERTIES,
+    },
+  },
+}

apps/sim/tools/fivetran/create_group.ts

@@ -0,0 +1,53 @@
+import {
+  type FivetranCreateGroupParams,
+  type FivetranGetGroupResponse,
+  GROUP_PROPERTIES,
+} from '@/tools/fivetran/types'
+import {
+  envelopeData,
+  FIVETRAN_AUTH_PARAMS,
+  FIVETRAN_ERROR_EXTRACTOR,
+  FIVETRAN_OAUTH,
+  fivetranHeaders,
+  fivetranUrl,
+  mapGroup,
+} from '@/tools/fivetran/utils'
+import type { ToolConfig } from '@/tools/types'
+
+export const fivetranCreateGroupTool: ToolConfig<
+  FivetranCreateGroupParams,
+  FivetranGetGroupResponse
+> = {
+  id: 'fivetran_create_group',
+  name: 'Fivetran Create Group',
+  description:
+    'Create a Fivetran group. Each group holds one destination, so create the group first, then its destination.',
+  version: '1.0.0',
+  errorExtractor: FIVETRAN_ERROR_EXTRACTOR,
+  oauth: FIVETRAN_OAUTH,
+
+  params: {
+    ...FIVETRAN_AUTH_PARAMS,
+    name: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'Name of the group',
+    },
+  },
+
+  request: {
+    url: () => fivetranUrl('/groups'),
+    method: 'POST',
+    headers: (params) => fivetranHeaders(params),
+    body: (params) => ({ name: params.name?.trim() }),
+  },
+
+  transformResponse: async (response: Response) => {
+    return { success: true, output: { group: mapGroup(envelopeData(await response.json())) } }
+  },
+
+  outputs: {
+    group: { type: 'object', description: 'The created group', properties: GROUP_PROPERTIES },
+  },
+}

apps/sim/tools/fivetran/delete_connection.ts

@@ -1,4 +1,8 @@
-import type { FivetranConnectionParams, FivetranMessageResponse } from '@/tools/fivetran/types'
+import {
+  type FivetranConnectionParams,
+  type FivetranMessageResponse,
+  MESSAGE_OUTPUTS,
+} from '@/tools/fivetran/types'
 import {
   FIVETRAN_AUTH_PARAMS,
   FIVETRAN_CONNECTION_ID_PARAM,
@@ -7,7 +11,6 @@ import {
   fivetranHeaders,
   fivetranId,
   fivetranUrl,
-  MESSAGE_OUTPUTS,
   transformMessageResponse,
 } from '@/tools/fivetran/utils'
 import type { ToolConfig } from '@/tools/types'

apps/sim/tools/fivetran/get_connection.ts

@@ -1,6 +1,9 @@
-import type { FivetranConnectionParams, FivetranConnectionResponse } from '@/tools/fivetran/types'
 import {
   CONNECTION_PROPERTIES,
+  type FivetranConnectionParams,
+  type FivetranConnectionResponse,
+} from '@/tools/fivetran/types'
+import {
   envelopeData,
   FIVETRAN_AUTH_PARAMS,
   FIVETRAN_CONNECTION_ID_PARAM,

apps/sim/tools/fivetran/get_connection_schema_config.ts

@@ -1,4 +1,8 @@
-import type { FivetranConnectionParams, FivetranSchemaConfigResponse } from '@/tools/fivetran/types'
+import {
+  type FivetranConnectionParams,
+  type FivetranSchemaConfigResponse,
+  SCHEMA_CONFIG_OUTPUTS,
+} from '@/tools/fivetran/types'
 import {
   envelopeData,
   FIVETRAN_AUTH_PARAMS,
@@ -9,7 +13,6 @@ import {
   fivetranId,
   fivetranUrl,
   mapSchemaConfig,
-  SCHEMA_CONFIG_OUTPUTS,
 } from '@/tools/fivetran/utils'
 import type { ToolConfig } from '@/tools/types'
 

apps/sim/tools/fivetran/get_connector_type.ts

@@ -0,0 +1,77 @@
+import { toRecordOrNull } from '@sim/utils/object'
+import {
+  CONNECTOR_TYPE_PROPERTIES,
+  type FivetranGetConnectorTypeParams,
+  type FivetranGetConnectorTypeResponse,
+} from '@/tools/fivetran/types'
+import {
+  envelopeData,
+  FIVETRAN_AUTH_PARAMS,
+  FIVETRAN_ERROR_EXTRACTOR,
+  FIVETRAN_OAUTH,
+  fivetranHeaders,
+  fivetranId,
+  fivetranUrl,
+  mapConnectorType,
+} from '@/tools/fivetran/utils'
+import type { ToolConfig } from '@/tools/types'
+
+export const fivetranGetConnectorTypeTool: ToolConfig<
+  FivetranGetConnectorTypeParams,
+  FivetranGetConnectorTypeResponse
+> = {
+  id: 'fivetran_get_connector_type',
+  name: 'Fivetran Get Connector Type',
+  description:
+    'Get the configuration and authorization fields a Fivetran connector type takes, to build the config for Create Connection.',
+  version: '1.0.0',
+  errorExtractor: FIVETRAN_ERROR_EXTRACTOR,
+  oauth: FIVETRAN_OAUTH,
+
+  params: {
+    ...FIVETRAN_AUTH_PARAMS,
+    service: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'Connector type ID, e.g. okta (see List Connector Types)',
+    },
+  },
+
+  request: {
+    url: (params) =>
+      fivetranUrl(`/metadata/connector-types/${fivetranId(params.service, 'service')}`),
+    method: 'GET',
+    headers: (params) => fivetranHeaders(params),
+  },
+
+  transformResponse: async (response: Response) => {
+    const data = envelopeData(await response.json())
+    return {
+      success: true,
+      output: {
+        connectorType: mapConnectorType(data),
+        config: toRecordOrNull(data.config),
+        auth: toRecordOrNull(data.auth),
+      },
+    }
+  },
+
+  outputs: {
+    connectorType: {
+      type: 'object',
+      description: 'The connector type',
+      properties: CONNECTOR_TYPE_PROPERTIES,
+    },
+    config: {
+      type: 'json',
+      description: 'JSON Schema of the configuration fields the connector takes in config',
+      nullable: true,
+    },
+    auth: {
+      type: 'json',
+      description: 'JSON Schema of the authorization fields the connector takes in auth, if any',
+      nullable: true,
+    },
+  },
+}

apps/sim/tools/fivetran/get_destination.ts

@@ -1,6 +1,9 @@
-import type { FivetranDestinationParams, FivetranDestinationResponse } from '@/tools/fivetran/types'
 import {
   DESTINATION_PROPERTIES,
+  type FivetranDestinationParams,
+  type FivetranDestinationResponse,
+} from '@/tools/fivetran/types'
+import {
   envelopeData,
   FIVETRAN_AUTH_PARAMS,
   FIVETRAN_DESTINATION_ID_PARAM,

apps/sim/tools/fivetran/get_group.ts

@@ -1,4 +1,8 @@
-import type { FivetranGetGroupResponse, FivetranGroupParams } from '@/tools/fivetran/types'
+import {
+  type FivetranGetGroupResponse,
+  type FivetranGroupParams,
+  GROUP_PROPERTIES,
+} from '@/tools/fivetran/types'
 import {
   envelopeData,
   FIVETRAN_AUTH_PARAMS,
@@ -7,7 +11,6 @@ import {
   fivetranHeaders,
   fivetranId,
   fivetranUrl,
-  GROUP_PROPERTIES,
   mapGroup,
 } from '@/tools/fivetran/utils'
 import type { ToolConfig } from '@/tools/types'

apps/sim/tools/fivetran/get_sync_history.ts

@@ -1,8 +1,9 @@
 import { toNumberOrNull, toStringOrNull } from '@sim/utils/coerce'
 import { toRecord } from '@sim/utils/object'
-import type {
-  FivetranGetSyncHistoryParams,
-  FivetranGetSyncHistoryResponse,
+import {
+  type FivetranGetSyncHistoryParams,
+  type FivetranGetSyncHistoryResponse,
+  NEXT_CURSOR_OUTPUT,
 } from '@/tools/fivetran/types'
 import {
   envelopePage,
@@ -15,7 +16,6 @@ import {
   fivetranId,
   fivetranPaginationQuery,
   fivetranUrl,
-  NEXT_CURSOR_OUTPUT,
   trimmed,
 } from '@/tools/fivetran/utils'
 import type { ToolConfig } from '@/tools/types'