From 311cb25a6a7dd36e43605e7a52b58a7c56573aed Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:30:42 +0000 Subject: [PATCH 1/4] Add Lakebase synced tables bundle example Co-authored-by: omnigent --- .../lakebase_synced_tables/.gitignore | 1 + .../lakebase_synced_tables/README.md | 51 +++++++++++++++++++ .../lakebase_synced_tables/databricks.yml | 33 ++++++++++++ .../resources/lakebase.postgres.yml | 43 ++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 knowledge_base/lakebase_synced_tables/.gitignore create mode 100644 knowledge_base/lakebase_synced_tables/README.md create mode 100644 knowledge_base/lakebase_synced_tables/databricks.yml create mode 100644 knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml diff --git a/knowledge_base/lakebase_synced_tables/.gitignore b/knowledge_base/lakebase_synced_tables/.gitignore new file mode 100644 index 00000000..15bcc6dd --- /dev/null +++ b/knowledge_base/lakebase_synced_tables/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/knowledge_base/lakebase_synced_tables/README.md b/knowledge_base/lakebase_synced_tables/README.md new file mode 100644 index 00000000..b21bcfb1 --- /dev/null +++ b/knowledge_base/lakebase_synced_tables/README.md @@ -0,0 +1,51 @@ +# Lakebase synced tables + +This Declarative Automation Bundles example syncs a Unity Catalog Delta table to Lakebase Postgres. It also creates a target-specific branch, endpoint, and disabled snapshot schedule. + +The `postgres_*` resources use Lakebase Autoscaling. Legacy `database_instances` resources configure Lakebase Provisioned. + +See the [Lakebase Autoscaling documentation](https://docs.databricks.com/aws/en/oltp/) for more information. + +## Prerequisites + +- Databricks CLI v1.20.0 or above. +- A workspace with Lakebase Autoscaling and serverless pipelines. +- A source Delta table with an `id` primary key column. +- A standard catalog and schema for pipeline storage. + +## Usage + +1. Set a unique project ID and your Unity Catalog object names. + + ```sh + export BUNDLE_VAR_project_id=my-synced-tables + export BUNDLE_VAR_source_table=main.example.source_table + export BUNDLE_VAR_destination_table=main.example.synced_source_table + export BUNDLE_VAR_pipeline_storage_catalog=main + export BUNDLE_VAR_pipeline_storage_schema=example + ``` + +2. Validate the bundle. + + ```sh + databricks bundle validate + ``` + +3. Deploy the development branch, endpoint, schedule, and synced table. + + ```sh + databricks bundle deploy + ``` + +4. Deploy the production target when required. + + ```sh + databricks bundle deploy -t prod + ``` + +5. Remove the resources for each deployed target. + + ```sh + databricks bundle destroy --auto-approve + databricks bundle destroy -t prod --auto-approve + ``` diff --git a/knowledge_base/lakebase_synced_tables/databricks.yml b/knowledge_base/lakebase_synced_tables/databricks.yml new file mode 100644 index 00000000..827f1fb5 --- /dev/null +++ b/knowledge_base/lakebase_synced_tables/databricks.yml @@ -0,0 +1,33 @@ +bundle: + name: lakebase-synced-tables + databricks_cli_version: ">= 1.20.0" + +include: + - resources/*.yml + +variables: + project_id: + description: Unique Lakebase project ID + source_table: + description: Three-part name of the source Delta table + destination_table: + description: Three-part name of the destination synced table + pipeline_storage_catalog: + description: Standard catalog for pipeline storage + pipeline_storage_schema: + description: Schema for pipeline storage + branch_id: + default: production + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + variables: + branch_id: development + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} diff --git a/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml b/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml new file mode 100644 index 00000000..1d632fb4 --- /dev/null +++ b/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml @@ -0,0 +1,43 @@ +resources: + postgres_projects: + synced_tables: + project_id: ${var.project_id} + display_name: Lakebase synced tables example + pg_version: 17 + purge_on_delete: true + + postgres_branches: + target_branch: + parent: ${resources.postgres_projects.synced_tables.id} + branch_id: ${var.branch_id} + no_expiry: true + replace_existing: true + purge_on_delete: true + + postgres_endpoints: + target_endpoint: + parent: ${resources.postgres_branches.target_branch.id} + endpoint_id: primary + endpoint_type: ENDPOINT_TYPE_READ_WRITE + replace_existing: true + autoscaling_limit_min_cu: 0.5 + autoscaling_limit_max_cu: 1 + suspend_timeout_duration: 60s + + postgres_snapshot_schedules: + target_schedule: + branch: ${resources.postgres_branches.target_branch.id} + schedule: [] + + postgres_synced_tables: + example: + synced_table_id: ${var.destination_table} + source_table_full_name: ${var.source_table} + branch: ${resources.postgres_branches.target_branch.id} + postgres_database: databricks_postgres + primary_key_columns: + - id + scheduling_policy: TRIGGERED + new_pipeline_spec: + storage_catalog: ${var.pipeline_storage_catalog} + storage_schema: ${var.pipeline_storage_schema} From 8a903d64f52928e686140c052795365a711e7907 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:37:09 +0000 Subject: [PATCH 2/4] Isolate Lakebase resources by target Co-authored-by: omnigent --- .../lakebase_synced_tables/README.md | 20 ++++++++++++------- .../resources/lakebase.postgres.yml | 9 ++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/knowledge_base/lakebase_synced_tables/README.md b/knowledge_base/lakebase_synced_tables/README.md index b21bcfb1..d5e01c13 100644 --- a/knowledge_base/lakebase_synced_tables/README.md +++ b/knowledge_base/lakebase_synced_tables/README.md @@ -1,8 +1,6 @@ # Lakebase synced tables -This Declarative Automation Bundles example syncs a Unity Catalog Delta table to Lakebase Postgres. It also creates a target-specific branch, endpoint, and disabled snapshot schedule. - -The `postgres_*` resources use Lakebase Autoscaling. Legacy `database_instances` resources configure Lakebase Provisioned. +This Declarative Automation Bundles example syncs a Unity Catalog Delta table to Lakebase Postgres. Each target creates its own project, branch, endpoint, and synced table. See the [Lakebase Autoscaling documentation](https://docs.databricks.com/aws/en/oltp/) for more information. @@ -10,8 +8,9 @@ See the [Lakebase Autoscaling documentation](https://docs.databricks.com/aws/en/ - Databricks CLI v1.20.0 or above. - A workspace with Lakebase Autoscaling and serverless pipelines. -- A source Delta table with an `id` primary key column. +- A source Delta table with change data feed enabled and an `id` primary key column. - A standard catalog and schema for pipeline storage. +- A new project ID. The bundle creates and purges its project during deployment and cleanup. ## Usage @@ -31,19 +30,26 @@ See the [Lakebase Autoscaling documentation](https://docs.databricks.com/aws/en/ databricks bundle validate ``` -3. Deploy the development branch, endpoint, schedule, and synced table. +3. Deploy the development project and synced table. ```sh databricks bundle deploy ``` -4. Deploy the production target when required. +4. Deploy the production project and synced table. ```sh databricks bundle deploy -t prod ``` -5. Remove the resources for each deployed target. +5. Verify the development and production sync status. + + ```sh + databricks postgres get-synced-table "synced_tables/${BUNDLE_VAR_destination_table}_dev" + databricks postgres get-synced-table "synced_tables/${BUNDLE_VAR_destination_table}_prod" + ``` + +6. Remove the resources for each deployed target. ```sh databricks bundle destroy --auto-approve diff --git a/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml b/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml index 1d632fb4..3915a8c1 100644 --- a/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml +++ b/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml @@ -1,7 +1,7 @@ resources: postgres_projects: synced_tables: - project_id: ${var.project_id} + project_id: ${var.project_id}-${bundle.target} display_name: Lakebase synced tables example pg_version: 17 purge_on_delete: true @@ -24,14 +24,9 @@ resources: autoscaling_limit_max_cu: 1 suspend_timeout_duration: 60s - postgres_snapshot_schedules: - target_schedule: - branch: ${resources.postgres_branches.target_branch.id} - schedule: [] - postgres_synced_tables: example: - synced_table_id: ${var.destination_table} + synced_table_id: ${var.destination_table}_${bundle.target} source_table_full_name: ${var.source_table} branch: ${resources.postgres_branches.target_branch.id} postgres_database: databricks_postgres From 595ac3e0931065a71ef8ab1bf7fdbe202ae9f09f Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:42:58 +0000 Subject: [PATCH 3/4] Add Lakebase daily snapshot schedule Co-authored-by: omnigent --- knowledge_base/lakebase_synced_tables/README.md | 1 + knowledge_base/lakebase_synced_tables/databricks.yml | 5 ----- .../resources/lakebase.postgres.yml | 10 +++++++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/knowledge_base/lakebase_synced_tables/README.md b/knowledge_base/lakebase_synced_tables/README.md index d5e01c13..98605630 100644 --- a/knowledge_base/lakebase_synced_tables/README.md +++ b/knowledge_base/lakebase_synced_tables/README.md @@ -47,6 +47,7 @@ See the [Lakebase Autoscaling documentation](https://docs.databricks.com/aws/en/ ```sh databricks postgres get-synced-table "synced_tables/${BUNDLE_VAR_destination_table}_dev" databricks postgres get-synced-table "synced_tables/${BUNDLE_VAR_destination_table}_prod" + databricks psql --project "${BUNDLE_VAR_project_id}-dev" --branch production --endpoint primary -- -d databricks_postgres -c "SELECT count(*) FROM example.synced_source_table_dev;" ``` 6. Remove the resources for each deployed target. diff --git a/knowledge_base/lakebase_synced_tables/databricks.yml b/knowledge_base/lakebase_synced_tables/databricks.yml index 827f1fb5..c9985d39 100644 --- a/knowledge_base/lakebase_synced_tables/databricks.yml +++ b/knowledge_base/lakebase_synced_tables/databricks.yml @@ -16,17 +16,12 @@ variables: description: Standard catalog for pipeline storage pipeline_storage_schema: description: Schema for pipeline storage - branch_id: - default: production - targets: dev: default: true mode: development workspace: root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} - variables: - branch_id: development prod: mode: production workspace: diff --git a/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml b/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml index 3915a8c1..8d0a50bb 100644 --- a/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml +++ b/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml @@ -9,7 +9,7 @@ resources: postgres_branches: target_branch: parent: ${resources.postgres_projects.synced_tables.id} - branch_id: ${var.branch_id} + branch_id: production no_expiry: true replace_existing: true purge_on_delete: true @@ -24,6 +24,14 @@ resources: autoscaling_limit_max_cu: 1 suspend_timeout_duration: 60s + postgres_snapshot_schedules: + daily_snapshots: + branch: ${resources.postgres_branches.target_branch.id} + schedule: + - daily_schedule: + hour: 0 + retention: 3600s + postgres_synced_tables: example: synced_table_id: ${var.destination_table}_${bundle.target} From 436bd2a93c1df9e234bac0fe77bc02cdda9b5b04 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:44:46 +0000 Subject: [PATCH 4/4] Use variables in Lakebase row count step Co-authored-by: omnigent --- knowledge_base/lakebase_synced_tables/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/knowledge_base/lakebase_synced_tables/README.md b/knowledge_base/lakebase_synced_tables/README.md index 98605630..80f8f354 100644 --- a/knowledge_base/lakebase_synced_tables/README.md +++ b/knowledge_base/lakebase_synced_tables/README.md @@ -11,6 +11,7 @@ See the [Lakebase Autoscaling documentation](https://docs.databricks.com/aws/en/ - A source Delta table with change data feed enabled and an `id` primary key column. - A standard catalog and schema for pipeline storage. - A new project ID. The bundle creates and purges its project during deployment and cleanup. +- The `psql` client, for the row count in step 5. ## Usage @@ -47,7 +48,7 @@ See the [Lakebase Autoscaling documentation](https://docs.databricks.com/aws/en/ ```sh databricks postgres get-synced-table "synced_tables/${BUNDLE_VAR_destination_table}_dev" databricks postgres get-synced-table "synced_tables/${BUNDLE_VAR_destination_table}_prod" - databricks psql --project "${BUNDLE_VAR_project_id}-dev" --branch production --endpoint primary -- -d databricks_postgres -c "SELECT count(*) FROM example.synced_source_table_dev;" + databricks psql --project "${BUNDLE_VAR_project_id}-dev" --branch production --endpoint primary -- -d databricks_postgres -c "SELECT count(*) FROM ${BUNDLE_VAR_destination_table#*.}_dev;" ``` 6. Remove the resources for each deployed target.