From bb3da0a6278b22e6a543caa2f29b92f1c23cbc37 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 14:42:34 +0000 Subject: [PATCH 1/3] Add direct engine migration example Co-authored-by: omnigent --- .../direct_engine_migration/.gitignore | 1 + .../direct_engine_migration/README.md | 57 +++++++++++++++++++ .../direct_engine_migration/databricks.yml | 13 +++++ .../resources/direct_engine_migration.job.yml | 13 +++++ .../direct_engine_migration/src/main.py | 1 + 5 files changed, 85 insertions(+) create mode 100644 knowledge_base/direct_engine_migration/.gitignore create mode 100644 knowledge_base/direct_engine_migration/README.md create mode 100644 knowledge_base/direct_engine_migration/databricks.yml create mode 100644 knowledge_base/direct_engine_migration/resources/direct_engine_migration.job.yml create mode 100644 knowledge_base/direct_engine_migration/src/main.py diff --git a/knowledge_base/direct_engine_migration/.gitignore b/knowledge_base/direct_engine_migration/.gitignore new file mode 100644 index 00000000..15bcc6dd --- /dev/null +++ b/knowledge_base/direct_engine_migration/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/knowledge_base/direct_engine_migration/README.md b/knowledge_base/direct_engine_migration/README.md new file mode 100644 index 00000000..68910386 --- /dev/null +++ b/knowledge_base/direct_engine_migration/README.md @@ -0,0 +1,57 @@ +# Direct engine migration + +This Declarative Automation Bundles example migrates a serverless job from the Terraform engine to the direct engine. See the [direct engine documentation](https://docs.databricks.com/dev-tools/bundles/direct). + +## Prerequisites + +* Databricks CLI v1.19.x for the initial Terraform deployment +* Databricks CLI v1.20.0 or above for migration and later commands +* A Databricks workspace configuration profile named `dabs-examples` + +## Usage + +| Engine | Local state | Workspace state | +| --- | --- | --- | +| Terraform | `.databricks/bundle/dev/terraform/terraform.tfstate` | `/state/terraform.tfstate` | +| Direct | `.databricks/bundle/dev/resources.json` | `/state/resources.json` | + +1. Use Databricks CLI v1.19.x to validate the bundle with the Terraform engine. + + ```text + DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle validate --profile dabs-examples + ``` + +2. Deploy and run the job with the Terraform engine. + + ```text + DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle deploy --profile dabs-examples + DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle run direct_engine_migration --profile dabs-examples + ``` + +3. Record the job ID and inspect `.databricks/bundle/dev/terraform/terraform.tfstate`. + + ```text + DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle summary --output json --profile dabs-examples + ``` + +4. Upgrade to Databricks CLI v1.20.0 or above. Migrate the local state. + + ```text + DATABRICKS_BUNDLE_ENGINE=direct databricks bundle deployment migrate --profile dabs-examples + ``` + +5. Inspect the direct state at `.databricks/bundle/dev/resources.json`. + +6. Confirm that the plan has zero changes. Deploy and run the same job ID. + + ```text + DATABRICKS_BUNDLE_ENGINE=direct databricks bundle plan --profile dabs-examples + DATABRICKS_BUNDLE_ENGINE=direct databricks bundle deploy --profile dabs-examples + DATABRICKS_BUNDLE_ENGINE=direct databricks bundle run direct_engine_migration --profile dabs-examples + ``` + +7. Remove the job and workspace files. + + ```text + DATABRICKS_BUNDLE_ENGINE=direct databricks bundle destroy --auto-approve --profile dabs-examples + ``` diff --git a/knowledge_base/direct_engine_migration/databricks.yml b/knowledge_base/direct_engine_migration/databricks.yml new file mode 100644 index 00000000..108c00ac --- /dev/null +++ b/knowledge_base/direct_engine_migration/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: direct_engine_migration + +include: + - resources/*.yml + +workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + +targets: + dev: + default: true + mode: development diff --git a/knowledge_base/direct_engine_migration/resources/direct_engine_migration.job.yml b/knowledge_base/direct_engine_migration/resources/direct_engine_migration.job.yml new file mode 100644 index 00000000..d3b5f4b5 --- /dev/null +++ b/knowledge_base/direct_engine_migration/resources/direct_engine_migration.job.yml @@ -0,0 +1,13 @@ +resources: + jobs: + direct_engine_migration: + name: Direct engine migration + tasks: + - task_key: hello + spark_python_task: + python_file: ../src/main.py + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "6" diff --git a/knowledge_base/direct_engine_migration/src/main.py b/knowledge_base/direct_engine_migration/src/main.py new file mode 100644 index 00000000..a6d2510e --- /dev/null +++ b/knowledge_base/direct_engine_migration/src/main.py @@ -0,0 +1 @@ +print("Hello from the direct engine migration example.") From 738fca8fe73d4efc3551e12eeb81f57511f3e5ff Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 14:50:44 +0000 Subject: [PATCH 2/3] Address direct engine migration review Co-authored-by: omnigent --- .../direct_engine_migration/README.md | 43 ++++++++++--------- .../direct_engine_migration/databricks.yml | 5 +-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/knowledge_base/direct_engine_migration/README.md b/knowledge_base/direct_engine_migration/README.md index 68910386..48a26bc0 100644 --- a/knowledge_base/direct_engine_migration/README.md +++ b/knowledge_base/direct_engine_migration/README.md @@ -1,12 +1,13 @@ # Direct engine migration -This Declarative Automation Bundles example migrates a serverless job from the Terraform engine to the direct engine. See the [direct engine documentation](https://docs.databricks.com/dev-tools/bundles/direct). +This Declarative Automation Bundles example migrates a serverless job from the Terraform engine to the direct engine. + +See the [direct engine documentation](https://docs.databricks.com/dev-tools/bundles/direct). ## Prerequisites -* Databricks CLI v1.19.x for the initial Terraform deployment -* Databricks CLI v1.20.0 or above for migration and later commands -* A Databricks workspace configuration profile named `dabs-examples` +* Databricks CLI v1.19.x for the Terraform deployment. Version 1.20.0 removes the Terraform engine. +* Databricks CLI v1.20.0 or above for the migration ## Usage @@ -14,44 +15,46 @@ This Declarative Automation Bundles example migrates a serverless job from the T | --- | --- | --- | | Terraform | `.databricks/bundle/dev/terraform/terraform.tfstate` | `/state/terraform.tfstate` | | Direct | `.databricks/bundle/dev/resources.json` | `/state/resources.json` | - -1. Use Databricks CLI v1.19.x to validate the bundle with the Terraform engine. +1. Use Databricks CLI v1.19.x to deploy with the Terraform engine. ```text - DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle validate --profile dabs-examples + DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle deploy ``` -2. Deploy and run the job with the Terraform engine. +2. Record the job ID with the Terraform engine. ```text - DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle deploy --profile dabs-examples - DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle run direct_engine_migration --profile dabs-examples + DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle summary --output json ``` -3. Record the job ID and inspect `.databricks/bundle/dev/terraform/terraform.tfstate`. +3. Upgrade to Databricks CLI v1.20.0 or above. Migrate the local state. `databricks bundle deploy` also migrates the state automatically. ```text - DATABRICKS_BUNDLE_ENGINE=terraform databricks bundle summary --output json --profile dabs-examples + databricks bundle deployment migrate ``` -4. Upgrade to Databricks CLI v1.20.0 or above. Migrate the local state. +4. Confirm that the migration plan has zero changes. ```text - DATABRICKS_BUNDLE_ENGINE=direct databricks bundle deployment migrate --profile dabs-examples + databricks bundle plan ``` -5. Inspect the direct state at `.databricks/bundle/dev/resources.json`. +5. Deploy and run the job. Confirm that the job ID is unchanged. + + ```text + databricks bundle deploy + databricks bundle run direct_engine_migration + databricks bundle summary --output json + ``` -6. Confirm that the plan has zero changes. Deploy and run the same job ID. +6. Confirm that the deployed bundle still has zero changes. ```text - DATABRICKS_BUNDLE_ENGINE=direct databricks bundle plan --profile dabs-examples - DATABRICKS_BUNDLE_ENGINE=direct databricks bundle deploy --profile dabs-examples - DATABRICKS_BUNDLE_ENGINE=direct databricks bundle run direct_engine_migration --profile dabs-examples + databricks bundle plan ``` 7. Remove the job and workspace files. ```text - DATABRICKS_BUNDLE_ENGINE=direct databricks bundle destroy --auto-approve --profile dabs-examples + databricks bundle destroy --auto-approve ``` diff --git a/knowledge_base/direct_engine_migration/databricks.yml b/knowledge_base/direct_engine_migration/databricks.yml index 108c00ac..ecc643bf 100644 --- a/knowledge_base/direct_engine_migration/databricks.yml +++ b/knowledge_base/direct_engine_migration/databricks.yml @@ -4,10 +4,9 @@ bundle: include: - resources/*.yml -workspace: - root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} - targets: dev: default: true mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/dev/${workspace.current_user.short_name} From e96de180059022809dd48599b148897fcecaaea9 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 14:51:55 +0000 Subject: [PATCH 3/3] knowledge_base/direct_engine_migration: separate state table from steps Co-authored-by: Isaac Co-authored-by: omnigent --- knowledge_base/direct_engine_migration/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/knowledge_base/direct_engine_migration/README.md b/knowledge_base/direct_engine_migration/README.md index 48a26bc0..4a2d278a 100644 --- a/knowledge_base/direct_engine_migration/README.md +++ b/knowledge_base/direct_engine_migration/README.md @@ -15,6 +15,7 @@ See the [direct engine documentation](https://docs.databricks.com/dev-tools/bund | --- | --- | --- | | Terraform | `.databricks/bundle/dev/terraform/terraform.tfstate` | `/state/terraform.tfstate` | | Direct | `.databricks/bundle/dev/resources.json` | `/state/resources.json` | + 1. Use Databricks CLI v1.19.x to deploy with the Terraform engine. ```text