From 75ead0d2f99d6f99b08a537f6f5c6b51b901e7d4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 12:48:41 +0200 Subject: [PATCH 1/4] acc: bundle run with terraform state uses exact job id; rejects engine: terraform Co-authored-by: Isaac --- .../migrate/auto/run-no-commit/databricks.yml | 3 +-- .../bundle/migrate/auto/run-no-commit/output.txt | 15 ++++++++++++++- .../bundle/migrate/auto/run-no-commit/script | 7 +++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/migrate/auto/run-no-commit/databricks.yml b/acceptance/bundle/migrate/auto/run-no-commit/databricks.yml index 40404a02b8d..9227b4b8b99 100644 --- a/acceptance/bundle/migrate/auto/run-no-commit/databricks.yml +++ b/acceptance/bundle/migrate/auto/run-no-commit/databricks.yml @@ -1,5 +1,4 @@ -bundle: - name: test-bundle +bundle: {name: test-bundle} resources: jobs: diff --git a/acceptance/bundle/migrate/auto/run-no-commit/output.txt b/acceptance/bundle/migrate/auto/run-no-commit/output.txt index 950329c8541..0ca8884ffd1 100644 --- a/acceptance/bundle/migrate/auto/run-no-commit/output.txt +++ b/acceptance/bundle/migrate/auto/run-no-commit/output.txt @@ -5,7 +5,7 @@ === bundle run on direct migrates the terraform state IN MEMORY (no commit) to resolve the job >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle run test_job --no-wait Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[NUMID]?o=[NUMID] >>> print_migration_telemetry @@ -18,3 +18,16 @@ Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] >>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state ID Type Language Path [NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate + +=== bundle run rejects engine: terraform +>>> update_file.py databricks.yml {name: test-bundle} {name: test-bundle, engine: terraform} + +>>> musterr [CLI] bundle run test_job --no-wait +Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + in databricks.yml:1:37 + +Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + + +>>> print_requests.py //jobs diff --git a/acceptance/bundle/migrate/auto/run-no-commit/script b/acceptance/bundle/migrate/auto/run-no-commit/script index 4d65ccbf135..78091948a12 100644 --- a/acceptance/bundle/migrate/auto/run-no-commit/script +++ b/acceptance/bundle/migrate/auto/run-no-commit/script @@ -2,6 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= title "Replay a prior terraform deployment" trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py rm -f "$OUT_REQUESTS" title "bundle run on direct migrates the terraform state IN MEMORY (no commit) to resolve the job" @@ -17,3 +18,9 @@ title "Nothing durable written: still on terraform (local state files + remote), # cache under .terraform/ is excluded by matching only state files (tfstate/.backup, resources*.json, WAL). trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" + +title "bundle run rejects engine: terraform" +# Set after the replay: replay_tfstate.py runs bundle validate, which rejects engine: terraform. +trace update_file.py databricks.yml "{name: test-bundle}" "{name: test-bundle, engine: terraform}" +trace musterr $CLI bundle run test_job --no-wait +trace print_requests.py //jobs From 74df17873ef5687ca6287050bb805b11198c5405 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 12:54:00 +0200 Subject: [PATCH 2/4] Allow bundle run with engine: terraform Co-authored-by: Isaac --- cmd/bundle/run.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 35061f23105..be05b604393 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -138,6 +138,7 @@ Example usage: if cmd.ArgsLenAtDash() == 0 && len(args) > 0 { b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ SkipInitialize: true, + InitFunc: allowTerraformEngineConfig, }) if err != nil { return err @@ -154,6 +155,7 @@ Example usage: _, _, err := utils.ProcessBundleRet(cmd, utils.ProcessOptions{ AlwaysPull: true, ErrorOnEmptyState: true, + InitFunc: allowTerraformEngineConfig, PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { var err error key, runArgs, err = resolveRunArgument(ctx, b, args) @@ -316,3 +318,8 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { Dir: dir, }) } + +// allowTerraformEngineConfig lets users run resources of a bundle that pins the removed Terraform engine. +func allowTerraformEngineConfig(b *bundle.Bundle) { + b.AllowTerraformEngineConfig = true +} From 0a6affeaeb03d87b2ddfa19c392b3ac7524df7ea Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 12:55:02 +0200 Subject: [PATCH 3/4] acc: bundle run works with engine: terraform Co-authored-by: Isaac --- .../migrate/auto/run-no-commit/output.txt | 24 ++++++++++++++++--- .../bundle/migrate/auto/run-no-commit/script | 5 ++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/migrate/auto/run-no-commit/output.txt b/acceptance/bundle/migrate/auto/run-no-commit/output.txt index 0ca8884ffd1..449bbd59f9a 100644 --- a/acceptance/bundle/migrate/auto/run-no-commit/output.txt +++ b/acceptance/bundle/migrate/auto/run-no-commit/output.txt @@ -19,15 +19,33 @@ Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[NUMID]?o=[NUMID] ID Type Language Path [NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate -=== bundle run rejects engine: terraform +=== bundle run works with engine: terraform: warns and runs the job from the terraform state >>> update_file.py databricks.yml {name: test-bundle} {name: test-bundle, engine: terraform} ->>> musterr [CLI] bundle run test_job --no-wait -Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 +>>> [CLI] bundle run test_job --no-wait +Warning: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 in databricks.yml:1:37 Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. See https://docs.databricks.com/dev-tools/bundles/direct for details +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[NUMID]?o=[NUMID] + >>> print_requests.py //jobs +{ + "headers": { + "User-Agent": [ + "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [TEST_JOB_ID] + } +} + +>>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) +.databricks/bundle/default/terraform/terraform.tfstate diff --git a/acceptance/bundle/migrate/auto/run-no-commit/script b/acceptance/bundle/migrate/auto/run-no-commit/script index 78091948a12..2640ea4e235 100644 --- a/acceptance/bundle/migrate/auto/run-no-commit/script +++ b/acceptance/bundle/migrate/auto/run-no-commit/script @@ -19,8 +19,9 @@ title "Nothing durable written: still on terraform (local state files + remote), trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" -title "bundle run rejects engine: terraform" +title "bundle run works with engine: terraform: warns and runs the job from the terraform state" # Set after the replay: replay_tfstate.py runs bundle validate, which rejects engine: terraform. trace update_file.py databricks.yml "{name: test-bundle}" "{name: test-bundle, engine: terraform}" -trace musterr $CLI bundle run test_job --no-wait +trace $CLI bundle run test_job --no-wait trace print_requests.py //jobs +trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' From 1f43509db04f8c6e5662dcd4732946cc59a17fcd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 13:06:28 +0200 Subject: [PATCH 4/4] Inline allowTerraformEngineConfig in bundle run Co-authored-by: Isaac --- cmd/bundle/run.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index be05b604393..17199a6cabf 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -138,7 +138,9 @@ Example usage: if cmd.ArgsLenAtDash() == 0 && len(args) > 0 { b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ SkipInitialize: true, - InitFunc: allowTerraformEngineConfig, + InitFunc: func(b *bundle.Bundle) { + b.AllowTerraformEngineConfig = true + }, }) if err != nil { return err @@ -155,7 +157,10 @@ Example usage: _, _, err := utils.ProcessBundleRet(cmd, utils.ProcessOptions{ AlwaysPull: true, ErrorOnEmptyState: true, - InitFunc: allowTerraformEngineConfig, + // Let users run a bundle that pins the removed Terraform engine. + InitFunc: func(b *bundle.Bundle) { + b.AllowTerraformEngineConfig = true + }, PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { var err error key, runArgs, err = resolveRunArgument(ctx, b, args) @@ -318,8 +323,3 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { Dir: dir, }) } - -// allowTerraformEngineConfig lets users run resources of a bundle that pins the removed Terraform engine. -func allowTerraformEngineConfig(b *bundle.Bundle) { - b.AllowTerraformEngineConfig = true -}