diff --git a/acceptance/bundle/migrate/auto/default/databricks.yml b/acceptance/bundle/migrate/auto/default/databricks.yml index 7acc0c3576e..5fa3205df9d 100644 --- a/acceptance/bundle/migrate/auto/default/databricks.yml +++ b/acceptance/bundle/migrate/auto/default/databricks.yml @@ -1,5 +1,4 @@ -bundle: - name: test-bundle +bundle: {name: test-bundle} resources: jobs: diff --git a/acceptance/bundle/migrate/auto/default/output.txt b/acceptance/bundle/migrate/auto/default/output.txt index e5a10caf919..3755ca5dcde 100644 --- a/acceptance/bundle/migrate/auto/default/output.txt +++ b/acceptance/bundle/migrate/auto/default/output.txt @@ -2,6 +2,51 @@ === Replay a prior terraform deployment >>> replay_tfstate.py terraform-requests.json terraform.tfstate +=== Summary works with engine: terraform and shows the deployed job's ID and URL +>>> update_file.py databricks.yml {name: test-bundle} {name: test-bundle, engine: terraform} + +>>> [CLI] bundle summary +Warning: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + in databricks.yml:1:37 + +Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Jobs: + test_job: + Name: Test Auto-Migrate Job + URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]?w=[NUMID] + +>>> [CLI] bundle summary -o json +Warning: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + in databricks.yml:1:37 + +Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +{ + "engine": "terraform", + "id": "[TEST_JOB_ID]", + "url": "[DATABRICKS_URL]/jobs/[TEST_JOB_ID]?w=[NUMID]" +} + +=== Summary does not migrate: no state writes, terraform state is left untouched + +>>> print_requests.py + +>>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) +.databricks/bundle/default/terraform/terraform.tfstate + +>>> update_file.py databricks.yml {name: test-bundle, engine: terraform} {name: test-bundle} + === Redeploy with nothing set: direct is the default, so auto-migration triggers >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). diff --git a/acceptance/bundle/migrate/auto/default/script b/acceptance/bundle/migrate/auto/default/script index 28f598a5dfc..e5a34df7e03 100644 --- a/acceptance/bundle/migrate/auto/default/script +++ b/acceptance/bundle/migrate/auto/default/script @@ -5,8 +5,20 @@ export DATABRICKS_BUNDLE_ENGINE= # must not deploy with terraform. title "Replay a prior terraform deployment" trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py rm -f "$OUT_REQUESTS" +# Set after the replay: replay_tfstate.py runs bundle validate, which rejects engine: terraform. +title "Summary works with engine: terraform and shows the deployed job's ID and URL" +trace update_file.py databricks.yml "{name: test-bundle}" "{name: test-bundle, engine: terraform}" +trace $CLI bundle summary +trace $CLI bundle summary -o json | jq '{engine: .bundle.engine, id: .resources.jobs.test_job.id, url: .resources.jobs.test_job.url}' + +title "Summary does not migrate: no state writes, terraform state is left untouched\n" +trace print_requests.py +trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' +trace update_file.py databricks.yml "{name: test-bundle, engine: terraform}" "{name: test-bundle}" + title "Redeploy with nothing set: direct is the default, so auto-migration triggers" trace $CLI bundle deploy trace print_migration_telemetry diff --git a/acceptance/bundle/state/bad_env/output.txt b/acceptance/bundle/state/bad_env/output.txt index fea87090ecb..68190faa62a 100644 --- a/acceptance/bundle/state/bad_env/output.txt +++ b/acceptance/bundle/state/bad_env/output.txt @@ -5,8 +5,17 @@ Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'direct') >>> musterr [CLI] bundle deploy Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'direct') ->>> musterr [CLI] bundle summary -Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'direct') +>>> [CLI] bundle summary +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Schemas: + foo: + Name: foo + URL: (not deployed) >>> musterr [CLI] bundle destroy --auto-approve Error: unexpected setting for DATABRICKS_BUNDLE_ENGINE="abc" (expected 'direct') diff --git a/acceptance/bundle/state/bad_env/script b/acceptance/bundle/state/bad_env/script index fce3a3a0f85..a509ad748c8 100644 --- a/acceptance/bundle/state/bad_env/script +++ b/acceptance/bundle/state/bad_env/script @@ -1,5 +1,6 @@ export DATABRICKS_BUNDLE_ENGINE=abc trace musterr $CLI bundle plan trace musterr $CLI bundle deploy -trace musterr $CLI bundle summary +# summary does not check DATABRICKS_BUNDLE_ENGINE +trace $CLI bundle summary trace musterr $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/summary/engine-terraform/databricks.yml b/acceptance/bundle/summary/engine-terraform/databricks.yml new file mode 100644 index 00000000000..06f4c135b48 --- /dev/null +++ b/acceptance/bundle/summary/engine-terraform/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test-bundle + engine: terraform + +resources: + jobs: + my_job: + name: my job diff --git a/acceptance/bundle/summary/engine-terraform/out.test.toml b/acceptance/bundle/summary/engine-terraform/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/summary/engine-terraform/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/summary/engine-terraform/output.txt b/acceptance/bundle/summary/engine-terraform/output.txt new file mode 100644 index 00000000000..815ccf11750 --- /dev/null +++ b/acceptance/bundle/summary/engine-terraform/output.txt @@ -0,0 +1,90 @@ + +>>> [CLI] bundle summary +Warning: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + in databricks.yml:3:11 + +Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Jobs: + my_job: + Name: my job + URL: (not deployed) + +>>> [CLI] bundle summary -o json +Warning: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + in databricks.yml:3:11 + +Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +"terraform" + +>>> musterr [CLI] bundle validate +Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + in databricks.yml:3:11 + +Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +Name: test-bundle +Target: default + +Found 1 error + +>>> musterr [CLI] bundle validate -o json +Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + in databricks.yml:3:11 + +Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +{ + "engine": "terraform", + "environment": "default", + "name": "test-bundle", + "target": "default" +} + +>>> update_file.py databricks.yml engine: terraform + +>>> [CLI] bundle summary +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Resources: + Jobs: + my_job: + Name: my job + URL: (not deployed) + +>>> musterr [CLI] bundle validate +Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + +Unset the DATABRICKS_BUNDLE_ENGINE environment variable (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +Name: test-bundle +Target: default + +Found 1 error + +>>> musterr [CLI] bundle validate -o json +Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0 + +Unset the DATABRICKS_BUNDLE_ENGINE environment variable (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x. +See https://docs.databricks.com/dev-tools/bundles/direct for details + +{ + "environment": "default", + "name": "test-bundle", + "target": "default" +} diff --git a/acceptance/bundle/summary/engine-terraform/script b/acceptance/bundle/summary/engine-terraform/script new file mode 100644 index 00000000000..bf03d3b291e --- /dev/null +++ b/acceptance/bundle/summary/engine-terraform/script @@ -0,0 +1,11 @@ +trace $CLI bundle summary +trace $CLI bundle summary -o json | jq .bundle.engine +# Other commands still reject engine: terraform. +trace musterr $CLI bundle validate +trace musterr $CLI bundle validate -o json | jq .bundle + +# The env var is not checked by summary (bundle.engine is removed first, since it takes priority over the env var). +trace update_file.py databricks.yml " engine: terraform" "" +DATABRICKS_BUNDLE_ENGINE=terraform trace $CLI bundle summary +DATABRICKS_BUNDLE_ENGINE=terraform trace musterr $CLI bundle validate +DATABRICKS_BUNDLE_ENGINE=terraform trace musterr $CLI bundle validate -o json | jq .bundle diff --git a/bundle/bundle.go b/bundle/bundle.go index 5082e8a6c6c..68654bead7c 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -194,6 +194,11 @@ type Bundle struct { // leave the terraform state untouched. MigratingToDirect bool + // AllowTerraformEngineConfig reports a terraform bundle.engine as a warning instead + // of an error and skips the DATABRICKS_BUNDLE_ENGINE check. Set by read-only commands + // (bundle summary) so users can still inspect a bundle that pins the removed engine. + AllowTerraformEngineConfig bool + // Quiet is the output verbosity reduction requested via -q/--quiet, which is // repeatable: QuietSummary drops the per-resource lines, QuietAll additionally // drops the summary and progress lines, leaving warnings and errors. diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index 6ebd53df50e..056e0bd50a6 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -51,8 +51,12 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos } if parsed == engine.EngineTerraform { + severity := diag.Error + if b.AllowTerraformEngineConfig { + severity = diag.Warning + } return diags.Append(diag.Diagnostic{ - Severity: diag.Error, + Severity: severity, Summary: engine.TerraformRemovedSummary, Detail: engine.TerraformRemovedConfigDetail, Locations: []dyn.Location{loc}, diff --git a/bundle/config/validate/validate_engine_test.go b/bundle/config/validate/validate_engine_test.go index a08e5c92bae..f74cb002650 100644 --- a/bundle/config/validate/validate_engine_test.go +++ b/bundle/config/validate/validate_engine_test.go @@ -81,3 +81,19 @@ func TestValidateEngineTerraformConfigDeprecated(t *testing.T) { assert.Contains(t, diags[0].Summary, "bundle.terraform is deprecated") assert.Equal(t, []dyn.Location{loc}, diags[0].Locations) } + +func TestValidateEngineTerraformAllowed(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + Engine: engine.EngineTerraform, + }, + }, + AllowTerraformEngineConfig: true, + } + bundletest.SetLocation(b, "bundle.engine", []dyn.Location{{File: "databricks.yml", Line: 5, Column: 3}}) + diags := ValidateEngine().Apply(t.Context(), b) + assert.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Contains(t, diags[0].Summary, "has been removed") +} diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go index b3a55a607cc..1a49a89c334 100644 --- a/cmd/bundle/summary.go +++ b/cmd/bundle/summary.go @@ -31,6 +31,10 @@ Useful after deployment to see what was created and where to find it.`, AlwaysPull: forcePull, IncludeLocations: includeLocations, InitIDs: true, + // Let users inspect a bundle that pins the removed Terraform engine. + InitFunc: func(b *bundle.Bundle) { + b.AllowTerraformEngineConfig = true + }, }) if err != nil { return err diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index a93e314c7df..7e60716ad6c 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -562,12 +562,16 @@ func ValidateEngineSetting(ctx context.Context, b *bundle.Bundle) error { if !ok { return fmt.Errorf("invalid value %q for bundle.engine (expected %q)", configEngine, engine.EngineDirect) } - if parsed == engine.EngineTerraform { + if parsed == engine.EngineTerraform && !b.AllowTerraformEngineConfig { return errors.New(engine.TerraformRemovedConfigMessage) } return nil } + if b.AllowTerraformEngineConfig { + return nil + } + envEngine, err := engine.FromEnv(ctx) if err != nil { return err