diff --git a/.nextchanges/bundles/migrate-empty-state.md b/.nextchanges/bundles/migrate-empty-state.md new file mode 100644 index 00000000000..f8ddc4ee860 --- /dev/null +++ b/.nextchanges/bundles/migrate-empty-state.md @@ -0,0 +1 @@ +* Fix `bundle deployment migrate` failing with "no such file or directory" when the Terraform state has no resources or the configuration no longer declares any of them. ([#6958](https://github.com/databricks/cli/pull/6958)) diff --git a/acceptance/bundle/migrate/command/empty-state/databricks.yml b/acceptance/bundle/migrate/command/empty-state/databricks.yml new file mode 100644 index 00000000000..fe3a0edaf1d --- /dev/null +++ b/acceptance/bundle/migrate/command/empty-state/databricks.yml @@ -0,0 +1,6 @@ +bundle: + name: test-bundle + +resources: + jobs: + job_a: {name: "Job A"} diff --git a/acceptance/bundle/migrate/command/empty-state/out.test.toml b/acceptance/bundle/migrate/command/empty-state/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/command/empty-state/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/command/empty-state/output.txt b/acceptance/bundle/migrate/command/empty-state/output.txt new file mode 100644 index 00000000000..d7a6dbf872e --- /dev/null +++ b/acceptance/bundle/migrate/command/empty-state/output.txt @@ -0,0 +1,17 @@ + +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +>>> [CLI] bundle deployment migrate +Success! Migrated 0 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json + +Validate the migration by running "databricks bundle plan", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy". + +To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate + + +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan +create jobs.job_a + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/migrate/command/empty-state/script b/acceptance/bundle/migrate/command/empty-state/script new file mode 100644 index 00000000000..ae14476c979 --- /dev/null +++ b/acceptance/bundle/migrate/command/empty-state/script @@ -0,0 +1,10 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# A terraform state that tracks no resources (e.g. everything was destroyed) +trace replay_tfstate.py terraform-requests.json terraform.tfstate + +trace $CLI bundle deployment migrate + +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 to add" + +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/command/empty-state/terraform-requests.json b/acceptance/bundle/migrate/command/empty-state/terraform-requests.json new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/acceptance/bundle/migrate/command/empty-state/terraform-requests.json @@ -0,0 +1 @@ +[] diff --git a/acceptance/bundle/migrate/command/empty-state/terraform.tfstate b/acceptance/bundle/migrate/command/empty-state/terraform.tfstate new file mode 100644 index 00000000000..fce53877298 --- /dev/null +++ b/acceptance/bundle/migrate/command/empty-state/terraform.tfstate @@ -0,0 +1,9 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 1, + "lineage": "37f6c843-91a5-3d5c-457c-2c43d9630444", + "outputs": {}, + "resources": [], + "check_results": null +} diff --git a/acceptance/bundle/migrate/command/removed-all/databricks.yml b/acceptance/bundle/migrate/command/removed-all/databricks.yml new file mode 100644 index 00000000000..3495c042502 --- /dev/null +++ b/acceptance/bundle/migrate/command/removed-all/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: migrate-removed-test + +resources: + jobs: + job_a: {name: "Job A"} + job_b: {name: "Job B"} diff --git a/acceptance/bundle/migrate/command/removed-all/out.test.toml b/acceptance/bundle/migrate/command/removed-all/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/command/removed-all/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/command/removed-all/output.txt b/acceptance/bundle/migrate/command/removed-all/output.txt new file mode 100644 index 00000000000..886c4091c22 --- /dev/null +++ b/acceptance/bundle/migrate/command/removed-all/output.txt @@ -0,0 +1,51 @@ + +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +>>> [CLI] bundle deployment migrate +Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json + +Validate the migration by running "databricks bundle plan", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy". + +To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate + + +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan +delete jobs.job_a +delete jobs.job_b + +Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged + +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-removed-test/default/files... +Deleted jobs.job_a +Deleted jobs.job_b +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 2 deleted, 0 unchanged + +>>> print_requests.py //jobs/delete +{ + "headers": { + "User-Agent": [ + "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [NUMID] + } +} +{ + "headers": { + "User-Agent": [ + "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [NUMID] + } +} diff --git a/acceptance/bundle/migrate/command/removed-all/script b/acceptance/bundle/migrate/command/removed-all/script new file mode 100644 index 00000000000..0a4cf0545e8 --- /dev/null +++ b/acceptance/bundle/migrate/command/removed-all/script @@ -0,0 +1,19 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# Replay a prior terraform deployment (job_a and job_b) +trace replay_tfstate.py terraform-requests.json terraform.tfstate + +# Remove every resource from config without deploying the deletion +cat > databricks.yml <<'YAML' +bundle: + name: migrate-removed-test +YAML + +# Both IDs are preserved in direct state for deletion on next deploy +trace $CLI bundle deployment migrate + +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "2 to delete" + +rm "$OUT_REQUESTS" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve +trace print_requests.py //jobs/delete diff --git a/acceptance/bundle/migrate/command/removed-all/terraform-requests.json b/acceptance/bundle/migrate/command/removed-all/terraform-requests.json new file mode 100644 index 00000000000..20d41dc71cf --- /dev/null +++ b/acceptance/bundle/migrate/command/removed-all/terraform-requests.json @@ -0,0 +1,36 @@ +[ + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Job A", + "queue": { + "enabled": true + } + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Job B", + "queue": { + "enabled": true + } + } + } +] diff --git a/acceptance/bundle/migrate/command/removed-all/terraform.tfstate b/acceptance/bundle/migrate/command/removed-all/terraform.tfstate new file mode 100644 index 00000000000..6a8ad85c9a1 --- /dev/null +++ b/acceptance/bundle/migrate/command/removed-all/terraform.tfstate @@ -0,0 +1,210 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 1, + "lineage": "37f6c843-91a5-3d5c-457c-2c43d9630444", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "job_a", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8790614038615498000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "Job A", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8790614038615498000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + }, + { + "mode": "managed", + "type": "databricks_job", + "name": "job_b", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8790614038615799000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "Job B", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8790614038615799000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + } + ], + "check_results": null +} diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index ac67bd02ee6..0f0776152eb 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -2,6 +2,7 @@ package migrate import ( "context" + "encoding/json" "fmt" "maps" "slices" @@ -20,6 +21,28 @@ import ( "github.com/databricks/cli/libs/structs/structvar" ) +// SeedState records every terraform-state resource in stateDB, which must be open for write, so +// the migrated state persists all deployed resources, including ones the current config no longer +// declares. BuildStateFromTF then overwrites the config-declared entries with their full state (the +// later WAL entry wins on replay); the rest keep this minimal entry, which is enough for the first +// direct plan to delete them. Without this, a config that dropped every resource would record no +// WAL entries at all and Finalize would persist no state file. An empty terraform state seeds +// nothing, so its base file is written directly; the header-only Finalize leaves it intact. +func SeedState(ctx context.Context, stateDB *dstate.DeploymentState, tfState *TFState) error { + if len(tfState.IDs) == 0 && len(tfState.Attrs) == 0 { + if err := stateDB.Persist(); err != nil { + return fmt.Errorf("persisting empty migrated state: %w", err) + } + return nil + } + for key, id := range tfState.IDs { + if err := stateDB.SaveState(ctx, key, id, json.RawMessage("{}"), nil); err != nil { + return fmt.Errorf("seeding migrated state for %s: %w", key, err) + } + } + return nil +} + // BuildStateFromTF iterates over bundle resources, resolves cross-resource // references using TF state attributes, and writes each resource's state entry. // configRoot should be an un-interpolated config (with ${resources.*} references). diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 02edcee9094..d51eff57ec4 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -258,16 +258,6 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr var stateDB dstate.DeploymentState stateDB.OpenWithData(tempStatePath, dstate.NewDatabase(tfState.Lineage, tfState.Serial+1)) - // An empty terraform state seeds and builds no WAL entries below, so Finalize would persist - // no file. Write the base file now so the migration always yields one; the deferred commit - // builds on it and a crash mid-apply stays recoverable. The header-only Finalize leaves it - // intact. - if len(tfState.IDs) == 0 && len(tfState.Attrs) == 0 { - if err := stateDB.Persist(); err != nil { - return tempStatePath, false, nil, fmt.Errorf("persisting empty migrated state: %w", err) - } - } - // Apply SecretScopeFixups so the config matches what the direct engine expects. // This adds MANAGE ACL for the current user to all secret scopes, ensuring // the migrated state and config agree on .permissions entries. @@ -300,17 +290,8 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr return tempStatePath, false, nil, fmt.Errorf("upgrading state for apply: %w", err) } - // Seed every terraform-state resource into the WAL so the migrated state persists - // all deployed resources, including ones the current config no longer declares. - // BuildStateFromTF overwrites the config-declared entries below with their full - // state (the later WAL entry wins on replay); the rest keep this minimal entry, - // which is enough for the first direct plan to delete them. Without this, a config - // that dropped every resource would record no WAL entries at all, so Finalize would - // persist no state file and the migration would fail with a missing resources.json. - for key, id := range tfState.IDs { - if err := stateDB.SaveState(ctx, key, id, json.RawMessage("{}"), nil); err != nil { - return tempStatePath, false, nil, fmt.Errorf("seeding migrated state for %s: %w", key, err) - } + if err := migrate.SeedState(ctx, &stateDB, tfState); err != nil { + return tempStatePath, false, nil, err } // warnPrefix labels the conversion's warnings as coming from the background dry run. diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index be7b07c9a67..6abca072861 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -1,7 +1,6 @@ package deployment import ( - "encoding/json" "errors" "fmt" "os" @@ -126,19 +125,8 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("state file %s already exists", localPath) } - state := make(map[string]dstate.ResourceEntry) - for key, id := range tfState.IDs { - state[key] = dstate.ResourceEntry{ - ID: id, - State: json.RawMessage("{}"), - } - } - - migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) - migratedDB.State = state - var stateDB dstate.DeploymentState - stateDB.OpenWithData(tempStatePath, migratedDB) + stateDB.OpenWithData(tempStatePath, dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1)) tempStatePathAutoRemove := true @@ -165,6 +153,10 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("upgrading state for apply: %w", err) } + if err := migrate.SeedState(ctx, &stateDB, tfState); err != nil { + return err + } + if _, err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfState.Attrs, tfState.IDs, ""); err != nil { return err } @@ -197,7 +189,7 @@ Validate the migration by running "databricks bundle plan%s", there should be no The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy%s". To undo the migration, remove %s and rename %s to %s -`, len(state), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath)) +`, len(tfState.IDs), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath)) return nil }