Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/migrate-empty-state.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Fix `bundle deployment migrate` failing with "no such file or directory" when the Terraform state has no resources or the configuration no longer declares any of them. ([#6958](https://github.com/databricks/cli/pull/6958))
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
bundle:
name: test-bundle

resources:
jobs:
job_a: {name: "Job A"}
2 changes: 2 additions & 0 deletions acceptance/bundle/migrate/command/empty-state/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = [""]
17 changes: 17 additions & 0 deletions acceptance/bundle/migrate/command/empty-state/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@

>>> replay_tfstate.py terraform-requests.json terraform.tfstate

>>> [CLI] bundle deployment migrate
Success! Migrated 0 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json

Validate the migration by running "databricks bundle plan", there should be no actions planned.

The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy".

To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate


>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
create jobs.job_a

Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
10 changes: 10 additions & 0 deletions acceptance/bundle/migrate/command/empty-state/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
export DATABRICKS_BUNDLE_ENGINE=

# A terraform state that tracks no resources (e.g. everything was destroyed)
trace replay_tfstate.py terraform-requests.json terraform.tfstate

trace $CLI bundle deployment migrate

trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 to add"

rm "$OUT_REQUESTS"
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[]
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"version": 4,
"terraform_version": "1.5.5",
"serial": 1,
"lineage": "37f6c843-91a5-3d5c-457c-2c43d9630444",
"outputs": {},
"resources": [],
"check_results": null
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
bundle:
name: migrate-removed-test

resources:
jobs:
job_a: {name: "Job A"}
job_b: {name: "Job B"}
2 changes: 2 additions & 0 deletions acceptance/bundle/migrate/command/removed-all/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = [""]
51 changes: 51 additions & 0 deletions acceptance/bundle/migrate/command/removed-all/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@

>>> replay_tfstate.py terraform-requests.json terraform.tfstate

>>> [CLI] bundle deployment migrate
Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json

Validate the migration by running "databricks bundle plan", there should be no actions planned.

The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy".

To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate


>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
delete jobs.job_a
delete jobs.job_b

Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged

>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-removed-test/default/files...
Deleted jobs.job_a
Deleted jobs.job_b
Files: 1 uploaded, 0 deleted
Resources: 0 created, 0 changed, 2 deleted, 0 unchanged

>>> print_requests.py //jobs/delete
{
"headers": {
"User-Agent": [
"cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat"
]
},
"method": "POST",
"path": "/api/2.2/jobs/delete",
"body": {
"job_id": [NUMID]
}
}
{
"headers": {
"User-Agent": [
"cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat"
]
},
"method": "POST",
"path": "/api/2.2/jobs/delete",
"body": {
"job_id": [NUMID]
}
}
19 changes: 19 additions & 0 deletions acceptance/bundle/migrate/command/removed-all/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
export DATABRICKS_BUNDLE_ENGINE=

# Replay a prior terraform deployment (job_a and job_b)
trace replay_tfstate.py terraform-requests.json terraform.tfstate

# Remove every resource from config without deploying the deletion
cat > databricks.yml <<'YAML'
bundle:
name: migrate-removed-test
YAML

# Both IDs are preserved in direct state for deletion on next deploy
trace $CLI bundle deployment migrate

trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "2 to delete"

rm "$OUT_REQUESTS"
trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve
trace print_requests.py //jobs/delete
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
[
{
"method": "POST",
"path": "/api/2.2/jobs/create",
"body": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "Job A",
"queue": {
"enabled": true
}
}
},
{
"method": "POST",
"path": "/api/2.2/jobs/create",
"body": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "Job B",
"queue": {
"enabled": true
}
}
}
]
210 changes: 210 additions & 0 deletions acceptance/bundle/migrate/command/removed-all/terraform.tfstate
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
{
"version": 4,
"terraform_version": "1.5.5",
"serial": 1,
"lineage": "37f6c843-91a5-3d5c-457c-2c43d9630444",
"outputs": {},
"resources": [
{
"mode": "managed",
"type": "databricks_job",
"name": "job_a",
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
"instances": [
{
"schema_version": 2,
"attributes": {
"always_running": false,
"budget_policy_id": null,
"continuous": [],
"control_run_state": false,
"dbt_task": [],
"deployment": [
{
"deployment_id": "",
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json",
"version_id": ""
}
],
"description": null,
"edit_mode": "UI_LOCKED",
"email_notifications": [
{
"no_alert_for_skipped_runs": false,
"on_duration_warning_threshold_exceeded": [],
"on_failure": [],
"on_start": [],
"on_streaming_backlog_exceeded": [],
"on_success": []
}
],
"environment": [],
"existing_cluster_id": null,
"format": "MULTI_TASK",
"git_source": [],
"health": [],
"id": "8790614038615498000",
"job_cluster": [],
"library": [],
"max_concurrent_runs": 1,
"max_retries": 0,
"min_retry_interval_millis": 0,
"name": "Job A",
"new_cluster": [],
"notebook_task": [],
"notification_settings": [],
"parameter": [],
"parent_path": null,
"performance_target": null,
"pipeline_task": [],
"provider_config": [
{
"workspace_id": "900800700600"
}
],
"python_wheel_task": [],
"queue": [
{
"enabled": true
}
],
"retry_on_timeout": false,
"run_as": [
{
"group_name": "",
"service_principal_name": "",
"user_name": "tester@databricks.com"
}
],
"run_job_task": [],
"schedule": [],
"spark_jar_task": [],
"spark_python_task": [],
"spark_submit_task": [],
"tags": null,
"task": [],
"timeout_seconds": 0,
"timeouts": null,
"trigger": [],
"triggers": [],
"url": "[DATABRICKS_URL]/#job/8790614038615498000",
"usage_policy_id": null,
"webhook_notifications": [
{
"on_duration_warning_threshold_exceeded": [],
"on_failure": [],
"on_start": [],
"on_streaming_backlog_exceeded": [],
"on_success": []
}
]
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
}
]
},
{
"mode": "managed",
"type": "databricks_job",
"name": "job_b",
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
"instances": [
{
"schema_version": 2,
"attributes": {
"always_running": false,
"budget_policy_id": null,
"continuous": [],
"control_run_state": false,
"dbt_task": [],
"deployment": [
{
"deployment_id": "",
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json",
"version_id": ""
}
],
"description": null,
"edit_mode": "UI_LOCKED",
"email_notifications": [
{
"no_alert_for_skipped_runs": false,
"on_duration_warning_threshold_exceeded": [],
"on_failure": [],
"on_start": [],
"on_streaming_backlog_exceeded": [],
"on_success": []
}
],
"environment": [],
"existing_cluster_id": null,
"format": "MULTI_TASK",
"git_source": [],
"health": [],
"id": "8790614038615799000",
"job_cluster": [],
"library": [],
"max_concurrent_runs": 1,
"max_retries": 0,
"min_retry_interval_millis": 0,
"name": "Job B",
"new_cluster": [],
"notebook_task": [],
"notification_settings": [],
"parameter": [],
"parent_path": null,
"performance_target": null,
"pipeline_task": [],
"provider_config": [
{
"workspace_id": "900800700600"
}
],
"python_wheel_task": [],
"queue": [
{
"enabled": true
}
],
"retry_on_timeout": false,
"run_as": [
{
"group_name": "",
"service_principal_name": "",
"user_name": "tester@databricks.com"
}
],
"run_job_task": [],
"schedule": [],
"spark_jar_task": [],
"spark_python_task": [],
"spark_submit_task": [],
"tags": null,
"task": [],
"timeout_seconds": 0,
"timeouts": null,
"trigger": [],
"triggers": [],
"url": "[DATABRICKS_URL]/#job/8790614038615799000",
"usage_policy_id": null,
"webhook_notifications": [
{
"on_duration_warning_threshold_exceeded": [],
"on_failure": [],
"on_start": [],
"on_streaming_backlog_exceeded": [],
"on_success": []
}
]
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
}
]
}
],
"check_results": null
}
Loading
Loading