diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 841ffabe13b..c0bfbeaa9b8 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -25,21 +25,29 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "features": null } -=== Enabling recording is refused because the existing deployment is not recorded +=== The environment flag enables best-effort history for the existing deployment >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: deployment history setting (true) does not match the existing state (false) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over - +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Updated jobs.one +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --oneline - -=== Wiping the local cache still refuses after pulling the non-DMS state from the workspace +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_INITIAL_REGISTER", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "3"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.one"}], "previous_version_id": "2", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"3\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +=== Wiping the local cache pulls history-enabled file state without repeating registration >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: deployment history setting (true) does not match the existing state (false) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over - +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === A leftover local WAL with the wrong lineage is still refused >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy @@ -47,7 +55,7 @@ Error: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.js === Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.one @@ -62,10 +70,14 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "5"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_DELETE", "resource_key": "jobs.one"}], "previous_version_id": "3", "version_type": "VERSION_TYPE_DESTROY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/operations/jobs.one", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "DELETE", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID_2]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID_2]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} >>> find.py resources[.]json[.]wal --expect 0 diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 837b8bc1d8f..be1fe055949 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -2,17 +2,20 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" trace $CLI bundle deploy +replace_ids.py dms_plan trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json -title "Enabling recording is refused because the existing deployment is not recorded" -musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +title "The environment flag enables best-effort history for the existing deployment" +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID trace print_requests.py --dms //api/2.0/bundle --oneline -title "Wiping the local cache still refuses after pulling the non-DMS state from the workspace" +title "Wiping the local cache pulls history-enabled file state without repeating registration" rm -rf .databricks -musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "A leftover local WAL with the wrong lineage is still refused" mkdir -p .databricks/bundle/default @@ -20,11 +23,11 @@ printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone" -# Clear the stale WAL first: it was refused, not deleted, and destroy runs without recording, so it +# Clear the stale WAL first: it was refused, not deleted, so destroy # would otherwise try to replay it and fail on the lineage mismatch. rm -f .databricks/bundle/default/resources.json.wal # Keep the mismatched setting here to verify that destroy ignores configuration and follows state. -trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle destroy --auto-approve trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline trace find.py 'resources[.]json[.]wal' --expect 0 diff --git a/acceptance/bundle/dms/migration-before-deploy/databricks.yml.tmpl b/acceptance/bundle/dms/migration-before-deploy/databricks.yml.tmpl new file mode 100644 index 00000000000..a0af2a05d62 --- /dev/null +++ b/acceptance/bundle/dms/migration-before-deploy/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: dms-migration-before-deploy-$UNIQUE_NAME + +experimental: + deployment_history: false + +sync: + exclude: + - LOG.* + +resources: + jobs: + job: + name: migration-before-deploy + schemas: + schema: + name: dms_migration_before_deploy_$UNIQUE_NAME + catalog_name: main diff --git a/acceptance/bundle/dms/migration-before-deploy/out.test.toml b/acceptance/bundle/dms/migration-before-deploy/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/migration-before-deploy/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/migration-before-deploy/output.txt b/acceptance/bundle/dms/migration-before-deploy/output.txt new file mode 100644 index 00000000000..7d7fc79c06a --- /dev/null +++ b/acceptance/bundle/dms/migration-before-deploy/output.txt @@ -0,0 +1,79 @@ + +=== Setup (LOG.setup): create a job and schema without history +Next deploy enables history and recreates the schema in another catalog, requiring approval. +=== 1. Fail deployment planning +Expect: migration version 2 stays complete; no deployment version is created. +>>> fault.py GET /api/2.2/jobs/get* 403 0 1 + +>>> musterr [CLI] bundle deploy -qq +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 +Error: cannot plan resources.jobs.job: reading id="[JOB_ID]": Fault injected by test. (403 INJECTED) + +Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[JOB_ID] +HTTP Status: 403 Forbidden +API error_code: INJECTED +API message: Fault injected by test. + +Error: planning failed + + +=== Request order: complete migration -> upload full file state -> plan jobs +>>> print_requests.py --dms --get //api/2.0/bundle //jobs/get //workspace/import //workspace-files/import-file --oneline +{"method":"POST","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete"} +{"method":"POST","path":"/Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json"} +{"method":"GET","path":"/api/2.2/jobs/get"} + +=== Remote state: version 2 file state with resources; original snapshot preserved in the backup +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.marker.json + +>>> jq -c {serial, features, state} .databricks/resources.marker.json +{"serial":2,"features":{"deployment_history_best_effort":{}},"state":{"resources.jobs.job":{"__id__":"[JOB_ID]","state":{"deployment":{"kind":"BUNDLE","metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/metadata.json"},"edit_mode":"UI_LOCKED","format":"MULTI_TASK","max_concurrent_runs":1,"name":"migration-before-deploy","queue":{"enabled":true}}},"resources.schemas.schema":{"__id__":"[SCHEMA_ID]","state":{"catalog_name":"main","name":"dms_migration_before_deploy_[UNIQUE_NAME]"}}}} + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.json + +>>> cmp .databricks/resources.before-migration.json .databricks/resources.backup.json + +=== Migration result: only completed version 2, with INITIAL_REGISTER operations +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json +[{"resource_key":"jobs.job","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"schemas.schema","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] + +=== 2. Retry without approval +Expect: deployment declined; migration stays at version 2 and is not repeated. +>>> musterr [CLI] bundle deploy -qq + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.schema +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + + +=== Local and remote markers must stay byte-for-byte unchanged +>>> cmp .databricks/resources.marker.json .databricks/bundle/default/resources.json + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.after-decline.json + +>>> cmp .databricks/resources.marker.json .databricks/resources.after-decline.json + +=== No DMS writes and no extra version on the declined retry +>>> print_requests.py --dms //api/2.0/bundle + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +=== 3. Approve deployment +Expect: ordinary deployment version 3; version 2 remains the only migration. +>>> [CLI] bundle deploy --auto-approve -qq + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.schema + +>>> [CLI] bundle summary -o json +{"latest_version_id":3} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] diff --git a/acceptance/bundle/dms/migration-before-deploy/script b/acceptance/bundle/dms/migration-before-deploy/script new file mode 100644 index 00000000000..02dd4659937 --- /dev/null +++ b/acceptance/bundle/dms/migration-before-deploy/script @@ -0,0 +1,50 @@ +envsubst < databricks.yml.tmpl > databricks.yml +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-before-deploy-${UNIQUE_NAME}/default/state" +other_catalog="dms_other_${UNIQUE_NAME}" + +title "Setup (LOG.setup): create a job and schema without history\nNext deploy enables history and recreates the schema in another catalog, requiring approval." +{ + $CLI bundle deploy -qq + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py + cp .databricks/bundle/default/resources.json .databricks/resources.before-migration.json + $CLI catalogs create "$other_catalog" + update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' + update_file.py databricks.yml 'catalog_name: main' "catalog_name: ${other_catalog}" +} &> LOG.setup +trap '$CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1; $CLI catalogs delete "$other_catalog" --force >> LOG.cleanup 2>&1; rm -f "$OUT_REQUESTS"' EXIT +rm -f "$OUT_REQUESTS" + +title "1. Fail deployment planning\nExpect: migration version 2 stays complete; no deployment version is created." +trace fault.py 'GET /api/2.2/jobs/get*' 403 0 1 +trace musterr $CLI bundle deploy -qq +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_path/resources.deployment.json" -o json | jq -r '.object_id') +add_repl "$deployment_id" DEPLOYMENT_ID +title "Request order: complete migration -> upload full file state -> plan jobs" +trace print_requests.py --dms --get //api/2.0/bundle //jobs/get //workspace/import //workspace-files/import-file --oneline | + MSYS_NO_PATHCONV=1 jq -c --arg marker "$state_path/resources.json" ' + (.body.multipart_form.path // .body.path // (.path | ltrimstr("/api/2.0/workspace-files/import-file"))) as $destination | + select((.path | endswith("/complete")) or (.path | endswith("/resources")) or .path == "/api/2.2/jobs/get" or $destination == $marker) | + {method, path: (if $destination == $marker then $destination else .path end)}' +title "Remote state: version 2 file state with resources; original snapshot preserved in the backup" +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.marker.json +trace jq -c '{serial, features, state}' .databricks/resources.marker.json +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.json +trace cmp .databricks/resources.before-migration.json .databricks/resources.backup.json +title "Migration result: only completed version 2, with INITIAL_REGISTER operations" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'map({version_id, status, completion_reason})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, action_type, status})' + +title "2. Retry without approval\nExpect: deployment declined; migration stays at version 2 and is not repeated." +trace musterr $CLI bundle deploy -qq +title "Local and remote markers must stay byte-for-byte unchanged" +trace cmp .databricks/resources.marker.json .databricks/bundle/default/resources.json +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.after-decline.json +trace cmp .databricks/resources.marker.json .databricks/resources.after-decline.json +title "No DMS writes and no extra version on the declined retry" +trace print_requests.py --dms //api/2.0/bundle +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'map({version_id, status, completion_reason})' + +title "3. Approve deployment\nExpect: ordinary deployment version 3; version 2 remains the only migration." +trace $CLI bundle deploy --auto-approve -qq +trace $CLI bundle summary -o json | jq -c '.bundle.deployment | {latest_version_id}' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, status, completion_reason})' diff --git a/acceptance/bundle/dms/migration-before-deploy/test.toml b/acceptance/bundle/dms/migration-before-deploy/test.toml new file mode 100644 index 00000000000..7123b8a9421 --- /dev/null +++ b/acceptance/bundle/dms/migration-before-deploy/test.toml @@ -0,0 +1,2 @@ +# Fault injection exercises a real planner failure against the local testserver. +Cloud = false diff --git a/acceptance/bundle/dms/migration-fails-open/databricks.yml.tmpl b/acceptance/bundle/dms/migration-fails-open/databricks.yml.tmpl new file mode 100644 index 00000000000..05cb4515bc8 --- /dev/null +++ b/acceptance/bundle/dms/migration-fails-open/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: dms-migration-fails-open-$UNIQUE_NAME + +experimental: + deployment_history: false + +resources: + jobs: + one: + name: one + two: + name: two + tasks: + - task_key: run_one + run_job_task: + job_id: ${resources.jobs.one.id} diff --git a/acceptance/bundle/dms/migration-fails-open/out.test.toml b/acceptance/bundle/dms/migration-fails-open/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/migration-fails-open/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/migration-fails-open/output.txt b/acceptance/bundle/dms/migration-fails-open/output.txt new file mode 100644 index 00000000000..184ada0a93b --- /dev/null +++ b/acceptance/bundle/dms/migration-fails-open/output.txt @@ -0,0 +1,51 @@ + +=== Create file-backed resources without history +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-migration-fails-open-[UNIQUE_NAME]/default/files... +Created jobs.one +Created jobs.two +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Fail one registration upload; enable history and deploy an update anyway +>>> fault.py PATCH /api/2.0/bundle/* 400 0 1 + +>>> update_file.py databricks.yml deployment_history: false deployment_history: true + +>>> update_file.py databricks.yml name: one name: updated + +>>> [CLI] bundle deploy +Enabling deployment history... +Warn: Initial history registration was incomplete; continuing with file state: recording migration version: recording operation for resources.jobs.one: Fault injected by test. +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-migration-fails-open-[UNIQUE_NAME]/default/files... +Updated jobs.one +Updated jobs.two +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + +=== The file still owns both resources, with the original IDs +>>> jq {serial, features, resources: (.state | keys), names: (.state | with_entries(.value = .value.state.name))} .databricks/bundle/default/resources.json +{ + "serial": 3, + "features": { + "deployment_history_best_effort": {} + }, + "resources": [ + "resources.jobs.one", + "resources.jobs.two" + ], + "names": { + "resources.jobs.one": "updated", + "resources.jobs.two": "two" + } +} + +>>> cmp .databricks/ids.before.json .databricks/ids.after.json + +=== Migration failed, but the following deployment completed successfully +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","completion_reason":"VERSION_COMPLETE_FAILURE"},{"version_id":"3","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json +[{"resource_key":"jobs.one","status":"OPERATION_STATUS_PENDING"},{"resource_key":"jobs.two","status":"OPERATION_STATUS_SUCCEEDED"}] diff --git a/acceptance/bundle/dms/migration-fails-open/script b/acceptance/bundle/dms/migration-fails-open/script new file mode 100644 index 00000000000..371e0f4d1aa --- /dev/null +++ b/acceptance/bundle/dms/migration-fails-open/script @@ -0,0 +1,25 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Create file-backed resources without history" +trace $CLI bundle deploy +DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py +jq '.state | with_entries(.value = .value.__id__)' .databricks/bundle/default/resources.json > .databricks/ids.before.json + +title "Fail one registration upload; enable history and deploy an update anyway" +trace fault.py "PATCH /api/2.0/bundle/*" 400 0 1 +trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' +trace update_file.py databricks.yml 'name: one' 'name: updated' +trace $CLI bundle deploy + +title "The file still owns both resources, with the original IDs" +trace jq '{serial, features, resources: (.state | keys), names: (.state | with_entries(.value = .value.state.name))}' .databricks/bundle/default/resources.json +jq '.state | with_entries(.value = .value.__id__)' .databricks/bundle/default/resources.json > .databricks/ids.after.json +trace cmp .databricks/ids.before.json .databricks/ids.after.json + +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-fails-open-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | jq -r '.object_id') +add_repl "$deployment_id" DEPLOYMENT_ID + +title "Migration failed, but the following deployment completed successfully" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, completion_reason})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, status})' +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/migration-fails-open/test.toml b/acceptance/bundle/dms/migration-fails-open/test.toml new file mode 100644 index 00000000000..814a8f36262 --- /dev/null +++ b/acceptance/bundle/dms/migration-fails-open/test.toml @@ -0,0 +1,2 @@ +# Fault injection verifies best-effort registration against the local fake. +Cloud = false diff --git a/acceptance/bundle/dms/migration/databricks.yml.tmpl b/acceptance/bundle/dms/migration/databricks.yml.tmpl new file mode 100644 index 00000000000..bb879a94254 --- /dev/null +++ b/acceptance/bundle/dms/migration/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: dms-migration-$UNIQUE_NAME + +experimental: + deployment_history: false + +resources: + jobs: + one: + name: one + two: + name: two + tasks: + - task_key: run_one + run_job_task: + job_id: ${resources.jobs.one.id} diff --git a/acceptance/bundle/dms/migration/out.test.toml b/acceptance/bundle/dms/migration/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/dms/migration/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/migration/output.txt b/acceptance/bundle/dms/migration/output.txt new file mode 100644 index 00000000000..69167c2441a --- /dev/null +++ b/acceptance/bundle/dms/migration/output.txt @@ -0,0 +1,174 @@ + +=== 1. Create two jobs without history (direct state serial 1) +Config disables history even though the environment enables it. +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/files... +Created jobs.one +Created jobs.two +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Save the original resources.json for the backup check +>>> cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json + +=== 2. Enable history and rename job one +Expect: version 2 registers old state; version 3 applies the new name; no new jobs. +>>> update_file.py databricks.yml deployment_history: false deployment_history: true + +>>> update_file.py databricks.yml name: one name: updated + +>>> musterr [CLI] bundle deploy --plan .databricks/before-history.plan.json +Error: enabling deployment history on existing state does not support --plan; run 'bundle deploy' without --plan + +>>> [CLI] bundle deploy +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/files... +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 +Updated jobs.one +Updated jobs.two +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + +=== 3. Check local state +Expect: serial=3; history enforced; full resource state remains in the file; no WAL. +>>> find.py resources[.]json[.]wal --expect 0 + +>>> jq {serial, features, state} .databricks/bundle/default/resources.json +{ + "serial": 3, + "features": { + "deployment_history_best_effort": {} + }, + "state": { + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "3" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "updated", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "3" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [ONE_ID] + }, + "task_key": "run_one" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.one", + "label": "${resources.jobs.one.id}" + } + ] + } + } +} + +=== 4. Check backup +Expect: original IDs, names and dependencies; exact pre-migration bytes. +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.first.json + +>>> cat .databricks/resources.backup.first.json +{ + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "state": { + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [ONE_ID] + }, + "task_key": "run_one" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.one", + "label": "${resources.jobs.one.id}" + } + ] + } + } +} +>>> cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json + +=== 5. Check DMS records +Expect: completed versions 2 and 3; original job IDs and dependency preserved. +=== Version history: migration 2 completes before deployment 3 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +=== Migration 2: register the original job IDs (INITIAL_REGISTER) +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] + +=== Deployment 3: record updates separately +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] + +=== Current resources: new name, original IDs, and job two still depends on job one +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] diff --git a/acceptance/bundle/dms/migration/script b/acceptance/bundle/dms/migration/script new file mode 100644 index 00000000000..784e051e751 --- /dev/null +++ b/acceptance/bundle/dms/migration/script @@ -0,0 +1,43 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "1. Create two jobs without history (direct state serial 1)\nConfig disables history even though the environment enables it." +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history' +replace_ids.py +title "Save the original resources.json for the backup check" +trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json +$CLI bundle plan -o json > .databricks/before-history.plan.json + +title "2. Enable history and rename job one\nExpect: version 2 registers old state; version 3 applies the new name; no new jobs." +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false +trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' +trace update_file.py databricks.yml 'name: one' 'name: updated' +# Initial registration changes state features and serial, so an older saved plan is refused. +trace musterr $CLI bundle deploy --plan .databricks/before-history.plan.json +trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' 'Resources: 0 created' + +title "3. Check local state\nExpect: serial=3; history enforced; full resource state remains in the file; no WAL." +trace find.py 'resources[.]json[.]wal' --expect 0 +trace jq '{serial, features, state}' .databricks/bundle/default/resources.json + +title "4. Check backup\nExpect: original IDs, names and dependencies; exact pre-migration bytes." +trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-${UNIQUE_NAME}/default/state/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json +trace cat .databricks/resources.backup.first.json +trace cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json + +title "5. Check DMS records\nExpect: completed versions 2 and 3; original job IDs and dependency preserved." +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID +title "Version history: migration 2 completes before deployment 3" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' +title "Migration 2: register the original job IDs (INITIAL_REGISTER)" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Deployment 3: record updates separately" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/3" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Current resources: new name, original IDs, and job two still depends on job one" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -c ' + sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, + state: (.state | fromjson | {name: .state.name, + tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}], + depends_on: (.depends_on // [])})})' + +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/migration/test.toml b/acceptance/bundle/dms/migration/test.toml new file mode 100644 index 00000000000..4da5c0d15a3 --- /dev/null +++ b/acceptance/bundle/dms/migration/test.toml @@ -0,0 +1,17 @@ +# This test checks config precedence with conflicting environment settings, +# so it drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY instead of the parent's matrix. +EnvMatrix.DMS = [""] + +# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens. +# Inherited Repls append and cannot be disabled here. +# Mask at Order -1 before the parent's default 0, then restore at +1. +# This lets these tests assert both migration and deployment links. +[[Repls]] +Old = '(?m)^Current Deployment Version: ' +New = '__CURRENT_DEPLOYMENT_VERSION__: ' +Order = -1 + +[[Repls]] +Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: ' +New = 'Current Deployment Version: ' +Order = 1 diff --git a/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl b/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl new file mode 100644 index 00000000000..6c29e76684e --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: dms-repeated-migration-$UNIQUE_NAME + +experimental: + deployment_history: false + +sync: + exclude: + - LOG.* + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/repeated-migration/out.test.toml b/acceptance/bundle/dms/repeated-migration/out.test.toml new file mode 100644 index 00000000000..84b22ba6735 --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/repeated-migration/output.txt b/acceptance/bundle/dms/repeated-migration/output.txt new file mode 100644 index 00000000000..420a829d71b --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/output.txt @@ -0,0 +1,106 @@ + +=== Setup (LOG.setup) +Current: job=updated; DMS version=3; old backup=one. + +=== 1. Compare the old backup with the latest saved state +Expect: old backup name=one; latest state name=updated. + +>>> gron.py .databricks/resources.backup.first.json +json.state.resources.jobs.one.state.name = "one"; + +>>> gron.py .databricks/resources.before-repeat.json +json.state.resources.jobs.one.state.name = "updated"; + +=== 2. Restore saved state without its history flag and deploy again +Expect: migration=4; deployment=5; name=repeated; no new jobs. + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json --file .databricks/resources.before-repeat.json --format AUTO --overwrite + +>>> cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json + +>>> update_file.py databricks.yml name: updated name: repeated + +>>> [CLI] bundle deploy +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/files... +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=5 +Updated jobs.one +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== 3. Check DMS records +Expect: completed migration 4 and deployment 5; original ID preserved; name=repeated. + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"4","previous_version_id":"3","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"5","previous_version_id":"4","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +=== Migration 4: register the existing job ID again (INITIAL_REGISTER) +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/4 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] + +=== Deployment 5: update the job name without creating a replacement +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/5 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] + +=== Current resource: original job ID, name=repeated, last version=5 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_version_id":"5","name":"repeated"}] + +=== 4. Check backup was overwritten +Expect: name=updated (before this deploy), not one or repeated. + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.repeat.json + +>>> cat .databricks/resources.backup.repeat.json +{ + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "state": { + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "updated", + "queue": { + "enabled": true + } + } + } + } +} + +=== Backup matches the restored snapshot byte for byte +>>> cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json + +=== Compare backups: serial 1 -> 3 and job name one -> updated +>>> diff.py .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json +--- .databricks/resources.backup.first.json ++++ .databricks/resources.backup.repeat.json +@@ -3,5 +3,5 @@ + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", +- "serial": 1, ++ "serial": 3, + "state": { + "resources.jobs.one": { +@@ -15,5 +15,5 @@ + "format": "MULTI_TASK", + "max_concurrent_runs": 1, +- "name": "one", ++ "name": "updated", + "queue": { + "enabled": true +@@ -22,3 +22,3 @@ + } + } +-} ++} diff --git a/acceptance/bundle/dms/repeated-migration/script b/acceptance/bundle/dms/repeated-migration/script new file mode 100644 index 00000000000..99d363df7f3 --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/script @@ -0,0 +1,48 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-repeated-migration-${UNIQUE_NAME}/default/state" +title "Setup (LOG.setup)\nCurrent: job=updated; DMS version=3; old backup=one.\n" +{ + trace $CLI bundle deploy + trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py + trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' + trace update_file.py databricks.yml 'name: one' 'name: updated' + trace $CLI bundle deploy + deployment_id=$(trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_path/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') + add_repl "$deployment_id" DEPLOYMENT_ID + trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json + trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.before-repeat.json +} &> LOG.setup + +title "1. Compare the old backup with the latest saved state\nExpect: old backup name=one; latest state name=updated.\n" +trace gron.py .databricks/resources.backup.first.json | grep '\.state\.name = ' | contains.py '"one"' +trace gron.py .databricks/resources.before-repeat.json | grep '\.state\.name = ' | contains.py '"updated"' + +title "2. Restore saved state without its history flag and deploy again\nExpect: migration=4; deployment=5; name=repeated; no new jobs.\n" +trace MSYS_NO_PATHCONV=1 $CLI workspace import "$state_path/resources.json" --file .databricks/resources.before-repeat.json --format AUTO --overwrite +trace cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json +trace update_file.py databricks.yml 'name: updated' 'name: repeated' +trace $CLI bundle deploy 2>&1 | contains.py 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=4' '?version=5' 'Resources: 0 created' '!Created ' + +title "3. Check DMS records\nExpect: completed migration 4 and deployment 5; original ID preserved; name=repeated.\n" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | + jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' +title "Migration 4: register the existing job ID again (INITIAL_REGISTER)" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/4" -o json | + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Deployment 5: update the job name without creating a replacement" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/5" -o json | + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Current resource: original job ID, name=repeated, last version=5" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | + jq -c 'map({resource_key, resource_id, last_version_id, name: (.state | fromjson | .state.name)})' + +title "4. Check backup was overwritten\nExpect: name=updated (before this deploy), not one or repeated.\n" +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.repeat.json +trace cat .databricks/resources.backup.repeat.json | contains.py '"name": "updated"' '!"name": "one"' '!"name": "repeated"' +title "Backup matches the restored snapshot byte for byte" +trace cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json +title "Compare backups: serial 1 -> 3 and job name one -> updated" +trace diff.py .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json + +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/repeated-migration/test.toml b/acceptance/bundle/dms/repeated-migration/test.toml new file mode 100644 index 00000000000..8182a51630b --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/test.toml @@ -0,0 +1,13 @@ +# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens. +# Inherited Repls append and cannot be disabled here. +# Mask at Order -1 before the parent's default 0, then restore at +1. +# This lets these tests assert both migration and deployment links. +[[Repls]] +Old = '(?m)^Current Deployment Version: ' +New = '__CURRENT_DEPLOYMENT_VERSION__: ' +Order = -1 + +[[Repls]] +Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: ' +New = 'Current Deployment Version: ' +Order = 1 diff --git a/acceptance/bundle/migrate/auto/dms-migration/output.txt b/acceptance/bundle/migrate/auto/dms-migration/output.txt index ba6f36cc8bf..0e0d6c26459 100644 --- a/acceptance/bundle/migrate/auto/dms-migration/output.txt +++ b/acceptance/bundle/migrate/auto/dms-migration/output.txt @@ -4,11 +4,7 @@ === Direct + deployment-history requested against a terraform state is rejected before commit >>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true errcode [CLI] bundle deploy -Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Error: deployment history setting (true) does not match the existing state (false) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over - +Error: deployment history is not supported with Terraform state; first migrate to direct deployment with deployment history disabled Exit code: 1 diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 2df01b7e226..b56487fb378 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -57,6 +57,7 @@ type Experimental struct { // Resource state remains in resources.json. History uploads are best effort: // DMS errors do not fail file-backed deployments. // Existing DMS-authoritative deployments keep their current state source. + // Terraform state must first be migrated to direct with history disabled. // An explicit value takes priority over the environment setting. DeploymentHistory *bool `json:"deployment_history,omitempty"` } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 57e10b8d4b4..c8f2a5fbee3 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/dms" ) @@ -16,6 +17,24 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } +// MarkFileHistory commits the protocol marker without replacing resource state. +// The caller holds the deployment lock and has attempted initial registration. +func (db *DeploymentState) MarkFileHistory(versionID int) error { + db.AssertOpenedForRead() + db.mu.Lock() + defer db.mu.Unlock() + + db.VersionID = max(db.VersionID, versionID) + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}) + } + db.Data.Features[FeatureDeploymentHistoryBestEffort] = struct{}{} + db.Data.Serial++ + db.Data.Lineage = db.GetOrInitLineage() + db.Data.CLIVersion = build.GetInfo().Version + return db.unlockedSave() +} + // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS // resources.json does not track state. The service is queried for state for resources and then they // are filled in-place in DeploymentState.Data.State: diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 1bf195a6034..684e4b0375f 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -163,7 +163,7 @@ func prepareDeploymentHistory(ctx context.Context, b *bundle.Bundle, plan *deplo if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { return err } - logDeploymentVersion(ctx, b) + logDeploymentVersion(ctx, b, "Current Deployment Version") return nil } @@ -237,7 +237,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version // logDeploymentVersion logs the deployment version URL. Workspace ID is omitted // so the page stays clickable in a terminal and redirects correctly without it. -func logDeploymentVersion(ctx context.Context, b *bundle.Bundle) { +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, message string) { deploymentID := b.DeploymentBundle.StateDB.DeploymentID version := b.DeploymentBundle.StateDB.VersionID if version == 0 { @@ -249,11 +249,11 @@ func logDeploymentVersion(ctx context.Context, b *bundle.Bundle) { // Only the link is lost, so report the version without it rather than failing // a deploy over it. log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) - cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", deploymentID, version)) + cmdio.LogString(ctx, fmt.Sprintf("%s: %s version %d", message, deploymentID, version)) return } - cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) + cmdio.LogString(ctx, message+": "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) } // deploymentMetadata describes the bundle this deploy came from and where it diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go new file mode 100644 index 00000000000..6796a64802e --- /dev/null +++ b/bundle/phases/dms_migration.go @@ -0,0 +1,160 @@ +package phases + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strconv" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/lock" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// ErrHistoryRequiresDirectState keeps Terraform conversion separate from history registration. +var ErrHistoryRequiresDirectState = errors.New("deployment history is not supported with Terraform state; first migrate to direct deployment with deployment history disabled") + +// MigrateToFileHistory registers the open file snapshot before normal deployment planning. +// History errors only warn; resources.json remains the authoritative resource state. +func MigrateToFileHistory(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateDesc) { + if !state.Engine.IsDirect() { + logdiag.LogError(ctx, ErrHistoryRequiresDirectState) + return + } + db := &b.DeploymentBundle.StateDB + recorded := state.RecordsHistory() + if db.IsOpen() { + recorded = db.RecordsHistory() + } + if recorded { + return + } + + bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalDeploy)) + if logdiag.HasError(ctx) { + return + } + defer bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) + + db.AssertOpenedForRead() + // Another deployment may have finished while this command built artifacts. + // Recheck under the lock before committing the snapshot loaded before the build. + latest := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) + if logdiag.HasError(ctx) { + return + } + if latest.Lineage != db.Data.Lineage || latest.Serial > db.Data.Serial { + logdiag.LogError(ctx, errors.New("deployment state changed before enabling history; run 'bundle deploy' again")) + return + } + w := b.WorkspaceClient(ctx) + id, deployment, err := dms.ReadDeployment(ctx, w, b.Config.Workspace.StatePath) + if err != nil { + log.Warnf(ctx, "Could not initialize deployment history; continuing with file state: %s", err) + return + } + lastVersion, err := dms.ParseLastVersionID(deployment) + if err != nil { + log.Warnf(ctx, "Could not initialize deployment history; continuing with file state: %s", err) + return + } + if err := statemgmt.BackupRemoteResourcesState(ctx, b); err != nil { + log.Warnf(ctx, "Could not back up resource state before history registration: %s", err) + } + db.DeploymentID, db.VersionID = id, lastVersion + if err := createOrUpdateDeployment(ctx, b, deployment); err != nil { + log.Warnf(ctx, "Could not initialize deployment history; continuing with file state: %s", err) + return + } + if !cmdctx.HasWorkspaceClient(ctx) { + ctx = cmdctx.SetWorkspaceClient(ctx, w) + } + cmdio.LogString(ctx, "Enabling deployment history...") + versionID, recordErr := registerMigrationState(ctx, db) + if recordErr != nil { + log.Warnf(ctx, "Initial history registration was incomplete; continuing with file state: %s", recordErr) + } + // Commit only the file marker. IDs, resource payloads and dependency edges stay intact. + if err := db.MarkFileHistory(versionID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("saving history-enabled file state: %w", err)) + return + } + logDeploymentVersion(ctx, b, "Deployment history enabled") + statemgmt.PushResourcesState(ctx, b) + state.Serial = db.Data.Serial +} + +// registerMigrationState records every resource from the open direct state in an initial registration version. +// The caller must hold the deployment lock and have created the DMS deployment. +func registerMigrationState(ctx context.Context, db *dstate.DeploymentState) (int, error) { + keys := slices.Sorted(maps.Keys(db.Data.State)) + if len(keys) == 0 { + return 0, nil + } + states := make([]json.RawMessage, len(keys)) + operations := make([]bundledeployments.StagedOperation, len(keys)) + for i, key := range keys { + entry := db.Data.State[key] + state, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + return 0, fmt.Errorf("serializing migration state for %s: %w", key, err) + } + states[i] = state + operations[i] = bundledeployments.StagedOperation{ + ResourceKey: strings.TrimPrefix(key, dms.StatePrefix), + ActionType: bundledeployments.OperationActionTypeOperationActionTypeInitialRegister, + } + } + + // Migration uses max(state serial, last version)+1; deployment follows at the next version. + versionID := max(db.Data.Serial, db.VersionID) + 1 + previousVersionID := "" + if db.VersionID > 0 { + previousVersionID = strconv.Itoa(db.VersionID) + } + w := cmdctx.WorkspaceClient(ctx) + _, err := w.BundleDeployments.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ + Parent: dms.DeploymentName(db.DeploymentID), + VersionId: strconv.Itoa(versionID), + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: dms.VersionTypeDeploy, + PreviousVersionId: previousVersionID, + Operations: operations, + }, + }) + if err != nil { + return 0, fmt.Errorf("creating migration version: %w", err) + } + + buffer := dms.StartOperationBuffer(ctx, db.DeploymentID, versionID, dms.BufferOptions{BestEffort: true, OperationCount: len(keys)}) + for i, key := range keys { + buffer.RecordOperation(ctx, key, false, db.Data.State[key].ID, states[i]) + } + recordErr := buffer.Drain(ctx) + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if recordErr != nil { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + _, completeErr := w.BundleDeployments.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: dms.VersionName(db.DeploymentID, versionID), + CompletionReason: reason, + }) + if err := errors.Join(recordErr, completeErr); err != nil { + return versionID, fmt.Errorf("recording migration version: %w", err) + } + + return versionID, nil +} diff --git a/bundle/phases/dms_migration_test.go b/bundle/phases/dms_migration_test.go new file mode 100644 index 00000000000..3e4297dc46c --- /dev/null +++ b/bundle/phases/dms_migration_test.go @@ -0,0 +1,137 @@ +package phases + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/apierr" + sdkconfig "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestMigrateToFileHistoryRejectsTerraformBeforeLock(t *testing.T) { + // 1. Try migration with Terraform state and locking enabled by default. + b := &bundle.Bundle{} + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + + MigrateToFileHistory(ctx, b, &statemgmt.StateDesc{Engine: engine.EngineTerraform}) + + // 2. Reject the state before creating a deployment lock. + require.True(t, logdiag.HasError(ctx)) + assert.Nil(t, b.Locker, "Terraform state must be rejected before locking") +} + +func TestMigrateToFileHistorySkipsAlreadyRecordedBeforeLock(t *testing.T) { + // 1. File state already records history; no open database or client is needed. + b := &bundle.Bundle{} + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + state := &statemgmt.StateDesc{ + Features: map[string]struct{}{dstate.FeatureDeploymentHistoryBestEffort: {}}, + } + + MigrateToFileHistory(ctx, b, state) + + // 2. Return successfully without acquiring a migration lock. + require.False(t, logdiag.HasError(ctx), "%v", logdiag.FlushCollected(ctx)) + assert.Nil(t, b.Locker, "Already-recorded state must not acquire a migration lock") +} + +func TestRegisterMigrationStateVersionNumbers(t *testing.T) { + // Acceptance tests cover registration and persistence. These cases cover unequal counters. + tests := []struct { + name string + stateSerial int + serviceVersion int + wantVersion int + }{ + { + name: "service version is newer", + stateSerial: 2, + serviceVersion: 5, + wantVersion: 6, + }, + { + name: "state serial is newer", + stateSerial: 8, + serviceVersion: 3, + wantVersion: 9, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // 1. Open state with deliberately different file and service counters. + statePath := filepath.Join(t.TempDir(), "resources.json") + var db dstate.DeploymentState + db.OpenWithData(statePath, dstate.NewDatabase("lineage", tt.stateSerial)) + db.Data.State["resources.jobs.job"] = dstate.ResourceEntry{ID: "123"} + db.DeploymentID = "123" + db.VersionID = tt.serviceVersion + + // 2. Registration uses max(file, service)+1, but its predecessor is the service version. + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockBundleDeploymentsAPI() + api.EXPECT().CreateVersion(mock.Anything, mock.Anything). + Run(func(_ context.Context, request bundledeployments.CreateVersionRequest) { + assert.Equal(t, strconv.Itoa(tt.wantVersion), request.VersionId, "Migration must follow the larger counter") + assert.Equal(t, strconv.Itoa(tt.serviceVersion), request.Version.PreviousVersionId, "Predecessor must be the actual DMS version") + }).Return(&bundledeployments.Version{}, nil).Once() + api.EXPECT().UpdateOperation(mock.Anything, mock.Anything). + Return(&bundledeployments.Operation{SequenceId: 1}, nil).Once() + api.EXPECT().CompleteVersion(mock.Anything, mock.Anything).Return(&bundledeployments.Version{}, nil).Once() + ctx := cmdctx.SetWorkspaceClient(t.Context(), m.WorkspaceClient) + + // 3. Run registration; the mock checks the version numbers above. + version, err := registerMigrationState(ctx, &db) + require.NoError(t, err) + assert.Equal(t, tt.wantVersion, version) + }) + } +} + +func TestMigrateToFileHistoryRejectsNewerRemoteState(t *testing.T) { + b := &bundle.Bundle{BundleRootPath: t.TempDir()} + b.Config.Bundle.Target = "default" + b.Config.Bundle.Deployment.Lock.Enabled = new(false) + b.Config.Workspace.StatePath = "/state" + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + _, localPath := b.StateFilenameDirect(ctx) + b.DeploymentBundle.StateDB.OpenWithData(localPath, dstate.NewDatabase("lineage", 1)) + + // The command opened serial 1, but another deploy committed serial 2 before migration. + newer := `{"lineage":"lineage","serial":2,"state":{"resources.jobs.new":{"__id__":"new-id"}}}` + m := mocks.NewMockWorkspaceClient(t) + m.WorkspaceClient.Config = &sdkconfig.Config{Host: "https://workspace.test", Token: "token"} + m.GetMockWorkspaceAPI().EXPECT().Download(mock.Anything, "/state/resources.json"). + Return(io.NopCloser(strings.NewReader(newer)), nil).Once() + m.GetMockWorkspaceAPI().EXPECT().Download(mock.Anything, "/state/terraform.tfstate"). + Return(nil, apierr.ErrNotFound).Once() + b.SetWorkpaceClient(m.WorkspaceClient) + + MigrateToFileHistory(ctx, b, &statemgmt.StateDesc{Engine: engine.EngineDirect}) + + // No DMS writes are expected. Keep the newer file; never replace it with serial 1. + require.True(t, logdiag.HasError(ctx)) + assert.Contains(t, fmt.Sprint(logdiag.FlushCollected(ctx)), "deployment state changed before enabling history") + content, err := os.ReadFile(localPath) + require.NoError(t, err) + assert.JSONEq(t, newer, string(content)) +} diff --git a/bundle/statemgmt/state_push.go b/bundle/statemgmt/state_push.go index 03036ba712d..622dbd6d925 100644 --- a/bundle/statemgmt/state_push.go +++ b/bundle/statemgmt/state_push.go @@ -3,6 +3,7 @@ package statemgmt import ( "context" "errors" + "fmt" "io/fs" "os" @@ -43,6 +44,27 @@ func PushResourcesState(ctx context.Context, b *bundle.Bundle) { } } +// BackupRemoteResourcesState preserves the remote state before enabling deployment history. +func BackupRemoteResourcesState(ctx context.Context, b *bundle.Bundle) error { + f, err := deploy.StateFiler(ctx, b) + if err != nil { + return fmt.Errorf("backing up resource state: creating state filer: %w", err) + } + + remotePath, _ := b.StateFilenameDirect(ctx) + reader, err := f.Read(ctx, remotePath) + if err != nil { + return fmt.Errorf("backing up resource state: reading %s: %w", remotePath, err) + } + defer reader.Close() + + backupPath := remotePath + ".backup" + if err := f.Write(ctx, backupPath, reader, filer.OverwriteIfExists); err != nil { + return fmt.Errorf("backing up resource state: writing %s: %w", backupPath, err) + } + return nil +} + func BackupRemoteTerraformState(ctx context.Context, b *bundle.Bundle) { f, err := deploy.StateFiler(ctx, b) if err != nil { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 4a5e666dabb..fcbbfa06298 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,9 +4,7 @@ import ( "context" "errors" "fmt" - "path" "path/filepath" - "strconv" "time" "github.com/databricks/cli/bundle" @@ -31,14 +29,10 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" - "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/databricks/databricks-sdk-go/useragent" "github.com/spf13/cobra" "golang.org/x/mod/semver" @@ -333,6 +327,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // was removed in v1.20.0, so a migration failure is fatal - there is no engine to fall // back to. Read-only commands that do not need state keep reading the Terraform state. if b.MigratingToDirect && needsState { + // Keep the Terraform conversion separate from enabling deployment history. + if configuresDeploymentHistory(ctx, b) && (opts.Deploy || opts.PreDeployChecks) { + return b, stateDesc, phases.ErrHistoryRequiresDirectState + } if err := migrateTerraformToDirect(ctx, b, stateDesc); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -369,6 +367,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } + if needDirectState && !stateDesc.IsDMS() { + // Registration updates the header; warn about its previous writer first. + warnIfNewerState(ctx, b) + } + stateMode := dstate.WritingFile switch { case stateDesc.IsDMS(): @@ -382,6 +385,17 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle case configuresDeploymentHistory(ctx, b): // Deploy and plan follow the current configuration. stateMode = dstate.WritingFileWithHistoryEnforced + // Commit registration before planning so later failures do not repeat migration. + // The file remains authoritative and stays open for planning. + if opts.Deploy && stateDesc.SourcePath != "" && !b.DeploymentBundle.StateDB.RecordsHistory() { + if opts.ReadPlanPath != "" { + return b, stateDesc, errors.New("enabling deployment history on existing state does not support --plan; run 'bundle deploy' without --plan") + } + phases.MigrateToFileHistory(ctx, b, stateDesc) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } if opts.Deploy || opts.PreDeployChecks { // Retain only this history feature for plan/deploy; overwrite all other flags. b.DeploymentBundle.StateDB.Data.Features = map[string]struct{}{dstate.FeatureDeploymentHistoryBestEffort: {}} @@ -401,7 +415,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if needDirectState && stateMode == dstate.WritingDMS { var err error - dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + dmsDeploymentID, dmsDeployment, err = dms.ReadDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -411,7 +425,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // the plan carries them. version_id is always known (last recorded + 1); deployment_id // does not exist until a first deploy creates it, so it is left off here and the deploy // phase stamps the created id. - lastVersionID, err := parseLastVersionID(dmsDeployment) + lastVersionID, err := dms.ParseLastVersionID(dmsDeployment) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -439,16 +453,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } - if needDirectState { - // Warn when the state was last written by a newer CLI than the one - // running now. The state schema version is a hard gate (dstate.Open - // rejects a too-new state_version), but a state can be written by a - // newer CLI that shares this schema; that is allowed, and this only - // hints that a downgrade may be unintended. - currentVersion := build.GetInfo().Version - if stateVersion := b.DeploymentBundle.StateDB.StateCLIVersion(); isNewerVersion(stateVersion, currentVersion) { - log.Warnf(ctx, "State was last deployed with CLI version %s but current version is %s", stateVersion, currentVersion) - } + if needDirectState && stateMode == dstate.WritingDMS { + warnIfNewerState(ctx, b) } // These are not safe in plan/deploy because they insert empty config settings for deleted resources. @@ -613,53 +619,6 @@ func ValidateEngineSetting(ctx context.Context, b *bundle.Bundle) error { return nil } -// Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json -// -// TODO: a deployment is only usable when both the node and the service's record exist, and a -// half-created one - node present, record missing - blocks the bundle here even though -// CreateDeployment already recovers from it. Move this behind a dms.ReadDeployment(ctx, statePath) -// that returns an empty id and version unless both halves are there, leaving the caller to call -// CreateDeployment to create or finalize it. -// -// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a -// workspace lookup to turn the node into an id, then a get by that id. -func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { - nodePath := path.Join(statePath, dms.DeploymentNodeName) - - obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) { - return w.Workspace.GetStatusByPath(ctx, nodePath) - }) - if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { - return "", nil, nil - } - if err != nil { - return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) - } - deploymentID := strconv.FormatInt(obj.ObjectId, 10) - deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) { - return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: dms.DeploymentName(deploymentID), - }) - }) - if err != nil { - return "", nil, err - } - return deploymentID, deployment, nil -} - -// parseLastVersionID parses the deployment's last recorded version, which the service reports -// as a string. It returns 0 when the deployment does not exist yet or has no recorded version. -func parseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error) { - if dmsDeployment == nil || dmsDeployment.LastVersionId == "" { - return 0, nil - } - v, err := strconv.Atoi(dmsDeployment.LastVersionId) - if err != nil { - return 0, fmt.Errorf("failed to parse last_version_id %q: %w", dmsDeployment.LastVersionId, err) - } - return v, nil -} - // OpenDirectStateForRead opens the direct-engine state database read-only. When the bundle // records deployment history the local state file is only a tombstone, so the resources are // read from the deployment metadata service instead. @@ -680,11 +639,11 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st return nil } - dmsDeploymentID, dmsDeployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + dmsDeploymentID, dmsDeployment, err := dms.ReadDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { return err } - lastVersionID, err := parseLastVersionID(dmsDeployment) + lastVersionID, err := dms.ParseLastVersionID(dmsDeployment) if err != nil { return err } @@ -707,7 +666,7 @@ func enforceDeploymentHistorySetting(ctx context.Context, b *bundle.Bundle, stat if b.DeploymentBundle.StateDB.IsOpen() { recorded = b.DeploymentBundle.StateDB.RecordsHistory() } - if configured == recorded || (!configured && !stateDesc.IsDMS()) { + if configured == recorded || !stateDesc.IsDMS() { return nil } if requireMatch { @@ -726,11 +685,11 @@ Run "databricks bundle destroy" first, then deploy again with deployment history // prepareFileHistory reads version metadata, never resource state, from DMS. func prepareFileHistory(ctx context.Context, b *bundle.Bundle) (*bundledeployments.Deployment, error) { - id, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + id, deployment, err := dms.ReadDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { return nil, err } - lastVersion, err := parseLastVersionID(deployment) + lastVersion, err := dms.ParseLastVersionID(deployment) if err != nil { return nil, err } @@ -755,6 +714,19 @@ func configuresDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { return bundleenv.RecordsDeploymentHistory(ctx, configured) } +// warnIfNewerState runs before a state-writing migration replaces the CLI version. +func warnIfNewerState(ctx context.Context, b *bundle.Bundle) { + // Warn when the state was last written by a newer CLI than the one + // running now. The state schema version is a hard gate (dstate.Open + // rejects a too-new state_version), but a state can be written by a + // newer CLI that shares this schema; that is allowed, and this only + // hints that a downgrade may be unintended. + currentVersion := build.GetInfo().Version + if stateVersion := b.DeploymentBundle.StateDB.StateCLIVersion(); isNewerVersion(stateVersion, currentVersion) { + log.Warnf(ctx, "State was last deployed with CLI version %s but current version is %s", stateVersion, currentVersion) + } +} + // isNewerVersion reports whether the state's recorded CLI version is strictly // newer than the running build. Both are bare versions without a leading "v". // An empty stateVersion (state not written by any CLI yet) or an unparseable diff --git a/cmd/bundle/utils/process_test.go b/cmd/bundle/utils/process_test.go index 21910039344..b05fb9b0a84 100644 --- a/cmd/bundle/utils/process_test.go +++ b/cmd/bundle/utils/process_test.go @@ -3,6 +3,7 @@ package utils import ( "testing" + "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -61,7 +62,7 @@ func TestParseLastVersionID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseLastVersionID(tt.deployment) + got, err := dms.ParseLastVersionID(tt.deployment) if tt.wantErr { require.Error(t, err) return diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go index f9e23a67db0..7ac7e9fed74 100644 --- a/libs/dms/metadata.go +++ b/libs/dms/metadata.go @@ -1,12 +1,19 @@ package dms import ( + "context" + "errors" "fmt" + "path" "strconv" "strings" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" ) // VersionType identifies the kind of deployment a version records. @@ -85,6 +92,53 @@ func NextVersion(lastVersionID string) (int, error) { // Keep it equal to the service's DEPLOYMENT_NODE_NAME. const DeploymentNodeName = "resources.deployment.json" +// ReadDeployment looks up the deployment object from its workspace state path. +// +// TODO: a deployment is only usable when both the node and the service's record exist, and a +// half-created one - node present, record missing - blocks the bundle here even though +// CreateDeployment already recovers from it. Move this behind a ReadDeployment(ctx, statePath) +// that returns an empty id and version unless both halves are there, leaving the caller to call +// CreateDeployment to create or finalize it. +// +// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a +// workspace lookup to turn the node into an id, then a get by that id. +func ReadDeployment(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { + nodePath := path.Join(statePath, DeploymentNodeName) + + obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) { + return w.Workspace.GetStatusByPath(ctx, nodePath) + }) + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil, nil + } + if err != nil { + return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + deploymentID := strconv.FormatInt(obj.ObjectId, 10) + deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) { + return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: DeploymentName(deploymentID), + }) + }) + if err != nil { + return "", nil, err + } + return deploymentID, deployment, nil +} + +// ParseLastVersionID parses the deployment's last recorded version, which the service reports +// as a string. It returns 0 when the deployment does not exist yet or has no recorded version. +func ParseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error) { + if dmsDeployment == nil || dmsDeployment.LastVersionId == "" { + return 0, nil + } + v, err := strconv.Atoi(dmsDeployment.LastVersionId) + if err != nil { + return 0, fmt.Errorf("failed to parse last_version_id %q: %w", dmsDeployment.LastVersionId, err) + } + return v, nil +} + // DeploymentName, VersionName and OperationName build the resource names the // service uses, so callers pass only ids. func DeploymentName(deploymentID string) string { diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index d5c190750c9..a2615c3c122 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -2,6 +2,7 @@ package testserver import ( "bytes" + "cmp" "encoding/json" "fmt" "path" @@ -624,6 +625,27 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return Response{Body: body} } +// ListVersions returns recorded versions in descending numeric order. +func (s *FakeWorkspace) ListVersions(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + versions := make([]bundledeployments.Version, 0, len(d.Versions)) + for _, version := range d.Versions { + versions = append(versions, *version) + } + slices.SortFunc(versions, func(a, b bundledeployments.Version) int { + aID, _ := strconv.ParseInt(a.VersionId, 10, 64) + bID, _ := strconv.ParseInt(b.VersionId, 10, 64) + return cmp.Compare(bID, aID) + }) + return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} +} + func (s *FakeWorkspace) ListResources(deploymentID string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8ea561ca2ba..83fd076d99a 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -468,6 +468,9 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.ListVersions(req.Vars["deployment_id"]) + }) server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) })