Register existing file state into best-effort deployment history

acceptance/bundle/dms/migration-before-deploy/databricks.yml.tmpl

@@ -0,0 +1,18 @@
+bundle:
+  name: dms-migration-before-deploy-$UNIQUE_NAME
+
+experimental:
+  deployment_history: false
+
+sync:
+  exclude:
+    - LOG.*
+
+resources:
+  jobs:
+    job:
+      name: migration-before-deploy
+  schemas:
+    schema:
+      name: dms_migration_before_deploy_$UNIQUE_NAME
+      catalog_name: main

acceptance/bundle/dms/migration-before-deploy/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]

acceptance/bundle/dms/migration-before-deploy/output.txt

@@ -0,0 +1,79 @@
+
+=== Setup (LOG.setup): create a job and schema without history
+Next deploy enables history and recreates the schema in another catalog, requiring approval.
+=== 1. Fail deployment planning
+Expect: migration version 2 stays complete; no deployment version is created.
+>>> fault.py GET /api/2.2/jobs/get* 403 0 1
+
+>>> musterr [CLI] bundle deploy -qq
+Enabling deployment history...
+Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2
+Error: cannot plan resources.jobs.job: reading id="[JOB_ID]": Fault injected by test. (403 INJECTED)
+
+Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[JOB_ID]
+HTTP Status: 403 Forbidden
+API error_code: INJECTED
+API message: Fault injected by test.
+
+Error: planning failed
+
+
+=== Request order: complete migration -> upload full file state -> plan jobs
+>>> print_requests.py --dms --get //api/2.0/bundle //jobs/get //workspace/import //workspace-files/import-file --oneline
+{"method":"POST","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete"}
+{"method":"POST","path":"/Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json"}
+{"method":"GET","path":"/api/2.2/jobs/get"}
+
+=== Remote state: version 2 file state with resources; original snapshot preserved in the backup
+>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.marker.json
+
+>>> jq -c {serial, features, state} .databricks/resources.marker.json
+{"serial":2,"features":{"deployment_history_best_effort":{}},"state":{"resources.jobs.job":{"__id__":"[JOB_ID]","state":{"deployment":{"kind":"BUNDLE","metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/metadata.json"},"edit_mode":"UI_LOCKED","format":"MULTI_TASK","max_concurrent_runs":1,"name":"migration-before-deploy","queue":{"enabled":true}}},"resources.schemas.schema":{"__id__":"[SCHEMA_ID]","state":{"catalog_name":"main","name":"dms_migration_before_deploy_[UNIQUE_NAME]"}}}}
+
+>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.json
+
+>>> cmp .databricks/resources.before-migration.json .databricks/resources.backup.json
+
+=== Migration result: only completed version 2, with INITIAL_REGISTER operations
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json
+[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}]
+
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json
+[{"resource_key":"jobs.job","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"schemas.schema","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}]
+
+=== 2. Retry without approval
+Expect: deployment declined; migration stays at version 2 and is not repeated.
+>>> musterr [CLI] bundle deploy -qq
+
+This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
+  recreate resources.schemas.schema
+Error: the deployment requires destructive actions, but the current console does not support prompting.
+Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed.
+To proceed, use --auto-approve after reviewing the plan above.
+
+
+=== Local and remote markers must stay byte-for-byte unchanged
+>>> cmp .databricks/resources.marker.json .databricks/bundle/default/resources.json
+
+>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.after-decline.json
+
+>>> cmp .databricks/resources.marker.json .databricks/resources.after-decline.json
+
+=== No DMS writes and no extra version on the declined retry
+>>> print_requests.py --dms //api/2.0/bundle
+
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json
+[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}]
+
+=== 3. Approve deployment
+Expect: ordinary deployment version 3; version 2 remains the only migration.
+>>> [CLI] bundle deploy --auto-approve -qq
+
+This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost:
+  recreate resources.schemas.schema
+
+>>> [CLI] bundle summary -o json
+{"latest_version_id":3}
+
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json
+[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}]

acceptance/bundle/dms/migration-before-deploy/script

@@ -0,0 +1,50 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-before-deploy-${UNIQUE_NAME}/default/state"
+other_catalog="dms_other_${UNIQUE_NAME}"
+
+title "Setup (LOG.setup): create a job and schema without history\nNext deploy enables history and recreates the schema in another catalog, requiring approval."
+{
+    $CLI bundle deploy -qq
+    DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py
+    cp .databricks/bundle/default/resources.json .databricks/resources.before-migration.json
+    $CLI catalogs create "$other_catalog"
+    update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true'
+    update_file.py databricks.yml 'catalog_name: main' "catalog_name: ${other_catalog}"
+} &> LOG.setup
+trap '$CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1; $CLI catalogs delete "$other_catalog" --force >> LOG.cleanup 2>&1; rm -f "$OUT_REQUESTS"' EXIT
+rm -f "$OUT_REQUESTS"
+
+title "1. Fail deployment planning\nExpect: migration version 2 stays complete; no deployment version is created."
+trace fault.py 'GET /api/2.2/jobs/get*' 403 0 1
+trace musterr $CLI bundle deploy -qq
+deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_path/resources.deployment.json" -o json | jq -r '.object_id')
+add_repl "$deployment_id" DEPLOYMENT_ID
+title "Request order: complete migration -> upload full file state -> plan jobs"
+trace print_requests.py --dms --get //api/2.0/bundle //jobs/get //workspace/import //workspace-files/import-file --oneline |
+    MSYS_NO_PATHCONV=1 jq -c --arg marker "$state_path/resources.json" '
+        (.body.multipart_form.path // .body.path // (.path | ltrimstr("/api/2.0/workspace-files/import-file"))) as $destination |
+        select((.path | endswith("/complete")) or (.path | endswith("/resources")) or .path == "/api/2.2/jobs/get" or $destination == $marker) |
+        {method, path: (if $destination == $marker then $destination else .path end)}'
+title "Remote state: version 2 file state with resources; original snapshot preserved in the backup"
+trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.marker.json
+trace jq -c '{serial, features, state}' .databricks/resources.marker.json
+trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.json
+trace cmp .databricks/resources.before-migration.json .databricks/resources.backup.json
+title "Migration result: only completed version 2, with INITIAL_REGISTER operations"
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'map({version_id, status, completion_reason})'
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, action_type, status})'
+
+title "2. Retry without approval\nExpect: deployment declined; migration stays at version 2 and is not repeated."
+trace musterr $CLI bundle deploy -qq
+title "Local and remote markers must stay byte-for-byte unchanged"
+trace cmp .databricks/resources.marker.json .databricks/bundle/default/resources.json
+trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.after-decline.json
+trace cmp .databricks/resources.marker.json .databricks/resources.after-decline.json
+title "No DMS writes and no extra version on the declined retry"
+trace print_requests.py --dms //api/2.0/bundle
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'map({version_id, status, completion_reason})'
+
+title "3. Approve deployment\nExpect: ordinary deployment version 3; version 2 remains the only migration."
+trace $CLI bundle deploy --auto-approve -qq
+trace $CLI bundle summary -o json | jq -c '.bundle.deployment | {latest_version_id}'
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, status, completion_reason})'

acceptance/bundle/dms/migration-before-deploy/test.toml

@@ -0,0 +1,2 @@
+# Fault injection exercises a real planner failure against the local testserver.
+Cloud = false

acceptance/bundle/dms/migration-fails-open/databricks.yml.tmpl

@@ -0,0 +1,16 @@
+bundle:
+  name: dms-migration-fails-open-$UNIQUE_NAME
+
+experimental:
+  deployment_history: false
+
+resources:
+  jobs:
+    one:
+      name: one
+    two:
+      name: two
+      tasks:
+        - task_key: run_one
+          run_job_task:
+            job_id: ${resources.jobs.one.id}

acceptance/bundle/dms/migration-fails-open/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]

acceptance/bundle/dms/migration-fails-open/output.txt

@@ -0,0 +1,51 @@
+
+=== Create file-backed resources without history
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-migration-fails-open-[UNIQUE_NAME]/default/files...
+Created jobs.one
+Created jobs.two
+Files: 2 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Fail one registration upload; enable history and deploy an update anyway
+>>> fault.py PATCH /api/2.0/bundle/* 400 0 1
+
+>>> update_file.py databricks.yml deployment_history: false deployment_history: true
+
+>>> update_file.py databricks.yml name: one name: updated
+
+>>> [CLI] bundle deploy
+Enabling deployment history...
+Warn: Initial history registration was incomplete; continuing with file state: recording migration version: recording operation for resources.jobs.one: Fault injected by test.
+Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-migration-fails-open-[UNIQUE_NAME]/default/files...
+Updated jobs.one
+Updated jobs.two
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 2 changed, 0 deleted, 0 unchanged
+
+=== The file still owns both resources, with the original IDs
+>>> jq {serial, features, resources: (.state | keys), names: (.state | with_entries(.value = .value.state.name))} .databricks/bundle/default/resources.json
+{
+  "serial": 3,
+  "features": {
+    "deployment_history_best_effort": {}
+  },
+  "resources": [
+    "resources.jobs.one",
+    "resources.jobs.two"
+  ],
+  "names": {
+    "resources.jobs.one": "updated",
+    "resources.jobs.two": "two"
+  }
+}
+
+>>> cmp .databricks/ids.before.json .databricks/ids.after.json
+
+=== Migration failed, but the following deployment completed successfully
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json
+[{"version_id":"2","completion_reason":"VERSION_COMPLETE_FAILURE"},{"version_id":"3","completion_reason":"VERSION_COMPLETE_SUCCESS"}]
+
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json
+[{"resource_key":"jobs.one","status":"OPERATION_STATUS_PENDING"},{"resource_key":"jobs.two","status":"OPERATION_STATUS_SUCCEEDED"}]

acceptance/bundle/dms/migration-fails-open/script

@@ -0,0 +1,25 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+title "Create file-backed resources without history"
+trace $CLI bundle deploy
+DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py
+jq '.state | with_entries(.value = .value.__id__)' .databricks/bundle/default/resources.json > .databricks/ids.before.json
+
+title "Fail one registration upload; enable history and deploy an update anyway"
+trace fault.py "PATCH /api/2.0/bundle/*" 400 0 1
+trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true'
+trace update_file.py databricks.yml 'name: one' 'name: updated'
+trace $CLI bundle deploy
+
+title "The file still owns both resources, with the original IDs"
+trace jq '{serial, features, resources: (.state | keys), names: (.state | with_entries(.value = .value.state.name))}' .databricks/bundle/default/resources.json
+jq '.state | with_entries(.value = .value.__id__)' .databricks/bundle/default/resources.json > .databricks/ids.after.json
+trace cmp .databricks/ids.before.json .databricks/ids.after.json
+
+deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-fails-open-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | jq -r '.object_id')
+add_repl "$deployment_id" DEPLOYMENT_ID
+
+title "Migration failed, but the following deployment completed successfully"
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, completion_reason})'
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, status})'
+rm -f "$OUT_REQUESTS"

acceptance/bundle/dms/migration-fails-open/test.toml

@@ -0,0 +1,2 @@
+# Fault injection verifies best-effort registration against the local fake.
+Cloud = false

acceptance/bundle/dms/migration/databricks.yml.tmpl

@@ -0,0 +1,16 @@
+bundle:
+  name: dms-migration-$UNIQUE_NAME
+
+experimental:
+  deployment_history: false
+
+resources:
+  jobs:
+    one:
+      name: one
+    two:
+      name: two
+      tasks:
+        - task_key: run_one
+          run_job_task:
+            job_id: ${resources.jobs.one.id}

acceptance/bundle/dms/migration/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = [""]