Use enforced best-effort history for the YAML opt-in

acceptance/bin/print_state.py

@@ -13,6 +13,7 @@
 import os
 import posixpath
 import subprocess
+from pathlib import Path
 
 from nostamp import scrub
 
@@ -123,9 +124,23 @@ def get_deployment_id(target):
 def get_resources(target):
     """Map every recorded resource key ("jobs.foo") to its {"id", "state"}.
 
-    Empty when the bundle has no deployment recorded yet. Cached because a lookup costs three
-    round trips and a script asks for one resource at a time.
+    File-history state is read locally. Legacy DMS state is read from the service,
+    or is empty when no deployment has been recorded. Cached for repeated resource lookups.
     """
+    filename = get_state_file(target, False)
+    if filename.endswith("resources.json") and os.path.exists(filename):
+        data = json.loads(Path(filename).read_text())
+        if "deployment_history_best_effort" in (data.get("features") or {}):
+            # DMS can miss history updates. The file remains authoritative in this mode.
+            return {
+                key.removeprefix("resources."): {
+                    "id": entry.get("__id__"),
+                    "state": entry.get("state") or {},
+                    "depends_on": entry.get("depends_on") or [],
+                }
+                for key, entry in data.get("state", {}).items()
+            }
+
     deployment_id = get_deployment_id(target)
     if not deployment_id:
         return {}
@@ -174,10 +189,9 @@ def get_last_version_id(target):
 
 
 def print_recorded_state(filename, target):
-    """Print the state file with its resources filled in from the deployment metadata service.
+    """Normalize history state for comparisons with ordinary file-backed deployments.
 
-    While recording, the file itself carries only the header - the service holds the resources - so
-    printing it raw would show an empty state and differ from the same test's non-recording run.
+    get_resources uses the file for best-effort history and the service for legacy DMS state.
     """
     data = json.loads(open(filename).read())
     # Recording stamps each resource payload with the deployment and version; drop it here so the
@@ -186,11 +200,18 @@ def print_recorded_state(filename, target):
 
     # The file's serial is a copy of the DMS version to ensure outdated resource state is not read.
     # Rebuild in header order with the authoritative serial from the deployment.
-    last_version_id = get_last_version_id(target)
-    serial = int(last_version_id) if last_version_id else 0
+    if "deployment_history_best_effort" in (data.get("features") or {}):
+        serial = data.get("serial", 0)
+    else:
+        last_version_id = get_last_version_id(target)
+        serial = int(last_version_id) if last_version_id else 0
 
     # Recording itself is not what these tests assert, so drop the feature that marks it.
-    features = {k: v for k, v in (data.get("features") or {}).items() if k != "deployment_history"}
+    features = {
+        k: v
+        for k, v in (data.get("features") or {}).items()
+        if k not in {"deployment_history", "deployment_history_best_effort"}
+    }
 
     rebuilt = {}
     for key in ("state_version", "cli_version", "lineage"):
@@ -213,7 +234,7 @@ def main():
         "--no-dms",
         action="store_true",
         help="Print resources.json as-is without fetching state from DMS. "
-        "Use in bundle/dms tests to assert the tombstone content of the file.",
+        "Use in bundle/dms tests to assert the file state and its feature markers.",
     )
     args = parser.parse_args()
 

acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl

@@ -1,8 +1,5 @@
 bundle:
   name: dms-bind-unbind-$UNIQUE_NAME
-experimental:
-  deployment_history: true
-
 resources:
   jobs:
     foo:

acceptance/bundle/dms/declined-deploy/output.txt

@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_best_effort": {}
   }
 }
 
@@ -79,18 +79,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
   "method": "GET",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
 }
-{
-  "method": "GET",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
-}
 {
   "method": "GET",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
 }
-{
-  "method": "GET",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
-}
 
 >>> print_state.py --no-dms
 {
@@ -99,9 +91,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
  "lineage": "[UUID]",
  "serial": 1,
  "features": {
-  "deployment_history": {}
+  "deployment_history_best_effort": {}
  },
- "state": {}
+ "state": {
+  "resources.schemas.foo": {
+   "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]",
+   "state": {
+    "catalog_name": "main",
+    "name": "dms_declined_deploy_schema_[UNIQUE_NAME]"
+   }
+  }
+ }
 }
 
 === A destructive change without --auto-approve is declined: this console cannot prompt
@@ -124,10 +124,6 @@ Files: 2 uploaded, 0 deleted
   "method": "GET",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
 }
-{
-  "method": "GET",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
-}
 
 === The recorded resources are unchanged by the decline: the schema is still at version 1
 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
@@ -163,10 +159,6 @@ false
   "method": "GET",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
 }
-{
-  "method": "GET",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
-}
 {
   "method": "POST",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",

acceptance/bundle/dms/depends-on/output.txt

@@ -3,10 +3,7 @@
 >>> [CLI] bundle plan -o json
 {
   "plan_version": [PLAN_VERSION],
-  "cli_version": "[CLI_VERSION]",
-  "features": {
-    "deployment_history": {}
-  }
+  "cli_version": "[CLI_VERSION]"
 }
 
 >>> [CLI] bundle plan
@@ -61,7 +58,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_best_effort": {}
   }
 }
 
@@ -91,9 +88,53 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
  "lineage": "[UUID]",
  "serial": 1,
  "features": {
-  "deployment_history": {}
+  "deployment_history_best_effort": {}
  },
- "state": {}
+ "state": {
+  "resources.jobs.child": {
+   "__id__": "[CHILD_ID]",
+   "state": {
+    "deployment": {
+     "deployment_id": "[DEPLOYMENT_ID]",
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json",
+     "version_id": "1"
+    },
+    "description": "depends on [PARENT_ID]",
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "child",
+    "queue": {
+     "enabled": true
+    }
+   },
+   "depends_on": [
+    {
+     "node": "resources.jobs.parent",
+     "label": "${resources.jobs.parent.id}"
+    }
+   ]
+  },
+  "resources.jobs.parent": {
+   "__id__": "[PARENT_ID]",
+   "state": {
+    "deployment": {
+     "deployment_id": "[DEPLOYMENT_ID]",
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json",
+     "version_id": "1"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "parent",
+    "queue": {
+     "enabled": true
+    }
+   }
+  }
+ }
 }
 
 >>> [CLI] bundle destroy --auto-approve

acceptance/bundle/dms/deployment-metadata-change/output.txt

@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_best_effort": {}
   }
 }
 
@@ -128,7 +128,7 @@ A common practice is to use a username or principal name in this path, i.e. use
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_best_effort": {}
   }
 }
 

acceptance/bundle/dms/empty-grants/output.txt

@@ -138,7 +138,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
   "lineage": "[UUID]",
   "serial": 2,
   "features": {
-    "deployment_history": {}
+    "deployment_history_best_effort": {}
   }
 }
 
@@ -163,9 +163,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
  "lineage": "[UUID]",
  "serial": 2,
  "features": {
-  "deployment_history": {}
+  "deployment_history_best_effort": {}
  },
- "state": {}
+ "state": {
+  "resources.schemas.foo": {
+   "__id__": "main.dms_empty_grants_[UNIQUE_NAME]",
+   "state": {
+    "catalog_name": "main",
+    "name": "dms_empty_grants_[UNIQUE_NAME]"
+   }
+  }
+ }
 }
 
 >>> [CLI] bundle destroy --auto-approve

acceptance/bundle/dms/failed-delete/output.txt

@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
   "lineage": "[UUID]",
   "serial": 1,
   "features": {
-    "deployment_history": {}
+    "deployment_history_best_effort": {}
   }
 }
 
@@ -92,9 +92,28 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
  "lineage": "[UUID]",
  "serial": 1,
  "features": {
-  "deployment_history": {}
+  "deployment_history_best_effort": {}
  },
- "state": {}
+ "state": {
+  "resources.jobs.stuck": {
+   "__id__": "[STUCK_ID]",
+   "state": {
+    "deployment": {
+     "deployment_id": "[DEPLOYMENT_ID]",
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json",
+     "version_id": "1"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "stuck",
+    "queue": {
+     "enabled": true
+    }
+   }
+  }
+ }
 }
 
 === A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone
@@ -176,22 +195,41 @@ API message: Fault injected by test.
   }
 ]
 
-=== The local state file records failed version 2
+=== The local state file keeps the resource and its serial after the failed delete
 >>> print_state.py --no-dms
 {
  "state_version": 3,
  "cli_version": "[CLI_VERSION]",
  "lineage": "[UUID]",
- "serial": 2,
+ "serial": 1,
  "features": {
-  "deployment_history": {}
+  "deployment_history_best_effort": {}
  },
- "state": {}
+ "state": {
+  "resources.jobs.stuck": {
+   "__id__": "[STUCK_ID]",
+   "state": {
+    "deployment": {
+     "deployment_id": "[DEPLOYMENT_ID]",
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json",
+     "version_id": "1"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "stuck",
+    "queue": {
+     "enabled": true
+    }
+   }
+  }
+ }
 }
 
 === The remote checkpoint stays at serial 1 while DMS advances to version 2
 >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/resources.json --format RAW
 {"serial":1}
 
->>> print_state.py
-{"serial":2}
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments get-deployment deployments/[DEPLOYMENT_ID] -o json
+{"last_version_id":"2"}

acceptance/bundle/dms/failed-delete/script

@@ -29,12 +29,12 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d
 title "The failed operation records the error message"
 trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -S 'map({resource_key, action_type, status, error_message})'
 
-title "The local state file records failed version 2"
+title "The local state file keeps the resource and its serial after the failed delete"
 trace print_state.py --no-dms
 
 title "The remote checkpoint stays at serial 1 while DMS advances to version 2"
 trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.json" --format RAW | jq -c '{serial}'
-trace print_state.py | jq -c '{serial}'
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments get-deployment "deployments/${deployment_id}" -o json | jq -c '{last_version_id}'
 
 # Drain any accumulated requests (plan, state reads) so the harness has nothing to diff.
 rm -f "$OUT_REQUESTS"