acc: Reproduce that file syncs are done despite failed resource deployments

acceptance/bundle/deploy/files/move-during-failed-deploy/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+  name: move-during-failed-deploy
+
+resources:
+  jobs:
+    foo:
+      name: test-job
+      tasks:
+        - task_key: my_task
+          spark_python_task:
+            python_file: ./src/main.py
+          new_cluster:
+            num_workers: 1
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge

acceptance/bundle/deploy/files/move-during-failed-deploy/output.txt

@@ -0,0 +1,46 @@
+
+=== Deploy 1: the job references src/main.py, which is uploaded to the workspace
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files...
+Created jobs.foo
+Files: 5 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files/src/main.py
+{
+  "object_type": "FILE",
+  "path": "/Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files/src/main.py"
+}
+
+=== Move the referenced file and point the job at the new location
+>>> mv src/main.py src/renamed.py
+
+>>> update_file.py databricks.yml src/main.py src/renamed.py
+
+=== Fail the job update. Files are still synced first, so main.py is deleted and
+renamed.py uploaded before the failing jobs/reset.
+>>> musterr [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files...
+Error: cannot update resources.jobs.foo: updating id=[FOO_ID]: Fault injected by test. (403 INJECTED)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/reset
+HTTP Status: 403 Forbidden
+API error_code: INJECTED
+API message: Fault injected by test.
+
+Files: 3 uploaded, 1 deleted
+
+=== Broken state: the failed deploy already deleted the old file from the workspace
+>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files/src/main.py
+Error: Path (/Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files/src/main.py) doesn't exist.
+
+=== The new file was uploaded
+>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files/src/renamed.py
+{
+  "object_type": "FILE",
+  "path": "/Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files/src/renamed.py"
+}
+
+=== But the deployed job still points at the deleted file, so a run fails file-not-found
+>>> [CLI] jobs get [FOO_ID]
+"/Workspace/Users/[USERNAME]/.bundle/move-during-failed-deploy/default/files/src/main.py"

acceptance/bundle/deploy/files/move-during-failed-deploy/script

@@ -0,0 +1,26 @@
+BUNDLE_FILES="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/move-during-failed-deploy/default/files"
+
+title "Deploy 1: the job references src/main.py, which is uploaded to the workspace"
+trace $CLI bundle deploy
+# MSYS_NO_PATHCONV=1 keeps Git Bash from rewriting the leading-slash workspace path
+# into a Windows path before the CLI sees it. Scoped per-command so it does not also
+# break the shebang-resolved .py helpers (update_file.py, read_id.py) on Windows.
+trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "$BUNDLE_FILES/src/main.py" | jq '{object_type,path}'
+
+title "Move the referenced file and point the job at the new location"
+trace mv src/main.py src/renamed.py
+trace update_file.py databricks.yml src/main.py src/renamed.py
+
+title "Fail the job update. Files are still synced first, so main.py is deleted and\nrenamed.py uploaded before the failing jobs/reset."
+fault.py "POST /api/2.2/jobs/reset" 403 0 1
+trace musterr $CLI bundle deploy
+
+title "Broken state: the failed deploy already deleted the old file from the workspace"
+trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "$BUNDLE_FILES/src/main.py"
+
+title "The new file was uploaded"
+trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "$BUNDLE_FILES/src/renamed.py" | jq '{object_type,path}'
+
+title "But the deployed job still points at the deleted file, so a run fails file-not-found"
+job_id=$(read_id.py foo)
+trace $CLI jobs get "$job_id" | jq '.settings.tasks[0].spark_python_task.python_file'

acceptance/bundle/deploy/files/move-during-failed-deploy/src/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/deploy/files/move-during-failed-deploy/test.toml

@@ -0,0 +1,11 @@
+Badness = "A deploy uploads and deletes bundle files before it applies resources, so a resource-update failure after a file move leaves the job pointing at a file that no longer exists in the workspace."
+
+# The injected fault (POST /api/2.2/jobs/reset) and the state inspection differ in
+# output between the terraform and direct engines, so pin the direct engine (same
+# precedent as ../partial-summary-on-push-fail). The ordering bug being demonstrated
+# is engine-independent: files.Upload runs before deployCore for both engines.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+# The script moves src/main.py to src/renamed.py, so the renamed file is not a
+# committed input and would otherwise be flagged as an unexpected leftover.
+Ignore = ["src/renamed.py"]