filer: switch workspace upload from import-file to /workspace/import

.nextchanges/bundles/6149.md

@@ -0,0 +1 @@
+* Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`, raising the per-workspace upload rate limit and improving deploy throughput under concurrency. ([#6149](https://github.com/databricks/cli/pull/6149))

acceptance/air/run-submit-deps/output.txt

@@ -10,9 +10,9 @@ Stream logs after submission using:
   databricks air logs 555
 
 === only config + command are uploaded; no requirements.yaml
->>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep
-{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"}
-{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n    accelerator_type: GPU_1xH100\n    num_accelerators: 1\n    pool_id: capacity-123\nenvironment:\n    version: 5\n    dependencies:\n        - numpy\n        - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"}
+>>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep
+{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n    accelerator_type: GPU_1xH100\n    num_accelerators: 1\n    pool_id: capacity-123\nenvironment:\n    version: 5\n    dependencies:\n        - numpy\n        - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml"}}}
+{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh"}}}
 
 === inline deps, artifact location, and capacity id reach the submit payload
 >>> print_requests.py //api/2.2/jobs/runs/submit

acceptance/air/run-submit-deps/script

@@ -2,7 +2,10 @@ title "submit with inline dependencies"
 trace $CLI air run -f run.yaml
 
 title "only config + command are uploaded; no requirements.yaml"
-trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep
+# Uploads go to POST /workspace/import, which carries the target filename in the
+# multipart body rather than the URL, so each uploaded path shows up under
+# body.multipart_form.path instead of in the request path.
+trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep
 
 title "inline deps, artifact location, and capacity id reach the submit payload"
 trace print_requests.py //api/2.2/jobs/runs/submit

acceptance/air/run-submit-overrides/output.txt

@@ -13,7 +13,7 @@ Stream logs after submission using:
   databricks air logs 555
 
 === the uploaded config preserves source order and contains overrides
->>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique
-{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"}
-{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml", "q": {"overwrite": "true"}, "raw_body": "model:\n    hidden_size: 2048\noptimizer:\n    learning_rate: 0.001\n    name: adamw\n"}
-{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n    accelerator_type: GPU_1xH100\n    num_accelerators: 2\nparameters:\n    model:\n        hidden_size: 2048\n    optimizer:\n        name: adamw\n        learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"}
+>>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique
+{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n    accelerator_type: GPU_1xH100\n    num_accelerators: 2\nparameters:\n    model:\n        hidden_size: 2048\n    optimizer:\n        name: adamw\n        learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml"}}}
+{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "model:\n    hidden_size: 2048\noptimizer:\n    learning_rate: 0.001\n    name: adamw\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml"}}}
+{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh"}}}

acceptance/air/run-submit-overrides/script

@@ -5,4 +5,4 @@ trace $CLI air run -f run.yaml \
   --override parameters.optimizer.learning_rate=0.001
 
 title "the uploaded config preserves source order and contains overrides"
-trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique
+trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique

acceptance/bin/fault.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 """Set up a fault rule on the testserver for the current test token.
 
-Usage: fault.py [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]
+Usage: fault.py [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]
 
   --after-handler  run the handler first; keep its effect and replace only the response
+  --body-contains  only fire when the request body contains SUBSTR, e.g. to target one
+                   file's /workspace/import upload (all uploads share that path)
   PATTERN     HTTP method and path, supports trailing * wildcard,
               e.g. "PUT /api/2.0/permissions/pipelines/*"
   STATUS_CODE HTTP status code to return, e.g. 504
@@ -33,8 +35,17 @@
 if after_handler:
     args.remove("--after-handler")
 
+body_contains = ""
+if "--body-contains" in args:
+    i = args.index("--body-contains")
+    body_contains = args[i + 1]
+    del args[i : i + 2]
+
 if len(args) not in (4, 5):
-    print(f"usage: {sys.argv[0]} [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr)
+    print(
+        f"usage: {sys.argv[0]} [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]",
+        file=sys.stderr,
+    )
     sys.exit(1)
 
 pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3])
@@ -49,6 +60,7 @@
         "offset": offset,
         "times": times,
         "after_handler": after_handler,
+        "body_contains": body_contains,
     }
 ).encode()
 

acceptance/bundle/apps/app_yaml/out.app.yml.txt

@@ -1,8 +1,12 @@
 {
   "method": "POST",
-  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml",
-  "q": {
-    "overwrite": "true"
-  },
-  "raw_body": "command:\n  - python\n  - app.py\n"
+  "path": "/api/2.0/workspace/import",
+  "body": {
+    "multipart_form": {
+      "content": "command:\n  - python\n  - app.py\n",
+      "format": "AUTO",
+      "overwrite": "true",
+      "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml"
+    }
+  }
 }

acceptance/bundle/apps/app_yaml/output.txt

@@ -19,7 +19,7 @@ Created apps.myapp
 Files: 3 uploaded, 0 deleted
 Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 
->>> jq select(.path | test("app.yml")) [OUTPUT_DIR]/out.requests.txt
+>>> jq select(.body.multipart_form.path | strings | test("app.yml")) [OUTPUT_DIR]/out.requests.txt
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/apps/app_yaml/script

@@ -1,7 +1,7 @@
 trace $CLI bundle validate
 trace $CLI bundle plan
 trace $CLI bundle deploy
-trace jq 'select(.path | test("app.yml"))' "$OUT_REQUESTS" | sed 's/\\r//g'  > out.app.yml.txt
+trace jq 'select(.body.multipart_form.path | strings | test("app.yml"))' "$OUT_REQUESTS" | sed 's/\\r//g'  > out.app.yml.txt
 #trace print_requests.py //apps  # currently fails due to TF inserting description=""
 rm "$OUT_REQUESTS"
 

acceptance/bundle/artifacts/ai_runtime_code_source/output.txt

@@ -16,5 +16,5 @@
 }
 
 === Expecting the code_source tarball to be uploaded
->>> jq .path
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz"
+>>> print_workspace_imports
+/Workspace/foo/bar/artifacts/.internal/code.tgz

acceptance/bundle/artifacts/ai_runtime_code_source/script

@@ -6,6 +6,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' "$OUT_REQUESTS"
 
 title "Expecting the code_source tarball to be uploaded"
-trace jq .path < "$OUT_REQUESTS" | grep import | grep code.tgz | sort
+trace print_workspace_imports | grep code.tgz | sort
 
 rm "$OUT_REQUESTS"

acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt

@@ -48,9 +48,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 ]
 
 === Expecting wheel to be uploaded
->>> jq .path
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl"
+>>> print_workspace_imports
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl
+/Workspace/foo/bar/artifacts/.internal/source.whl
 
 === Expecting environment dependencies to be updated
 >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt

acceptance/bundle/artifacts/artifact_upload_for_workspace/script

@@ -9,7 +9,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create"
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS"
 
 title "Expecting wheel to be uploaded"
-trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort
+trace print_workspace_imports | grep whl | sort
 
 title "Expecting environment dependencies to be updated"
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS"

acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt

@@ -4,9 +4,9 @@ Files: 4 uploaded, 0 deleted
 Resources: 0 created, 0 changed, 0 deleted, 0 unchanged
 
 === Expecting wheel to be uploaded
->>> jq .path
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl"
+>>> print_workspace_imports
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl
+/Workspace/foo/bar/artifacts/.internal/source.whl
 
 === Expecting delete request to artifact_path/.internal folder
 >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt

acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script

@@ -6,7 +6,7 @@ $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan
 $CLI bundle deploy $(readplanarg tmp.plan.json)
 
 title "Expecting wheel to be uploaded"
-trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort
+trace print_workspace_imports | grep whl | sort
 
 title "Expecting delete request to artifact_path/.internal folder"
 trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS"

acceptance/bundle/artifacts/upload_multiple_libraries/output.txt

@@ -38,15 +38,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 ]
 
 === Expecting 4 wheels to be uploaded
->>> jq .path
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source1.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source2.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source3.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl"
+>>> print_workspace_imports
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl
+/Workspace/foo/bar/artifacts/.internal/source1.whl
+/Workspace/foo/bar/artifacts/.internal/source2.whl
+/Workspace/foo/bar/artifacts/.internal/source3.whl
+/Workspace/foo/bar/artifacts/.internal/source4.whl
 
 === Expecting environment dependencies to be updated
 >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt

acceptance/bundle/artifacts/upload_multiple_libraries/script

@@ -12,7 +12,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create"
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS"
 
 title "Expecting 4 wheels to be uploaded"
-trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort
+trace print_workspace_imports | grep whl | sort
 
 title "Expecting environment dependencies to be updated"
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS"