diff --git a/.nextchanges/bundles/6149.md b/.nextchanges/bundles/6149.md new file mode 100644 index 00000000000..12bf478ca8d --- /dev/null +++ b/.nextchanges/bundles/6149.md @@ -0,0 +1 @@ +* Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`, raising the per-workspace upload rate limit and improving deploy throughput under concurrency. ([#6149](https://github.com/databricks/cli/pull/6149)) diff --git a/acceptance/air/run-submit-deps/output.txt b/acceptance/air/run-submit-deps/output.txt index 3346fcd06d7..de4c49e1b7f 100644 --- a/acceptance/air/run-submit-deps/output.txt +++ b/acceptance/air/run-submit-deps/output.txt @@ -10,9 +10,9 @@ Stream logs after submission using: databricks air logs 555 === only config + command are uploaded; no requirements.yaml ->>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep -{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} -{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} +>>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh"}}} === inline deps, artifact location, and capacity id reach the submit payload >>> print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/air/run-submit-deps/script b/acceptance/air/run-submit-deps/script index 422980c99b3..0e8951cae0f 100644 --- a/acceptance/air/run-submit-deps/script +++ b/acceptance/air/run-submit-deps/script @@ -2,7 +2,10 @@ title "submit with inline dependencies" trace $CLI air run -f run.yaml title "only config + command are uploaded; no requirements.yaml" -trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep +# Uploads go to POST /workspace/import, which carries the target filename in the +# multipart body rather than the URL, so each uploaded path shows up under +# body.multipart_form.path instead of in the request path. +trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep title "inline deps, artifact location, and capacity id reach the submit payload" trace print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/air/run-submit-overrides/output.txt b/acceptance/air/run-submit-overrides/output.txt index 0be49262d1b..19323ff9a50 100644 --- a/acceptance/air/run-submit-overrides/output.txt +++ b/acceptance/air/run-submit-overrides/output.txt @@ -13,7 +13,7 @@ Stream logs after submission using: databricks air logs 555 === the uploaded config preserves source order and contains overrides ->>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique -{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} -{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml", "q": {"overwrite": "true"}, "raw_body": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n"} -{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 2\nparameters:\n model:\n hidden_size: 2048\n optimizer:\n name: adamw\n learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} +>>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 2\nparameters:\n model:\n hidden_size: 2048\n optimizer:\n name: adamw\n learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh"}}} diff --git a/acceptance/air/run-submit-overrides/script b/acceptance/air/run-submit-overrides/script index 263c1920e6c..59234b5e874 100644 --- a/acceptance/air/run-submit-overrides/script +++ b/acceptance/air/run-submit-overrides/script @@ -5,4 +5,4 @@ trace $CLI air run -f run.yaml \ --override parameters.optimizer.learning_rate=0.001 title "the uploaded config preserves source order and contains overrides" -trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique +trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique diff --git a/acceptance/bin/fault.py b/acceptance/bin/fault.py index 0aed44d58d1..5b75c0a5907 100755 --- a/acceptance/bin/fault.py +++ b/acceptance/bin/fault.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 """Set up a fault rule on the testserver for the current test token. -Usage: fault.py [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] +Usage: fault.py [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] --after-handler run the handler first; keep its effect and replace only the response + --body-contains only fire when the request body contains SUBSTR, e.g. to target one + file's /workspace/import upload (all uploads share that path) PATTERN HTTP method and path, supports trailing * wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*" STATUS_CODE HTTP status code to return, e.g. 504 @@ -33,8 +35,17 @@ if after_handler: args.remove("--after-handler") +body_contains = "" +if "--body-contains" in args: + i = args.index("--body-contains") + body_contains = args[i + 1] + del args[i : i + 2] + if len(args) not in (4, 5): - print(f"usage: {sys.argv[0]} [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr) + print( + f"usage: {sys.argv[0]} [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", + file=sys.stderr, + ) sys.exit(1) pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3]) @@ -49,6 +60,7 @@ "offset": offset, "times": times, "after_handler": after_handler, + "body_contains": body_contains, } ).encode() diff --git a/acceptance/bundle/apps/app_yaml/out.app.yml.txt b/acceptance/bundle/apps/app_yaml/out.app.yml.txt index 29cb7da70fa..e79c7a4f57c 100644 --- a/acceptance/bundle/apps/app_yaml/out.app.yml.txt +++ b/acceptance/bundle/apps/app_yaml/out.app.yml.txt @@ -1,8 +1,12 @@ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml", - "q": { - "overwrite": "true" - }, - "raw_body": "command:\n - python\n - app.py\n" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "command:\n - python\n - app.py\n", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml" + } + } } diff --git a/acceptance/bundle/apps/app_yaml/output.txt b/acceptance/bundle/apps/app_yaml/output.txt index 3554b68f44e..50baf9b4559 100644 --- a/acceptance/bundle/apps/app_yaml/output.txt +++ b/acceptance/bundle/apps/app_yaml/output.txt @@ -19,7 +19,7 @@ Created apps.myapp Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> jq select(.path | test("app.yml")) [OUTPUT_DIR]/out.requests.txt +>>> jq select(.body.multipart_form.path | strings | test("app.yml")) [OUTPUT_DIR]/out.requests.txt >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/apps/app_yaml/script b/acceptance/bundle/apps/app_yaml/script index 3db6c9e8969..3267d66c598 100644 --- a/acceptance/bundle/apps/app_yaml/script +++ b/acceptance/bundle/apps/app_yaml/script @@ -1,7 +1,7 @@ trace $CLI bundle validate trace $CLI bundle plan trace $CLI bundle deploy -trace jq 'select(.path | test("app.yml"))' "$OUT_REQUESTS" | sed 's/\\r//g' > out.app.yml.txt +trace jq 'select(.body.multipart_form.path | strings | test("app.yml"))' "$OUT_REQUESTS" | sed 's/\\r//g' > out.app.yml.txt #trace print_requests.py //apps # currently fails due to TF inserting description="" rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt index 70bbb6eb108..1a3ffc22b37 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt @@ -16,5 +16,5 @@ } === Expecting the code_source tarball to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz" +>>> print_workspace_imports +/Workspace/foo/bar/artifacts/.internal/code.tgz diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/script b/acceptance/bundle/artifacts/ai_runtime_code_source/script index 396f907033e..32460827ec5 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/script +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/script @@ -6,6 +6,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' "$OUT_REQUESTS" title "Expecting the code_source tarball to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep code.tgz | sort +trace print_workspace_imports | grep code.tgz | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt index 2a10679b430..9997818f806 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt @@ -48,9 +48,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl +/Workspace/foo/bar/artifacts/.internal/source.whl === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script index 70c9dffac83..b3c226bc752 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script @@ -9,7 +9,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting wheel to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt index 2419cc5afbc..ca39c787992 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt @@ -4,9 +4,9 @@ Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Expecting wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl +/Workspace/foo/bar/artifacts/.internal/source.whl === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script index 7ac87760449..0d665cfbc5f 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script @@ -6,7 +6,7 @@ $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) title "Expecting wheel to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt index ee40963cbaa..e500ebd0342 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt @@ -38,15 +38,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 4 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source1.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source2.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source3.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl +/Workspace/foo/bar/artifacts/.internal/source1.whl +/Workspace/foo/bar/artifacts/.internal/source2.whl +/Workspace/foo/bar/artifacts/.internal/source3.whl +/Workspace/foo/bar/artifacts/.internal/source4.whl === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/script b/acceptance/bundle/artifacts/upload_multiple_libraries/script index f65f8154f2a..89d0ba55291 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/script +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/script @@ -12,7 +12,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 4 wheels to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 34b272c0f08..30134175168 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -29,19 +29,19 @@ dist/my_test_code-0.1.0-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path [OUTPUT_DIR]/out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME >>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0 @@ -62,14 +62,14 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> jq .path [OUTPUT_DIR]/out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME === Restore config to target old wheel >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl @@ -92,12 +92,12 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> jq .path [OUTPUT_DIR]/out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME diff --git a/acceptance/bundle/artifacts/whl_change_version/script b/acceptance/bundle/artifacts/whl_change_version/script index 49d50f0b0b5..4b97cb54259 100644 --- a/acceptance/bundle/artifacts/whl_change_version/script +++ b/acceptance/bundle/artifacts/whl_change_version/script @@ -6,7 +6,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace jq .path "$OUT_REQUESTS" | grep import | sort +trace print_workspace_imports | sort rm "$OUT_REQUESTS" @@ -20,7 +20,7 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace jq .path "$OUT_REQUESTS" | grep import | sort +trace print_workspace_imports | sort rm "$OUT_REQUESTS" @@ -34,6 +34,6 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace jq .path "$OUT_REQUESTS" | grep import | sort +trace print_workspace_imports | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index b4974a6f5ca..04b3a0f58fa 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -25,6 +25,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting no wheels to be uploaded ->>> errcode sh -c jq .path < "$OUT_REQUESTS" | grep import | grep whl +>>> print_workspace_imports Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script index 944ecf709f9..bd7b88b5806 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/script +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting no wheels to be uploaded" -trace errcode sh -c 'jq .path < "$OUT_REQUESTS" | grep import | grep whl' +trace print_workspace_imports | errcode grep whl rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index 51be24e7e43..3461014160d 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -48,11 +48,11 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 patched wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl === Updating the local wheel and deploying again @@ -74,7 +74,7 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice) ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index b73202a65ec..d031d02715c 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -22,7 +22,7 @@ title "Expecting 2 patched wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 2 patched wheels to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" @@ -40,4 +40,4 @@ title "Expecting 2 patched wheels in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 55aa3ff229a..14358c0ab32 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -21,9 +21,9 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index f55d7da467a..a1227cd125d 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -8,7 +8,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index d589ebc5d74..c610058f32f 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -21,6 +21,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index a42ce2844db..8124726f4ff 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -8,6 +8,6 @@ title "Expecting 1 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheels to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 04683b58ffc..1a93fb8c9bd 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -39,6 +39,6 @@ package/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl +/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 410957d8233..4d5fd31d6e7 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -7,6 +7,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index 4172ddf8f51..6a49d1e1889 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -11,6 +11,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index 72d0ce55360..41bae0e4237 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index 8bf5bef423d..96edc8e5c85 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -25,8 +25,8 @@ my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index df8ad98b3c4..8c06cd32454 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -8,6 +8,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm -fr "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt index a61388c76f9..b80a0145ad1 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt +++ b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt @@ -3,9 +3,9 @@ dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl === No calls to delete internal folder expected >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/script b/acceptance/bundle/artifacts/whl_no_cleanup/script index f3ef86ecf70..6a4a10bc75e 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/script +++ b/acceptance/bundle/artifacts/whl_no_cleanup/script @@ -5,7 +5,7 @@ cat LOG.deploy | contains.py "my_test_code-0.0.1-py3-none-any.whl" > /dev/null trace find.py --expect 1 whl title "Expecting 1 wheels to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index e43ce01f38c..c18e0b2d6da 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -36,8 +36,8 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index 2b9240de98a..8034c3171cf 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -10,6 +10,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt index 9611c9541f2..a034c80a088 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt @@ -32,6 +32,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/script b/acceptance/bundle/artifacts/whl_prebuilt_outside/script index ecfd7bea799..23a2e0fc1ef 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/script @@ -11,6 +11,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt index 324b0a1b18a..b26bae880f5 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt @@ -56,6 +56,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script index b53c1dc55b2..f972cf2504e 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script @@ -14,6 +14,6 @@ trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_RE trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index eb0d23431f3..f6ba380bc11 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -41,6 +41,6 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index d7657fa4b4f..09f9ed82c47 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in environments section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt index 39e7ead5144..e99a248c94b 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt @@ -41,5 +41,5 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded under its bare filename (no extras) ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index dadef34e181..157472b5f55 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -8,6 +8,6 @@ title "Expecting the environments dependency to keep its [train] extras suffix" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" -trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt index a97e8642981..4610f302b33 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt @@ -3,7 +3,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/script b/acceptance/bundle/deploy/partial-summary-on-push-fail/script index bc36c59cacc..d2fef85a1d7 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/script +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/script @@ -9,7 +9,7 @@ trap cleanup EXIT # Inject a 403 on the resource-state upload (PushResourcesState). The files are # uploaded by then, so "Files:" still reports them; "Resources:" must NOT print, # because this fails before planning and there are no resource counts to report. -fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 +fault.py --body-contains "state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 # -q: the state-push 403 races resource apply, so whether "Created jobs.my_job" is # emitted before the failure is platform-dependent. This test asserts Files is reported diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml b/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml index 01dc2713a99..64cb4896dce 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml @@ -1,5 +1,5 @@ # The resource summary must not print when pushing resource state fails, so the -# fault is injected on the state upload (POST import-file). The path is fixed, so +# fault is injected on the state upload (POST /workspace/import). The path is fixed, so # the bundle name must be stable (test-bundle, not $UNIQUE_NAME). # # The injected path is the direct-engine state file (state/resources.json), so diff --git a/acceptance/bundle/deploy/readplan/whl-upload/output.txt b/acceptance/bundle/deploy/readplan/whl-upload/output.txt index 48212c57e37..6b21d59c6b0 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-upload/output.txt @@ -7,14 +7,8 @@ Created jobs.my_job Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //artifacts/.internal --del-field raw_body -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", - "q": { - "overwrite": "true" - } -} +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -32,14 +26,8 @@ Created jobs.my_job Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //artifacts/.internal --del-field raw_body -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", - "q": { - "overwrite": "true" - } -} +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/readplan/whl-upload/script b/acceptance/bundle/deploy/readplan/whl-upload/script index c06707c5dd1..15b0c9aa6c3 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/script +++ b/acceptance/bundle/deploy/readplan/whl-upload/script @@ -5,9 +5,8 @@ title "bundle deploy uploads the wheel" trace $CLI bundle deploy -# An import-file request uploaded the wheel to its .../artifacts/.internal/ workspace path. -# raw_body is the wheel's binary payload; drop it to keep the golden readable. -trace print_requests.py //artifacts/.internal --del-field raw_body +# The wheel was uploaded to its .../artifacts/.internal/ workspace path. +trace print_workspace_imports | grep artifacts/.internal trace $CLI bundle destroy --auto-approve # Discard destroy's requests so the saved-plan deploy below starts from a clean recording. @@ -19,9 +18,9 @@ title "bundle deploy --plan also uploads the wheel" $CLI bundle plan -o json > tmp.plan.json $CLI bundle plan -o json | nostamp > out.plan.json trace $CLI bundle deploy --plan tmp.plan.json -# The same import-file request uploads the wheel to the .../artifacts/.internal/ path the -# job task points at, matching the plain `bundle deploy` above. -trace print_requests.py //artifacts/.internal --del-field raw_body +# The wheel is uploaded to the same .../artifacts/.internal/ path the job task points at, +# matching the plain `bundle deploy` above. +trace print_workspace_imports | grep artifacts/.internal trace $CLI bundle destroy --auto-approve rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt index d641b59fc93..f114763c887 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//import-file/ ^//workspace/ +>>> print_requests.py ^//workspace/ >>> [CLI] bundle summary -o json @@ -34,4 +34,4 @@ Recreated pipelines.foo Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/script b/acceptance/bundle/deployment/bind/pipelines/recreate/script index 9fd964fee14..8d02246da3b 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/script +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/script @@ -6,11 +6,11 @@ add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f "$OUT_REQUESTS" trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.txt -print_requests.py '^//import-file/' '^//workspace/' +print_requests.py '^//workspace/' rm -f "$OUT_REQUESTS" trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt -trace print_requests.py '^//import-file/' '^//workspace/' +trace print_requests.py '^//workspace/' trace $CLI bundle summary -o json | jq .resources > out.summary.json trace $CLI bundle plan @@ -19,4 +19,4 @@ trace musterr $CLI bundle deploy rm -f "$OUT_REQUESTS" trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/deployment/bind/pipelines/update/output.txt b/acceptance/bundle/deployment/bind/pipelines/update/output.txt index 5e1a9257fc1..732d4b8795c 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/update/output.txt @@ -19,4 +19,4 @@ Updated pipelines.foo Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/delete ^//telemetry-ext +>>> print_requests.py ^//workspace/import ^//workspace/delete ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/update/script b/acceptance/bundle/deployment/bind/pipelines/update/script index 68ed3eca1f2..36c80f00f5d 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/script +++ b/acceptance/bundle/deployment/bind/pipelines/update/script @@ -3,15 +3,15 @@ add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f "$OUT_REQUESTS" trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.txt -print_requests.py '^//import-file/' '^//workspace/delete' +print_requests.py '^//workspace/import' '^//workspace/delete' rm -f "$OUT_REQUESTS" trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt -print_requests.py '^//import-file/' '^//workspace/delete' +print_requests.py '^//workspace/import' '^//workspace/delete' trace $CLI bundle summary -o json > out.summary.json trace $CLI bundle plan rm -f "$OUT_REQUESTS" trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//import-file/' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//workspace/import' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/destroy/force-lock-node-limit/script b/acceptance/bundle/destroy/force-lock-node-limit/script index c7fffbc16a2..616b626f2ef 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/script +++ b/acceptance/bundle/destroy/force-lock-node-limit/script @@ -7,9 +7,9 @@ title "Deploy (creates the job and the deployment)" trace $CLI bundle deploy title "destroy without --force-lock (lock write hits the node limit: aborts)" -fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace errcode $CLI bundle destroy --auto-approve title "destroy with --force-lock (tolerates the node limit: proceeds and completes)" -fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace $CLI bundle destroy --force-lock --auto-approve diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt index 43e860f5d8e..976a2f7e3e6 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt +++ b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt @@ -54,8 +54,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> cat [OUTPUT_DIR]/out.requests.txt { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl" + } } } diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/script b/acceptance/bundle/libraries/outside_of_bundle_root/script index df54672eee7..7c4616890cc 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/script +++ b/acceptance/bundle/libraries/outside_of_bundle_root/script @@ -11,6 +11,6 @@ title "Check that the job libraries are uploaded and the path is correct in the trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.tasks[0].libraries' trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.environments[0].spec.dependencies' trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.0/pipelines" and .method == "POST")' | jq '.body.environment.dependencies' -trace cat "$OUT_REQUESTS" | jq 'select(.path | test("/api/2.0/workspace-files/import-file/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl"))' +trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // "" | test("/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl")))' | jq 'del(.body.multipart_form.content)' rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/output.txt b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt index 54447ea8ecc..2569d95ccfb 100644 --- a/acceptance/bundle/migrate/auto/empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt @@ -20,16 +20,16 @@ direct_migrated_via_config true === The remote had no terraform state, so nothing terraform-related is backed up remotely ->>> print_requests.py //workspace-files/import-file //workspace/delete --sort -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" +>>> print_requests.py //workspace/import //workspace/delete --sort "POST /api/2.0/workspace/delete" "POST /api/2.0/workspace/delete" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" === debug states confirms no terraform.tfstate anywhere; only the local .backup remains diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/script b/acceptance/bundle/migrate/auto/empty-tfstate/script index 225092954a3..c0df01e54ec 100644 --- a/acceptance/bundle/migrate/auto/empty-tfstate/script +++ b/acceptance/bundle/migrate/auto/empty-tfstate/script @@ -19,7 +19,7 @@ title "Migration is recorded in telemetry as via-config\n" trace print_migration_telemetry title "The remote had no terraform state, so nothing terraform-related is backed up remotely\n" -trace print_requests.py //workspace-files/import-file //workspace/delete --sort | jq '.method + " " + .path' | contains.py '!terraform.tfstate' +trace print_requests.py //workspace/import //workspace/delete --sort | jq '.method + " " + (.body.multipart_form.path // .path)' | contains.py '!terraform.tfstate' title "debug states confirms no terraform.tfstate anywhere; only the local .backup remains\n" trace $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/migrate/auto/push-failure/output.txt b/acceptance/bundle/migrate/auto/push-failure/output.txt index 6e9a1c61b78..bd638d6e829 100644 --- a/acceptance/bundle/migrate/auto/push-failure/output.txt +++ b/acceptance/bundle/migrate/auto/push-failure/output.txt @@ -8,7 +8,7 @@ Notice: migrating your bundle to direct deployment engine (https://github.com/da Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/migrate/auto/push-failure/script b/acceptance/bundle/migrate/auto/push-failure/script index da3ffaad3d9..4231617175e 100644 --- a/acceptance/bundle/migrate/auto/push-failure/script +++ b/acceptance/bundle/migrate/auto/push-failure/script @@ -10,7 +10,7 @@ title "Opt in via env var; inject 403 on the resources.json push so the deploy f # push fails the deploy - unlike a declined deploy, the migration is not discarded, so the # local direct state remains and a retry finishes the push. (A push failure is a hard error on # any direct deploy, not just a migrating one.) -fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 +fault.py --body-contains "Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 23808c5a896..503055861e5 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -18,7 +18,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py --nostamp ^//import-file/ ^//api/2.0/bundle +>>> print_requests.py --nostamp ^//workspace/import ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -34,7 +34,7 @@ Created pipelines.mypipeline Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --nostamp ^//import-file/ ^//api/2.0/bundle +>>> print_requests.py --nostamp ^//workspace/import ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -102,7 +102,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted ->>> print_requests.py --nostamp --sort ^//import-file/ ^//api/2.0/bundle +>>> print_requests.py --nostamp --sort ^//workspace/import ^//api/2.0/bundle { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index 6d1e4fd5839..ccd83d2f06d 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py --nostamp '^//import-file/' '^//api/2.0/bundle' +trace print_requests.py --nostamp '^//workspace/import' '^//api/2.0/bundle' trace $CLI bundle deploy -trace print_requests.py --nostamp '^//import-file/' '^//api/2.0/bundle' +trace print_requests.py --nostamp '^//workspace/import' '^//api/2.0/bundle' trace $CLI bundle destroy --auto-approve -trace print_requests.py --nostamp --sort '^//import-file/' '^//api/2.0/bundle' +trace print_requests.py --nostamp --sort '^//workspace/import' '^//api/2.0/bundle' diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index b53609dd402..9795d322b3e 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 5ea603ee6fe..70fe4992c16 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -31,5 +31,5 @@ nostamp < .databricks/tmp.plan.json > out.plan.json rm "$OUT_REQUESTS" trace errcode $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json trace errcode $CLI bundle plan &> out.plan_after_deploy.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt index e82dc779ba2..a23a2445c8b 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -36,7 +36,7 @@ Updated quality_monitors.monitor1 Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index fbd62ade8df..a4104e4d387 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -27,6 +27,6 @@ trace $CLI bundle plan -o json | nostamp > out.plan.json rm "$OUT_REQUESTS" trace $CLI bundle deploy # dashboard_id is output only field that terraform adds -trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt index 77656aa1913..beba3487430 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index ca49568b38a..a17d8b114d4 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -31,7 +31,7 @@ nostamp < .databricks/tmp.plan.json > out.plan.json rm "$OUT_REQUESTS" trace errcode $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json trace errcode $CLI bundle plan &> out.plan_after_deploy.txt trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.json diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt index fbe444e1eb7..5d1b3497525 100644 --- a/acceptance/bundle/resources/quality_monitors/create/output.txt +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -16,7 +16,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle plan -o json ->>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index b45b8226f4e..1a3a60b1381 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -21,7 +21,7 @@ trace $CLI bundle plan -o json | nostamp > out.plan_create.json rm "$OUT_REQUESTS" trace $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json # store state to ensure we have table_name there print_state.py | grep name > out.state.txt diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt index e21f54fcf1f..1943374afeb 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext { "method": "POST", "path": "/api/2.0/secrets/scopes/delete", diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/script b/acceptance/bundle/resources/secret_scopes/delete_scope/script index 6cd30fffdae..8d04d13a33f 100755 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/script +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/script @@ -15,4 +15,4 @@ trace $CLI bundle plan &> out.plan.txt rm "$OUT_REQUESTS" trace $CLI bundle deploy -qq -trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt +trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt diff --git a/acceptance/bundle/sync-upload-edge-cases/databricks.yml b/acceptance/bundle/sync-upload-edge-cases/databricks.yml new file mode 100644 index 00000000000..2261ff177c7 --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: upload-edge-cases diff --git a/acceptance/bundle/sync-upload-edge-cases/out.test.toml b/acceptance/bundle/sync-upload-edge-cases/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-upload-edge-cases/output.txt b/acceptance/bundle/sync-upload-edge-cases/output.txt new file mode 100644 index 00000000000..8ce0c31d5dc --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/output.txt @@ -0,0 +1,136 @@ + +>>> [CLI] bundle sync --output text +Action: PUT: .gitignore, dashboard.lvdash.json, databricks.yml, empty.txt, héllo.txt, large.bin, plain-script.py, pyNb.py, scalaNb.scala, sqlNb.sql, with spaces.txt +Initial Sync Complete +Uploaded .gitignore +Uploaded dashboard.lvdash.json +Uploaded databricks.yml +Uploaded empty.txt +Uploaded héllo.txt +Uploaded large.bin +Uploaded plain-script.py +Uploaded pyNb.py +Uploaded scalaNb.scala +Uploaded sqlNb.sql +Uploaded with spaces.txt + +=== workspace state after sync — assert object_type and language for each upload +>>> MSYS_NO_PATHCONV=1 [CLI] workspace list -o json /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files +{ + "name": ".gitignore", + "object_type": "FILE", + "language": null +} +{ + "name": "dashboard.lvdash.json", + "object_type": "FILE", + "language": null +} +{ + "name": "databricks.yml", + "object_type": "FILE", + "language": null +} +{ + "name": "empty.txt", + "object_type": "FILE", + "language": null +} +{ + "name": "héllo.txt", + "object_type": "FILE", + "language": null +} +{ + "name": "large.bin", + "object_type": "FILE", + "language": null +} +{ + "name": "plain-script.py", + "object_type": "FILE", + "language": null +} +{ + "name": "pyNb", + "object_type": "NOTEBOOK", + "language": "PYTHON" +} +{ + "name": "scalaNb", + "object_type": "NOTEBOOK", + "language": "SCALA" +} +{ + "name": "sqlNb", + "object_type": "NOTEBOOK", + "language": "SQL" +} +{ + "name": "with spaces.txt", + "object_type": "FILE", + "language": null +} + +=== uploaded paths from each multipart POST /workspace/import (sorted) +>>> print_workspace_imports +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/.gitignore +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/dashboard.lvdash.json +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/empty.txt +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/héllo.txt +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/large.bin +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/plain-script.py +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/pyNb.py +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/scalaNb.scala +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/sqlNb.sql +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/with spaces.txt + +=== every upload set format=AUTO +>>> jq -rs + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.format) | unique + [OUTPUT_DIR]/out.requests.txt +[ + "AUTO" +] + +=== recorded content for the small text inputs +>>> jq -rs + map(select(.path == "/api/2.0/workspace/import")) | + map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | + map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | + sort_by(.path) | .[] + [OUTPUT_DIR]/out.requests.txt +{ + "path": "dashboard.lvdash.json", + "content": "{\"datasets\":[],\"pages\":[]}\n" +} +{ + "path": "empty.txt", + "content": "" +} +{ + "path": "héllo.txt", + "content": "hello, naïve world\n" +} +{ + "path": "plain-script.py", + "content": "print(\"plain script, no header\")\n" +} +{ + "path": "pyNb.py", + "content": "# Databricks notebook source\nprint(\"hello\")\n" +} +{ + "path": "scalaNb.scala", + "content": "// Databricks notebook source\nprintln(\"hello\")\n" +} +{ + "path": "sqlNb.sql", + "content": "-- Databricks notebook source\nSELECT 1\n" +} +{ + "path": "with spaces.txt", + "content": "hello, world\n" +} diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script new file mode 100644 index 00000000000..bfd3abcaa03 --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/script @@ -0,0 +1,92 @@ +# Confidence checks that /workspace/import is a drop-in for the legacy import-file: +# verify the CLI's multipart upload pipeline handles the cases that differ between the +# two endpoints — large content (>10 MB), empty content, varied extensions/object types, +# and non-ASCII / spaced filenames in the multipart `path` field. + +# Exclude framework-generated and script-internal files from the sync. We only want the +# inputs we explicitly generate below to show up in the recorded uploads. +cat > .gitignore <<'EOF' +script +output.txt +out.requests.txt +repls.json +test.toml +.databricks +EOF + +# 12 MiB binary file, over the 10 MB cap on the JSON-body variant's base64 `content` +# field. The multipart variant this test exercises has no such cap. Generated +# dynamically to keep the repo small. +python3 -c "open('large.bin', 'wb').write(b'\\0' * (12 * 1024 * 1024))" + +# Empty file (multipart encodes an empty `content` part — distinct from JSON's empty string). +touch empty.txt + +# Python notebook: auto-detected as NOTEBOOK by /workspace/import; testserver mirrors +# this and stores the object at the path with .py stripped. +cat > pyNb.py <<'EOF' +# Databricks notebook source +print("hello") +EOF + +# SQL notebook: -- comment header, .sql extension, Language=SQL on the workspace side. +cat > sqlNb.sql <<'EOF' +-- Databricks notebook source +SELECT 1 +EOF + +# Scala notebook: // comment header, .scala extension, Language=SCALA. +cat > scalaNb.scala <<'EOF' +// Databricks notebook source +println("hello") +EOF + +# Plain .py without the source header — should land as FILE, not NOTEBOOK, +# even though the extension is .py. +cat > plain-script.py <<'EOF' +print("plain script, no header") +EOF + +# Lakeview dashboard descriptor: the real /workspace/import assigns object_type=DASHBOARD +# and preserves the .lvdash.json extension. The testserver doesn't emulate that and stores +# it as a generic FILE — but the upload-side request shape (which is what this test asserts) +# is identical. +echo '{"datasets":[],"pages":[]}' > dashboard.lvdash.json + +# Non-ASCII filename. Multipart encodes filenames with RFC 5987 / quoted-string rules, +# distinct from URL-encoding in the legacy import-file endpoint. +echo "hello, naïve world" > "héllo.txt" + +# Filename with a space. +echo "hello, world" > "with spaces.txt" + +trace $CLI bundle sync --output text 2>&1 | sort + +title "workspace state after sync — assert object_type and language for each upload" +remote_files=$($CLI bundle summary -o json 2>/dev/null | jq -r .workspace.file_path) +# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-slash +# workspace path into a C:/Program Files/... filesystem path before it +# reaches the CLI. +trace MSYS_NO_PATHCONV=1 $CLI workspace list -o json "$remote_files" | + jq 'sort_by(.path) | .[] | {name: (.path | sub(".*/"; "")), object_type, language}' + +title "uploaded paths from each multipart POST /workspace/import (sorted)" +trace print_workspace_imports | sort + +title "every upload set format=AUTO" +trace jq -rs ' + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.format) | unique +' "$OUT_REQUESTS" + +title "recorded content for the small text inputs" +trace jq -rs ' + map(select(.path == "/api/2.0/workspace/import")) | + map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | + map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | + sort_by(.path) | .[] +' "$OUT_REQUESTS" + +# Drop the recorded requests; their workspace-prefixed paths are noisy and the +# assertions above already pin the relevant fields. +rm "$OUT_REQUESTS" .gitignore diff --git a/acceptance/bundle/sync-upload-edge-cases/test.toml b/acceptance/bundle/sync-upload-edge-cases/test.toml new file mode 100644 index 00000000000..8bfde26201e --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/test.toml @@ -0,0 +1,18 @@ +RecordRequests = true + +# All sync uploads go through the same /workspace/import multipart code path; running +# both engine variants is unnecessary here. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "large.bin", + "empty.txt", + "pyNb.py", + "sqlNb.sql", + "scalaNb.scala", + "plain-script.py", + "dashboard.lvdash.json", + "héllo.txt", + "with spaces.txt", + ".databricks", +] diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt index 17eb89280ce..df5252aa3d7 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt @@ -1,9 +1,9 @@ >>> [CLI] bundle deploy -Error: Failed to acquire deployment lock: Internal error -Error: Internal error (500 INTERNAL_ERROR) +Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal error +Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal error (500 INTERNAL_ERROR) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fdeploy.lock?overwrite=false +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import HTTP Status: 500 Internal Server Error API error_code: INTERNAL_ERROR API message: Internal error diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml index 86d366ba8c2..1e685cca603 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml @@ -12,6 +12,6 @@ Cloud = false [[Server]] -Pattern = "POST /api/2.0/workspace-files/import-file/{path...}" +Pattern = "POST /api/2.0/workspace/import" Response.StatusCode = 500 Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 3029c012d05..2f91efecbe1 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -13,13 +13,13 @@ $CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.json rm "$OUT_REQUESTS" # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.txt read_id.py -t dev sample_job > /dev/null trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.txt read_id.py -t prod sample_job > /dev/null # Extracted with python, not jq: the id exceeds 2^53 and jq 1.6 rounds it. root_dir_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/my_default_python/prod" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index 93e5048afec..a5d2a395976 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -13,13 +13,13 @@ $CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.json rm "$OUT_REQUESTS" trace $CLI bundle deploy -t dev replace_ids.py -t dev -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.txt trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.json trace $CLI bundle deploy -t prod replace_ids.py -t prod -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.txt trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json | nostamp > ../../out.plan_after_deploy_prod.json diff --git a/acceptance/bundle/upload/internal_server_error/output.txt b/acceptance/bundle/upload/internal_server_error/output.txt index 5cde6aad650..794a0ed2072 100644 --- a/acceptance/bundle/upload/internal_server_error/output.txt +++ b/acceptance/bundle/upload/internal_server_error/output.txt @@ -1,8 +1,8 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: Internal Server Error (500) +Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt: Fault injected by test. (500 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import HTTP Status: 500 Internal Server Error -API error_code: -API message: Internal Server Error +API error_code: INJECTED +API message: Fault injected by test. diff --git a/acceptance/bundle/upload/internal_server_error/script b/acceptance/bundle/upload/internal_server_error/script index c1a222958e1..78c57ef8a94 100644 --- a/acceptance/bundle/upload/internal_server_error/script +++ b/acceptance/bundle/upload/internal_server_error/script @@ -1 +1,2 @@ +fault.py --body-contains "Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/files/file_to_upload.txt" "POST /api/2.0/workspace/import" 500 0 1000 musterr $CLI bundle deploy diff --git a/acceptance/bundle/upload/internal_server_error/test.toml b/acceptance/bundle/upload/internal_server_error/test.toml deleted file mode 100644 index 8ff8501b6de..00000000000 --- a/acceptance/bundle/upload/internal_server_error/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -[[Server]] -Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" -Response.StatusCode = 500 diff --git a/acceptance/bundle/upload/timeout/output.txt b/acceptance/bundle/upload/timeout/output.txt index c314a247707..0c779d73dc2 100644 --- a/acceptance/bundle/upload/timeout/output.txt +++ b/acceptance/bundle/upload/timeout/output.txt @@ -1,3 +1,3 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity +Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity +Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity diff --git a/acceptance/bundle/upload/timeout/test.toml b/acceptance/bundle/upload/timeout/test.toml index d6564ef9659..f0abd08ecbc 100644 --- a/acceptance/bundle/upload/timeout/test.toml +++ b/acceptance/bundle/upload/timeout/test.toml @@ -3,5 +3,5 @@ Env.DATABRICKS_BUNDLE_HTTP_TIMEOUT_SECONDS = "5" [[Server]] # CLI aborts after a single attempt when the HTTP timeout fires. Delay = "30s" -Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" +Pattern = "POST /api/2.0/workspace/import" Response.StatusCode = 200 diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 947a3167408..d4faf40804a 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -8,14 +8,14 @@ OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct -OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py engine/direct -OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct -OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct -OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct -OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json engine/direct -OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json engine/direct OK deploy /api/2.0/workspace/delete engine/direct OK deploy /api/2.0/workspace/delete engine/direct +OK deploy /api/2.0/workspace/import engine/direct +OK deploy /api/2.0/workspace/import engine/direct +OK deploy /api/2.0/workspace/import engine/direct +OK deploy /api/2.0/workspace/import engine/direct +OK deploy /api/2.0/workspace/import engine/direct +OK deploy /api/2.0/workspace/import engine/direct OK deploy /api/2.0/workspace/mkdirs engine/direct OK deploy /api/2.0/workspace/mkdirs engine/direct OK deploy /api/2.1/unity-catalog/schemas engine/direct @@ -30,9 +30,9 @@ OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct -OK destroy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct OK destroy /api/2.0/workspace/delete engine/direct OK destroy /api/2.0/workspace/delete engine/direct +OK destroy /api/2.0/workspace/import engine/direct MISS destroy /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS plan /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.json index f0c60602386..6617ac3b1ac 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.json @@ -130,9 +130,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", + "recursive": true } } { @@ -142,15 +143,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/delete", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } { @@ -160,15 +155,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/import", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "multipart_form": { + "content": "", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py" + } } } { @@ -178,22 +172,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "seq": 1, - "cli_version": "[CLI_VERSION]", - "timestamp": "[TIMESTAMP]", - "files": [ - { - "local_path": "empty.py", - "is_notebook": false - } - ], - "id": "[UUID]" + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -203,29 +188,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "config": { - "bundle": { - "name": "test-bundle", - "target": "default", - "git": { - "bundle_root_path": "." - } - }, - "workspace": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" - }, - "resources": {}, - "presets": { - "source_linked_deployment": false - } - }, - "extra": {} + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -235,23 +204,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "state_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "state": { - "resources.schemas.foo": { - "__id__": "mycatalog.myschema", - "state": { - "catalog_name": "mycatalog", - "name": "myschema" - } - } + "multipart_form": { + "content": "{\"version\":1,\"seq\":1,\"cli_version\":\"[CLI_VERSION]\",\"timestamp\":\"[TIMESTAMP]\",\"files\":[{\"local_path\":\"empty.py\",\"is_notebook\":false}],\"id\":\"[UUID]\"}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" } } } @@ -262,10 +221,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", - "recursive": true + "multipart_form": { + "content": "{\n \"version\": 1,\n \"config\": {\n \"bundle\": {\n \"name\": \"test-bundle\",\n \"target\": \"default\",\n \"git\": {\n \"bundle_root_path\": \".\"\n }\n },\n \"workspace\": {\n \"file_path\": \"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"\n },\n \"resources\": {},\n \"presets\": {\n \"source_linked_deployment\": false\n }\n },\n \"extra\": {}\n}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + } } } { @@ -275,9 +238,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + "multipart_form": { + "content": "{\n \"state_version\": 3,\n \"cli_version\": \"[CLI_VERSION]\",\n \"lineage\": \"[UUID]\",\n \"serial\": 1,\n \"state\": {\n \"resources.schemas.foo\": {\n \"__id__\": \"mycatalog.myschema\",\n \"state\": {\n \"catalog_name\": \"mycatalog\",\n \"name\": \"myschema\"\n }\n }\n }\n}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" + } } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.json index 43de64cf21f..49ae5e2cfc9 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.json @@ -125,15 +125,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/delete", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" } } { @@ -145,7 +139,8 @@ "method": "POST", "path": "/api/2.0/workspace/delete", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "recursive": true } } { @@ -155,10 +150,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "recursive": true + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { diff --git a/acceptance/bundle/validate/sync_patterns/out.sync.txt b/acceptance/bundle/validate/sync_patterns/out.sync.txt index ac5aba9ab29..fd939c9721b 100644 --- a/acceptance/bundle/validate/sync_patterns/out.sync.txt +++ b/acceptance/bundle/validate/sync_patterns/out.sync.txt @@ -1,7 +1,12 @@ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml" + } } } diff --git a/acceptance/bundle/validate/sync_patterns/script b/acceptance/bundle/validate/sync_patterns/script index d4cf7169b1b..f44ee310433 100644 --- a/acceptance/bundle/validate/sync_patterns/script +++ b/acceptance/bundle/validate/sync_patterns/script @@ -1,5 +1,5 @@ trace $CLI bundle validate trace $CLI bundle validate -o json | jq '.sync' trace $CLI bundle deploy -jq 'select(.path | test("dir/test.yml"))' "$OUT_REQUESTS" > out.sync.txt +jq 'select(.body.multipart_form.path | strings | test("dir/test.yml"))' "$OUT_REQUESTS" > out.sync.txt rm "$OUT_REQUESTS" diff --git a/acceptance/cmd/sync/dryrun-missing-remote/output.txt b/acceptance/cmd/sync/dryrun-missing-remote/output.txt index 4848aae1c2d..b7453994b58 100644 --- a/acceptance/cmd/sync/dryrun-missing-remote/output.txt +++ b/acceptance/cmd/sync/dryrun-missing-remote/output.txt @@ -15,11 +15,11 @@ Uploaded hello.py >>> print_requests.py --sort //api/ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/.gitignore" + "path": "/api/2.0/workspace/import" } { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/hello.py" + "path": "/api/2.0/workspace/import" } { "method": "POST", diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 239a4a904a1..468aa02dc9e 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -1,8 +1,12 @@ package internal import ( + "bytes" "encoding/json" "fmt" + "io" + "mime" + "mime/multipart" "net/http" "os" "path/filepath" @@ -331,6 +335,11 @@ func getLoggedRequest(req *testserver.Request, includedHeaders []string) LoggedR if json.Valid(req.Body) { result.Body = json.RawMessage(req.Body) + } else if normalized, ok := normalizeMultipartBody(req); ok { + // Multipart bodies contain a randomly generated boundary string and binary + // content; record a normalized form (sorted form-field names with sizes for + // file parts) so recorded requests stay deterministic and reviewable. + result.Body = normalized } else { result.RawBody = string(req.Body) } @@ -338,6 +347,55 @@ func getLoggedRequest(req *testserver.Request, includedHeaders []string) LoggedR return result } +// Multipart parts larger than this limit are summarized as a size placeholder +// to keep recorded fixtures small. Reviewers care about *what* was uploaded +// (path, format, overwrite flag), not the bytes themselves, for large blobs. +const multipartContentLimit = 4096 + +// normalizeMultipartBody returns a deterministic representation of a multipart +// form body if the request's Content-Type is multipart/*. The second return +// value is false if the body is not multipart or cannot be parsed. +// +// Text parts are recorded as their literal content so reviewers can read what +// was uploaded; existing global UNIX_TIME / UUID / USERNAME replacements +// normalize the timestamp / id / email fields embedded in deploy.lock and +// similar payloads. Non-UTF-8 (binary) parts and parts whose serialized +// content would bloat the recording past multipartContentLimit are +// summarized as "[binary content N bytes]". +func normalizeMultipartBody(req *testserver.Request) (any, bool) { + contentType := req.Headers.Get("Content-Type") + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil || !strings.HasPrefix(mediaType, "multipart/") { + return nil, false + } + boundary := params["boundary"] + if boundary == "" { + return nil, false + } + mr := multipart.NewReader(bytes.NewReader(req.Body), boundary) + parts := map[string]any{} + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false + } + name := part.FormName() + if !utf8.Valid(data) || len(data) > multipartContentLimit { + parts[name] = fmt.Sprintf("[binary content %d bytes]", len(data)) + continue + } + parts[name] = string(data) + } + return map[string]any{"multipart_form": parts}, true +} + func filterHeaders(h http.Header, includedHeaders []string) http.Header { headers := make(http.Header) for k, v := range h { diff --git a/acceptance/script.prepare b/acceptance/script.prepare index a53627b231d..66080eb1fcb 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -128,6 +128,12 @@ print_telemetry_bool_values() { jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental.bool_values // []) ) | map("\(.key) \(.value)") | .[]' "$OUT_REQUESTS" | grep -Ev '^(engine_terraform_|deployment_history_enabled )' | sort } +# Workspace paths uploaded via /workspace/import, one per line. The path is a +# multipart form field, not part of the URL, so print_requests.py can't filter on it. +print_workspace_imports() { + jq -r 'select(.path? == "/api/2.0/workspace/import") | .body.multipart_form.path | strings' "$OUT_REQUESTS" +} + sethome() { local home="$1" mkdir -p "$home" diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index b24740317af..5c5a2e7f23a 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -411,9 +411,11 @@ func TestSubmitWorkloadValidationOutcome(t *testing.T) { t.Cleanup(server.Close) var uploads, submissions atomic.Int32 server.Handle("POST", validateConfigPath, func(req testserver.Request) any { return tt.response }) - server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { uploads.Add(1) - return req.Workspace.WorkspaceFilesImportFile(req.Vars["path"], req.Body, true) + form, err := testserver.ParseImportForm(req) + assert.NoError(t, err) + return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, true) }) server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { submissions.Add(1) @@ -730,18 +732,19 @@ func TestSubmitWorkloadPlainTarContentAddressed(t *testing.T) { server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { return jobs.SubmitRunResponse{RunId: 555} }) - // Count snapshot import-file calls, preserving fake-workspace persistence so the + // Count snapshot import calls, preserving fake-workspace persistence so the // second submit's existence Stat sees the first upload. A count (not a set keyed by // path) is what proves the skip: both submits resolve to the same content-addressed // name, so a set could not tell a skipped second submit from one that re-uploaded to // that same path. snapshotUploads := 0 - server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { - p := req.Vars["path"] - if strings.Contains(p, "/.air/repo_snapshots/") { + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + form, err := testserver.ParseImportForm(req) + assert.NoError(t, err) + if strings.Contains(form.Path, "/.air/repo_snapshots/") { snapshotUploads++ } - return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) }) stubValidateConfig(server) testserver.AddDefaultHandlers(server) @@ -778,7 +781,7 @@ code_source: assert.Regexp(t, `^src_[0-9a-f]{16}\.tar\.gz$`, base) // Same unchanged tree → identical remote path, and the second submit moved no bytes: - // zero new import-file calls (a real skip), not a re-upload to the same name. + // zero new import calls (a real skip), not a re-upload to the same name. assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Equal(t, afterFirst, snapshotUploads, "unchanged plain_tar should skip the second upload") require.NotNil(t, first.SizeBytes) @@ -808,12 +811,13 @@ func TestSubmitWorkloadGitArchiveCaching(t *testing.T) { // Dedupe by path: the filer mkdirs-and-retries the import on a missing // parent dir, so one logical upload can hit this route more than once. uploaded := map[string]bool{} - server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { - p := req.Vars["path"] - if strings.Contains(p, "/.air/repo_snapshots/") { - uploaded[p] = true + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + form, err := testserver.ParseImportForm(req) + assert.NoError(t, err) + if strings.Contains(form.Path, "/.air/repo_snapshots/") { + uploaded[form.Path] = true } - return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) }) stubValidateConfig(server) testserver.AddDefaultHandlers(server) diff --git a/cmd/air/telemetry_test.go b/cmd/air/telemetry_test.go index 35a0cad984d..49760d24ad9 100644 --- a/cmd/air/telemetry_test.go +++ b/cmd/air/telemetry_test.go @@ -61,10 +61,11 @@ func TestAirRunTelemetry(t *testing.T) { return jobs.SubmitRunResponse{RunId: 555} }) var uploadedSize int64 - server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { - p := req.Vars["path"] - if strings.HasSuffix(p, ".tar.gz") { - uploadedSize = int64(len(req.Body)) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + form, err := testserver.ParseImportForm(req) + assert.NoError(t, err) + if strings.HasSuffix(form.Path, ".tar.gz") { + uploadedSize = int64(len(form.Content)) time.Sleep(20 * time.Millisecond) if tc.failUpload { return testserver.Response{StatusCode: http.StatusForbidden, Body: map[string]string{ @@ -72,7 +73,7 @@ func TestAirRunTelemetry(t *testing.T) { }} } } - return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") + return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) }) stubValidateConfig(server) w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index e1b62481243..24188974ed4 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -9,10 +9,8 @@ import ( "io" "io/fs" "net/http" - "net/url" "path" "slices" - "strings" "time" "github.com/databricks/cli/libs/auth" @@ -154,35 +152,29 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - // Remove leading "/" so we can use it in the URL. - overwrite := slices.Contains(mode, OverwriteIfExists) - urlPath := fmt.Sprintf( - "/api/2.0/workspace-files/import-file/%s?overwrite=%t", - url.PathEscape(strings.TrimLeft(absPath, "/")), - overwrite, - ) - // Buffer the file contents because we may need to retry below and we cannot read twice. body, err := io.ReadAll(reader) if err != nil { return err } - err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) + // Upload with the SDK's multipart Workspace.Upload; format=AUTO lets the server + // classify each payload as file or notebook. The JSON-body workspace.Import caps + // content at 10 MB, the multipart form at the 500 MB workspace file limit. + opts := []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatAuto)} + if slices.Contains(mode, OverwriteIfExists) { + opts = append(opts, workspace.UploadOverwrite()) + } + + err = w.workspaceClient.Workspace.Upload(ctx, absPath, bytes.NewReader(body), opts...) // Return early on success. if err == nil { return nil } - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return err - } - - // This API returns a 404 if the parent directory does not exist. - if aerr.StatusCode == http.StatusNotFound { + // Parent directory does not exist. + if errors.Is(err, apierr.ErrNotFound) { if !slices.Contains(mode, CreateParentDirectories) { return noSuchDirectoryError{path.Dir(absPath)} } @@ -190,8 +182,8 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // Create parent directory. err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath)) if err != nil { - if mkdirErr, ok := errors.AsType[*apierr.APIError](err); ok && mkdirErr.StatusCode == http.StatusForbidden { - return permissionError{absPath, mkdirErr} + if errors.Is(err, apierr.ErrPermissionDenied) { + return permissionError{absPath, err} } return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err) } @@ -200,27 +192,32 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return w.Write(ctx, name, bytes.NewReader(body), sliceWithout(mode, CreateParentDirectories)...) } - // This API returns 409 if the file already exists, when the object type is file - if aerr.StatusCode == http.StatusConflict { + // Path already taken. ErrResourceConflict covers every 409 (including the + // bare 409 with only a message); ErrResourceAlreadyExists covers the 400 + // RESOURCE_ALREADY_EXISTS, which no status maps to a conflict. + if errors.Is(err, apierr.ErrResourceConflict) || errors.Is(err, apierr.ErrResourceAlreadyExists) { return fileAlreadyExistsError{absPath} } - - // This API returns 400 if the file already exists when the object type is notebook. - // Both the historical "Path () already exists." format and the newer - // "RESOURCE_ALREADY_EXISTS: already exists. ..." format end with the same - // "already exists." marker; the JSON error_code is empty in both. The new format - // might not have been rolled out to all workspaces yet, so we anchor on the shared - // marker and return absPath rather than parsing the message. - if aerr.StatusCode == http.StatusBadRequest && strings.Contains(aerr.Message, "already exists.") { - return fileAlreadyExistsError{absPath} + // Overwrite rejected because the existing object's node type differs from + // the upload. The collision carries an AIP-193 ErrorInfo with a stable + // reason, so branch on it rather than the message text. + if errors.Is(err, apierr.ErrInvalidParameterValue) { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == "WORKSPACE_OBJECT_TYPE_MISMATCH" { + return fileAlreadyExistsError{absPath} + } + } } - // This API returns StatusForbidden when you have read access but don't have write access to a file - if aerr.StatusCode == http.StatusForbidden { - return permissionError{absPath, aerr} + // Caller has read access but no write access. + if errors.Is(err, apierr.ErrPermissionDenied) { + return permissionError{absPath, err} } - return err + // Any other failure (e.g. a server-side error with an empty message) is + // surfaced with the target path so the user can tell which upload failed; + // %w keeps the original error inspectable by callers. + return fmt.Errorf("failed to upload %s: %w", absPath, err) } func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 2915af60436..81832694d15 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -1,8 +1,11 @@ package filer import ( + "bytes" "encoding/json" "io/fs" + "net/http" + "strings" "testing" "time" @@ -15,6 +18,8 @@ import ( "github.com/stretchr/testify/require" ) +// Confirms directory entries derived from workspace object infos report the +// name, type, and size of each object type the workspace can return. func TestWorkspaceFilesDirEntry(t *testing.T) { entries := wsfsDirEntriesFromObjectInfos([]workspace.ObjectInfo{ { @@ -61,6 +66,8 @@ func TestWorkspaceFilesDirEntry(t *testing.T) { assert.True(t, i2.IsDir()) } +// Confirms the workspace ID routing header is set only when the client config +// carries a workspace ID, and that a nil workspace client yields no headers. func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { tests := []struct { name string @@ -97,6 +104,188 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } +// Confirms each error /workspace/import returns is translated to the filer's +// own error type, and that unrecognized errors pass through unchanged. The +// endpoint signals an occupied path with three different status/error_code +// combinations (see the comment on Write), so all three are covered here to +// pin the mapping the rest of the CLI relies on. +func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { + tests := []struct { + name string + mode []WriteMode + apiErr *apierr.APIError + expectErrTarget any + }{ + { + name: "404 without create-parent maps to noSuchDirectoryError", + apiErr: &apierr.APIError{StatusCode: http.StatusNotFound, Message: "not found"}, + expectErrTarget: noSuchDirectoryError{}, + }, + { + name: "400 RESOURCE_ALREADY_EXISTS maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "RESOURCE_ALREADY_EXISTS", + Message: "/dir/file.txt already exists. Please pass overwrite=true to overwrite it.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "409 ALREADY_EXISTS (concurrent contention) maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusConflict, + ErrorCode: "ALREADY_EXISTS", + Message: "Node with name /dir/file.txt already exists. Please pass overwrite=true to update it.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + // A 409 carrying no error_code unwraps to ErrResourceConflict only, + // not to ErrAlreadyExists; the testserver returns exactly this shape. + name: "409 without error_code maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusConflict, + Message: "Node with name /dir/file.txt already exists.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "400 INVALID_PARAMETER_VALUE other message passes through", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "some other validation failure", + }, + expectErrTarget: nil, + }, + { + name: "403 maps to permissionError", + apiErr: &apierr.APIError{StatusCode: http.StatusForbidden, Message: "denied"}, + expectErrTarget: permissionError{}, + }, + { + name: "500 passes through", + apiErr: &apierr.APIError{StatusCode: http.StatusInternalServerError, Message: "boom"}, + expectErrTarget: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := testserver.New(t) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + return testserver.Response{ + StatusCode: tc.apiErr.StatusCode, + Body: map[string]string{ + "error_code": tc.apiErr.ErrorCode, + "message": tc.apiErr.Message, + }, + } + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewWorkspaceFilesClient(client, "/dir") + require.NoError(t, err) + + err = f.Write(t.Context(), "file.txt", bytes.NewReader([]byte("data")), tc.mode...) + require.Error(t, err) + switch target := tc.expectErrTarget.(type) { + case noSuchDirectoryError: + assert.ErrorAs(t, err, &target) + case fileAlreadyExistsError: + assert.ErrorAs(t, err, &target) + case permissionError: + assert.ErrorAs(t, err, &target) + case nil: + // passthrough — the underlying APIError stays inspectable, + // and the message names the file that failed to upload. + var aerr *apierr.APIError + require.ErrorAs(t, err, &aerr) + assert.Equal(t, tc.apiErr.StatusCode, aerr.StatusCode) + assert.Contains(t, err.Error(), "/dir/file.txt") + } + }) + } +} + +// writeWithImportError exercises Write through a real HTTP roundtrip so the +// SDK parses AIP-193 error details from the response body (the errorDetails +// field on apierr.APIError is unexported and only populated during response +// parsing, so it cannot be set on a directly constructed APIError). +func writeWithImportError(t *testing.T, body map[string]any) error { + t.Helper() + + server := testserver.New(t) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + return testserver.Response{ + StatusCode: http.StatusBadRequest, + Body: body, + } + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewWorkspaceFilesClient(client, "/dir") + require.NoError(t, err) + + err = f.Write(t.Context(), "file.txt", strings.NewReader("data"), OverwriteIfExists) + require.Error(t, err) + return err +} + +// Confirms a type-mismatch collision is recognized from the structured +// AIP-193 ErrorInfo reason, independent of the error message wording. +func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { + // The message is deliberately unrelated to any collision wording, to prove + // the branch fires on the structured reason alone. + err := writeWithImportError(t, map[string]any{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "some future wording for the same condition", + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "WORKSPACE_OBJECT_TYPE_MISMATCH", + "domain": "workspace.databricks.com", + "metadata": map[string]string{"existing_type": "NOTEBOOK"}, + }, + }, + }) + var target fileAlreadyExistsError + assert.ErrorAs(t, err, &target) +} + +// Confirms an ErrorInfo carrying some other reason is not mistaken for a path +// collision, so the reason check above cannot swallow unrelated failures. +func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { + err := writeWithImportError(t, map[string]any{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "some other validation failure", + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "SOME_OTHER_REASON", + "domain": "workspace.databricks.com", + }, + }, + }) + var aerr *apierr.APIError + require.ErrorAs(t, err, &aerr) + assert.Equal(t, http.StatusBadRequest, aerr.StatusCode) +} + +// Confirms a workspace get-status payload unmarshals into wsfsFileInfo and that +// the fs.FileInfo methods derived from it report the expected values. func TestWorkspaceFilesClient_wsfsUnmarshal(t *testing.T) { payload := ` { @@ -163,6 +352,8 @@ func statWithError(t *testing.T, statusCode int, errorCode string) error { return err } +// Confirms a 403 from get-status surfaces as an APIError rather than being +// remapped, so callers can distinguish it from a missing file. func TestWorkspaceFilesClientStatForbidden(t *testing.T) { err := statWithError(t, 403, "PERMISSION_DENIED") var apiErr *apierr.APIError @@ -170,6 +361,8 @@ func TestWorkspaceFilesClientStatForbidden(t *testing.T) { assert.Equal(t, 403, apiErr.StatusCode) } +// Confirms a 500 from get-status surfaces as an APIError and is not treated as +// a missing file. func TestWorkspaceFilesClientStatInternalError(t *testing.T) { err := statWithError(t, 500, "INTERNAL_ERROR") var apiErr *apierr.APIError @@ -177,6 +370,8 @@ func TestWorkspaceFilesClientStatInternalError(t *testing.T) { assert.Equal(t, 500, apiErr.StatusCode) } +// Confirms RESOURCE_DOES_NOT_EXIST maps to fs.ErrNotExist so the filer plugs +// into the io/fs error conventions callers check against. func TestWorkspaceFilesClientStatNotFound(t *testing.T) { err := statWithError(t, 404, "RESOURCE_DOES_NOT_EXIST") assert.ErrorIs(t, err, fs.ErrNotExist) diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go index d3bb57662d9..fa7ff0baceb 100644 --- a/libs/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -10,7 +10,10 @@ import ( "golang.org/x/sync/errgroup" ) -// Maximum number of concurrent requests during sync. +// Maximum number of concurrent requests during sync. Chosen against the +// per-endpoint rate limits documented at +// https://docs.databricks.com/aws/en/resources/limits ("API rate limits" - +// Workspace API), which sync stays within for the endpoints it calls. const MaxRequestsInFlight = 20 // Delete the specified path. diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index f478f25c3a2..9daf7dee96f 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -882,6 +882,35 @@ func (s *FakeWorkspace) hasChildren(dirPath string) bool { return false } +// detectNotebookLanguage mirrors the real /workspace/import format=AUTO logic: +// a file whose extension is .py / .sql / .scala / .r and whose content starts +// with the language-appropriate "Databricks notebook source" line-comment is +// stored as a NOTEBOOK with the corresponding language; otherwise it's a FILE. +// +// The extension is lowercased because the real endpoint matches it +// case-insensitively, and ".R" is the conventional spelling for R sources. +func detectNotebookLanguage(extension string, body []byte) (workspace.Language, bool) { + switch strings.ToLower(extension) { + case ".py": + if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { + return workspace.LanguagePython, true + } + case ".sql": + if bytes.HasPrefix(body, []byte("-- Databricks notebook source")) { + return workspace.LanguageSql, true + } + case ".scala": + if bytes.HasPrefix(body, []byte("// Databricks notebook source")) { + return workspace.LanguageScala, true + } + case ".r": + if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { + return workspace.LanguageR, true + } + } + return "", false +} + func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, overwrite bool) Response { if !strings.HasPrefix(filePath, "/") { filePath = "/" + filePath @@ -904,18 +933,19 @@ func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, o } } - // Note: Files with .py, .scala, .r or .sql extension can - // be notebooks if they contain a magical "Databricks notebook source" - // header comment. We omit support non-python extensions for now for simplicity. + // Files with .py / .sql / .scala / .r extension can be notebooks if they + // carry the "Databricks notebook source" header comment in the language's + // line-comment syntax. Mirror the real workspace's auto-detection here so + // acceptance tests can assert the resulting object_type via /workspace/list. extension := filepath.Ext(filePath) - if extension == ".py" && strings.HasPrefix(string(body), "# Databricks notebook source") { + if lang, isNotebook := detectNotebookLanguage(extension, body); isNotebook { // Notebooks are stripped of their extension by the workspace import API. workspacePath = strings.TrimSuffix(filePath, extension) s.files[workspacePath] = FileEntry{ Info: workspace.ObjectInfo{ ObjectType: "NOTEBOOK", Path: workspacePath, - Language: "PYTHON", + Language: lang, ObjectId: nextID(), }, Data: body, diff --git a/libs/testserver/fault.go b/libs/testserver/fault.go index 75d3092bb70..43d46f3f28c 100644 --- a/libs/testserver/fault.go +++ b/libs/testserver/fault.go @@ -1,6 +1,7 @@ package testserver import ( + "bytes" "encoding/json" "strings" "sync" @@ -19,6 +20,12 @@ type FaultRule struct { AfterHandler bool offset int times int + + // bodyContains, if non-empty, additionally requires the request body to + // contain this substring for the rule to fire. /workspace/import routes + // every upload through the same method+path, so the only way to target a + // single file's upload is by matching its multipart "path" form field. + bodyContains string } // FaultRules holds the active fault injection rules for a test server. @@ -33,16 +40,16 @@ func NewFaultRules() *FaultRules { } // Set registers or replaces a fault rule for the given token and pattern. -func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int) { - fr.set(token, pattern, statusCode, body, offset, times, false) +func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { + fr.set(token, pattern, statusCode, body, offset, times, false, bodyContains) } // SetAfterHandler is like Set, but the handler runs first so its effect is kept. -func (fr *FaultRules) SetAfterHandler(token, pattern string, statusCode int, body string, offset, times int) { - fr.set(token, pattern, statusCode, body, offset, times, true) +func (fr *FaultRules) SetAfterHandler(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { + fr.set(token, pattern, statusCode, body, offset, times, true, bodyContains) } -func (fr *FaultRules) set(token, pattern string, statusCode int, body string, offset, times int, afterHandler bool) { +func (fr *FaultRules) set(token, pattern string, statusCode int, body string, offset, times int, afterHandler bool, bodyContains string) { fr.mu.Lock() defer fr.mu.Unlock() fr.rules[faultRuleKey{token: token, pattern: pattern}] = &FaultRule{ @@ -51,12 +58,14 @@ func (fr *FaultRules) set(token, pattern string, statusCode int, body string, of AfterHandler: afterHandler, offset: offset, times: times, + bodyContains: bodyContains, } } // Check returns a matching fault rule and advances its counters, or nil if no rule matches. // Pattern supports a trailing "*" wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*". -func (fr *FaultRules) Check(method, path, token string) *FaultRule { +// A rule with a non-empty bodyContains only matches when body contains that substring. +func (fr *FaultRules) Check(method, path, token string, body []byte) *FaultRule { requestPattern := method + " " + path fr.mu.Lock() @@ -76,6 +85,11 @@ func (fr *FaultRules) Check(method, path, token string) *FaultRule { if !matched { continue } + // A non-matching body must not consume the rule's offset/times budget, + // so this check precedes the counter updates below. + if rule.bodyContains != "" && !bytes.Contains(body, []byte(rule.bodyContains)) { + continue + } if rule.offset > 0 { rule.offset-- return nil @@ -105,14 +119,15 @@ func faultEndpointHandler(fr *FaultRules) HandlerFunc { Offset int `json:"offset"` Times int `json:"times"` AfterHandler bool `json:"after_handler"` + BodyContains string `json:"body_contains"` } if err := json.Unmarshal(req.Body, &body); err != nil { return Response{StatusCode: 400, Body: map[string]string{"error": err.Error()}} } if body.AfterHandler { - fr.SetAfterHandler(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) + fr.SetAfterHandler(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) } else { - fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) + fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) } return Response{StatusCode: 200} } diff --git a/libs/testserver/fault_test.go b/libs/testserver/fault_test.go index db41b74d3ba..ae27b954f93 100644 --- a/libs/testserver/fault_test.go +++ b/libs/testserver/fault_test.go @@ -14,18 +14,18 @@ import ( func TestFaultRulesNoMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 1) + fr.Set("tok", "GET /foo", 504, "body", 0, 1, "") - assert.Nil(t, fr.Check("POST", "/foo", "tok")) - assert.Nil(t, fr.Check("GET", "/bar", "tok")) - assert.Nil(t, fr.Check("GET", "/foo", "other")) + assert.Nil(t, fr.Check("POST", "/foo", "tok", nil)) + assert.Nil(t, fr.Check("GET", "/bar", "tok", nil)) + assert.Nil(t, fr.Check("GET", "/foo", "other", nil)) } func TestFaultRulesExactMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1) + fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1, "") - rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok") + rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok", nil) require.NotNil(t, rule) assert.Equal(t, 504, rule.StatusCode) assert.Equal(t, "body", rule.Body) @@ -33,31 +33,43 @@ func TestFaultRulesExactMatch(t *testing.T) { func TestFaultRulesWildcardMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2) + fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2, "") - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok")) - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) - assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) // exhausted + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok", nil)) + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) + assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) // exhausted } func TestFaultRulesOffset(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 2, 1) + fr.Set("tok", "GET /foo", 504, "body", 2, 1, "") - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 2→1 - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 1→0 - assert.NotNil(t, fr.Check("GET", "/foo", "tok")) // fires - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 2→1 + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 1→0 + assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) // fires + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted } func TestFaultRulesTimes(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 3) + fr.Set("tok", "GET /foo", 504, "body", 0, 3, "") for range 3 { - assert.NotNil(t, fr.Check("GET", "/foo", "tok")) + assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) } - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted +} + +func TestFaultRulesBodyContains(t *testing.T) { + fr := testserver.NewFaultRules() + fr.Set("tok", "POST /api/2.0/workspace/import", 403, "body", 0, 1, "state/resources.json") + + // Body without the substring must not match and must not consume the budget. + assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/deploy.lock"))) + // Body with the substring fires. + assert.NotNil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) + // Budget was only consumed by the matching request. + assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) } func TestServerFaultAfterHandlerKeepsTheHandlersEffect(t *testing.T) { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a8217e3bb20..9e060e288b4 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -29,6 +29,47 @@ var TestMetastore = catalog.MetastoreAssignment{ WorkspaceId: TestWorkspaceID, } +// ImportForm holds the fields of a multipart /workspace/import request. +type ImportForm struct { + Path string + Content []byte + Format string + Overwrite bool +} + +// ParseImportForm parses the multipart body of a /workspace/import request. +func ParseImportForm(req Request) (ImportForm, error) { + var form ImportForm + _, params, err := mime.ParseMediaType(req.Headers.Get("Content-Type")) + if err != nil { + return form, err + } + mr := multipart.NewReader(bytes.NewReader(req.Body), params["boundary"]) + for { + part, err := mr.NextPart() + if err == io.EOF { + return form, nil + } + if err != nil { + return form, err + } + data, err := io.ReadAll(part) + if err != nil { + return form, err + } + switch part.FormName() { + case "path": + form.Path = string(data) + case "content": + form.Content = data + case "format": + form.Format = string(data) + case "overwrite": + form.Overwrite = string(data) == "true" + } + } +} + func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/policies/clusters/list", func(req Request) any { return req.Workspace.ClusterPoliciesList(req) @@ -153,6 +194,54 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("POST", "/api/2.0/workspace/import", func(req Request) any { + // /workspace/import accepts both a JSON body (matching workspace.Import) and a + // multipart form body. The multipart variant is what databricks-sdk-go's + // Workspace.Upload uses; the JSON variant is kept for back-compat with anything + // that still hits Workspace.Import directly. + mediaType, _, _ := mime.ParseMediaType(req.Headers.Get("Content-Type")) + if strings.HasPrefix(mediaType, "multipart/") { + form, err := ParseImportForm(req) + if err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + // Require format explicitly: the real endpoint defaults to SOURCE, which + // would import every upload as a notebook, so a caller that drops the + // field is a bug rather than a request for the fake's usual behavior. + if form.Format != string(workspace.ImportFormatAuto) { + return Response{ + Body: fmt.Sprintf("internal error: The test server only supports auto format, got %q.", form.Format), + StatusCode: http.StatusInternalServerError, + } + } + + // Translate any 409 from the shared fake into the 400 + errorCode + // RESOURCE_ALREADY_EXISTS shape returned by the real /workspace/import + // endpoint, including the AIP-193 ErrorInfo detail it attaches to path + // collisions (universe PR #2019174, WP-6031). + resp := req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + if resp.StatusCode == http.StatusConflict { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]any{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Path (%s) already exists.", form.Path), + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "RESOURCE_ALREADY_EXISTS", + "domain": "workspace.databricks.com", + }, + }, + }, + } + } + return resp + } + var request workspace.Import err := json.Unmarshal(req.Body, &request) if err != nil { diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 1f56717c50a..5170b318818 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -372,7 +372,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, handler HandlerFu var resp EncodedResponse - fault := s.faults.Check(r.Method, r.URL.Path, token) + fault := s.faults.Check(r.Method, r.URL.Path, token, request.Body) if fault == nil || fault.AfterHandler { if bytes.Contains(request.Body, []byte("INJECT_ERROR")) { diff --git a/libs/testserver/workspace_test.go b/libs/testserver/workspace_test.go index cde1c9741c6..401758c6648 100644 --- a/libs/testserver/workspace_test.go +++ b/libs/testserver/workspace_test.go @@ -88,6 +88,24 @@ func TestWorkspaceDeleteNonRecursiveRequiresEmptyDirectory(t *testing.T) { assert.Equal(t, 404, getStatus(t, server.URL, "/a/b")) } +// format=AUTO matches the extension case-insensitively, so ".R" — the +// conventional spelling for R sources — is detected as a notebook just like +// ".r". Notebooks are stored with the extension stripped. +func TestWorkspaceImportDetectsNotebookExtensionCaseInsensitively(t *testing.T) { + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + + mkdirs(t, server.URL, "/test-dir") + + for _, name := range []string{"lower.r", "upper.R"} { + require.Equal(t, 200, importFile(t, server.URL, "/test-dir/"+name, "# Databricks notebook source\n"), name) + } + + // Both land at the extension-stripped path, which only happens for notebooks. + assert.Equal(t, 200, getStatus(t, server.URL, "/test-dir/lower")) + assert.Equal(t, 200, getStatus(t, server.URL, "/test-dir/upper")) +} + // mkdirs creates all intermediate directories, matching "mkdir -p". func TestWorkspaceMkdirsRecursive(t *testing.T) { server := testserver.New(t)