diff --git a/.nextchanges/bundles/revert-workspace-import.md b/.nextchanges/bundles/revert-workspace-import.md new file mode 100644 index 00000000000..01784b58052 --- /dev/null +++ b/.nextchanges/bundles/revert-workspace-import.md @@ -0,0 +1 @@ +* Revert bundle file uploads to the legacy `/workspace-files/import-file` endpoint, rolling back [#6149](https://github.com/databricks/cli/pull/6149) and [#7034](https://github.com/databricks/cli/pull/7034) released in [v1.21.0](https://github.com/databricks/cli/releases/tag/v1.21.0). This fixes notebooks being uploaded as plain files and deployments failing (for example SQL notebooks in [#7216](https://github.com/databricks/cli/issues/7216)). A remote file or notebook whose type differs from the local one is now deleted and replaced instead of blocking the deploy. ([#7218](https://github.com/databricks/cli/pull/7218)) diff --git a/acceptance/air/run-submit-deps/output.txt b/acceptance/air/run-submit-deps/output.txt index 44f1ae35d86..3346fcd06d7 100644 --- a/acceptance/air/run-submit-deps/output.txt +++ b/acceptance/air/run-submit-deps/output.txt @@ -10,9 +10,9 @@ Stream logs after submission using: databricks air logs 555 === only config + command are uploaded; no requirements.yaml ->>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh"}}} +>>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} === inline deps, artifact location, and capacity id reach the submit payload >>> print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/air/run-submit-deps/script b/acceptance/air/run-submit-deps/script index 0e8951cae0f..422980c99b3 100644 --- a/acceptance/air/run-submit-deps/script +++ b/acceptance/air/run-submit-deps/script @@ -2,10 +2,7 @@ title "submit with inline dependencies" trace $CLI air run -f run.yaml title "only config + command are uploaded; no requirements.yaml" -# Uploads go to POST /workspace/import, which carries the target filename in the -# multipart body rather than the URL, so each uploaded path shows up under -# body.multipart_form.path instead of in the request path. -trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep +trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep title "inline deps, artifact location, and capacity id reach the submit payload" trace print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/air/run-submit-overrides/output.txt b/acceptance/air/run-submit-overrides/output.txt index 5ab207a7e3b..0be49262d1b 100644 --- a/acceptance/air/run-submit-overrides/output.txt +++ b/acceptance/air/run-submit-overrides/output.txt @@ -13,7 +13,7 @@ Stream logs after submission using: databricks air logs 555 === the uploaded config preserves source order and contains overrides ->>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 2\nparameters:\n model:\n hidden_size: 2048\n optimizer:\n name: adamw\n learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh"}}} +>>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml", "q": {"overwrite": "true"}, "raw_body": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 2\nparameters:\n model:\n hidden_size: 2048\n optimizer:\n name: adamw\n learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} diff --git a/acceptance/air/run-submit-overrides/script b/acceptance/air/run-submit-overrides/script index 59234b5e874..263c1920e6c 100644 --- a/acceptance/air/run-submit-overrides/script +++ b/acceptance/air/run-submit-overrides/script @@ -5,4 +5,4 @@ trace $CLI air run -f run.yaml \ --override parameters.optimizer.learning_rate=0.001 title "the uploaded config preserves source order and contains overrides" -trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique +trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique diff --git a/acceptance/bundle/apps/app_yaml/out.app.yml.txt b/acceptance/bundle/apps/app_yaml/out.app.yml.txt index 51431f28439..29cb7da70fa 100644 --- a/acceptance/bundle/apps/app_yaml/out.app.yml.txt +++ b/acceptance/bundle/apps/app_yaml/out.app.yml.txt @@ -1,12 +1,8 @@ { "method": "POST", - "path": "/api/2.0/workspace/import", - "body": { - "multipart_form": { - "content": "command:\n - python\n - app.py\n", - "format": "RAW", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml" - } - } + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml", + "q": { + "overwrite": "true" + }, + "raw_body": "command:\n - python\n - app.py\n" } diff --git a/acceptance/bundle/apps/app_yaml/output.txt b/acceptance/bundle/apps/app_yaml/output.txt index 50baf9b4559..3554b68f44e 100644 --- a/acceptance/bundle/apps/app_yaml/output.txt +++ b/acceptance/bundle/apps/app_yaml/output.txt @@ -19,7 +19,7 @@ Created apps.myapp Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> jq select(.body.multipart_form.path | strings | test("app.yml")) [OUTPUT_DIR]/out.requests.txt +>>> jq select(.path | test("app.yml")) [OUTPUT_DIR]/out.requests.txt >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/apps/app_yaml/script b/acceptance/bundle/apps/app_yaml/script index 3267d66c598..3db6c9e8969 100644 --- a/acceptance/bundle/apps/app_yaml/script +++ b/acceptance/bundle/apps/app_yaml/script @@ -1,7 +1,7 @@ trace $CLI bundle validate trace $CLI bundle plan trace $CLI bundle deploy -trace jq 'select(.body.multipart_form.path | strings | test("app.yml"))' "$OUT_REQUESTS" | sed 's/\\r//g' > out.app.yml.txt +trace jq 'select(.path | test("app.yml"))' "$OUT_REQUESTS" | sed 's/\\r//g' > out.app.yml.txt #trace print_requests.py //apps # currently fails due to TF inserting description="" rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt index 1a3ffc22b37..70bbb6eb108 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt @@ -16,5 +16,5 @@ } === Expecting the code_source tarball to be uploaded ->>> print_workspace_imports -/Workspace/foo/bar/artifacts/.internal/code.tgz +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz" diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/script b/acceptance/bundle/artifacts/ai_runtime_code_source/script index 32460827ec5..396f907033e 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/script +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/script @@ -6,6 +6,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' "$OUT_REQUESTS" title "Expecting the code_source tarball to be uploaded" -trace print_workspace_imports | grep code.tgz | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep code.tgz | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt index 9997818f806..2a10679b430 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt @@ -48,9 +48,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl -/Workspace/foo/bar/artifacts/.internal/source.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script index b3c226bc752..70c9dffac83 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script @@ -9,7 +9,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt index ca39c787992..2419cc5afbc 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt @@ -4,9 +4,9 @@ Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Expecting wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl -/Workspace/foo/bar/artifacts/.internal/source.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script index 0d665cfbc5f..7ac87760449 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script @@ -6,7 +6,7 @@ $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) title "Expecting wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt index e500ebd0342..ee40963cbaa 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt @@ -38,15 +38,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 4 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl -/Workspace/foo/bar/artifacts/.internal/source1.whl -/Workspace/foo/bar/artifacts/.internal/source2.whl -/Workspace/foo/bar/artifacts/.internal/source3.whl -/Workspace/foo/bar/artifacts/.internal/source4.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source1.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source2.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source3.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl" === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/script b/acceptance/bundle/artifacts/upload_multiple_libraries/script index 89d0ba55291..f65f8154f2a 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/script +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/script @@ -12,7 +12,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 4 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 30134175168..34b272c0f08 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -29,19 +29,19 @@ dist/my_test_code-0.1.0-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME +>>> jq .path [OUTPUT_DIR]/out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" >>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0 @@ -62,14 +62,14 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME +>>> jq .path [OUTPUT_DIR]/out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" === Restore config to target old wheel >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl @@ -92,12 +92,12 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME +>>> jq .path [OUTPUT_DIR]/out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" diff --git a/acceptance/bundle/artifacts/whl_change_version/script b/acceptance/bundle/artifacts/whl_change_version/script index 4b97cb54259..49d50f0b0b5 100644 --- a/acceptance/bundle/artifacts/whl_change_version/script +++ b/acceptance/bundle/artifacts/whl_change_version/script @@ -6,7 +6,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | sort +trace jq .path "$OUT_REQUESTS" | grep import | sort rm "$OUT_REQUESTS" @@ -20,7 +20,7 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | sort +trace jq .path "$OUT_REQUESTS" | grep import | sort rm "$OUT_REQUESTS" @@ -34,6 +34,6 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | sort +trace jq .path "$OUT_REQUESTS" | grep import | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index 04b3a0f58fa..b4974a6f5ca 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -25,6 +25,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting no wheels to be uploaded ->>> print_workspace_imports +>>> errcode sh -c jq .path < "$OUT_REQUESTS" | grep import | grep whl Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script index bd7b88b5806..944ecf709f9 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/script +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting no wheels to be uploaded" -trace print_workspace_imports | errcode grep whl +trace errcode sh -c 'jq .path < "$OUT_REQUESTS" | grep import | grep whl' rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index 3461014160d..51be24e7e43 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -48,11 +48,11 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 patched wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl" === Updating the local wheel and deploying again @@ -74,7 +74,7 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice) ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index d031d02715c..b73202a65ec 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -22,7 +22,7 @@ title "Expecting 2 patched wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 2 patched wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" @@ -40,4 +40,4 @@ title "Expecting 2 patched wheels in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 14358c0ab32..55aa3ff229a 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -21,9 +21,9 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index a1227cd125d..f55d7da467a 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -8,7 +8,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index c610058f32f..d589ebc5d74 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -21,6 +21,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index 8124726f4ff..a42ce2844db 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -8,6 +8,6 @@ title "Expecting 1 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 1a93fb8c9bd..04683b58ffc 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -39,6 +39,6 @@ package/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl -/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 4d5fd31d6e7..410957d8233 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -7,6 +7,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index 6a49d1e1889..4172ddf8f51 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -11,6 +11,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index 41bae0e4237..72d0ce55360 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index 96edc8e5c85..8bf5bef423d 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -25,8 +25,8 @@ my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index 8c06cd32454..df8ad98b3c4 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -8,6 +8,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm -fr "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt index b80a0145ad1..a61388c76f9 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt +++ b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt @@ -3,9 +3,9 @@ dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" === No calls to delete internal folder expected >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/script b/acceptance/bundle/artifacts/whl_no_cleanup/script index 6a4a10bc75e..f3ef86ecf70 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/script +++ b/acceptance/bundle/artifacts/whl_no_cleanup/script @@ -5,7 +5,7 @@ cat LOG.deploy | contains.py "my_test_code-0.0.1-py3-none-any.whl" > /dev/null trace find.py --expect 1 whl title "Expecting 1 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index c18e0b2d6da..e43ce01f38c 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -36,8 +36,8 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index 8034c3171cf..2b9240de98a 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -10,6 +10,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt index a034c80a088..9611c9541f2 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt @@ -32,6 +32,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/script b/acceptance/bundle/artifacts/whl_prebuilt_outside/script index 23a2e0fc1ef..ecfd7bea799 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/script @@ -11,6 +11,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt index b26bae880f5..324b0a1b18a 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt @@ -56,6 +56,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script index f972cf2504e..b53c1dc55b2 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script @@ -14,6 +14,6 @@ trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_RE trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index f6ba380bc11..eb0d23431f3 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -41,6 +41,6 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index 09f9ed82c47..d7657fa4b4f 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in environments section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt index e99a248c94b..39e7ead5144 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt @@ -41,5 +41,5 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded under its bare filename (no extras) ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index 157472b5f55..dadef34e181 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -8,6 +8,6 @@ title "Expecting the environments dependency to keep its [train] extras suffix" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deploy/files/upload-504/output.txt b/acceptance/bundle/deploy/files/upload-504/output.txt index 52ff524b127..7e6db166c5c 100644 --- a/acceptance/bundle/deploy/files/upload-504/output.txt +++ b/acceptance/bundle/deploy/files/upload-504/output.txt @@ -1,6 +1,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace/import: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml: Fault injected by test. +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml diff --git a/acceptance/bundle/deploy/files/upload-504/script b/acceptance/bundle/deploy/files/upload-504/script index 78740b9becc..a6bfa10a541 100644 --- a/acceptance/bundle/deploy/files/upload-504/script +++ b/acceptance/bundle/deploy/files/upload-504/script @@ -1,8 +1,8 @@ # Let the two deploy lock requests through, then inject a single 504 on the next import request, which is a file upload. # Uploads overwrite, so they are retried and the deploy succeeds. -fault.py "POST /api/2.0/workspace/import" 504 2 1 +fault.py "POST /api/2.0/workspace-files/import-file/**" 504 2 1 $CLI bundle deploy # databricks.yml is the only file. Two requests for it should appear: the initial 504 and the successful retry. -print_requests.py //api/2.0/workspace/import | jq -r .body.multipart_form.path | grep '/files/' +print_requests.py //api/2.0/workspace-files/import-file | jq -r .path | grep '/files/' diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt index 4610f302b33..a97e8642981 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt @@ -3,7 +3,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/script b/acceptance/bundle/deploy/partial-summary-on-push-fail/script index d2fef85a1d7..bc36c59cacc 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/script +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/script @@ -9,7 +9,7 @@ trap cleanup EXIT # Inject a 403 on the resource-state upload (PushResourcesState). The files are # uploaded by then, so "Files:" still reports them; "Resources:" must NOT print, # because this fails before planning and there are no resource counts to report. -fault.py --body-contains "state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 # -q: the state-push 403 races resource apply, so whether "Created jobs.my_job" is # emitted before the failure is platform-dependent. This test asserts Files is reported diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml b/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml index 64cb4896dce..01dc2713a99 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml @@ -1,5 +1,5 @@ # The resource summary must not print when pushing resource state fails, so the -# fault is injected on the state upload (POST /workspace/import). The path is fixed, so +# fault is injected on the state upload (POST import-file). The path is fixed, so # the bundle name must be stable (test-bundle, not $UNIQUE_NAME). # # The injected path is the direct-engine state file (state/resources.json), so diff --git a/acceptance/bundle/deploy/readplan/whl-upload/output.txt b/acceptance/bundle/deploy/readplan/whl-upload/output.txt index 6b21d59c6b0..48212c57e37 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-upload/output.txt @@ -7,8 +7,14 @@ Created jobs.my_job Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +>>> print_requests.py //artifacts/.internal --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "q": { + "overwrite": "true" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -26,8 +32,14 @@ Created jobs.my_job Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +>>> print_requests.py //artifacts/.internal --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "q": { + "overwrite": "true" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/readplan/whl-upload/script b/acceptance/bundle/deploy/readplan/whl-upload/script index 15b0c9aa6c3..c06707c5dd1 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/script +++ b/acceptance/bundle/deploy/readplan/whl-upload/script @@ -5,8 +5,9 @@ title "bundle deploy uploads the wheel" trace $CLI bundle deploy -# The wheel was uploaded to its .../artifacts/.internal/ workspace path. -trace print_workspace_imports | grep artifacts/.internal +# An import-file request uploaded the wheel to its .../artifacts/.internal/ workspace path. +# raw_body is the wheel's binary payload; drop it to keep the golden readable. +trace print_requests.py //artifacts/.internal --del-field raw_body trace $CLI bundle destroy --auto-approve # Discard destroy's requests so the saved-plan deploy below starts from a clean recording. @@ -18,9 +19,9 @@ title "bundle deploy --plan also uploads the wheel" $CLI bundle plan -o json > tmp.plan.json $CLI bundle plan -o json | nostamp > out.plan.json trace $CLI bundle deploy --plan tmp.plan.json -# The wheel is uploaded to the same .../artifacts/.internal/ path the job task points at, -# matching the plain `bundle deploy` above. -trace print_workspace_imports | grep artifacts/.internal +# The same import-file request uploads the wheel to the .../artifacts/.internal/ path the +# job task points at, matching the plain `bundle deploy` above. +trace print_requests.py //artifacts/.internal --del-field raw_body trace $CLI bundle destroy --auto-approve rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt index e837de7f5f0..cb70b043077 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//workspace/ +>>> print_requests.py ^//import-file/ ^//workspace/ >>> [CLI] bundle summary -o json @@ -34,4 +34,4 @@ Recreated pipelines.foo Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged ->>> print_requests.py --nostamp ^//workspace/ ^//telemetry-ext +>>> print_requests.py --nostamp ^//import-file/ ^//workspace/ ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/script b/acceptance/bundle/deployment/bind/pipelines/recreate/script index a8cddf2dc5f..a9aa3403a35 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/script +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/script @@ -6,11 +6,11 @@ add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f "$OUT_REQUESTS" trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.txt -print_requests.py '^//workspace/' +print_requests.py '^//import-file/' '^//workspace/' rm -f "$OUT_REQUESTS" trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt -trace print_requests.py '^//workspace/' +trace print_requests.py '^//import-file/' '^//workspace/' trace $CLI bundle summary -o json | jq .resources > out.summary.json trace $CLI bundle plan @@ -19,4 +19,4 @@ trace musterr $CLI bundle deploy rm -f "$OUT_REQUESTS" trace $CLI bundle deploy --auto-approve -trace print_requests.py --nostamp '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py --nostamp '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/deployment/bind/pipelines/update/output.txt b/acceptance/bundle/deployment/bind/pipelines/update/output.txt index 29e3c78f22b..e384e521790 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/update/output.txt @@ -19,4 +19,4 @@ Updated pipelines.foo Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py --nostamp ^//workspace/import ^//workspace/delete ^//telemetry-ext +>>> print_requests.py --nostamp ^//import-file/ ^//workspace/delete ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/update/script b/acceptance/bundle/deployment/bind/pipelines/update/script index 68d98a43e6b..6bafa319765 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/script +++ b/acceptance/bundle/deployment/bind/pipelines/update/script @@ -3,15 +3,15 @@ add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f "$OUT_REQUESTS" trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.txt -print_requests.py '^//workspace/import' '^//workspace/delete' +print_requests.py '^//import-file/' '^//workspace/delete' rm -f "$OUT_REQUESTS" trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt -print_requests.py '^//workspace/import' '^//workspace/delete' +print_requests.py '^//import-file/' '^//workspace/delete' trace $CLI bundle summary -o json > out.summary.json trace $CLI bundle plan rm -f "$OUT_REQUESTS" trace $CLI bundle deploy --auto-approve -trace print_requests.py --nostamp '^//workspace/import' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py --nostamp '^//import-file/' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/destroy/force-lock-node-limit/script b/acceptance/bundle/destroy/force-lock-node-limit/script index 616b626f2ef..ccb6ae3d143 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/script +++ b/acceptance/bundle/destroy/force-lock-node-limit/script @@ -7,9 +7,9 @@ title "Deploy (creates the job and the deployment)" trace $CLI bundle deploy title "destroy without --force-lock (lock write hits the node limit: aborts)" -fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py "POST /api/2.0/workspace-files/import-file/**" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace errcode $CLI bundle destroy --auto-approve title "destroy with --force-lock (tolerates the node limit: proceeds and completes)" -fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py "POST /api/2.0/workspace-files/import-file/**" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace $CLI bundle destroy --force-lock --auto-approve diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt index 72411ab8dbb..43e860f5d8e 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt +++ b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt @@ -54,12 +54,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> cat [OUTPUT_DIR]/out.requests.txt { "method": "POST", - "path": "/api/2.0/workspace/import", - "body": { - "multipart_form": { - "format": "RAW", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl" - } + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl", + "q": { + "overwrite": "true" } } diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/script b/acceptance/bundle/libraries/outside_of_bundle_root/script index 7c4616890cc..df54672eee7 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/script +++ b/acceptance/bundle/libraries/outside_of_bundle_root/script @@ -11,6 +11,6 @@ title "Check that the job libraries are uploaded and the path is correct in the trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.tasks[0].libraries' trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.environments[0].spec.dependencies' trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.0/pipelines" and .method == "POST")' | jq '.body.environment.dependencies' -trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // "" | test("/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl")))' | jq 'del(.body.multipart_form.content)' +trace cat "$OUT_REQUESTS" | jq 'select(.path | test("/api/2.0/workspace-files/import-file/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl"))' rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/output.txt b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt index 2569d95ccfb..54447ea8ecc 100644 --- a/acceptance/bundle/migrate/auto/empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt @@ -20,16 +20,16 @@ direct_migrated_via_config true === The remote had no terraform state, so nothing terraform-related is backed up remotely ->>> print_requests.py //workspace/import //workspace/delete --sort +>>> print_requests.py //workspace-files/import-file //workspace/delete --sort +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" "POST /api/2.0/workspace/delete" "POST /api/2.0/workspace/delete" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" === debug states confirms no terraform.tfstate anywhere; only the local .backup remains diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/script b/acceptance/bundle/migrate/auto/empty-tfstate/script index c0df01e54ec..225092954a3 100644 --- a/acceptance/bundle/migrate/auto/empty-tfstate/script +++ b/acceptance/bundle/migrate/auto/empty-tfstate/script @@ -19,7 +19,7 @@ title "Migration is recorded in telemetry as via-config\n" trace print_migration_telemetry title "The remote had no terraform state, so nothing terraform-related is backed up remotely\n" -trace print_requests.py //workspace/import //workspace/delete --sort | jq '.method + " " + (.body.multipart_form.path // .path)' | contains.py '!terraform.tfstate' +trace print_requests.py //workspace-files/import-file //workspace/delete --sort | jq '.method + " " + .path' | contains.py '!terraform.tfstate' title "debug states confirms no terraform.tfstate anywhere; only the local .backup remains\n" trace $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/migrate/auto/push-failure/output.txt b/acceptance/bundle/migrate/auto/push-failure/output.txt index bd638d6e829..6e9a1c61b78 100644 --- a/acceptance/bundle/migrate/auto/push-failure/output.txt +++ b/acceptance/bundle/migrate/auto/push-failure/output.txt @@ -8,7 +8,7 @@ Notice: migrating your bundle to direct deployment engine (https://github.com/da Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/migrate/auto/push-failure/script b/acceptance/bundle/migrate/auto/push-failure/script index 4231617175e..da3ffaad3d9 100644 --- a/acceptance/bundle/migrate/auto/push-failure/script +++ b/acceptance/bundle/migrate/auto/push-failure/script @@ -10,7 +10,7 @@ title "Opt in via env var; inject 403 on the resources.json push so the deploy f # push fails the deploy - unlike a declined deploy, the migration is not discarded, so the # local direct state remains and a retry finishes the push. (A push failure is a hard error on # any direct deploy, not just a migrating one.) -fault.py --body-contains "Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 503055861e5..23808c5a896 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -18,7 +18,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py --nostamp ^//workspace/import ^//api/2.0/bundle +>>> print_requests.py --nostamp ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -34,7 +34,7 @@ Created pipelines.mypipeline Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --nostamp ^//workspace/import ^//api/2.0/bundle +>>> print_requests.py --nostamp ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -102,7 +102,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted ->>> print_requests.py --nostamp --sort ^//workspace/import ^//api/2.0/bundle +>>> print_requests.py --nostamp --sort ^//import-file/ ^//api/2.0/bundle { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index ccd83d2f06d..6d1e4fd5839 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py --nostamp '^//workspace/import' '^//api/2.0/bundle' +trace print_requests.py --nostamp '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle deploy -trace print_requests.py --nostamp '^//workspace/import' '^//api/2.0/bundle' +trace print_requests.py --nostamp '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle destroy --auto-approve -trace print_requests.py --nostamp --sort '^//workspace/import' '^//api/2.0/bundle' +trace print_requests.py --nostamp --sort '^//import-file/' '^//api/2.0/bundle' diff --git a/acceptance/bundle/resources/jobs/request-summary/output.txt b/acceptance/bundle/resources/jobs/request-summary/output.txt index 0f0514c2e5c..a05f10b342f 100644 --- a/acceptance/bundle/resources/jobs/request-summary/output.txt +++ b/acceptance/bundle/resources/jobs/request-summary/output.txt @@ -28,14 +28,14 @@ GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/reque GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/files/databricks.yml overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock overwrite=false +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock overwrite=false +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deployment.json overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/metadata.json overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/resources.json overwrite=true POST /api/2.0/workspace/delete POST /api/2.0/workspace/delete -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import POST /api/2.0/workspace/mkdirs POST /api/2.0/workspace/mkdirs POST /api/2.0/workspace/mkdirs @@ -57,12 +57,12 @@ GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/reque GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.2/jobs/get job_id=[FOO_ID] +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock overwrite=false +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deployment.json overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/metadata.json overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/resources.json overwrite=true POST /api/2.0/workspace/delete POST /api/2.0/workspace/delete -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import POST /api/2.0/workspace/mkdirs >>> [CLI] bundle destroy --auto-approve @@ -81,7 +81,7 @@ GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/reque GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.2/jobs/get job_id=[FOO_ID] +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock overwrite=false POST /api/2.0/workspace/delete POST /api/2.0/workspace/delete -POST /api/2.0/workspace/import POST /api/2.2/jobs/delete diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 9795d322b3e..b53609dd402 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 70fe4992c16..5ea603ee6fe 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -31,5 +31,5 @@ nostamp < .databricks/tmp.plan.json > out.plan.json rm "$OUT_REQUESTS" trace errcode $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json trace errcode $CLI bundle plan &> out.plan_after_deploy.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt index a23a2445c8b..e82dc779ba2 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -36,7 +36,7 @@ Updated quality_monitors.monitor1 Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index a4104e4d387..fbd62ade8df 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -27,6 +27,6 @@ trace $CLI bundle plan -o json | nostamp > out.plan.json rm "$OUT_REQUESTS" trace $CLI bundle deploy # dashboard_id is output only field that terraform adds -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt index beba3487430..77656aa1913 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index a17d8b114d4..ca49568b38a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -31,7 +31,7 @@ nostamp < .databricks/tmp.plan.json > out.plan.json rm "$OUT_REQUESTS" trace errcode $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json trace errcode $CLI bundle plan &> out.plan_after_deploy.txt trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.json diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt index 5d1b3497525..fbe444e1eb7 100644 --- a/acceptance/bundle/resources/quality_monitors/create/output.txt +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -16,7 +16,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle plan -o json ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 1a3a60b1381..b45b8226f4e 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -21,7 +21,7 @@ trace $CLI bundle plan -o json | nostamp > out.plan_create.json rm "$OUT_REQUESTS" trace $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json # store state to ensure we have table_name there print_state.py | grep name > out.state.txt diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt index 1943374afeb..e21f54fcf1f 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext { "method": "POST", "path": "/api/2.0/secrets/scopes/delete", diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/script b/acceptance/bundle/resources/secret_scopes/delete_scope/script index 8d04d13a33f..6cd30fffdae 100755 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/script +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/script @@ -15,4 +15,4 @@ trace $CLI bundle plan &> out.plan.txt rm "$OUT_REQUESTS" trace $CLI bundle deploy -qq -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt diff --git a/acceptance/bundle/sync-upload-edge-cases/databricks.yml b/acceptance/bundle/sync-upload-edge-cases/databricks.yml deleted file mode 100644 index 2261ff177c7..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/databricks.yml +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: upload-edge-cases diff --git a/acceptance/bundle/sync-upload-edge-cases/ipyNb.ipynb b/acceptance/bundle/sync-upload-edge-cases/ipyNb.ipynb deleted file mode 100644 index d2d581029fb..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/ipyNb.ipynb +++ /dev/null @@ -1,35 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "05030541-8399-4226-a499-f11b4910888c", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Hello world!" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "py_ipynb", - "notebookOrigID": 4221142147066610, - "widgets": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/sync-upload-edge-cases/output.txt b/acceptance/bundle/sync-upload-edge-cases/output.txt deleted file mode 100644 index 9cda650785a..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/output.txt +++ /dev/null @@ -1,194 +0,0 @@ - ->>> [CLI] bundle sync --output text -Action: PUT: .gitignore, archive.zip, dashboard.lvdash.json, databricks.yml, empty.txt, héllo.txt, ipyNb.ipynb, job.dbjob.json, large.bin, plain-script.py, pyNb.py, query.lwquery.ipynb, scalaNb.scala, space.geniespace.json, sqlNb.sql, with spaces.txt -Initial Sync Complete -Uploaded .gitignore -Uploaded archive.zip -Uploaded dashboard.lvdash.json -Uploaded databricks.yml -Uploaded empty.txt -Uploaded héllo.txt -Uploaded ipyNb.ipynb -Uploaded job.dbjob.json -Uploaded large.bin -Uploaded plain-script.py -Uploaded pyNb.py -Uploaded query.lwquery.ipynb -Uploaded scalaNb.scala -Uploaded space.geniespace.json -Uploaded sqlNb.sql -Uploaded with spaces.txt - -=== workspace state after sync — assert object_type and language for each upload ->>> MSYS_NO_PATHCONV=1 [CLI] workspace list -o json /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files -{ - "name": ".gitignore", - "object_type": "FILE", - "language": null -} -{ - "name": "archive.zip", - "object_type": "FILE", - "language": null -} -{ - "name": "dashboard.lvdash.json", - "object_type": "FILE", - "language": null -} -{ - "name": "databricks.yml", - "object_type": "FILE", - "language": null -} -{ - "name": "empty.txt", - "object_type": "FILE", - "language": null -} -{ - "name": "héllo.txt", - "object_type": "FILE", - "language": null -} -{ - "name": "ipyNb", - "object_type": "NOTEBOOK", - "language": "PYTHON" -} -{ - "name": "job.dbjob.json", - "object_type": "FILE", - "language": null -} -{ - "name": "large.bin", - "object_type": "FILE", - "language": null -} -{ - "name": "plain-script.py", - "object_type": "FILE", - "language": null -} -{ - "name": "pyNb", - "object_type": "NOTEBOOK", - "language": "PYTHON" -} -{ - "name": "query.lwquery", - "object_type": "NOTEBOOK", - "language": "PYTHON" -} -{ - "name": "scalaNb", - "object_type": "NOTEBOOK", - "language": "SCALA" -} -{ - "name": "space.geniespace.json", - "object_type": "FILE", - "language": null -} -{ - "name": "sqlNb", - "object_type": "NOTEBOOK", - "language": "SQL" -} -{ - "name": "with spaces.txt", - "object_type": "FILE", - "language": null -} - -=== the .zip is uploaded with the legacy import-file endpoint ->>> print_requests.py //workspace-files/import-file --del-field raw_body --keep -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/archive.zip", - "q": { - "overwrite": "true" - } -} - -=== uploaded paths from each multipart POST /workspace/import (sorted) ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/.gitignore -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/dashboard.lvdash.json -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/empty.txt -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/héllo.txt -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/ipyNb -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/job.dbjob.json -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/large.bin -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/plain-script.py -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/pyNb -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/query.lwquery -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/scalaNb -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/space.geniespace.json -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/sqlNb -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/with spaces.txt - -=== format per upload: SOURCE/JUPYTER for notebooks, AUTO for dashboards, RAW for everything else ->>> jq -rs - map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/files/"))) | - map({name: (.body.multipart_form.path | sub(".*/"; "")), format: .body.multipart_form.format}) | - sort_by(.name) | .[] | "\(.format) \(.name)" - [OUTPUT_DIR]/out.requests.txt -RAW .gitignore -AUTO dashboard.lvdash.json -RAW databricks.yml -RAW empty.txt -RAW héllo.txt -JUPYTER ipyNb -RAW job.dbjob.json -RAW large.bin -RAW plain-script.py -SOURCE pyNb -JUPYTER query.lwquery -SOURCE scalaNb -RAW space.geniespace.json -SOURCE sqlNb -RAW with spaces.txt - -=== recorded content for the small text inputs ->>> jq -rs - map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/(pyNb|sqlNb|scalaNb|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | - map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | - sort_by(.path) | .[] - [OUTPUT_DIR]/out.requests.txt -{ - "path": "dashboard.lvdash.json", - "content": "{\"datasets\":[],\"pages\":[]}\n" -} -{ - "path": "empty.txt", - "content": "" -} -{ - "path": "héllo.txt", - "content": "hello, naïve world\n" -} -{ - "path": "plain-script.py", - "content": "print(\"plain script, no header\")\n" -} -{ - "path": "pyNb", - "content": "# Databricks notebook source\nprint(\"hello\")\n" -} -{ - "path": "scalaNb", - "content": "// Databricks notebook source\nprintln(\"hello\")\n" -} -{ - "path": "sqlNb", - "content": "-- Databricks notebook source\nSELECT 1\n" -} -{ - "path": "with spaces.txt", - "content": "hello, world\n" -} diff --git a/acceptance/bundle/sync-upload-edge-cases/query.lwquery.ipynb b/acceptance/bundle/sync-upload-edge-cases/query.lwquery.ipynb deleted file mode 100644 index d2d581029fb..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/query.lwquery.ipynb +++ /dev/null @@ -1,35 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "05030541-8399-4226-a499-f11b4910888c", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Hello world!" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "py_ipynb", - "notebookOrigID": 4221142147066610, - "widgets": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script deleted file mode 100644 index 77c6708d27f..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/script +++ /dev/null @@ -1,112 +0,0 @@ -# Confidence checks that /workspace/import is a drop-in for the legacy import-file: -# verify the CLI's multipart upload pipeline handles the cases that differ between the -# two endpoints — large content (>10 MB), empty content, varied extensions/object types, -# and non-ASCII / spaced filenames in the multipart `path` field. - -# Exclude framework-generated and script-internal files from the sync. We only want the -# inputs we explicitly generate below to show up in the recorded uploads. -cat > .gitignore <<'EOF' -script -output.txt -out.requests.txt -repls.json -test.toml -.databricks -EOF - -# 12 MiB binary file, over the 10 MB cap on the JSON-body variant's base64 `content` -# field. The multipart variant this test exercises has no such cap. Generated -# dynamically to keep the repo small. -python3 -c "open('large.bin', 'wb').write(b'\\0' * (12 * 1024 * 1024))" - -# Empty file (multipart encodes an empty `content` part — distinct from JSON's empty string). -touch empty.txt - -# Python notebook: the CLI detects the header and imports it with format=SOURCE at the -# path with .py stripped. -cat > pyNb.py <<'EOF' -# Databricks notebook source -print("hello") -EOF - -# SQL notebook: -- comment header, .sql extension, Language=SQL on the workspace side. -cat > sqlNb.sql <<'EOF' --- Databricks notebook source -SELECT 1 -EOF - -# Scala notebook: // comment header, .scala extension, Language=SCALA. -cat > scalaNb.scala <<'EOF' -// Databricks notebook source -println("hello") -EOF - -# ipyNb.ipynb and query.lwquery.ipynb are committed Jupyter notebooks. The second name -# ends in a suffix /workspace/import reserves for another asset type; it must still -# import as a notebook. - -# Plain .py without the source header — should land as FILE, not NOTEBOOK, -# even though the extension is .py. -cat > plain-script.py <<'EOF' -print("plain script, no header") -EOF - -# Lakeview dashboard descriptor: uploaded with format=AUTO, which the real /workspace/import -# imports as object_type=DASHBOARD and preserves the .lvdash.json extension. The testserver -# doesn't emulate that and stores it as a generic FILE — but the upload-side request shape -# (which is what this test asserts) is identical. -echo '{"datasets":[],"pages":[]}' > dashboard.lvdash.json - -# Names format=AUTO would map to asset types /workspace/import cannot create -# ("Requested node type [UNKNOWN] is not supported"). They must still sync as files. -echo '{}' > space.geniespace.json -echo '{}' > job.dbjob.json - -# A .zip goes through the legacy import-file endpoint, which extracts it into its parent -# directory (inner.txt next to the archive). The testserver doesn't emulate that and stores -# the archive as a FILE. /workspace/import would instead create a folder at the archive's -# path that cannot be overwritten on the next sync. -python3 -c "import zipfile; zipfile.ZipFile('archive.zip', 'w').writestr('inner.txt', 'hello')" - -# Non-ASCII filename. Multipart encodes filenames with RFC 5987 / quoted-string rules, -# distinct from URL-encoding in the legacy import-file endpoint. -echo "hello, naïve world" > "héllo.txt" - -# Filename with a space. -echo "hello, world" > "with spaces.txt" - -trace $CLI bundle sync --output text 2>&1 | sort - -title "workspace state after sync — assert object_type and language for each upload" -remote_files=$($CLI bundle summary -o json 2>/dev/null | jq -r .workspace.file_path) -# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-slash -# workspace path into a C:/Program Files/... filesystem path before it -# reaches the CLI. -trace MSYS_NO_PATHCONV=1 $CLI workspace list -o json "$remote_files" | - jq 'sort_by(.path) | .[] | {name: (.path | sub(".*/"; "")), object_type, language}' - -title "the .zip is uploaded with the legacy import-file endpoint" -trace print_requests.py //workspace-files/import-file --del-field raw_body --keep - -title "uploaded paths from each multipart POST /workspace/import (sorted)" -trace print_workspace_imports | sort - -title "format per upload: SOURCE/JUPYTER for notebooks, AUTO for dashboards, RAW for everything else" -trace jq -rs ' - map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/files/"))) | - map({name: (.body.multipart_form.path | sub(".*/"; "")), format: .body.multipart_form.format}) | - sort_by(.name) | .[] | "\(.format) \(.name)" -' "$OUT_REQUESTS" - -title "recorded content for the small text inputs" -trace jq -rs ' - map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/(pyNb|sqlNb|scalaNb|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | - map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | - sort_by(.path) | .[] -' "$OUT_REQUESTS" - -# Drop the recorded requests; their workspace-prefixed paths are noisy and the -# assertions above already pin the relevant fields. -rm "$OUT_REQUESTS" .gitignore diff --git a/acceptance/bundle/sync-upload-edge-cases/test.toml b/acceptance/bundle/sync-upload-edge-cases/test.toml deleted file mode 100644 index db1e57486a1..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/test.toml +++ /dev/null @@ -1,17 +0,0 @@ -RecordRequests = true - -Ignore = [ - "large.bin", - "empty.txt", - "pyNb.py", - "sqlNb.sql", - "scalaNb.scala", - "plain-script.py", - "dashboard.lvdash.json", - "space.geniespace.json", - "job.dbjob.json", - "archive.zip", - "héllo.txt", - "with spaces.txt", - ".databricks", -] diff --git a/acceptance/bundle/sync/output.txt b/acceptance/bundle/sync/output.txt index ca0b5b7258d..276972e0d5d 100644 --- a/acceptance/bundle/sync/output.txt +++ b/acceptance/bundle/sync/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle sync --output text -Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py, upload-504/databricks.yml, upload-504/out.test.toml +Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py, replace-node-type/bom.sql, replace-node-type/databricks.yml.tmpl, replace-node-type/out.test.toml, replace-node-type/plain.sql, upload-504/databricks.yml, upload-504/out.test.toml, zip-extract/archive.zip, zip-extract/archive_v2.zip, zip-extract/databricks.yml.tmpl, zip-extract/out.test.toml Initial Sync Complete Uploaded .gitignore Uploaded databricks.yml @@ -10,6 +10,16 @@ Uploaded dryrun/out.test.toml Uploaded ignored-folder/folder1 Uploaded ignored-folder/folder1/script.py Uploaded ignored-folder/script.py +Uploaded replace-node-type +Uploaded replace-node-type/bom.sql +Uploaded replace-node-type/databricks.yml.tmpl +Uploaded replace-node-type/out.test.toml +Uploaded replace-node-type/plain.sql Uploaded upload-504 Uploaded upload-504/databricks.yml Uploaded upload-504/out.test.toml +Uploaded zip-extract +Uploaded zip-extract/archive.zip +Uploaded zip-extract/archive_v2.zip +Uploaded zip-extract/databricks.yml.tmpl +Uploaded zip-extract/out.test.toml diff --git a/acceptance/bundle/sync/replace-node-type/bom.sql b/acceptance/bundle/sync/replace-node-type/bom.sql new file mode 100644 index 00000000000..4aeddae0fd3 --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/bom.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +SELECT 1 diff --git a/acceptance/bundle/sync/replace-node-type/databricks.yml.tmpl b/acceptance/bundle/sync/replace-node-type/databricks.yml.tmpl new file mode 100644 index 00000000000..8c9201dd352 --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: replace-node-type-$UNIQUE_NAME + +# Only sync bom.sql and plain.sql. +sync: + exclude: + - databricks.yml + - databricks.yml.tmpl + - script + - test.toml + - output.txt + - out.* + - repls.json diff --git a/acceptance/bundle/sync-upload-edge-cases/out.test.toml b/acceptance/bundle/sync/replace-node-type/out.test.toml similarity index 67% rename from acceptance/bundle/sync-upload-edge-cases/out.test.toml rename to acceptance/bundle/sync/replace-node-type/out.test.toml index a927a5fbc06..0f167effef1 100644 --- a/acceptance/bundle/sync-upload-edge-cases/out.test.toml +++ b/acceptance/bundle/sync/replace-node-type/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync/replace-node-type/output.txt b/acceptance/bundle/sync/replace-node-type/output.txt new file mode 100644 index 00000000000..9e482f6a28e --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/output.txt @@ -0,0 +1,22 @@ + +=== A file holds the name of notebook bom.sql, a notebook holds the name of file plain.sql + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/replace-node-type-[UNIQUE_NAME]/default/files/bom.sql --file bom.sql --format RAW + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/replace-node-type-[UNIQUE_NAME]/default/files/plain --file plain.sql --format SOURCE --language SQL + +>>> list +FILE bom.sql +NOTEBOOK plain + +=== Sync replaces both + +>>> [CLI] bundle sync --output text +Action: PUT: bom.sql, plain.sql +Initial Sync Complete +Uploaded bom.sql +Uploaded plain.sql + +>>> list +NOTEBOOK bom +FILE plain.sql diff --git a/acceptance/bundle/sync/replace-node-type/plain.sql b/acceptance/bundle/sync/replace-node-type/plain.sql new file mode 100644 index 00000000000..2e3761f7a2c --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/plain.sql @@ -0,0 +1 @@ +SELECT 1 diff --git a/acceptance/bundle/sync/replace-node-type/script b/acceptance/bundle/sync/replace-node-type/script new file mode 100644 index 00000000000..32dd2c3cc53 --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/script @@ -0,0 +1,25 @@ +# Overwrite can't change a node's type, so when the node at a path has the other type, +# sync deletes it and uploads again. bom.sql is a notebook: its first line starts with a +# byte order mark, and the first line only needs to contain "Databricks notebook source". +# CLI v1.21 and v1.22 uploaded it as a file. +envsubst < databricks.yml.tmpl > databricks.yml +remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) + +cleanup() { + $CLI workspace delete "$($CLI bundle summary -o json | jq -r .workspace.root_path)" --recursive +} +trap cleanup EXIT + +list() { + $CLI workspace list -o json "$remote" | jq -r 'sort_by(.path)[] | "\(.object_type) \(.path | sub(".*/"; ""))"' +} + +title "A file holds the name of notebook bom.sql, a notebook holds the name of file plain.sql\n" +$CLI workspace mkdirs "$remote" +trace $CLI workspace import "$remote/bom.sql" --file bom.sql --format RAW +trace $CLI workspace import "$remote/plain" --file plain.sql --format SOURCE --language SQL +trace list + +title "Sync replaces both\n" +trace $CLI bundle sync --output text 2>&1 | sort +trace list diff --git a/acceptance/bundle/sync/replace-node-type/test.toml b/acceptance/bundle/sync/replace-node-type/test.toml new file mode 100644 index 00000000000..aab5e85edef --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/test.toml @@ -0,0 +1,3 @@ +Cloud = true +Env.MSYS_NO_PATHCONV = "1" +Ignore = [".databricks"] diff --git a/acceptance/bundle/sync/upload-504/output.txt b/acceptance/bundle/sync/upload-504/output.txt index af7934522d1..d05ed02a23e 100644 --- a/acceptance/bundle/sync/upload-504/output.txt +++ b/acceptance/bundle/sync/upload-504/output.txt @@ -1,6 +1,6 @@ Action: PUT: databricks.yml -Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace/import: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml: Fault injected by test. +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml Uploaded databricks.yml Initial Sync Complete -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml diff --git a/acceptance/bundle/sync/upload-504/script b/acceptance/bundle/sync/upload-504/script index d4e9e5d1b51..f3ca9aff682 100644 --- a/acceptance/bundle/sync/upload-504/script +++ b/acceptance/bundle/sync/upload-504/script @@ -1,7 +1,7 @@ # Inject a single 504 on the first file upload. Uploads overwrite, so they are retried and the sync succeeds. -fault.py "POST /api/2.0/workspace/import" 504 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/**" 504 0 1 $CLI bundle sync # databricks.yml is the only file. Two requests for it should appear: the initial 504 and the successful retry. -print_requests.py //api/2.0/workspace/import | jq -r .body.multipart_form.path +print_requests.py //api/2.0/workspace-files/import-file | jq -r .path diff --git a/acceptance/bundle/sync/zip-extract/archive.zip b/acceptance/bundle/sync/zip-extract/archive.zip new file mode 100644 index 00000000000..0452f3ee8bf Binary files /dev/null and b/acceptance/bundle/sync/zip-extract/archive.zip differ diff --git a/acceptance/bundle/sync/zip-extract/archive_v2.zip b/acceptance/bundle/sync/zip-extract/archive_v2.zip new file mode 100644 index 00000000000..a88139009e3 Binary files /dev/null and b/acceptance/bundle/sync/zip-extract/archive_v2.zip differ diff --git a/acceptance/bundle/sync/zip-extract/databricks.yml.tmpl b/acceptance/bundle/sync/zip-extract/databricks.yml.tmpl new file mode 100644 index 00000000000..061beca4d2d --- /dev/null +++ b/acceptance/bundle/sync/zip-extract/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: zip-extract-$UNIQUE_NAME + +# Only sync archive.zip. +sync: + exclude: + - archive_v2.zip + - databricks.yml + - databricks.yml.tmpl + - script + - test.toml + - output.txt + - out.* + - repls.json diff --git a/acceptance/bundle/sync/zip-extract/out.test.toml b/acceptance/bundle/sync/zip-extract/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/sync/zip-extract/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync/zip-extract/output.txt b/acceptance/bundle/sync/zip-extract/output.txt new file mode 100644 index 00000000000..7b9b7b113c8 --- /dev/null +++ b/acceptance/bundle/sync/zip-extract/output.txt @@ -0,0 +1,24 @@ + +=== Sync extracts the archive into the bundle's file path + +>>> [CLI] bundle sync +Action: PUT: archive.zip +Uploaded archive.zip +Initial Sync Complete + +>>> list /Workspace/Users/[USERNAME]/.bundle/zip-extract-[UNIQUE_NAME]/default/files +FILE /inner.txt +DIRECTORY /pkg + +>>> list /Workspace/Users/[USERNAME]/.bundle/zip-extract-[UNIQUE_NAME]/default/files/pkg +FILE /pkg/a.txt + +=== Syncing a changed archive overwrites the extracted files + +>>> [CLI] bundle sync +Action: PUT: archive.zip +Uploaded archive.zip +Initial Sync Complete + +>>> [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/zip-extract-[UNIQUE_NAME]/default/files/inner.txt --format AUTO +v2 diff --git a/acceptance/bundle/sync/zip-extract/script b/acceptance/bundle/sync/zip-extract/script new file mode 100644 index 00000000000..16f6d6df107 --- /dev/null +++ b/acceptance/bundle/sync/zip-extract/script @@ -0,0 +1,24 @@ +# The import-file endpoint extracts a .zip into its parent folder instead of storing the +# archive, and overwrites the extracted files on the next upload. archive.zip holds +# inner.txt ("v1") and pkg/a.txt; archive_v2.zip is the same with inner.txt "v2". +envsubst < databricks.yml.tmpl > databricks.yml +remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) + +cleanup() { + $CLI workspace delete "$($CLI bundle summary -o json | jq -r .workspace.root_path)" --recursive +} +trap cleanup EXIT + +list() { + $CLI workspace list -o json "$1" | jq -r 'sort_by(.path)[] | "\(.object_type) \(.path | sub(".*/files"; ""))"' +} + +title "Sync extracts the archive into the bundle's file path\n" +trace $CLI bundle sync +trace list "$remote" +trace list "$remote/pkg" + +title "Syncing a changed archive overwrites the extracted files\n" +cp archive_v2.zip archive.zip +trace $CLI bundle sync +trace $CLI workspace export "$remote/inner.txt" --format AUTO diff --git a/acceptance/bundle/sync/zip-extract/test.toml b/acceptance/bundle/sync/zip-extract/test.toml new file mode 100644 index 00000000000..aab5e85edef --- /dev/null +++ b/acceptance/bundle/sync/zip-extract/test.toml @@ -0,0 +1,3 @@ +Cloud = true +Env.MSYS_NO_PATHCONV = "1" +Ignore = [".databricks"] diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt index df5252aa3d7..17eb89280ce 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt @@ -1,9 +1,9 @@ >>> [CLI] bundle deploy -Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal error -Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal error (500 INTERNAL_ERROR) +Error: Failed to acquire deployment lock: Internal error +Error: Internal error (500 INTERNAL_ERROR) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fdeploy.lock?overwrite=false HTTP Status: 500 Internal Server Error API error_code: INTERNAL_ERROR API message: Internal error diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml index 1e685cca603..86d366ba8c2 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml @@ -12,6 +12,6 @@ Cloud = false [[Server]] -Pattern = "POST /api/2.0/workspace/import" +Pattern = "POST /api/2.0/workspace-files/import-file/{path...}" Response.StatusCode = 500 Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 2f91efecbe1..3029c012d05 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -13,13 +13,13 @@ $CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.json rm "$OUT_REQUESTS" # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' -print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.txt +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.txt read_id.py -t dev sample_job > /dev/null trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' -print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.txt +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.txt read_id.py -t prod sample_job > /dev/null # Extracted with python, not jq: the id exceeds 2^53 and jq 1.6 rounds it. root_dir_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/my_default_python/prod" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index a5d2a395976..93e5048afec 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -13,13 +13,13 @@ $CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.json rm "$OUT_REQUESTS" trace $CLI bundle deploy -t dev replace_ids.py -t dev -print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.txt +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.txt trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.json trace $CLI bundle deploy -t prod replace_ids.py -t prod -print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.txt +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.txt trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json | nostamp > ../../out.plan_after_deploy_prod.json diff --git a/acceptance/bundle/upload/internal_server_error/output.txt b/acceptance/bundle/upload/internal_server_error/output.txt index 2ae3634f94b..d75ee2a71b1 100644 --- a/acceptance/bundle/upload/internal_server_error/output.txt +++ b/acceptance/bundle/upload/internal_server_error/output.txt @@ -1,9 +1,9 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Warn: retrying after 500 Internal Server Error from POST /api/2.0/workspace/import: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt: Fault injected by test. -Warn: retrying after 500 Internal Server Error from POST /api/2.0/workspace/import: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt: Fault injected by test. -Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt: Fault injected by test. (500 INJECTED) +Warn: retrying after 500 Internal Server Error from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt +Warn: retrying after 500 Internal Server Error from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt +Error: Fault injected by test. (500 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true HTTP Status: 500 Internal Server Error API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/upload/internal_server_error/script b/acceptance/bundle/upload/internal_server_error/script index 78c57ef8a94..e8a831e20af 100644 --- a/acceptance/bundle/upload/internal_server_error/script +++ b/acceptance/bundle/upload/internal_server_error/script @@ -1,2 +1,2 @@ -fault.py --body-contains "Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/files/file_to_upload.txt" "POST /api/2.0/workspace/import" 500 0 1000 +fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/files/file_to_upload.txt" 500 0 1000 musterr $CLI bundle deploy diff --git a/acceptance/bundle/upload/timeout/output.txt b/acceptance/bundle/upload/timeout/output.txt index 0c779d73dc2..c314a247707 100644 --- a/acceptance/bundle/upload/timeout/output.txt +++ b/acceptance/bundle/upload/timeout/output.txt @@ -1,3 +1,3 @@ -Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity -Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity diff --git a/acceptance/bundle/upload/timeout/test.toml b/acceptance/bundle/upload/timeout/test.toml index f0abd08ecbc..d6564ef9659 100644 --- a/acceptance/bundle/upload/timeout/test.toml +++ b/acceptance/bundle/upload/timeout/test.toml @@ -3,5 +3,5 @@ Env.DATABRICKS_BUNDLE_HTTP_TIMEOUT_SECONDS = "5" [[Server]] # CLI aborts after a single attempt when the HTTP timeout fires. Delay = "30s" -Pattern = "POST /api/2.0/workspace/import" +Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" Response.StatusCode = 200 diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 4f84546f174..458fcaa9638 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -8,14 +8,14 @@ OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json engine/direct OK deploy /api/2.0/workspace/delete engine/direct OK deploy /api/2.0/workspace/delete engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct OK deploy /api/2.0/workspace/mkdirs engine/direct OK deploy /api/2.0/workspace/mkdirs engine/direct OK deploy /api/2.1/unity-catalog/schemas engine/direct @@ -29,9 +29,9 @@ OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct +OK destroy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct OK destroy /api/2.0/workspace/delete engine/direct OK destroy /api/2.0/workspace/delete engine/direct -OK destroy /api/2.0/workspace/import engine/direct MISS destroy /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS plan /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.json index 43b23a566c3..31bc7b0724a 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.json @@ -130,10 +130,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", - "recursive": true + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py", + "q": { + "overwrite": "true" } } { @@ -143,9 +142,15 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", + "q": { + "overwrite": "false" + }, "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" } } { @@ -155,14 +160,15 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", + "q": { + "overwrite": "false" + }, "body": { - "multipart_form": { - "content": "", - "format": "RAW", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py" - } + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" } } { @@ -172,13 +178,22 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json", + "q": { + "overwrite": "true" + }, "body": { - "multipart_form": { - "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "RAW", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" - } + "version": 1, + "seq": 1, + "cli_version": "[CLI_VERSION]", + "timestamp": "[TIMESTAMP]", + "files": [ + { + "local_path": "empty.py", + "is_notebook": false + } + ], + "id": "[UUID]" } } { @@ -188,13 +203,29 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json", + "q": { + "overwrite": "true" + }, "body": { - "multipart_form": { - "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "RAW", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" - } + "version": 1, + "config": { + "bundle": { + "name": "test-bundle", + "target": "default", + "git": { + "bundle_root_path": "." + } + }, + "workspace": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" + }, + "resources": {}, + "presets": { + "source_linked_deployment": false + } + }, + "extra": {} } } { @@ -204,13 +235,23 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json", + "q": { + "overwrite": "true" + }, "body": { - "multipart_form": { - "content": "{\"version\":1,\"seq\":1,\"cli_version\":\"[CLI_VERSION]\",\"timestamp\":\"[TIMESTAMP]\",\"files\":[{\"local_path\":\"empty.py\",\"is_notebook\":false}],\"id\":\"[UUID]\"}", - "format": "RAW", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "state": { + "resources.schemas.foo": { + "__id__": "mycatalog.myschema", + "state": { + "catalog_name": "mycatalog", + "name": "myschema" + } + } } } } @@ -221,14 +262,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace/delete", "body": { - "multipart_form": { - "content": "{\n \"version\": 1,\n \"config\": {\n \"bundle\": {\n \"name\": \"test-bundle\",\n \"target\": \"default\",\n \"git\": {\n \"bundle_root_path\": \".\"\n }\n },\n \"workspace\": {\n \"file_path\": \"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"\n },\n \"resources\": {},\n \"presets\": {\n \"source_linked_deployment\": false\n }\n },\n \"extra\": {}\n}", - "format": "RAW", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - } + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", + "recursive": true } } { @@ -238,14 +275,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace/delete", "body": { - "multipart_form": { - "content": "{\n \"state_version\": 3,\n \"cli_version\": \"[CLI_VERSION]\",\n \"lineage\": \"[UUID]\",\n \"serial\": 1,\n \"state\": {\n \"resources.schemas.foo\": {\n \"__id__\": \"mycatalog.myschema\",\n \"state\": {\n \"catalog_name\": \"mycatalog\",\n \"name\": \"myschema\"\n }\n }\n }\n}", - "format": "RAW", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" - } + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.json index 90328199953..87006c49f30 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.json @@ -112,9 +112,15 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", + "q": { + "overwrite": "false" + }, "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" } } { @@ -126,8 +132,7 @@ "method": "POST", "path": "/api/2.0/workspace/delete", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "recursive": true + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" } } { @@ -137,13 +142,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace/delete", "body": { - "multipart_form": { - "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "RAW", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" - } + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "recursive": true } } { diff --git a/acceptance/bundle/validate/sync_patterns/out.sync.txt b/acceptance/bundle/validate/sync_patterns/out.sync.txt index bbac212feb3..ac5aba9ab29 100644 --- a/acceptance/bundle/validate/sync_patterns/out.sync.txt +++ b/acceptance/bundle/validate/sync_patterns/out.sync.txt @@ -1,12 +1,7 @@ { "method": "POST", - "path": "/api/2.0/workspace/import", - "body": { - "multipart_form": { - "content": "", - "format": "RAW", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml" - } + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml", + "q": { + "overwrite": "true" } } diff --git a/acceptance/bundle/validate/sync_patterns/script b/acceptance/bundle/validate/sync_patterns/script index f44ee310433..d4cf7169b1b 100644 --- a/acceptance/bundle/validate/sync_patterns/script +++ b/acceptance/bundle/validate/sync_patterns/script @@ -1,5 +1,5 @@ trace $CLI bundle validate trace $CLI bundle validate -o json | jq '.sync' trace $CLI bundle deploy -jq 'select(.body.multipart_form.path | strings | test("dir/test.yml"))' "$OUT_REQUESTS" > out.sync.txt +jq 'select(.path | test("dir/test.yml"))' "$OUT_REQUESTS" > out.sync.txt rm "$OUT_REQUESTS" diff --git a/acceptance/cmd/sync/dryrun-missing-remote/output.txt b/acceptance/cmd/sync/dryrun-missing-remote/output.txt index b7453994b58..4848aae1c2d 100644 --- a/acceptance/cmd/sync/dryrun-missing-remote/output.txt +++ b/acceptance/cmd/sync/dryrun-missing-remote/output.txt @@ -15,11 +15,11 @@ Uploaded hello.py >>> print_requests.py --sort //api/ { "method": "POST", - "path": "/api/2.0/workspace/import" + "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/.gitignore" } { "method": "POST", - "path": "/api/2.0/workspace/import" + "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/hello.py" } { "method": "POST", diff --git a/acceptance/cmd/sync/upload-504/output.txt b/acceptance/cmd/sync/upload-504/output.txt index cb9b61d2b0c..669f986c419 100644 --- a/acceptance/cmd/sync/upload-504/output.txt +++ b/acceptance/cmd/sync/upload-504/output.txt @@ -1,6 +1,6 @@ Action: PUT: hello.py -Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace/import: failed to upload /Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py: Fault injected by test. +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py Uploaded hello.py Initial Sync Complete -/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py -/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py +/api/2.0/workspace-files/import-file/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py +/api/2.0/workspace-files/import-file/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py diff --git a/acceptance/cmd/sync/upload-504/script b/acceptance/cmd/sync/upload-504/script index 3be71acff67..b7000ed9385 100644 --- a/acceptance/cmd/sync/upload-504/script +++ b/acceptance/cmd/sync/upload-504/script @@ -15,10 +15,10 @@ cleanup() { trap cleanup EXIT # Inject a single 504 on the first file upload. Uploads overwrite, so they are retried and the sync succeeds. -fault.py "POST /api/2.0/workspace/import" 504 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/**" 504 0 1 # MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /Users/... to C:/Program Files/Git/Users/... MSYS_NO_PATHCONV=1 $CLI sync . "$remote_dir" # Two requests for hello.py should appear: the initial 504 and the successful retry. -print_requests.py //api/2.0/workspace/import | jq -r .body.multipart_form.path +print_requests.py //api/2.0/workspace-files/import-file | jq -r .path diff --git a/acceptance/cmd/workspace/import-zip-auto/archive.zip b/acceptance/cmd/workspace/import-zip-auto/archive.zip deleted file mode 100644 index fabadf6fa08..00000000000 Binary files a/acceptance/cmd/workspace/import-zip-auto/archive.zip and /dev/null differ diff --git a/acceptance/cmd/workspace/import-zip-auto/out.test.toml b/acceptance/cmd/workspace/import-zip-auto/out.test.toml deleted file mode 100644 index c7c6f58ed6e..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/out.test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = true diff --git a/acceptance/cmd/workspace/import-zip-auto/output.txt b/acceptance/cmd/workspace/import-zip-auto/output.txt deleted file mode 100644 index 6f9f2709742..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/output.txt +++ /dev/null @@ -1,40 +0,0 @@ - -=== AUTO extracts a .zip into a folder named after the archive, without .zip - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive.zip --file archive.zip --format AUTO - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME] -DIRECTORY /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive/a.txt -DIRECTORY /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive/sub - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive/sub -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive/sub/c.txt - -=== Importing the archive again is rejected, with or without overwrite - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive.zip --file archive.zip --format AUTO --overwrite -Error: Overwrite cannot be used for AUTO format when importing a folder. - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive.zip --file archive.zip --format AUTO -Error: /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive already exists. Please pass overwrite=true to overwrite it. - -=== RAW stores the archive as a file, and overwrites it - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/raw.zip --file archive.zip --format RAW - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/raw.zip --file archive.zip --format RAW --overwrite - -=== AUTO rejects names reserved for asset types it cannot import; RAW stores them as files - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/space.geniespace.json --file space.geniespace.json --format AUTO -Error: Requested node type [UNKNOWN] is not supported - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/space.geniespace.json --file space.geniespace.json --format RAW - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME] -DIRECTORY /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/raw.zip -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/space.geniespace.json diff --git a/acceptance/cmd/workspace/import-zip-auto/script b/acceptance/cmd/workspace/import-zip-auto/script deleted file mode 100644 index 8c48e84ee04..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/script +++ /dev/null @@ -1,34 +0,0 @@ -# Documents how POST /api/2.0/workspace/import handles a .zip and names reserved for -# other asset types. archive.zip holds a single top-level folder: pkg/a.txt, pkg/sub/c.txt. -DIR="/Workspace/Users/$CURRENT_USER_NAME/$UNIQUE_NAME" -$CLI workspace mkdirs "$DIR" - -cleanup() { - $CLI workspace delete "$DIR" --recursive -} -trap cleanup EXIT - -list() { - $CLI workspace list "$1" -o json | jq -r 'sort_by(.path)[] | "\(.object_type) \(.path)"' -} - -title "AUTO extracts a .zip into a folder named after the archive, without .zip\n" -trace $CLI workspace import "$DIR/archive.zip" --file archive.zip --format AUTO -trace list "$DIR" -# The archive's single top-level folder (pkg/) is replaced by the target folder. -trace list "$DIR/archive" -trace list "$DIR/archive/sub" - -title "Importing the archive again is rejected, with or without overwrite\n" -trace musterr $CLI workspace import "$DIR/archive.zip" --file archive.zip --format AUTO --overwrite -trace musterr $CLI workspace import "$DIR/archive.zip" --file archive.zip --format AUTO - -title "RAW stores the archive as a file, and overwrites it\n" -trace $CLI workspace import "$DIR/raw.zip" --file archive.zip --format RAW -trace $CLI workspace import "$DIR/raw.zip" --file archive.zip --format RAW --overwrite - -title "AUTO rejects names reserved for asset types it cannot import; RAW stores them as files\n" -trace musterr $CLI workspace import "$DIR/space.geniespace.json" --file space.geniespace.json --format AUTO -trace $CLI workspace import "$DIR/space.geniespace.json" --file space.geniespace.json --format RAW - -trace list "$DIR" diff --git a/acceptance/cmd/workspace/import-zip-auto/space.geniespace.json b/acceptance/cmd/workspace/import-zip-auto/space.geniespace.json deleted file mode 100644 index cb608f6e9c4..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/space.geniespace.json +++ /dev/null @@ -1 +0,0 @@ -{"version": 1} diff --git a/acceptance/cmd/workspace/import-zip-auto/test.toml b/acceptance/cmd/workspace/import-zip-auto/test.toml deleted file mode 100644 index 6b74aa538f0..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -Cloud = true - -# Workspace paths start with "/"; keep Git Bash on Windows from rewriting them. -Env.MSYS_NO_PATHCONV = "1" diff --git a/acceptance/script.prepare b/acceptance/script.prepare index 66080eb1fcb..a53627b231d 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -128,12 +128,6 @@ print_telemetry_bool_values() { jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental.bool_values // []) ) | map("\(.key) \(.value)") | .[]' "$OUT_REQUESTS" | grep -Ev '^(engine_terraform_|deployment_history_enabled )' | sort } -# Workspace paths uploaded via /workspace/import, one per line. The path is a -# multipart form field, not part of the URL, so print_requests.py can't filter on it. -print_workspace_imports() { - jq -r 'select(.path? == "/api/2.0/workspace/import") | .body.multipart_form.path | strings' "$OUT_REQUESTS" -} - sethome() { local home="$1" mkdir -p "$home" diff --git a/acceptance/ssh/bulk-transfer/script b/acceptance/ssh/bulk-transfer/script index 7e043dedf28..7272d7d56f1 100644 --- a/acceptance/ssh/bulk-transfer/script +++ b/acceptance/ssh/bulk-transfer/script @@ -30,10 +30,10 @@ else exit 0 fi - # No release artifacts locally and they're never executed; stand in empty archives. + # No release artifacts locally and they're never executed; stand in placeholder archives. mkdir -p releases - : >releases/databricks_cli_linux_amd64.zip - : >releases/databricks_cli_linux_arm64.zip + fake_release releases/databricks_cli_linux_amd64.zip + fake_release releases/databricks_cli_linux_arm64.zip CONNECT_FLAGS="--releases-dir=$PWD/releases --known-hosts-dir=known-hosts" fi diff --git a/acceptance/ssh/connect-serverless-cpu/script b/acceptance/ssh/connect-serverless-cpu/script index 02b0068f3da..14c787e58ea 100644 --- a/acceptance/ssh/connect-serverless-cpu/script +++ b/acceptance/ssh/connect-serverless-cpu/script @@ -15,10 +15,10 @@ else exit 0 fi - # No release artifacts locally and they're never executed; stand in empty archives. + # No release artifacts locally and they're never executed; stand in placeholder archives. mkdir -p releases - touch releases/databricks_cli_linux_amd64.zip - touch releases/databricks_cli_linux_arm64.zip + fake_release releases/databricks_cli_linux_amd64.zip + fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" title "CLI ssh connect (local sshd)\n" diff --git a/acceptance/ssh/connect-serverless-feature-disabled/output.txt b/acceptance/ssh/connect-serverless-feature-disabled/output.txt index 9957a1c7e18..226d0030a71 100644 --- a/acceptance/ssh/connect-serverless-feature-disabled/output.txt +++ b/acceptance/ssh/connect-serverless-feature-disabled/output.txt @@ -10,7 +10,7 @@ DRIVER_PROXY_FORBIDDEN === Each release archive is uploaded once, and the submitted run is cancelled ->>> print_requests.py //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel +>>> print_requests.py --del-field raw_body //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel { "method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.databricks/ssh-tunnel/[CLI_VERSION]/databricks_cli_linux_amd64/databricks.zip", diff --git a/acceptance/ssh/connect-serverless-feature-disabled/script b/acceptance/ssh/connect-serverless-feature-disabled/script index 9f8fe5c1b92..67901dc76ef 100644 --- a/acceptance/ssh/connect-serverless-feature-disabled/script +++ b/acceptance/ssh/connect-serverless-feature-disabled/script @@ -6,10 +6,10 @@ # The body repeats the name: with only {"value": false} in it, the API sees an empty message. trace $CLI workspace-settings-v2 patch-public-workspace-setting port_fwd_serverless_enabled --json '{"name": "port_fwd_serverless_enabled", "boolean_val": {"value": false}}' >/dev/null -# No release artifacts locally and they're never executed; stand in empty archives. +# No release artifacts locally and they're never executed; stand in placeholder archives. mkdir -p releases -: >releases/databricks_cli_linux_amd64.zip -: >releases/databricks_cli_linux_arm64.zip +fake_release releases/databricks_cli_linux_amd64.zip +fake_release releases/databricks_cli_linux_arm64.zip title "CLI ssh connect\n" musterr $CLI ssh connect --name feature-disabled --releases-dir="$PWD/releases" -- "echo 'Connection successful'" >out.connect.txt 2>&1 @@ -19,4 +19,4 @@ title "Reported error category\n" trace jq -r 'select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.ssh_tunnel_event.error_category' "$OUT_REQUESTS" title "Each release archive is uploaded once, and the submitted run is cancelled\n" -trace print_requests.py //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel +trace print_requests.py --del-field raw_body //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel diff --git a/acceptance/ssh/connect-serverless-gpu/script b/acceptance/ssh/connect-serverless-gpu/script index b2e5e3d40fa..5aab2af0ac0 100644 --- a/acceptance/ssh/connect-serverless-gpu/script +++ b/acceptance/ssh/connect-serverless-gpu/script @@ -1,9 +1,9 @@ if [ -z "${CLOUD_ENV:-}" ]; then # Local runs have no release artifacts and never execute the uploaded - # binaries, so stand in empty archives (dev name omits the version). + # binaries, so stand in placeholder archives (dev name omits the version). mkdir -p releases - : >releases/databricks_cli_linux_amd64.zip - : >releases/databricks_cli_linux_arm64.zip + fake_release releases/databricks_cli_linux_amd64.zip + fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" fi diff --git a/acceptance/ssh/connection/script b/acceptance/ssh/connection/script index 3858cd1b6ba..25a63feca78 100644 --- a/acceptance/ssh/connection/script +++ b/acceptance/ssh/connection/script @@ -1,9 +1,9 @@ if [ -z "${CLOUD_ENV:-}" ]; then # Local runs have no release artifacts and never execute the uploaded - # binaries, so stand in empty archives (dev name omits the version). + # binaries, so stand in placeholder archives (dev name omits the version). mkdir -p releases - : >releases/databricks_cli_linux_amd64.zip - : >releases/databricks_cli_linux_arm64.zip + fake_release releases/databricks_cli_linux_amd64.zip + fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" fi diff --git a/acceptance/ssh/reconnect-reused-name/script b/acceptance/ssh/reconnect-reused-name/script index b07e4304746..a0ac016f93f 100644 --- a/acceptance/ssh/reconnect-reused-name/script +++ b/acceptance/ssh/reconnect-reused-name/script @@ -11,10 +11,10 @@ if [ -z "$(command -v sshd || ls /usr/sbin/sshd /usr/local/sbin/sshd /sbin/sshd exit 0 fi -# No release artifacts locally and they're never executed; stand in empty archives. +# No release artifacts locally and they're never executed; stand in placeholder archives. mkdir -p releases -: >releases/databricks_cli_linux_amd64.zip -: >releases/databricks_cli_linux_arm64.zip +fake_release releases/databricks_cli_linux_amd64.zip +fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" # The state a user reconnecting with a reused name is in: an entry for this connection diff --git a/acceptance/ssh/script.prepare b/acceptance/ssh/script.prepare new file mode 100644 index 00000000000..f8b26a3cd2a --- /dev/null +++ b/acceptance/ssh/script.prepare @@ -0,0 +1,5 @@ +# Writes a stand-in release archive. The import-file endpoint unzips uploaded .zip files, +# so it has to be a valid archive holding the `databricks` binary. +fake_release() { + python3 -c "import sys, zipfile; zipfile.ZipFile(sys.argv[1], 'w').writestr('databricks', '')" "$1" +} diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index bbd6718c264..d722edac163 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -430,11 +430,9 @@ func TestSubmitWorkloadValidationOutcome(t *testing.T) { t.Cleanup(server.Close) var uploads, submissions atomic.Int32 server.Handle("POST", validateConfigPath, func(req testserver.Request) any { return tt.response }) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { uploads.Add(1) - form, err := testserver.ParseImportForm(req) - assert.NoError(t, err) - return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, true) + return req.Workspace.WorkspaceFilesImportFile(req.Vars["path"], req.Body, true) }) server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { submissions.Add(1) @@ -751,19 +749,18 @@ func TestSubmitWorkloadPlainTarContentAddressed(t *testing.T) { server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { return jobs.SubmitRunResponse{RunId: 555} }) - // Count snapshot import calls, preserving fake-workspace persistence so the + // Count snapshot import-file calls, preserving fake-workspace persistence so the // second submit's existence Stat sees the first upload. A count (not a set keyed by // path) is what proves the skip: both submits resolve to the same content-addressed // name, so a set could not tell a skipped second submit from one that re-uploaded to // that same path. snapshotUploads := 0 - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - form, err := testserver.ParseImportForm(req) - assert.NoError(t, err) - if strings.Contains(form.Path, "/.air/repo_snapshots/") { + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { snapshotUploads++ } - return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) stubValidateConfig(server) testserver.AddDefaultHandlers(server) @@ -800,7 +797,7 @@ code_source: assert.Regexp(t, `^src_[0-9a-f]{16}\.tar\.gz$`, base) // Same unchanged tree → identical remote path, and the second submit moved no bytes: - // zero new import calls (a real skip), not a re-upload to the same name. + // zero new import-file calls (a real skip), not a re-upload to the same name. assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Equal(t, afterFirst, snapshotUploads, "unchanged plain_tar should skip the second upload") require.NotNil(t, first.SizeBytes) @@ -830,13 +827,12 @@ func TestSubmitWorkloadGitArchiveCaching(t *testing.T) { // Dedupe by path: the filer mkdirs-and-retries the import on a missing // parent dir, so one logical upload can hit this route more than once. uploaded := map[string]bool{} - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - form, err := testserver.ParseImportForm(req) - assert.NoError(t, err) - if strings.Contains(form.Path, "/.air/repo_snapshots/") { - uploaded[form.Path] = true + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { + uploaded[p] = true } - return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) stubValidateConfig(server) testserver.AddDefaultHandlers(server) diff --git a/cmd/air/telemetry_test.go b/cmd/air/telemetry_test.go index 49760d24ad9..35a0cad984d 100644 --- a/cmd/air/telemetry_test.go +++ b/cmd/air/telemetry_test.go @@ -61,11 +61,10 @@ func TestAirRunTelemetry(t *testing.T) { return jobs.SubmitRunResponse{RunId: 555} }) var uploadedSize int64 - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - form, err := testserver.ParseImportForm(req) - assert.NoError(t, err) - if strings.HasSuffix(form.Path, ".tar.gz") { - uploadedSize = int64(len(form.Content)) + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.HasSuffix(p, ".tar.gz") { + uploadedSize = int64(len(req.Body)) time.Sleep(20 * time.Millisecond) if tc.failUpload { return testserver.Response{StatusCode: http.StatusForbidden, Body: map[string]string{ @@ -73,7 +72,7 @@ func TestAirRunTelemetry(t *testing.T) { }} } } - return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) stubValidateConfig(server) w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) diff --git a/experimental/ssh/internal/client/releases_test.go b/experimental/ssh/internal/client/releases_test.go index e0e2c384d9d..51b6018b614 100644 --- a/experimental/ssh/internal/client/releases_test.go +++ b/experimental/ssh/internal/client/releases_test.go @@ -1,6 +1,8 @@ package client import ( + "archive/zip" + "bytes" "context" "errors" "fmt" @@ -146,6 +148,17 @@ func TestUploadReleasesWithExistingBinary(t *testing.T) { } } +// releaseArchive returns a zip holding the databricks binary. The import-file +// endpoint unzips uploaded archives, so it has to be valid. +func releaseArchive(t *testing.T) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + _, err := zw.Create("databricks") + require.NoError(t, err) + require.NoError(t, zw.Close()) + return buf.Bytes() +} + func TestUploadReleasesSendsEachArchiveOnce(t *testing.T) { tests := []struct { name string @@ -197,8 +210,9 @@ func TestUploadReleasesSendsEachArchiveOnce(t *testing.T) { workspaceFiler, err := filer.NewWorkspaceFilesClient(client, "/Workspace/Users/tester@databricks.test/.databricks/ssh-tunnel/"+tt.version) require.NoError(t, err) + archive := releaseArchive(t) getRelease := func(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { - return io.NopCloser(strings.NewReader("archive")), nil + return io.NopCloser(bytes.NewReader(archive)), nil } for range tt.uploads { err = uploadReleases(t.Context(), workspaceFiler, getRelease, tt.version, "") diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index f6a32369f88..30ca51f03d9 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "path" + "regexp" "slices" "strings" "time" @@ -111,39 +112,6 @@ type apiClient interface { visitors ...func(*http.Request) error) error } -// Extensions that format=AUTO imports as the same asset type the legacy import-file -// endpoint created: dashboards, alerts and Lakeflow Designer files. -var autoImportAssetExtensions = []string{".lvdash.json", ".dbalert.json", ".designer.py", ".designer.ipynb"} - -// importRequest returns the workspace path and /workspace/import options for a file, -// matching how the legacy import-file endpoint stored it. Other files avoid format=AUTO -// because it types them by name and rejects names reserved for other assets -// (e.g. .geniespace.json, .lwquery.ipynb). -func importRequest(absPath string, content []byte) (string, []workspace.UploadOption) { - lower := strings.ToLower(absPath) - for _, ext := range autoImportAssetExtensions { - if strings.HasSuffix(lower, ext) { - return absPath, []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatAuto)} - } - } - - isNotebook, language, _ := notebook.DetectBytes(absPath, content) - if !isNotebook { - return absPath, []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatRaw)} - } - - // Notebooks are imported without their extension and with an explicit format. - ext := path.Ext(absPath) - notebookPath := strings.TrimSuffix(absPath, ext) - if strings.EqualFold(ext, notebook.ExtensionJupyter) { - return notebookPath, []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatJupyter)} - } - return notebookPath, []workspace.UploadOption{ - workspace.UploadFormat(workspace.ImportFormatSource), - workspace.UploadLanguage(language), - } -} - // WorkspaceFilesClient implements the files-in-workspace API. // NOTE: This API is available for files under /Repos if a workspace has files-in-repos enabled. @@ -188,31 +156,32 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } + // Remove leading "/" so we can use it in the URL. + overwrite := slices.Contains(mode, OverwriteIfExists) + urlPath := fmt.Sprintf( + "/api/2.0/workspace-files/import-file/%s?overwrite=%t", + url.PathEscape(strings.TrimLeft(absPath, "/")), + overwrite, + ) + // Buffer the file contents because we may need to retry below and we cannot read twice. body, err := io.ReadAll(reader) if err != nil { return err } - overwrite := slices.Contains(mode, OverwriteIfExists) - if strings.HasSuffix(absPath, ".zip") { - // The legacy import-file endpoint extracts a .zip into its parent directory and - // overwrites existing entries. /workspace/import imports it as a new folder that - // cannot be overwritten, so zips stay on import-file until it supports that. - urlPath := fmt.Sprintf( - "/api/2.0/workspace-files/import-file/%s?overwrite=%t", - url.PathEscape(strings.TrimLeft(absPath, "/")), - overwrite, - ) - err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) - } else { - // Upload with the SDK's multipart Workspace.Upload. The JSON-body workspace.Import - // caps content at 10 MB, the multipart form at the 500 MB workspace file limit. - uploadPath, opts := importRequest(absPath, body) - if overwrite { - opts = append(opts, workspace.UploadOverwrite()) + err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) + + // Overwrite can't change a node's type. If a node of the other type holds the + // path, delete it and upload again so the remote matches the local file. + if err != nil && overwrite { + deleted, derr := w.deleteNodeOfOtherType(ctx, absPath, err) + if derr != nil { + return derr + } + if deleted { + err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) } - err = w.workspaceClient.Workspace.Upload(ctx, uploadPath, bytes.NewReader(body), opts...) } // Return early on success. @@ -220,8 +189,14 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return nil } - // Parent directory does not exist. - if errors.Is(err, apierr.ErrNotFound) { + // Special handling of this error only if it is an API error. + aerr, ok := errors.AsType[*apierr.APIError](err) + if !ok { + return err + } + + // This API returns a 404 if the parent directory does not exist. + if aerr.StatusCode == http.StatusNotFound { if !slices.Contains(mode, CreateParentDirectories) { return noSuchDirectoryError{path.Dir(absPath)} } @@ -229,8 +204,8 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // Create parent directory. err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath)) if err != nil { - if errors.Is(err, apierr.ErrPermissionDenied) { - return permissionError{absPath, err} + if mkdirErr, ok := errors.AsType[*apierr.APIError](err); ok && mkdirErr.StatusCode == http.StatusForbidden { + return permissionError{absPath, mkdirErr} } return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err) } @@ -239,32 +214,89 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return w.Write(ctx, name, bytes.NewReader(body), sliceWithout(mode, CreateParentDirectories)...) } - // Path already taken. ErrResourceConflict covers every 409 (including the - // bare 409 with only a message); ErrResourceAlreadyExists covers the 400 - // RESOURCE_ALREADY_EXISTS, which no status maps to a conflict. - if errors.Is(err, apierr.ErrResourceConflict) || errors.Is(err, apierr.ErrResourceAlreadyExists) { + // This API returns 409 if the file already exists, when the object type is file + if aerr.StatusCode == http.StatusConflict { return fileAlreadyExistsError{absPath} } - // Overwrite rejected because the existing object's node type differs from - // the upload. The collision carries an AIP-193 ErrorInfo with a stable - // reason, so branch on it rather than the message text. - if errors.Is(err, apierr.ErrInvalidParameterValue) { - if aerr, ok := errors.AsType[*apierr.APIError](err); ok { - if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == "WORKSPACE_OBJECT_TYPE_MISMATCH" { - return fileAlreadyExistsError{absPath} - } - } + + // This API returns 400 if the file already exists when the object type is notebook. + // Both the historical "Path () already exists." format and the newer + // "RESOURCE_ALREADY_EXISTS: already exists. ..." format end with the same + // "already exists." marker; the JSON error_code is empty in both. The new format + // might not have been rolled out to all workspaces yet, so we anchor on the shared + // marker and return absPath rather than parsing the message. + if aerr.StatusCode == http.StatusBadRequest && strings.Contains(aerr.Message, "already exists.") { + return fileAlreadyExistsError{absPath} + } + + // This API returns StatusForbidden when you have read access but don't have write access to a file + if aerr.StatusCode == http.StatusForbidden { + return permissionError{absPath, aerr} + } + + return err +} + +// Messages import-file returns when an overwrite would change a node's type. The +// endpoint sets no error code for them. Matching is best-effort: if the wording +// changes, the original error is returned as before. +var ( + // A file holds the name of a notebook upload. Captures the file's name. + fileHoldsNotebookName = regexp.MustCompile(`Node named '(.+)' already exists`) + // A notebook holds the name of a file upload (SQL notebook "x" holds "x.sql"). + // Captures the upload's path. + notebookHoldsFileName = regexp.MustCompile(`Creating file failed\. An item with path (.+) already exists`) +) + +// messageNames reports whether msg matches re and names name. +func messageNames(re *regexp.Regexp, msg, name string) bool { + m := re.FindStringSubmatch(msg) + return m != nil && m[1] == name +} + +// deleteNodeOfOtherType deletes the node that blocks an overwrite of absPath +// because its type differs from the upload's, and reports whether it did. +// CLI v1.21 and v1.22 uploaded some notebooks as files this way. +func (w *WorkspaceFilesClient) deleteNodeOfOtherType(ctx context.Context, absPath string, uploadErr error) (bool, error) { + aerr, ok := errors.AsType[*apierr.APIError](uploadErr) + if !ok { + return false, nil + } + + var nodePath string + var nodeType workspace.ObjectType + switch { + case messageNames(fileHoldsNotebookName, aerr.Message, path.Base(absPath)): + nodePath, nodeType = absPath, workspace.ObjectTypeFile + case messageNames(notebookHoldsFileName, aerr.Message, absPath): + nodePath, nodeType = strings.TrimSuffix(absPath, path.Ext(absPath)), workspace.ObjectTypeNotebook + default: + return false, nil } - // Caller has read access but no write access. - if errors.Is(err, apierr.ErrPermissionDenied) { - return permissionError{absPath, err} + // The second message is also returned when a directory holds the path or + // when two writers create it at once, so only delete a node of the expected + // type. A notebook only holds the name with its language's extension: SQL + // notebook "x" holds "x.sql", not "x.py". + info, err := w.workspaceClient.Workspace.GetStatusByPath(ctx, nodePath) + if errors.Is(err, apierr.ErrNotFound) { + return false, nil + } + if err != nil { + return false, err + } + if info.ObjectType != nodeType { + return false, nil + } + if nodeType == workspace.ObjectTypeNotebook && info.Language != notebook.ExtensionToLanguage[strings.ToLower(path.Ext(absPath))] { + return false, nil } - // Any other failure (e.g. a server-side error with an empty message) is - // surfaced with the target path so the user can tell which upload failed; - // %w keeps the original error inspectable by callers. - return fmt.Errorf("failed to upload %s: %w", absPath, err) + err = w.workspaceClient.Workspace.Delete(ctx, workspace.Delete{Path: nodePath}) + if err != nil { + return false, fmt.Errorf("failed to delete %s to replace it with %s: %w", nodePath, absPath, err) + } + return true, nil } func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 81832694d15..2915af60436 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -1,11 +1,8 @@ package filer import ( - "bytes" "encoding/json" "io/fs" - "net/http" - "strings" "testing" "time" @@ -18,8 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -// Confirms directory entries derived from workspace object infos report the -// name, type, and size of each object type the workspace can return. func TestWorkspaceFilesDirEntry(t *testing.T) { entries := wsfsDirEntriesFromObjectInfos([]workspace.ObjectInfo{ { @@ -66,8 +61,6 @@ func TestWorkspaceFilesDirEntry(t *testing.T) { assert.True(t, i2.IsDir()) } -// Confirms the workspace ID routing header is set only when the client config -// carries a workspace ID, and that a nil workspace client yields no headers. func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { tests := []struct { name string @@ -104,188 +97,6 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } -// Confirms each error /workspace/import returns is translated to the filer's -// own error type, and that unrecognized errors pass through unchanged. The -// endpoint signals an occupied path with three different status/error_code -// combinations (see the comment on Write), so all three are covered here to -// pin the mapping the rest of the CLI relies on. -func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { - tests := []struct { - name string - mode []WriteMode - apiErr *apierr.APIError - expectErrTarget any - }{ - { - name: "404 without create-parent maps to noSuchDirectoryError", - apiErr: &apierr.APIError{StatusCode: http.StatusNotFound, Message: "not found"}, - expectErrTarget: noSuchDirectoryError{}, - }, - { - name: "400 RESOURCE_ALREADY_EXISTS maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusBadRequest, - ErrorCode: "RESOURCE_ALREADY_EXISTS", - Message: "/dir/file.txt already exists. Please pass overwrite=true to overwrite it.", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, - { - name: "409 ALREADY_EXISTS (concurrent contention) maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusConflict, - ErrorCode: "ALREADY_EXISTS", - Message: "Node with name /dir/file.txt already exists. Please pass overwrite=true to update it.", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, - { - // A 409 carrying no error_code unwraps to ErrResourceConflict only, - // not to ErrAlreadyExists; the testserver returns exactly this shape. - name: "409 without error_code maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusConflict, - Message: "Node with name /dir/file.txt already exists.", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, - { - name: "400 INVALID_PARAMETER_VALUE other message passes through", - apiErr: &apierr.APIError{ - StatusCode: http.StatusBadRequest, - ErrorCode: "INVALID_PARAMETER_VALUE", - Message: "some other validation failure", - }, - expectErrTarget: nil, - }, - { - name: "403 maps to permissionError", - apiErr: &apierr.APIError{StatusCode: http.StatusForbidden, Message: "denied"}, - expectErrTarget: permissionError{}, - }, - { - name: "500 passes through", - apiErr: &apierr.APIError{StatusCode: http.StatusInternalServerError, Message: "boom"}, - expectErrTarget: nil, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - server := testserver.New(t) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - return testserver.Response{ - StatusCode: tc.apiErr.StatusCode, - Body: map[string]string{ - "error_code": tc.apiErr.ErrorCode, - "message": tc.apiErr.Message, - }, - } - }) - testserver.AddDefaultHandlers(server) - - client, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "testtoken", - }) - require.NoError(t, err) - - f, err := NewWorkspaceFilesClient(client, "/dir") - require.NoError(t, err) - - err = f.Write(t.Context(), "file.txt", bytes.NewReader([]byte("data")), tc.mode...) - require.Error(t, err) - switch target := tc.expectErrTarget.(type) { - case noSuchDirectoryError: - assert.ErrorAs(t, err, &target) - case fileAlreadyExistsError: - assert.ErrorAs(t, err, &target) - case permissionError: - assert.ErrorAs(t, err, &target) - case nil: - // passthrough — the underlying APIError stays inspectable, - // and the message names the file that failed to upload. - var aerr *apierr.APIError - require.ErrorAs(t, err, &aerr) - assert.Equal(t, tc.apiErr.StatusCode, aerr.StatusCode) - assert.Contains(t, err.Error(), "/dir/file.txt") - } - }) - } -} - -// writeWithImportError exercises Write through a real HTTP roundtrip so the -// SDK parses AIP-193 error details from the response body (the errorDetails -// field on apierr.APIError is unexported and only populated during response -// parsing, so it cannot be set on a directly constructed APIError). -func writeWithImportError(t *testing.T, body map[string]any) error { - t.Helper() - - server := testserver.New(t) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - return testserver.Response{ - StatusCode: http.StatusBadRequest, - Body: body, - } - }) - testserver.AddDefaultHandlers(server) - - client, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "testtoken", - }) - require.NoError(t, err) - - f, err := NewWorkspaceFilesClient(client, "/dir") - require.NoError(t, err) - - err = f.Write(t.Context(), "file.txt", strings.NewReader("data"), OverwriteIfExists) - require.Error(t, err) - return err -} - -// Confirms a type-mismatch collision is recognized from the structured -// AIP-193 ErrorInfo reason, independent of the error message wording. -func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { - // The message is deliberately unrelated to any collision wording, to prove - // the branch fires on the structured reason alone. - err := writeWithImportError(t, map[string]any{ - "error_code": "INVALID_PARAMETER_VALUE", - "message": "some future wording for the same condition", - "details": []map[string]any{ - { - "@type": "type.googleapis.com/google.rpc.ErrorInfo", - "reason": "WORKSPACE_OBJECT_TYPE_MISMATCH", - "domain": "workspace.databricks.com", - "metadata": map[string]string{"existing_type": "NOTEBOOK"}, - }, - }, - }) - var target fileAlreadyExistsError - assert.ErrorAs(t, err, &target) -} - -// Confirms an ErrorInfo carrying some other reason is not mistaken for a path -// collision, so the reason check above cannot swallow unrelated failures. -func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { - err := writeWithImportError(t, map[string]any{ - "error_code": "INVALID_PARAMETER_VALUE", - "message": "some other validation failure", - "details": []map[string]any{ - { - "@type": "type.googleapis.com/google.rpc.ErrorInfo", - "reason": "SOME_OTHER_REASON", - "domain": "workspace.databricks.com", - }, - }, - }) - var aerr *apierr.APIError - require.ErrorAs(t, err, &aerr) - assert.Equal(t, http.StatusBadRequest, aerr.StatusCode) -} - -// Confirms a workspace get-status payload unmarshals into wsfsFileInfo and that -// the fs.FileInfo methods derived from it report the expected values. func TestWorkspaceFilesClient_wsfsUnmarshal(t *testing.T) { payload := ` { @@ -352,8 +163,6 @@ func statWithError(t *testing.T, statusCode int, errorCode string) error { return err } -// Confirms a 403 from get-status surfaces as an APIError rather than being -// remapped, so callers can distinguish it from a missing file. func TestWorkspaceFilesClientStatForbidden(t *testing.T) { err := statWithError(t, 403, "PERMISSION_DENIED") var apiErr *apierr.APIError @@ -361,8 +170,6 @@ func TestWorkspaceFilesClientStatForbidden(t *testing.T) { assert.Equal(t, 403, apiErr.StatusCode) } -// Confirms a 500 from get-status surfaces as an APIError and is not treated as -// a missing file. func TestWorkspaceFilesClientStatInternalError(t *testing.T) { err := statWithError(t, 500, "INTERNAL_ERROR") var apiErr *apierr.APIError @@ -370,8 +177,6 @@ func TestWorkspaceFilesClientStatInternalError(t *testing.T) { assert.Equal(t, 500, apiErr.StatusCode) } -// Confirms RESOURCE_DOES_NOT_EXIST maps to fs.ErrNotExist so the filer plugs -// into the io/fs error conventions callers check against. func TestWorkspaceFilesClientStatNotFound(t *testing.T) { err := statWithError(t, 404, "RESOURCE_DOES_NOT_EXIST") assert.ErrorIs(t, err, fs.ErrNotExist) diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go index fa7ff0baceb..d3bb57662d9 100644 --- a/libs/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -10,10 +10,7 @@ import ( "golang.org/x/sync/errgroup" ) -// Maximum number of concurrent requests during sync. Chosen against the -// per-endpoint rate limits documented at -// https://docs.databricks.com/aws/en/resources/limits ("API rate limits" - -// Workspace API), which sync stays within for the endpoints it calls. +// Maximum number of concurrent requests during sync. const MaxRequestsInFlight = 20 // Delete the specified path. diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 647bd4c38cd..dd5050f6f49 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "path" - "path/filepath" "reflect" "slices" "strconv" @@ -974,37 +973,181 @@ func (s *FakeWorkspace) hasChildren(dirPath string) bool { return false } -// detectNotebookLanguage mirrors the real /workspace/import format=AUTO logic: -// a file whose extension is .py / .sql / .scala / .r and whose content starts -// with the language-appropriate "Databricks notebook source" line-comment is -// stored as a NOTEBOOK with the corresponding language; otherwise it's a FILE. -// -// The extension is lowercased because the real endpoint matches it -// case-insensitively, and ".R" is the conventional spelling for R sources. -func detectNotebookLanguage(extension string, body []byte) (workspace.Language, bool) { - switch strings.ToLower(extension) { - case ".py": - if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { - return workspace.LanguagePython, true +// notebookSourceMarker is the text that makes the workspace import a source +// file as a notebook. +const notebookSourceMarker = "Databricks notebook source" + +// notebookDetectionLength is how many leading bytes the workspace reads to +// classify a source file (NOTEBOOK_IDENTIFIER_BUFFER_LENGTH in universe). +const notebookDetectionLength = 200 + +// Source notebook languages by extension. The workspace matches the extension +// case-insensitively, so ".R", the conventional spelling for R sources, counts too. +var sourceNotebookLanguages = map[string]workspace.Language{ + ".py": workspace.LanguagePython, + ".sql": workspace.LanguageSql, + ".scala": workspace.LanguageScala, + ".r": workspace.LanguageR, +} + +// notebookDetector reports whether body uploaded at filePath is imported as a +// notebook, and in which language. +type notebookDetector func(filePath string, body []byte) (workspace.Language, bool) + +// detectLegacyNotebook mirrors the legacy import-file endpoint, which looks for +// the marker on the first line only (NotebookUtils.checkSourceFileNotebook in +// universe). +func detectLegacyNotebook(filePath string, body []byte) (workspace.Language, bool) { + language, ok := sourceNotebookLanguages[strings.ToLower(path.Ext(filePath))] + firstLine, _, _ := bytes.Cut(notebookDetectionPrefix(body), []byte("\n")) + return language, ok && bytes.Contains(firstLine, []byte(notebookSourceMarker)) +} + +// detectAutoNotebook mirrors /workspace/import with format=AUTO, which finds the +// marker on any line of the leading bytes. +func detectAutoNotebook(filePath string, body []byte) (workspace.Language, bool) { + language, ok := sourceNotebookLanguages[strings.ToLower(path.Ext(filePath))] + return language, ok && bytes.Contains(notebookDetectionPrefix(body), []byte(notebookSourceMarker)) +} + +func notebookDetectionPrefix(body []byte) []byte { + return body[:min(len(body), notebookDetectionLength)] +} + +// notebookSourceName returns the name a notebook reserves next to it: SQL +// notebook "x" reserves "x.sql", so a file "x.sql" and notebook "x" can't coexist. +func notebookSourceName(notebookPath string, language workspace.Language) string { + for ext, l := range sourceNotebookLanguages { + if l == language { + return notebookPath + ext + } + } + return "" +} + +// importConflicts holds an endpoint's responses for an import that would +// replace an object of another type. Overwrite never changes an object's type. +type importConflicts struct { + // nameTaken is returned when a notebook's source name is taken by a file or directory. + nameTaken func(requestPath, name string) Response + // notebookTaken is returned when a file's path is an existing notebook's source name. + notebookTaken func(filePath string) Response +} + +var workspaceImportConflicts = importConflicts{ + nameTaken: workspaceImportNameTaken, + notebookTaken: workspaceImportNotebookTaken, +} + +func workspaceImportNameTaken(_, name string) Response { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Node named '%s' already exists", name), + }, + } +} + +func workspaceImportNotebookTaken(string) Response { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]any{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Requested node type [FILE] is different from the existing node type [NOTEBOOK]", + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "WORKSPACE_OBJECT_TYPE_MISMATCH", + "domain": "workspace.databricks.com", + }, + }, + }, + } +} + +var importFileConflicts = importConflicts{ + nameTaken: importFileNameTaken, + notebookTaken: importFileNotebookTaken, +} + +// importFileNameTaken wraps the notebook import error the way the legacy +// endpoint does: in the message, without an error code. +func importFileNameTaken(requestPath, name string) Response { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "message": fmt.Sprintf("Request failed for POST %s: RESOURCE_ALREADY_EXISTS: Node named '%s' already exists", requestPath, name), + }, + } +} + +func importFileNotebookTaken(filePath string) Response { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "message": fmt.Sprintf("Creating file failed. An item with path %s already exists", filePath), + }, + } +} + +// WorkspaceFilesImportFile mirrors the legacy import-file endpoint. +func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, overwrite bool) Response { + // The endpoint matches this extension case-sensitively. + if path.Ext(filePath) == ".zip" { + return s.importFileZip(filePath, body, overwrite) + } + return s.importFile(filePath, body, overwrite, detectLegacyNotebook, importFileConflicts) +} + +// importFileZip mirrors import-file for a .zip (importZipFile in universe): each +// entry is imported into the archive's parent folder, the archive itself is not +// stored, and directories must have their own entry. +func (s *FakeWorkspace) importFileZip(zipPath string, body []byte, overwrite bool) Response { + if !strings.HasPrefix(zipPath, "/") { + zipPath = "/" + zipPath + } + + zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{"message": "Failed to unzip with error: " + err.Error()}, + } + } + + parent := path.Dir(zipPath) + for _, entry := range zr.File { + // macOS archives carry an extra __MACOSX directory. + if strings.HasPrefix(entry.Name, "__MACOSX") { + continue + } + entryPath := path.Join(parent, entry.Name) + if entry.FileInfo().IsDir() { + s.WorkspaceMkdirs(workspace.Mkdirs{Path: entryPath}) + continue } - case ".sql": - if bytes.HasPrefix(body, []byte("-- Databricks notebook source")) { - return workspace.LanguageSql, true + if strings.HasSuffix(entry.Name, ".zip") { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{"message": fmt.Sprintf("Request failed for POST %s: don't support importing nested zip files", zipPath)}, + } } - case ".scala": - if bytes.HasPrefix(body, []byte("// Databricks notebook source")) { - return workspace.LanguageScala, true + data, err := readZipEntry(entry) + if err != nil { + return Response{StatusCode: http.StatusBadRequest, Body: map[string]string{"message": err.Error()}} } - case ".r": - if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { - return workspace.LanguageR, true + if resp := s.importFile(entryPath, data, overwrite, detectLegacyNotebook, importFileConflicts); resp.StatusCode != 0 { + return resp } } - return "", false + return Response{} } -func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, overwrite bool) Response { - return s.importFile(filePath, body, overwrite, true) +// WorkspaceImportAutoFile mirrors /workspace/import with format=AUTO for a +// file that is not an archive or another asset type. +func (s *FakeWorkspace) WorkspaceImportAutoFile(filePath string, body []byte, overwrite bool) Response { + return s.importFile(filePath, body, overwrite, detectAutoNotebook, workspaceImportConflicts) } // WorkspaceImportZipFolder mirrors /workspace/import with format=AUTO for a .zip path @@ -1082,7 +1225,7 @@ func (s *FakeWorkspace) WorkspaceImportZipFolder(zipPath string, body []byte, ov if err != nil { return Response{StatusCode: http.StatusBadRequest, Body: map[string]string{"message": err.Error()}} } - if resp := s.importFile(entryPath, data, false, true); resp.StatusCode != 0 { + if resp := s.importFile(entryPath, data, false, detectAutoNotebook, workspaceImportConflicts); resp.StatusCode != 0 { return resp } } @@ -1101,64 +1244,92 @@ func readZipEntry(entry *zip.File) ([]byte, error) { // WorkspaceImportRawFile stores body as a file without notebook detection, as // /workspace/import does for format=RAW. func (s *FakeWorkspace) WorkspaceImportRawFile(filePath string, body []byte, overwrite bool) Response { - return s.importFile(filePath, body, overwrite, false) + return s.importFile(filePath, body, overwrite, nil, workspaceImportConflicts) } -func (s *FakeWorkspace) importFile(filePath string, body []byte, overwrite, detectNotebooks bool) Response { +// importFile stores body at filePath, as a notebook if detect says so. A nil +// detect always stores a file. +func (s *FakeWorkspace) importFile(filePath string, body []byte, overwrite bool, detect notebookDetector, conflicts importConflicts) Response { if !strings.HasPrefix(filePath, "/") { filePath = "/" + filePath } defer s.LockUnlock()() - workspacePath := filePath - - if resp, ok := s.requireParentDirectory(workspacePath); !ok { + if resp, ok := s.requireParentDirectory(filePath); !ok { return resp } if !overwrite { - if _, exists := s.files[workspacePath]; exists { + if _, exists := s.files[filePath]; exists { return Response{ StatusCode: 409, - Body: map[string]string{"message": fmt.Sprintf("File already exists at (%s).", workspacePath)}, + Body: map[string]string{"message": fmt.Sprintf("File already exists at (%s).", filePath)}, } } } - // Files with .py / .sql / .scala / .r extension can be notebooks if they - // carry the "Databricks notebook source" header comment in the language's - // line-comment syntax. Mirror the real workspace's auto-detection here so - // acceptance tests can assert the resulting object_type via /workspace/list. - extension := filepath.Ext(filePath) - if lang, isNotebook := detectNotebookLanguage(extension, body); detectNotebooks && isNotebook { - // Notebooks are stripped of their extension by the workspace import API. - workspacePath = strings.TrimSuffix(filePath, extension) - s.files[workspacePath] = FileEntry{ - Info: workspace.ObjectInfo{ - ObjectType: "NOTEBOOK", - Path: workspacePath, - Language: lang, - ObjectId: nextTreenodeID(), - }, - Data: body, - } - } else { - // The endpoint does not set language for files, so we omit that - // here as well. - // ref: https://docs.databricks.com/api/workspace/workspace/getstatus#language - s.files[workspacePath] = FileEntry{ - Info: workspace.ObjectInfo{ - ObjectType: "FILE", - Path: workspacePath, - ObjectId: nextTreenodeID(), - }, - Data: body, + if detect != nil { + if language, isNotebook := detect(filePath, body); isNotebook { + // Notebooks are stripped of their extension by the workspace import API. + notebookPath := strings.TrimSuffix(filePath, path.Ext(filePath)) + return s.storeNotebook(filePath, notebookPath, body, language, conflicts) } } + + if s.isNotebookSourceName(filePath) { + return conflicts.notebookTaken(filePath) + } + + // The endpoint does not set language for files, so we omit that + // here as well. + // ref: https://docs.databricks.com/api/workspace/workspace/getstatus#language + s.files[filePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: filePath, + ObjectId: nextTreenodeID(), + }, + Data: body, + } + return Response{} +} + +// storeNotebook stores a notebook at notebookPath unless a file or directory +// holds its source name. The caller must hold the lock. +func (s *FakeWorkspace) storeNotebook(requestPath, notebookPath string, body []byte, language workspace.Language, conflicts importConflicts) Response { + sourceName := notebookSourceName(notebookPath, language) + _, isFile := s.files[sourceName] + _, isDir := s.directories[sourceName] + if isFile || isDir { + return conflicts.nameTaken(requestPath, path.Base(sourceName)) + } + + s.files[notebookPath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "NOTEBOOK", + Path: notebookPath, + Language: language, + ObjectId: nextTreenodeID(), + }, + Data: body, + } return Response{} } +// isNotebookSourceName reports whether filePath is the source name of an +// existing notebook. The caller must hold the lock. +func (s *FakeWorkspace) isNotebookSourceName(filePath string) bool { + ext := path.Ext(filePath) + if ext == "" { + return false + } + notebookPath := strings.TrimSuffix(filePath, ext) + entry, ok := s.files[notebookPath] + return ok && entry.Info.ObjectType == workspace.ObjectTypeNotebook && + strings.EqualFold(notebookSourceName(notebookPath, entry.Info.Language), filePath) +} + // requireParentDirectory returns a 404 response when objectPath's parent // directory does not exist. The real import API does not create missing parents; // callers get "mkdir -p" semantics only by first calling /workspace/mkdirs (see @@ -1203,16 +1374,7 @@ func (s *FakeWorkspace) WorkspaceImportNotebook(filePath string, body []byte, la } } - s.files[filePath] = FileEntry{ - Info: workspace.ObjectInfo{ - ObjectType: "NOTEBOOK", - Path: filePath, - Language: language, - ObjectId: nextTreenodeID(), - }, - Data: body, - } - return Response{} + return s.storeNotebook(filePath, filePath, body, language, workspaceImportConflicts) } func (s *FakeWorkspace) WorkspaceFilesExportFile(path string) []byte { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 73839619c3a..50ff904a573 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -92,7 +92,7 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { if unsupported, ok := autoImportUnsupported(form.Path); ok { return unsupported } - resp = w.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + resp = w.WorkspaceImportAutoFile(form.Path, form.Content, form.Overwrite) default: return Response{ Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW, SOURCE and JUPYTER formats, got %q.", form.Format),