From c7d28bc1f9c561953c8b2592cc63e8ac9e26017b Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 12:33:47 +0000 Subject: [PATCH 01/16] acceptance: capture notebook-vs-file classification gaps behind #7216 Since v1.21 the CLI decides itself whether a source file is a notebook (exact first-line header). The legacy import-file endpoint used up to v1.20 imports a file as a notebook if its first line contains "Databricks notebook source", and always imports .Rmd as an R notebook. Files where the two disagree break deployments that switch between CLI versions, which is the error reported in #7216. Add acceptance tests, with Badness, for the CLI side: - near-miss headers (BOM, trailing space, ...) are uploaded as files - .Rmd is uploaded as a file - syncing over a legacy-created notebook fails with "file already exists" - the legacy endpoint fails over a file synced by this CLI with RESOURCE_ALREADY_EXISTS: Node named '...' already exists and a Cloud test documenting /workspace/import: AUTO's 200-byte, any-line marker rule, R_MARKDOWN, and that overwrite never changes an object's type. The testserver now mirrors the legacy and AUTO classification rules, the R_MARKDOWN format and the collision responses observed on a real workspace. Co-authored-by: Isaac --- .../downgrade-over-file/bom.sql | 2 + .../downgrade-over-file/databricks.yml | 12 + .../downgrade-over-file/out.test.toml | 2 + .../downgrade-over-file/output.txt | 14 + .../downgrade-over-file/script | 8 + .../downgrade-over-file/test.toml | 1 + .../near-miss-header/bom.scala | 2 + .../near-miss-header/bom.sql | 2 + .../near-miss-header/databricks.yml | 11 + .../near-miss-header/double_hash.py | 2 + .../near-miss-header/exact.sql | 2 + .../near-miss-header/hash_comment.sql | 2 + .../near-miss-header/leading_tab.py | 2 + .../near-miss-header/no_comment.py | 2 + .../near-miss-header/no_space.sql | 2 + .../near-miss-header/out.test.toml | 2 + .../near-miss-header/output.txt | 48 ++++ .../near-miss-header/script | 13 + .../near-miss-header/second_line.py | 3 + .../near-miss-header/suffix.sql | 2 + .../near-miss-header/test.toml | 2 + .../near-miss-header/trailing_space.sql | 2 + .../rmd/databricks.yml | 11 + .../sync-notebook-detection/rmd/doc.Rmd | 6 + .../sync-notebook-detection/rmd/lower.rmd | 6 + .../sync-notebook-detection/rmd/out.test.toml | 2 + .../sync-notebook-detection/rmd/output.txt | 21 ++ .../bundle/sync-notebook-detection/rmd/script | 11 + .../sync-notebook-detection/rmd/test.toml | 2 + .../sync-notebook-detection/script.prepare | 29 ++ .../bundle/sync-notebook-detection/test.toml | 4 + .../upgrade-over-legacy-notebook/bom.sql | 2 + .../databricks.yml | 12 + .../out.test.toml | 2 + .../upgrade-over-legacy-notebook/output.txt | 13 + .../upgrade-over-legacy-notebook/script | 8 + .../upgrade-over-legacy-notebook/test.toml | 1 + .../import-notebook-detection/bom.sql | 2 + .../import-notebook-detection/doc.Rmd | 6 + .../import-notebook-detection/exact.sql | 2 + .../marker_past_200_bytes.py | 2 + .../import-notebook-detection/out.test.toml | 1 + .../import-notebook-detection/output.txt | 66 +++++ .../import-notebook-detection/plain.sql | 1 + .../import-notebook-detection/script | 54 ++++ .../import-notebook-detection/second_line.py | 3 + .../import-notebook-detection/test.toml | 4 + libs/testserver/fake_workspace.go | 264 +++++++++++++----- libs/testserver/handlers.go | 6 +- ruff.toml | 4 + 50 files changed, 608 insertions(+), 75 deletions(-) create mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/bom.sql create mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/databricks.yml create mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/out.test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/output.txt create mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/script create mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/bom.scala create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/bom.sql create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/databricks.yml create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/double_hash.py create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/exact.sql create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/hash_comment.sql create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/leading_tab.py create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/no_comment.py create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/no_space.sql create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/out.test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/output.txt create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/script create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/second_line.py create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/suffix.sql create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/trailing_space.sql create mode 100644 acceptance/bundle/sync-notebook-detection/rmd/databricks.yml create mode 100644 acceptance/bundle/sync-notebook-detection/rmd/doc.Rmd create mode 100644 acceptance/bundle/sync-notebook-detection/rmd/lower.rmd create mode 100644 acceptance/bundle/sync-notebook-detection/rmd/out.test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/rmd/output.txt create mode 100644 acceptance/bundle/sync-notebook-detection/rmd/script create mode 100644 acceptance/bundle/sync-notebook-detection/rmd/test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/script.prepare create mode 100644 acceptance/bundle/sync-notebook-detection/test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/bom.sql create mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/databricks.yml create mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/out.test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/output.txt create mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/script create mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/test.toml create mode 100644 acceptance/cmd/workspace/import-notebook-detection/bom.sql create mode 100644 acceptance/cmd/workspace/import-notebook-detection/doc.Rmd create mode 100644 acceptance/cmd/workspace/import-notebook-detection/exact.sql create mode 100644 acceptance/cmd/workspace/import-notebook-detection/marker_past_200_bytes.py create mode 100644 acceptance/cmd/workspace/import-notebook-detection/out.test.toml create mode 100644 acceptance/cmd/workspace/import-notebook-detection/output.txt create mode 100644 acceptance/cmd/workspace/import-notebook-detection/plain.sql create mode 100644 acceptance/cmd/workspace/import-notebook-detection/script create mode 100644 acceptance/cmd/workspace/import-notebook-detection/second_line.py create mode 100644 acceptance/cmd/workspace/import-notebook-detection/test.toml diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/bom.sql b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/bom.sql new file mode 100644 index 00000000000..4aeddae0fd3 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/bom.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/databricks.yml b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/databricks.yml new file mode 100644 index 00000000000..652152d00d6 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: sync-notebook-detection-downgrade-over-file + +# Only sync the committed inputs, not the test harness files. +sync: + exclude: + - script + - test.toml + - output.txt + - out.* + - repls.json + - databricks.yml diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/out.test.toml b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/output.txt b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/output.txt new file mode 100644 index 00000000000..7ca5fc15cc8 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/output.txt @@ -0,0 +1,14 @@ + +>>> [CLI] bundle sync --output text +Action: PUT: bom.sql +Initial Sync Complete +Uploaded bom.sql + +>>> remote_objects +FILE - bom.sql + +>>> musterr legacy_import_bom +Error: Request failed for POST /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection-downgrade-over-file/default/files/bom.sql: RESOURCE_ALREADY_EXISTS: Node named 'bom.sql' already exists + +>>> remote_objects +FILE - bom.sql diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/script b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/script new file mode 100644 index 00000000000..907d15619ac --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/script @@ -0,0 +1,8 @@ +# bom.sql starts with a UTF-8 byte order mark. This CLI uploads it as file bom.sql; the +# legacy import-file endpoint then tries to import it as notebook "bom", whose name the +# file already holds. +trace $CLI bundle sync --output text 2>&1 | sort +trace remote_objects + +trace musterr legacy_import_bom +trace remote_objects diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/test.toml b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/test.toml new file mode 100644 index 00000000000..5f0ffaf3ca9 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/test.toml @@ -0,0 +1 @@ +Badness = "After this CLI uploads a file it no longer detects as a notebook, v1.20 or earlier fails to upload it with RESOURCE_ALREADY_EXISTS: Node named '...' already exists, the error reported in #7216." diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.scala b/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.scala new file mode 100644 index 00000000000..b8ee8bff261 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.scala @@ -0,0 +1,2 @@ +// Databricks notebook source +println(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.sql new file mode 100644 index 00000000000..4aeddae0fd3 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/databricks.yml b/acceptance/bundle/sync-notebook-detection/near-miss-header/databricks.yml new file mode 100644 index 00000000000..580b5fa8072 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: sync-notebook-detection-near-miss-header + +# Only sync the committed inputs, not the test harness files. +sync: + exclude: + - script + - test.toml + - output.txt + - out.* + - repls.json diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/double_hash.py b/acceptance/bundle/sync-notebook-detection/near-miss-header/double_hash.py new file mode 100644 index 00000000000..9355a12f1bd --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/double_hash.py @@ -0,0 +1,2 @@ +## Databricks notebook source +print(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/exact.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/exact.sql new file mode 100644 index 00000000000..3b831911443 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/exact.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/hash_comment.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/hash_comment.sql new file mode 100644 index 00000000000..1f26f31b652 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/hash_comment.sql @@ -0,0 +1,2 @@ +# Databricks notebook source +SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/leading_tab.py b/acceptance/bundle/sync-notebook-detection/near-miss-header/leading_tab.py new file mode 100644 index 00000000000..b2c5cfb6442 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/leading_tab.py @@ -0,0 +1,2 @@ + # Databricks notebook source +print(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/no_comment.py b/acceptance/bundle/sync-notebook-detection/near-miss-header/no_comment.py new file mode 100644 index 00000000000..d4b4a6fb946 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/no_comment.py @@ -0,0 +1,2 @@ +Databricks notebook source +print(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/no_space.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/no_space.sql new file mode 100644 index 00000000000..eb5f005861f --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/no_space.sql @@ -0,0 +1,2 @@ +--Databricks notebook source +SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/out.test.toml b/acceptance/bundle/sync-notebook-detection/near-miss-header/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/output.txt b/acceptance/bundle/sync-notebook-detection/near-miss-header/output.txt new file mode 100644 index 00000000000..f79fa808753 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/output.txt @@ -0,0 +1,48 @@ + +>>> [CLI] bundle sync --output text +Action: PUT: bom.scala, bom.sql, databricks.yml, double_hash.py, exact.sql, hash_comment.sql, leading_tab.py, no_comment.py, no_space.sql, second_line.py, suffix.sql, trailing_space.sql +Initial Sync Complete +Uploaded bom.scala +Uploaded bom.sql +Uploaded databricks.yml +Uploaded double_hash.py +Uploaded exact.sql +Uploaded hash_comment.sql +Uploaded leading_tab.py +Uploaded no_comment.py +Uploaded no_space.sql +Uploaded second_line.py +Uploaded suffix.sql +Uploaded trailing_space.sql + +=== Format per upload: SOURCE only for the exact header + +>>> upload_formats +RAW bom.scala +RAW bom.sql +RAW databricks.yml +RAW double_hash.py +SOURCE exact +RAW hash_comment.sql +RAW leading_tab.py +RAW no_comment.py +RAW no_space.sql +RAW second_line.py +RAW suffix.sql +RAW trailing_space.sql + +=== Synced objects + +>>> remote_objects +FILE - bom.scala +FILE - bom.sql +FILE - databricks.yml +FILE - double_hash.py +NOTEBOOK SQL exact +FILE - hash_comment.sql +FILE - leading_tab.py +FILE - no_comment.py +FILE - no_space.sql +FILE - second_line.py +FILE - suffix.sql +FILE - trailing_space.sql diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/script b/acceptance/bundle/sync-notebook-detection/near-miss-header/script new file mode 100644 index 00000000000..a4ff5e843de --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/script @@ -0,0 +1,13 @@ +# bom.sql and bom.scala start with a UTF-8 byte order mark, trailing_space.sql ends its +# first line with a space and leading_tab.py starts with a tab. The legacy import-file +# endpoint imported all of these as notebooks, except second_line.py, whose marker is not +# on the first line. exact.sql is a notebook for both. +trace $CLI bundle sync --output text 2>&1 | sort + +title "Format per upload: SOURCE only for the exact header\n" +trace upload_formats + +title "Synced objects\n" +trace remote_objects + +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/second_line.py b/acceptance/bundle/sync-notebook-detection/near-miss-header/second_line.py new file mode 100644 index 00000000000..cbcf42a24f9 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/second_line.py @@ -0,0 +1,3 @@ + +# Databricks notebook source +print(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/suffix.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/suffix.sql new file mode 100644 index 00000000000..815c08eac2a --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/suffix.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source (daily) +SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/test.toml b/acceptance/bundle/sync-notebook-detection/near-miss-header/test.toml new file mode 100644 index 00000000000..07deac45c31 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/test.toml @@ -0,0 +1,2 @@ +Badness = "Source files whose first line contains \"Databricks notebook source\" but is not the exact header are uploaded with format=RAW and stored as files. v1.20 and earlier imported them as notebooks, so deploying the same bundle with both fails (#7216)." +RecordRequests = true diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/trailing_space.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/trailing_space.sql new file mode 100644 index 00000000000..71a808d64a2 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/near-miss-header/trailing_space.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/rmd/databricks.yml b/acceptance/bundle/sync-notebook-detection/rmd/databricks.yml new file mode 100644 index 00000000000..a91aecc8086 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/rmd/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: sync-notebook-detection-rmd + +# Only sync the committed inputs, not the test harness files. +sync: + exclude: + - script + - test.toml + - output.txt + - out.* + - repls.json diff --git a/acceptance/bundle/sync-notebook-detection/rmd/doc.Rmd b/acceptance/bundle/sync-notebook-detection/rmd/doc.Rmd new file mode 100644 index 00000000000..16cf3fe51cc --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/rmd/doc.Rmd @@ -0,0 +1,6 @@ +--- +title: doc +--- +```{r} +1 +``` diff --git a/acceptance/bundle/sync-notebook-detection/rmd/lower.rmd b/acceptance/bundle/sync-notebook-detection/rmd/lower.rmd new file mode 100644 index 00000000000..16cf3fe51cc --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/rmd/lower.rmd @@ -0,0 +1,6 @@ +--- +title: doc +--- +```{r} +1 +``` diff --git a/acceptance/bundle/sync-notebook-detection/rmd/out.test.toml b/acceptance/bundle/sync-notebook-detection/rmd/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/rmd/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-notebook-detection/rmd/output.txt b/acceptance/bundle/sync-notebook-detection/rmd/output.txt new file mode 100644 index 00000000000..b72ce123dcd --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/rmd/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle sync --output text +Action: PUT: databricks.yml, doc.Rmd, lower.rmd +Initial Sync Complete +Uploaded databricks.yml +Uploaded doc.Rmd +Uploaded lower.rmd + +=== Format per upload + +>>> upload_formats +RAW databricks.yml +RAW doc.Rmd +RAW lower.rmd + +=== Synced objects + +>>> remote_objects +FILE - databricks.yml +FILE - doc.Rmd +FILE - lower.rmd diff --git a/acceptance/bundle/sync-notebook-detection/rmd/script b/acceptance/bundle/sync-notebook-detection/rmd/script new file mode 100644 index 00000000000..150f77b7774 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/rmd/script @@ -0,0 +1,11 @@ +# The legacy import-file endpoint imported doc.Rmd as R notebook "doc". It matched the +# extension case-sensitively, so lower.rmd was a file there too. +trace $CLI bundle sync --output text 2>&1 | sort + +title "Format per upload\n" +trace upload_formats + +title "Synced objects\n" +trace remote_objects + +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/sync-notebook-detection/rmd/test.toml b/acceptance/bundle/sync-notebook-detection/rmd/test.toml new file mode 100644 index 00000000000..2280fc8f5de --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/rmd/test.toml @@ -0,0 +1,2 @@ +Badness = "R Markdown (.Rmd) files are uploaded with format=RAW and stored as files. v1.20 and earlier imported them as R notebooks." +RecordRequests = true diff --git a/acceptance/bundle/sync-notebook-detection/script.prepare b/acceptance/bundle/sync-notebook-detection/script.prepare new file mode 100644 index 00000000000..d6f2bda231d --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/script.prepare @@ -0,0 +1,29 @@ +# Since v1.21 the CLI decides itself whether a source file is a notebook (libs/notebook): +# the first line must be exactly the header for the file's language. Up to v1.20 it +# uploaded through the legacy import-file endpoint, which instead imports a file as a +# notebook if its first line contains "Databricks notebook source" (and always imports +# .Rmd as an R notebook). The tests below capture where the two disagree. + +remote_files() { + $CLI bundle summary -o json | jq -r .workspace.file_path +} + +# Lists the objects synced to the bundle's file path. +remote_objects() { + $CLI workspace list -o json "$(remote_files)" | jq -r 'sort_by(.path)[] | "\(.object_type) \(.language // "-") \(.path | sub(".*/"; ""))"' +} + +# Prints the /workspace/import format the CLI used for each synced file. +upload_formats() { + jq -rs ' + map(select(.path == "/api/2.0/workspace/import") | .body.multipart_form) | + sort_by(.path) | .[] | "\(.format) \(.path | sub(".*/"; ""))" + ' "$OUT_REQUESTS" +} + +# Uploads bom.sql the way v1.20 and earlier did, through the legacy import-file endpoint. +# `api post` only sends JSON bodies; a JSON string whose first line contains the marker +# takes the same notebook path as the file itself. +legacy_import_bom() { + $CLI api post "/api/2.0/workspace-files/import-file$(remote_files)/bom.sql?overwrite=true" --json '"-- Databricks notebook source"' +} diff --git a/acceptance/bundle/sync-notebook-detection/test.toml b/acceptance/bundle/sync-notebook-detection/test.toml new file mode 100644 index 00000000000..d02f0e2c468 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/test.toml @@ -0,0 +1,4 @@ +# Workspace paths start with "/"; keep Git Bash on Windows from rewriting them. +Env.MSYS_NO_PATHCONV = "1" + +Ignore = [".databricks"] diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/bom.sql b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/bom.sql new file mode 100644 index 00000000000..4aeddae0fd3 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/bom.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/databricks.yml b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/databricks.yml new file mode 100644 index 00000000000..f71ed3dc3ae --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: sync-notebook-detection-upgrade-over-legacy-notebook + +# Only sync the committed inputs, not the test harness files. +sync: + exclude: + - script + - test.toml + - output.txt + - out.* + - repls.json + - databricks.yml diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/out.test.toml b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/output.txt b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/output.txt new file mode 100644 index 00000000000..1a4d637b8a3 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/output.txt @@ -0,0 +1,13 @@ + +>>> legacy_import_bom +null + +>>> remote_objects +NOTEBOOK SQL bom + +>>> musterr [CLI] bundle sync --output text +Action: PUT: bom.sql +Error: file already exists: /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection-upgrade-over-legacy-notebook/default/files/bom.sql + +>>> remote_objects +NOTEBOOK SQL bom diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/script b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/script new file mode 100644 index 00000000000..631e2287885 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/script @@ -0,0 +1,8 @@ +# bom.sql starts with a UTF-8 byte order mark. v1.20 and earlier imported it as notebook +# "bom"; this CLI uploads it as file bom.sql, which the notebook already holds the name of. +$CLI workspace mkdirs "$(remote_files)" +trace legacy_import_bom +trace remote_objects + +trace musterr $CLI bundle sync --output text +trace remote_objects diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/test.toml b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/test.toml new file mode 100644 index 00000000000..c8917d1e454 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/test.toml @@ -0,0 +1 @@ +Badness = "Syncing over a notebook that v1.20 or earlier created fails with \"file already exists\" when the CLI no longer detects the file as a notebook (#7216)." diff --git a/acceptance/cmd/workspace/import-notebook-detection/bom.sql b/acceptance/cmd/workspace/import-notebook-detection/bom.sql new file mode 100644 index 00000000000..4aeddae0fd3 --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/bom.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +SELECT 1 diff --git a/acceptance/cmd/workspace/import-notebook-detection/doc.Rmd b/acceptance/cmd/workspace/import-notebook-detection/doc.Rmd new file mode 100644 index 00000000000..16cf3fe51cc --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/doc.Rmd @@ -0,0 +1,6 @@ +--- +title: doc +--- +```{r} +1 +``` diff --git a/acceptance/cmd/workspace/import-notebook-detection/exact.sql b/acceptance/cmd/workspace/import-notebook-detection/exact.sql new file mode 100644 index 00000000000..3b831911443 --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/exact.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +SELECT 1 diff --git a/acceptance/cmd/workspace/import-notebook-detection/marker_past_200_bytes.py b/acceptance/cmd/workspace/import-notebook-detection/marker_past_200_bytes.py new file mode 100644 index 00000000000..57b44dea5dc --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/marker_past_200_bytes.py @@ -0,0 +1,2 @@ +######################################################################################################################################################################################################## Databricks notebook source +print(1) diff --git a/acceptance/cmd/workspace/import-notebook-detection/out.test.toml b/acceptance/cmd/workspace/import-notebook-detection/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/workspace/import-notebook-detection/output.txt b/acceptance/cmd/workspace/import-notebook-detection/output.txt new file mode 100644 index 00000000000..b8bcc68fd81 --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/output.txt @@ -0,0 +1,66 @@ + +=== AUTO imports a source file as a notebook if "Databricks notebook source" is anywhere in its first 200 bytes + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/exact.sql --file exact.sql --format AUTO + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/bom.sql --file bom.sql --format AUTO + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/second_line.py --file second_line.py --format AUTO + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/marker_past_200_bytes.py --file marker_past_200_bytes.py --format AUTO + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/doc.Rmd --file doc.Rmd --format AUTO + +>>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto +NOTEBOOK SQL /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/bom +FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/doc.Rmd +NOTEBOOK SQL /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/exact +FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/marker_past_200_bytes.py +NOTEBOOK PYTHON /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/second_line + +=== R_MARKDOWN imports R Markdown as an R notebook at the given path + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/rmd/doc --file doc.Rmd --format R_MARKDOWN + +>>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/rmd +NOTEBOOK R /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/rmd/doc + +=== Overwrite replaces an object of the same type + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/exact.sql --file exact.sql --format AUTO + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/exact.sql --file exact.sql --format AUTO --overwrite + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/plain.sql --file plain.sql --format RAW + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/plain.sql --file plain.sql --format RAW --overwrite + +>>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same +NOTEBOOK SQL /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/exact +FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/plain.sql + +=== Overwrite cannot turn a file into a notebook + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook/x.sql --file plain.sql --format RAW + +>>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook/x.sql --file exact.sql --format AUTO --overwrite +Error: Node named 'x.sql' already exists + +>>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook/x --file exact.sql --format SOURCE --language SQL --overwrite +Error: Node named 'x.sql' already exists + +>>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook +FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook/x.sql + +=== Overwrite cannot turn a notebook into a file + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x --file exact.sql --format SOURCE --language SQL + +>>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x.sql --file plain.sql --format RAW --overwrite +Error: Requested node type [FILE] is different from the existing node type [NOTEBOOK] + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x.py --file plain.sql --format RAW --overwrite + +>>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file +NOTEBOOK SQL /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x +FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x.py diff --git a/acceptance/cmd/workspace/import-notebook-detection/plain.sql b/acceptance/cmd/workspace/import-notebook-detection/plain.sql new file mode 100644 index 00000000000..2e3761f7a2c --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/plain.sql @@ -0,0 +1 @@ +SELECT 1 diff --git a/acceptance/cmd/workspace/import-notebook-detection/script b/acceptance/cmd/workspace/import-notebook-detection/script new file mode 100644 index 00000000000..6c26a8c9cf8 --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/script @@ -0,0 +1,54 @@ +# Documents how POST /api/2.0/workspace/import classifies source notebooks, and that +# overwrite never changes an existing object's type. +DIR="/Workspace/Users/$CURRENT_USER_NAME/$UNIQUE_NAME" +$CLI workspace mkdirs "$DIR" + +cleanup() { + $CLI workspace delete "$DIR" --recursive +} +trap cleanup EXIT + +list() { + $CLI workspace list "$1" -o json | jq -r 'sort_by(.path)[] | "\(.object_type) \(.language // "-") \(.path)"' +} + +# bom.sql starts with a UTF-8 byte order mark. marker_past_200_bytes.py has the marker +# on its first line, after 200 bytes of "#". + +title "AUTO imports a source file as a notebook if \"Databricks notebook source\" is anywhere in its first 200 bytes\n" +# The legacy import-file endpoint only checks the first line, so second_line.py was a file +# there. AUTO imports R Markdown as a file; import-file imported it as an R notebook. +$CLI workspace mkdirs "$DIR/auto" +for f in exact.sql bom.sql second_line.py marker_past_200_bytes.py doc.Rmd; do + trace $CLI workspace import "$DIR/auto/$f" --file $f --format AUTO +done +trace list "$DIR/auto" + +title "R_MARKDOWN imports R Markdown as an R notebook at the given path\n" +$CLI workspace mkdirs "$DIR/rmd" +trace $CLI workspace import "$DIR/rmd/doc" --file doc.Rmd --format R_MARKDOWN +trace list "$DIR/rmd" + +title "Overwrite replaces an object of the same type\n" +$CLI workspace mkdirs "$DIR/same" +trace $CLI workspace import "$DIR/same/exact.sql" --file exact.sql --format AUTO +trace $CLI workspace import "$DIR/same/exact.sql" --file exact.sql --format AUTO --overwrite +trace $CLI workspace import "$DIR/same/plain.sql" --file plain.sql --format RAW +trace $CLI workspace import "$DIR/same/plain.sql" --file plain.sql --format RAW --overwrite +trace list "$DIR/same" + +title "Overwrite cannot turn a file into a notebook\n" +# Notebook "x" would take the name x.sql, which the file already holds. +$CLI workspace mkdirs "$DIR/to_notebook" +trace $CLI workspace import "$DIR/to_notebook/x.sql" --file plain.sql --format RAW +trace musterr $CLI workspace import "$DIR/to_notebook/x.sql" --file exact.sql --format AUTO --overwrite +trace musterr $CLI workspace import "$DIR/to_notebook/x" --file exact.sql --format SOURCE --language SQL --overwrite +trace list "$DIR/to_notebook" + +title "Overwrite cannot turn a notebook into a file\n" +# SQL notebook "x" holds the name x.sql, but not x.py. +$CLI workspace mkdirs "$DIR/to_file" +trace $CLI workspace import "$DIR/to_file/x" --file exact.sql --format SOURCE --language SQL +trace musterr $CLI workspace import "$DIR/to_file/x.sql" --file plain.sql --format RAW --overwrite +trace $CLI workspace import "$DIR/to_file/x.py" --file plain.sql --format RAW --overwrite +trace list "$DIR/to_file" diff --git a/acceptance/cmd/workspace/import-notebook-detection/second_line.py b/acceptance/cmd/workspace/import-notebook-detection/second_line.py new file mode 100644 index 00000000000..cbcf42a24f9 --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/second_line.py @@ -0,0 +1,3 @@ + +# Databricks notebook source +print(1) diff --git a/acceptance/cmd/workspace/import-notebook-detection/test.toml b/acceptance/cmd/workspace/import-notebook-detection/test.toml new file mode 100644 index 00000000000..6b74aa538f0 --- /dev/null +++ b/acceptance/cmd/workspace/import-notebook-detection/test.toml @@ -0,0 +1,4 @@ +Cloud = true + +# Workspace paths start with "/"; keep Git Bash on Windows from rewriting them. +Env.MSYS_NO_PATHCONV = "1" diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 647bd4c38cd..856f5856fd6 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "path" - "path/filepath" "reflect" "slices" "strconv" @@ -974,37 +973,137 @@ func (s *FakeWorkspace) hasChildren(dirPath string) bool { return false } -// detectNotebookLanguage mirrors the real /workspace/import format=AUTO logic: -// a file whose extension is .py / .sql / .scala / .r and whose content starts -// with the language-appropriate "Databricks notebook source" line-comment is -// stored as a NOTEBOOK with the corresponding language; otherwise it's a FILE. -// -// The extension is lowercased because the real endpoint matches it -// case-insensitively, and ".R" is the conventional spelling for R sources. -func detectNotebookLanguage(extension string, body []byte) (workspace.Language, bool) { - switch strings.ToLower(extension) { - case ".py": - if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { - return workspace.LanguagePython, true - } - case ".sql": - if bytes.HasPrefix(body, []byte("-- Databricks notebook source")) { - return workspace.LanguageSql, true - } - case ".scala": - if bytes.HasPrefix(body, []byte("// Databricks notebook source")) { - return workspace.LanguageScala, true - } - case ".r": - if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { - return workspace.LanguageR, true +// notebookSourceMarker is the text that makes the workspace import a source +// file as a notebook. +const notebookSourceMarker = "Databricks notebook source" + +// notebookDetectionLength is how many leading bytes the workspace reads to +// classify a source file (NOTEBOOK_IDENTIFIER_BUFFER_LENGTH in universe). +const notebookDetectionLength = 200 + +// Source notebook languages by extension. The workspace matches the extension +// case-insensitively, so ".R", the conventional spelling for R sources, counts too. +var sourceNotebookLanguages = map[string]workspace.Language{ + ".py": workspace.LanguagePython, + ".sql": workspace.LanguageSql, + ".scala": workspace.LanguageScala, + ".r": workspace.LanguageR, +} + +// notebookDetector reports whether body uploaded at filePath is imported as a +// notebook, and in which language. +type notebookDetector func(filePath string, body []byte) (workspace.Language, bool) + +// detectLegacyNotebook mirrors the legacy import-file endpoint, which looks for +// the marker on the first line only (NotebookUtils.checkSourceFileNotebook in +// universe) and always imports R Markdown as an R notebook. +func detectLegacyNotebook(filePath string, body []byte) (workspace.Language, bool) { + // The endpoint matches this extension case-sensitively. + if path.Ext(filePath) == ".Rmd" { + return workspace.LanguageR, true + } + language, ok := sourceNotebookLanguages[strings.ToLower(path.Ext(filePath))] + firstLine, _, _ := bytes.Cut(notebookDetectionPrefix(body), []byte("\n")) + return language, ok && bytes.Contains(firstLine, []byte(notebookSourceMarker)) +} + +// detectAutoNotebook mirrors /workspace/import with format=AUTO, which finds the +// marker on any line of the leading bytes and imports R Markdown as a file. +func detectAutoNotebook(filePath string, body []byte) (workspace.Language, bool) { + language, ok := sourceNotebookLanguages[strings.ToLower(path.Ext(filePath))] + return language, ok && bytes.Contains(notebookDetectionPrefix(body), []byte(notebookSourceMarker)) +} + +func notebookDetectionPrefix(body []byte) []byte { + return body[:min(len(body), notebookDetectionLength)] +} + +// notebookSourceName returns the name a notebook reserves next to it: SQL +// notebook "x" reserves "x.sql", so a file "x.sql" and notebook "x" can't coexist. +func notebookSourceName(notebookPath string, language workspace.Language) string { + for ext, l := range sourceNotebookLanguages { + if l == language { + return notebookPath + ext } } - return "", false + return "" +} + +// importConflicts holds an endpoint's responses for an import that would +// replace an object of another type. Overwrite never changes an object's type. +type importConflicts struct { + // nameTaken is returned when a notebook's source name is taken by a file or directory. + nameTaken func(requestPath, name string) Response + // notebookTaken is returned when a file's path is an existing notebook's source name. + notebookTaken func(filePath string) Response +} + +var workspaceImportConflicts = importConflicts{ + nameTaken: workspaceImportNameTaken, + notebookTaken: workspaceImportNotebookTaken, +} + +func workspaceImportNameTaken(_, name string) Response { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Node named '%s' already exists", name), + }, + } +} + +func workspaceImportNotebookTaken(string) Response { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]any{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Requested node type [FILE] is different from the existing node type [NOTEBOOK]", + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "WORKSPACE_OBJECT_TYPE_MISMATCH", + "domain": "workspace.databricks.com", + }, + }, + }, + } +} + +var importFileConflicts = importConflicts{ + nameTaken: importFileNameTaken, + notebookTaken: importFileNotebookTaken, +} + +// importFileNameTaken wraps the notebook import error the way the legacy +// endpoint does: in the message, without an error code. +func importFileNameTaken(requestPath, name string) Response { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "message": fmt.Sprintf("Request failed for POST %s: RESOURCE_ALREADY_EXISTS: Node named '%s' already exists", requestPath, name), + }, + } +} + +func importFileNotebookTaken(filePath string) Response { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "message": fmt.Sprintf("Creating file failed. An item with path %s already exists", filePath), + }, + } } +// WorkspaceFilesImportFile mirrors the legacy import-file endpoint. func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, overwrite bool) Response { - return s.importFile(filePath, body, overwrite, true) + return s.importFile(filePath, body, overwrite, detectLegacyNotebook, importFileConflicts) +} + +// WorkspaceImportAutoFile mirrors /workspace/import with format=AUTO for a +// file that is not an archive or another asset type. +func (s *FakeWorkspace) WorkspaceImportAutoFile(filePath string, body []byte, overwrite bool) Response { + return s.importFile(filePath, body, overwrite, detectAutoNotebook, workspaceImportConflicts) } // WorkspaceImportZipFolder mirrors /workspace/import with format=AUTO for a .zip path @@ -1082,7 +1181,7 @@ func (s *FakeWorkspace) WorkspaceImportZipFolder(zipPath string, body []byte, ov if err != nil { return Response{StatusCode: http.StatusBadRequest, Body: map[string]string{"message": err.Error()}} } - if resp := s.importFile(entryPath, data, false, true); resp.StatusCode != 0 { + if resp := s.importFile(entryPath, data, false, detectAutoNotebook, workspaceImportConflicts); resp.StatusCode != 0 { return resp } } @@ -1101,64 +1200,92 @@ func readZipEntry(entry *zip.File) ([]byte, error) { // WorkspaceImportRawFile stores body as a file without notebook detection, as // /workspace/import does for format=RAW. func (s *FakeWorkspace) WorkspaceImportRawFile(filePath string, body []byte, overwrite bool) Response { - return s.importFile(filePath, body, overwrite, false) + return s.importFile(filePath, body, overwrite, nil, workspaceImportConflicts) } -func (s *FakeWorkspace) importFile(filePath string, body []byte, overwrite, detectNotebooks bool) Response { +// importFile stores body at filePath, as a notebook if detect says so. A nil +// detect always stores a file. +func (s *FakeWorkspace) importFile(filePath string, body []byte, overwrite bool, detect notebookDetector, conflicts importConflicts) Response { if !strings.HasPrefix(filePath, "/") { filePath = "/" + filePath } defer s.LockUnlock()() - workspacePath := filePath - - if resp, ok := s.requireParentDirectory(workspacePath); !ok { + if resp, ok := s.requireParentDirectory(filePath); !ok { return resp } if !overwrite { - if _, exists := s.files[workspacePath]; exists { + if _, exists := s.files[filePath]; exists { return Response{ StatusCode: 409, - Body: map[string]string{"message": fmt.Sprintf("File already exists at (%s).", workspacePath)}, + Body: map[string]string{"message": fmt.Sprintf("File already exists at (%s).", filePath)}, } } } - // Files with .py / .sql / .scala / .r extension can be notebooks if they - // carry the "Databricks notebook source" header comment in the language's - // line-comment syntax. Mirror the real workspace's auto-detection here so - // acceptance tests can assert the resulting object_type via /workspace/list. - extension := filepath.Ext(filePath) - if lang, isNotebook := detectNotebookLanguage(extension, body); detectNotebooks && isNotebook { - // Notebooks are stripped of their extension by the workspace import API. - workspacePath = strings.TrimSuffix(filePath, extension) - s.files[workspacePath] = FileEntry{ - Info: workspace.ObjectInfo{ - ObjectType: "NOTEBOOK", - Path: workspacePath, - Language: lang, - ObjectId: nextTreenodeID(), - }, - Data: body, - } - } else { - // The endpoint does not set language for files, so we omit that - // here as well. - // ref: https://docs.databricks.com/api/workspace/workspace/getstatus#language - s.files[workspacePath] = FileEntry{ - Info: workspace.ObjectInfo{ - ObjectType: "FILE", - Path: workspacePath, - ObjectId: nextTreenodeID(), - }, - Data: body, + if detect != nil { + if language, isNotebook := detect(filePath, body); isNotebook { + // Notebooks are stripped of their extension by the workspace import API. + notebookPath := strings.TrimSuffix(filePath, path.Ext(filePath)) + return s.storeNotebook(filePath, notebookPath, body, language, conflicts) } } + + if s.isNotebookSourceName(filePath) { + return conflicts.notebookTaken(filePath) + } + + // The endpoint does not set language for files, so we omit that + // here as well. + // ref: https://docs.databricks.com/api/workspace/workspace/getstatus#language + s.files[filePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: filePath, + ObjectId: nextTreenodeID(), + }, + Data: body, + } + return Response{} +} + +// storeNotebook stores a notebook at notebookPath unless a file or directory +// holds its source name. The caller must hold the lock. +func (s *FakeWorkspace) storeNotebook(requestPath, notebookPath string, body []byte, language workspace.Language, conflicts importConflicts) Response { + sourceName := notebookSourceName(notebookPath, language) + _, isFile := s.files[sourceName] + _, isDir := s.directories[sourceName] + if isFile || isDir { + return conflicts.nameTaken(requestPath, path.Base(sourceName)) + } + + s.files[notebookPath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "NOTEBOOK", + Path: notebookPath, + Language: language, + ObjectId: nextTreenodeID(), + }, + Data: body, + } return Response{} } +// isNotebookSourceName reports whether filePath is the source name of an +// existing notebook. The caller must hold the lock. +func (s *FakeWorkspace) isNotebookSourceName(filePath string) bool { + ext := path.Ext(filePath) + if ext == "" { + return false + } + notebookPath := strings.TrimSuffix(filePath, ext) + entry, ok := s.files[notebookPath] + return ok && entry.Info.ObjectType == workspace.ObjectTypeNotebook && + strings.EqualFold(notebookSourceName(notebookPath, entry.Info.Language), filePath) +} + // requireParentDirectory returns a 404 response when objectPath's parent // directory does not exist. The real import API does not create missing parents; // callers get "mkdir -p" semantics only by first calling /workspace/mkdirs (see @@ -1203,16 +1330,7 @@ func (s *FakeWorkspace) WorkspaceImportNotebook(filePath string, body []byte, la } } - s.files[filePath] = FileEntry{ - Info: workspace.ObjectInfo{ - ObjectType: "NOTEBOOK", - Path: filePath, - Language: language, - ObjectId: nextTreenodeID(), - }, - Data: body, - } - return Response{} + return s.storeNotebook(filePath, filePath, body, language, workspaceImportConflicts) } func (s *FakeWorkspace) WorkspaceFilesExportFile(path string) []byte { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 73839619c3a..a0245e1fc13 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -81,6 +81,8 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { resp = w.WorkspaceImportRawFile(form.Path, form.Content, form.Overwrite) case string(workspace.ImportFormatSource): resp = w.WorkspaceImportNotebook(form.Path, form.Content, workspace.Language(form.Language), form.Overwrite) + case string(workspace.ImportFormatRMarkdown): + resp = w.WorkspaceImportNotebook(form.Path, form.Content, workspace.LanguageR, form.Overwrite) case string(workspace.ImportFormatJupyter): // The real endpoint reads the language from the notebook metadata. _, language, _ := notebook.DetectBytes(form.Path+notebook.ExtensionJupyter, form.Content) @@ -92,10 +94,10 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { if unsupported, ok := autoImportUnsupported(form.Path); ok { return unsupported } - resp = w.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + resp = w.WorkspaceImportAutoFile(form.Path, form.Content, form.Overwrite) default: return Response{ - Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW, SOURCE and JUPYTER formats, got %q.", form.Format), + Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW, SOURCE, JUPYTER and R_MARKDOWN formats, got %q.", form.Format), StatusCode: http.StatusInternalServerError, } } diff --git a/ruff.toml b/ruff.toml index db6ffa98604..6108b0b51fc 100644 --- a/ruff.toml +++ b/ruff.toml @@ -16,6 +16,10 @@ extend-exclude = [ # traversal and shields output/ from the acceptance/**/output prune above. "acceptance/bundle/templates/default-sql", "**/testdata", + # Byte-exact notebook headers (leading blank line, tab, invalid Python) that + # formatting would rewrite. + "acceptance/bundle/sync-notebook-detection", + "acceptance/cmd/workspace/import-notebook-detection", ] [lint] From b9057f9044acba84fec2299c1dd33d2cc10761dc Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 12:46:20 +0000 Subject: [PATCH 02/16] acceptance: reduce #7216 tests to the one file shape that triggers it Keep a single SQL notebook with a byte order mark. One bundle test covers both version-switching failures, and the endpoint test covers AUTO vs RAW and that overwrite can't change an object's type. Drop the other header variants, .Rmd and AUTO's 200-byte rule, along with the testserver R_MARKDOWN/.Rmd handling and the ruff exclusion they needed. Co-authored-by: Isaac --- .../{downgrade-over-file => }/bom.sql | 0 .../{downgrade-over-file => }/databricks.yml | 6 +- .../downgrade-over-file/output.txt | 14 ---- .../downgrade-over-file/script | 8 --- .../downgrade-over-file/test.toml | 1 - .../near-miss-header/bom.scala | 2 - .../near-miss-header/bom.sql | 2 - .../near-miss-header/databricks.yml | 11 --- .../near-miss-header/double_hash.py | 2 - .../near-miss-header/exact.sql | 2 - .../near-miss-header/hash_comment.sql | 2 - .../near-miss-header/leading_tab.py | 2 - .../near-miss-header/no_comment.py | 2 - .../near-miss-header/no_space.sql | 2 - .../near-miss-header/out.test.toml | 2 - .../near-miss-header/output.txt | 48 ------------- .../near-miss-header/script | 13 ---- .../near-miss-header/second_line.py | 3 - .../near-miss-header/suffix.sql | 2 - .../near-miss-header/test.toml | 2 - .../near-miss-header/trailing_space.sql | 2 - .../{downgrade-over-file => }/out.test.toml | 0 .../bundle/sync-notebook-detection/output.txt | 25 +++++++ .../rmd/databricks.yml | 11 --- .../sync-notebook-detection/rmd/doc.Rmd | 6 -- .../sync-notebook-detection/rmd/lower.rmd | 6 -- .../sync-notebook-detection/rmd/out.test.toml | 2 - .../sync-notebook-detection/rmd/output.txt | 21 ------ .../bundle/sync-notebook-detection/rmd/script | 11 --- .../sync-notebook-detection/rmd/test.toml | 2 - .../bundle/sync-notebook-detection/script | 27 +++++++ .../sync-notebook-detection/script.prepare | 29 -------- .../bundle/sync-notebook-detection/test.toml | 2 + .../upgrade-over-legacy-notebook/bom.sql | 2 - .../databricks.yml | 12 ---- .../out.test.toml | 2 - .../upgrade-over-legacy-notebook/output.txt | 13 ---- .../upgrade-over-legacy-notebook/script | 8 --- .../upgrade-over-legacy-notebook/test.toml | 1 - .../import-notebook-detection/doc.Rmd | 6 -- .../import-notebook-detection/exact.sql | 2 - .../marker_past_200_bytes.py | 2 - .../import-notebook-detection/output.txt | 70 ++++--------------- .../import-notebook-detection/plain.sql | 1 - .../import-notebook-detection/script | 57 ++++----------- .../import-notebook-detection/second_line.py | 3 - libs/testserver/fake_workspace.go | 8 +-- libs/testserver/handlers.go | 4 +- ruff.toml | 4 -- 49 files changed, 84 insertions(+), 381 deletions(-) rename acceptance/bundle/sync-notebook-detection/{downgrade-over-file => }/bom.sql (100%) rename acceptance/bundle/sync-notebook-detection/{downgrade-over-file => }/databricks.yml (51%) delete mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/output.txt delete mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/script delete mode 100644 acceptance/bundle/sync-notebook-detection/downgrade-over-file/test.toml delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/bom.scala delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/bom.sql delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/databricks.yml delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/double_hash.py delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/exact.sql delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/hash_comment.sql delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/leading_tab.py delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/no_comment.py delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/no_space.sql delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/out.test.toml delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/output.txt delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/script delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/second_line.py delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/suffix.sql delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/test.toml delete mode 100644 acceptance/bundle/sync-notebook-detection/near-miss-header/trailing_space.sql rename acceptance/bundle/sync-notebook-detection/{downgrade-over-file => }/out.test.toml (100%) create mode 100644 acceptance/bundle/sync-notebook-detection/output.txt delete mode 100644 acceptance/bundle/sync-notebook-detection/rmd/databricks.yml delete mode 100644 acceptance/bundle/sync-notebook-detection/rmd/doc.Rmd delete mode 100644 acceptance/bundle/sync-notebook-detection/rmd/lower.rmd delete mode 100644 acceptance/bundle/sync-notebook-detection/rmd/out.test.toml delete mode 100644 acceptance/bundle/sync-notebook-detection/rmd/output.txt delete mode 100644 acceptance/bundle/sync-notebook-detection/rmd/script delete mode 100644 acceptance/bundle/sync-notebook-detection/rmd/test.toml create mode 100644 acceptance/bundle/sync-notebook-detection/script delete mode 100644 acceptance/bundle/sync-notebook-detection/script.prepare delete mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/bom.sql delete mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/databricks.yml delete mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/out.test.toml delete mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/output.txt delete mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/script delete mode 100644 acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/test.toml delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/doc.Rmd delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/exact.sql delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/marker_past_200_bytes.py delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/plain.sql delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/second_line.py diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/bom.sql b/acceptance/bundle/sync-notebook-detection/bom.sql similarity index 100% rename from acceptance/bundle/sync-notebook-detection/downgrade-over-file/bom.sql rename to acceptance/bundle/sync-notebook-detection/bom.sql diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/databricks.yml b/acceptance/bundle/sync-notebook-detection/databricks.yml similarity index 51% rename from acceptance/bundle/sync-notebook-detection/downgrade-over-file/databricks.yml rename to acceptance/bundle/sync-notebook-detection/databricks.yml index 652152d00d6..1aac7e9796b 100644 --- a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/databricks.yml +++ b/acceptance/bundle/sync-notebook-detection/databricks.yml @@ -1,12 +1,12 @@ bundle: - name: sync-notebook-detection-downgrade-over-file + name: sync-notebook-detection -# Only sync the committed inputs, not the test harness files. +# Only sync bom.sql. sync: exclude: + - databricks.yml - script - test.toml - output.txt - out.* - repls.json - - databricks.yml diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/output.txt b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/output.txt deleted file mode 100644 index 7ca5fc15cc8..00000000000 --- a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/output.txt +++ /dev/null @@ -1,14 +0,0 @@ - ->>> [CLI] bundle sync --output text -Action: PUT: bom.sql -Initial Sync Complete -Uploaded bom.sql - ->>> remote_objects -FILE - bom.sql - ->>> musterr legacy_import_bom -Error: Request failed for POST /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection-downgrade-over-file/default/files/bom.sql: RESOURCE_ALREADY_EXISTS: Node named 'bom.sql' already exists - ->>> remote_objects -FILE - bom.sql diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/script b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/script deleted file mode 100644 index 907d15619ac..00000000000 --- a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/script +++ /dev/null @@ -1,8 +0,0 @@ -# bom.sql starts with a UTF-8 byte order mark. This CLI uploads it as file bom.sql; the -# legacy import-file endpoint then tries to import it as notebook "bom", whose name the -# file already holds. -trace $CLI bundle sync --output text 2>&1 | sort -trace remote_objects - -trace musterr legacy_import_bom -trace remote_objects diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/test.toml b/acceptance/bundle/sync-notebook-detection/downgrade-over-file/test.toml deleted file mode 100644 index 5f0ffaf3ca9..00000000000 --- a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/test.toml +++ /dev/null @@ -1 +0,0 @@ -Badness = "After this CLI uploads a file it no longer detects as a notebook, v1.20 or earlier fails to upload it with RESOURCE_ALREADY_EXISTS: Node named '...' already exists, the error reported in #7216." diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.scala b/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.scala deleted file mode 100644 index b8ee8bff261..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.scala +++ /dev/null @@ -1,2 +0,0 @@ -// Databricks notebook source -println(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.sql deleted file mode 100644 index 4aeddae0fd3..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/bom.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Databricks notebook source -SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/databricks.yml b/acceptance/bundle/sync-notebook-detection/near-miss-header/databricks.yml deleted file mode 100644 index 580b5fa8072..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/databricks.yml +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: sync-notebook-detection-near-miss-header - -# Only sync the committed inputs, not the test harness files. -sync: - exclude: - - script - - test.toml - - output.txt - - out.* - - repls.json diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/double_hash.py b/acceptance/bundle/sync-notebook-detection/near-miss-header/double_hash.py deleted file mode 100644 index 9355a12f1bd..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/double_hash.py +++ /dev/null @@ -1,2 +0,0 @@ -## Databricks notebook source -print(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/exact.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/exact.sql deleted file mode 100644 index 3b831911443..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/exact.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Databricks notebook source -SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/hash_comment.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/hash_comment.sql deleted file mode 100644 index 1f26f31b652..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/hash_comment.sql +++ /dev/null @@ -1,2 +0,0 @@ -# Databricks notebook source -SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/leading_tab.py b/acceptance/bundle/sync-notebook-detection/near-miss-header/leading_tab.py deleted file mode 100644 index b2c5cfb6442..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/leading_tab.py +++ /dev/null @@ -1,2 +0,0 @@ - # Databricks notebook source -print(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/no_comment.py b/acceptance/bundle/sync-notebook-detection/near-miss-header/no_comment.py deleted file mode 100644 index d4b4a6fb946..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/no_comment.py +++ /dev/null @@ -1,2 +0,0 @@ -Databricks notebook source -print(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/no_space.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/no_space.sql deleted file mode 100644 index eb5f005861f..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/no_space.sql +++ /dev/null @@ -1,2 +0,0 @@ ---Databricks notebook source -SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/out.test.toml b/acceptance/bundle/sync-notebook-detection/near-miss-header/out.test.toml deleted file mode 100644 index a927a5fbc06..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/output.txt b/acceptance/bundle/sync-notebook-detection/near-miss-header/output.txt deleted file mode 100644 index f79fa808753..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/output.txt +++ /dev/null @@ -1,48 +0,0 @@ - ->>> [CLI] bundle sync --output text -Action: PUT: bom.scala, bom.sql, databricks.yml, double_hash.py, exact.sql, hash_comment.sql, leading_tab.py, no_comment.py, no_space.sql, second_line.py, suffix.sql, trailing_space.sql -Initial Sync Complete -Uploaded bom.scala -Uploaded bom.sql -Uploaded databricks.yml -Uploaded double_hash.py -Uploaded exact.sql -Uploaded hash_comment.sql -Uploaded leading_tab.py -Uploaded no_comment.py -Uploaded no_space.sql -Uploaded second_line.py -Uploaded suffix.sql -Uploaded trailing_space.sql - -=== Format per upload: SOURCE only for the exact header - ->>> upload_formats -RAW bom.scala -RAW bom.sql -RAW databricks.yml -RAW double_hash.py -SOURCE exact -RAW hash_comment.sql -RAW leading_tab.py -RAW no_comment.py -RAW no_space.sql -RAW second_line.py -RAW suffix.sql -RAW trailing_space.sql - -=== Synced objects - ->>> remote_objects -FILE - bom.scala -FILE - bom.sql -FILE - databricks.yml -FILE - double_hash.py -NOTEBOOK SQL exact -FILE - hash_comment.sql -FILE - leading_tab.py -FILE - no_comment.py -FILE - no_space.sql -FILE - second_line.py -FILE - suffix.sql -FILE - trailing_space.sql diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/script b/acceptance/bundle/sync-notebook-detection/near-miss-header/script deleted file mode 100644 index a4ff5e843de..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/script +++ /dev/null @@ -1,13 +0,0 @@ -# bom.sql and bom.scala start with a UTF-8 byte order mark, trailing_space.sql ends its -# first line with a space and leading_tab.py starts with a tab. The legacy import-file -# endpoint imported all of these as notebooks, except second_line.py, whose marker is not -# on the first line. exact.sql is a notebook for both. -trace $CLI bundle sync --output text 2>&1 | sort - -title "Format per upload: SOURCE only for the exact header\n" -trace upload_formats - -title "Synced objects\n" -trace remote_objects - -rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/second_line.py b/acceptance/bundle/sync-notebook-detection/near-miss-header/second_line.py deleted file mode 100644 index cbcf42a24f9..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/second_line.py +++ /dev/null @@ -1,3 +0,0 @@ - -# Databricks notebook source -print(1) diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/suffix.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/suffix.sql deleted file mode 100644 index 815c08eac2a..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/suffix.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Databricks notebook source (daily) -SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/test.toml b/acceptance/bundle/sync-notebook-detection/near-miss-header/test.toml deleted file mode 100644 index 07deac45c31..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Badness = "Source files whose first line contains \"Databricks notebook source\" but is not the exact header are uploaded with format=RAW and stored as files. v1.20 and earlier imported them as notebooks, so deploying the same bundle with both fails (#7216)." -RecordRequests = true diff --git a/acceptance/bundle/sync-notebook-detection/near-miss-header/trailing_space.sql b/acceptance/bundle/sync-notebook-detection/near-miss-header/trailing_space.sql deleted file mode 100644 index 71a808d64a2..00000000000 --- a/acceptance/bundle/sync-notebook-detection/near-miss-header/trailing_space.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Databricks notebook source -SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/downgrade-over-file/out.test.toml b/acceptance/bundle/sync-notebook-detection/out.test.toml similarity index 100% rename from acceptance/bundle/sync-notebook-detection/downgrade-over-file/out.test.toml rename to acceptance/bundle/sync-notebook-detection/out.test.toml diff --git a/acceptance/bundle/sync-notebook-detection/output.txt b/acceptance/bundle/sync-notebook-detection/output.txt new file mode 100644 index 00000000000..141ac1ca448 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/output.txt @@ -0,0 +1,25 @@ + +=== Upgrade: this CLI syncs over the notebook v1.20 created + +>>> legacy_import +null + +>>> list +NOTEBOOK bom + +>>> musterr [CLI] bundle sync +Action: PUT: bom.sql +Error: file already exists: /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection/default/files/bom.sql + +=== Downgrade: v1.20 uploads over the file this CLI created + +>>> [CLI] bundle sync --full +Action: PUT: bom.sql +Uploaded bom.sql +Initial Sync Complete + +>>> list +FILE bom.sql + +>>> musterr legacy_import +Error: Request failed for POST /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection/default/files/bom.sql: RESOURCE_ALREADY_EXISTS: Node named 'bom.sql' already exists diff --git a/acceptance/bundle/sync-notebook-detection/rmd/databricks.yml b/acceptance/bundle/sync-notebook-detection/rmd/databricks.yml deleted file mode 100644 index a91aecc8086..00000000000 --- a/acceptance/bundle/sync-notebook-detection/rmd/databricks.yml +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: sync-notebook-detection-rmd - -# Only sync the committed inputs, not the test harness files. -sync: - exclude: - - script - - test.toml - - output.txt - - out.* - - repls.json diff --git a/acceptance/bundle/sync-notebook-detection/rmd/doc.Rmd b/acceptance/bundle/sync-notebook-detection/rmd/doc.Rmd deleted file mode 100644 index 16cf3fe51cc..00000000000 --- a/acceptance/bundle/sync-notebook-detection/rmd/doc.Rmd +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: doc ---- -```{r} -1 -``` diff --git a/acceptance/bundle/sync-notebook-detection/rmd/lower.rmd b/acceptance/bundle/sync-notebook-detection/rmd/lower.rmd deleted file mode 100644 index 16cf3fe51cc..00000000000 --- a/acceptance/bundle/sync-notebook-detection/rmd/lower.rmd +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: doc ---- -```{r} -1 -``` diff --git a/acceptance/bundle/sync-notebook-detection/rmd/out.test.toml b/acceptance/bundle/sync-notebook-detection/rmd/out.test.toml deleted file mode 100644 index a927a5fbc06..00000000000 --- a/acceptance/bundle/sync-notebook-detection/rmd/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-notebook-detection/rmd/output.txt b/acceptance/bundle/sync-notebook-detection/rmd/output.txt deleted file mode 100644 index b72ce123dcd..00000000000 --- a/acceptance/bundle/sync-notebook-detection/rmd/output.txt +++ /dev/null @@ -1,21 +0,0 @@ - ->>> [CLI] bundle sync --output text -Action: PUT: databricks.yml, doc.Rmd, lower.rmd -Initial Sync Complete -Uploaded databricks.yml -Uploaded doc.Rmd -Uploaded lower.rmd - -=== Format per upload - ->>> upload_formats -RAW databricks.yml -RAW doc.Rmd -RAW lower.rmd - -=== Synced objects - ->>> remote_objects -FILE - databricks.yml -FILE - doc.Rmd -FILE - lower.rmd diff --git a/acceptance/bundle/sync-notebook-detection/rmd/script b/acceptance/bundle/sync-notebook-detection/rmd/script deleted file mode 100644 index 150f77b7774..00000000000 --- a/acceptance/bundle/sync-notebook-detection/rmd/script +++ /dev/null @@ -1,11 +0,0 @@ -# The legacy import-file endpoint imported doc.Rmd as R notebook "doc". It matched the -# extension case-sensitively, so lower.rmd was a file there too. -trace $CLI bundle sync --output text 2>&1 | sort - -title "Format per upload\n" -trace upload_formats - -title "Synced objects\n" -trace remote_objects - -rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/sync-notebook-detection/rmd/test.toml b/acceptance/bundle/sync-notebook-detection/rmd/test.toml deleted file mode 100644 index 2280fc8f5de..00000000000 --- a/acceptance/bundle/sync-notebook-detection/rmd/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Badness = "R Markdown (.Rmd) files are uploaded with format=RAW and stored as files. v1.20 and earlier imported them as R notebooks." -RecordRequests = true diff --git a/acceptance/bundle/sync-notebook-detection/script b/acceptance/bundle/sync-notebook-detection/script new file mode 100644 index 00000000000..514f2d36725 --- /dev/null +++ b/acceptance/bundle/sync-notebook-detection/script @@ -0,0 +1,27 @@ +# bom.sql is a SQL notebook whose first line starts with a UTF-8 byte order mark. The legacy +# import-file endpoint, used up to v1.20, imports it as notebook "bom" because its first line +# contains "Databricks notebook source". This CLI requires the exact header, so it uploads +# file "bom.sql". Overwrite can't turn one into the other. +remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) + +list() { + $CLI workspace list -o json "$remote" | jq -r '.[] | "\(.object_type) \(.path | sub(".*/"; ""))"' +} + +# Uploads bom.sql the way v1.20 did. `api post` only sends JSON; a JSON string whose +# first line contains the marker is imported as a notebook, just like bom.sql. +legacy_import() { + $CLI api post "/api/2.0/workspace-files/import-file$remote/bom.sql?overwrite=true" --json '"-- Databricks notebook source"' +} + +title "Upgrade: this CLI syncs over the notebook v1.20 created\n" +$CLI workspace mkdirs "$remote" +trace legacy_import +trace list +trace musterr $CLI bundle sync + +title "Downgrade: v1.20 uploads over the file this CLI created\n" +$CLI workspace delete "$remote" --recursive +trace $CLI bundle sync --full +trace list +trace musterr legacy_import diff --git a/acceptance/bundle/sync-notebook-detection/script.prepare b/acceptance/bundle/sync-notebook-detection/script.prepare deleted file mode 100644 index d6f2bda231d..00000000000 --- a/acceptance/bundle/sync-notebook-detection/script.prepare +++ /dev/null @@ -1,29 +0,0 @@ -# Since v1.21 the CLI decides itself whether a source file is a notebook (libs/notebook): -# the first line must be exactly the header for the file's language. Up to v1.20 it -# uploaded through the legacy import-file endpoint, which instead imports a file as a -# notebook if its first line contains "Databricks notebook source" (and always imports -# .Rmd as an R notebook). The tests below capture where the two disagree. - -remote_files() { - $CLI bundle summary -o json | jq -r .workspace.file_path -} - -# Lists the objects synced to the bundle's file path. -remote_objects() { - $CLI workspace list -o json "$(remote_files)" | jq -r 'sort_by(.path)[] | "\(.object_type) \(.language // "-") \(.path | sub(".*/"; ""))"' -} - -# Prints the /workspace/import format the CLI used for each synced file. -upload_formats() { - jq -rs ' - map(select(.path == "/api/2.0/workspace/import") | .body.multipart_form) | - sort_by(.path) | .[] | "\(.format) \(.path | sub(".*/"; ""))" - ' "$OUT_REQUESTS" -} - -# Uploads bom.sql the way v1.20 and earlier did, through the legacy import-file endpoint. -# `api post` only sends JSON bodies; a JSON string whose first line contains the marker -# takes the same notebook path as the file itself. -legacy_import_bom() { - $CLI api post "/api/2.0/workspace-files/import-file$(remote_files)/bom.sql?overwrite=true" --json '"-- Databricks notebook source"' -} diff --git a/acceptance/bundle/sync-notebook-detection/test.toml b/acceptance/bundle/sync-notebook-detection/test.toml index d02f0e2c468..90bee878f2e 100644 --- a/acceptance/bundle/sync-notebook-detection/test.toml +++ b/acceptance/bundle/sync-notebook-detection/test.toml @@ -1,3 +1,5 @@ +Badness = "A notebook whose first line isn't exactly the header (here: a byte order mark) is uploaded as a file, while v1.20 and earlier uploaded it as a notebook. Deploying the same bundle with both fails in either order (#7216)." + # Workspace paths start with "/"; keep Git Bash on Windows from rewriting them. Env.MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/bom.sql b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/bom.sql deleted file mode 100644 index 4aeddae0fd3..00000000000 --- a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/bom.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Databricks notebook source -SELECT 1 diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/databricks.yml b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/databricks.yml deleted file mode 100644 index f71ed3dc3ae..00000000000 --- a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/databricks.yml +++ /dev/null @@ -1,12 +0,0 @@ -bundle: - name: sync-notebook-detection-upgrade-over-legacy-notebook - -# Only sync the committed inputs, not the test harness files. -sync: - exclude: - - script - - test.toml - - output.txt - - out.* - - repls.json - - databricks.yml diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/out.test.toml b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/out.test.toml deleted file mode 100644 index a927a5fbc06..00000000000 --- a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/output.txt b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/output.txt deleted file mode 100644 index 1a4d637b8a3..00000000000 --- a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/output.txt +++ /dev/null @@ -1,13 +0,0 @@ - ->>> legacy_import_bom -null - ->>> remote_objects -NOTEBOOK SQL bom - ->>> musterr [CLI] bundle sync --output text -Action: PUT: bom.sql -Error: file already exists: /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection-upgrade-over-legacy-notebook/default/files/bom.sql - ->>> remote_objects -NOTEBOOK SQL bom diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/script b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/script deleted file mode 100644 index 631e2287885..00000000000 --- a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/script +++ /dev/null @@ -1,8 +0,0 @@ -# bom.sql starts with a UTF-8 byte order mark. v1.20 and earlier imported it as notebook -# "bom"; this CLI uploads it as file bom.sql, which the notebook already holds the name of. -$CLI workspace mkdirs "$(remote_files)" -trace legacy_import_bom -trace remote_objects - -trace musterr $CLI bundle sync --output text -trace remote_objects diff --git a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/test.toml b/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/test.toml deleted file mode 100644 index c8917d1e454..00000000000 --- a/acceptance/bundle/sync-notebook-detection/upgrade-over-legacy-notebook/test.toml +++ /dev/null @@ -1 +0,0 @@ -Badness = "Syncing over a notebook that v1.20 or earlier created fails with \"file already exists\" when the CLI no longer detects the file as a notebook (#7216)." diff --git a/acceptance/cmd/workspace/import-notebook-detection/doc.Rmd b/acceptance/cmd/workspace/import-notebook-detection/doc.Rmd deleted file mode 100644 index 16cf3fe51cc..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/doc.Rmd +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: doc ---- -```{r} -1 -``` diff --git a/acceptance/cmd/workspace/import-notebook-detection/exact.sql b/acceptance/cmd/workspace/import-notebook-detection/exact.sql deleted file mode 100644 index 3b831911443..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/exact.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Databricks notebook source -SELECT 1 diff --git a/acceptance/cmd/workspace/import-notebook-detection/marker_past_200_bytes.py b/acceptance/cmd/workspace/import-notebook-detection/marker_past_200_bytes.py deleted file mode 100644 index 57b44dea5dc..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/marker_past_200_bytes.py +++ /dev/null @@ -1,2 +0,0 @@ -######################################################################################################################################################################################################## Databricks notebook source -print(1) diff --git a/acceptance/cmd/workspace/import-notebook-detection/output.txt b/acceptance/cmd/workspace/import-notebook-detection/output.txt index b8bcc68fd81..9b59bf41029 100644 --- a/acceptance/cmd/workspace/import-notebook-detection/output.txt +++ b/acceptance/cmd/workspace/import-notebook-detection/output.txt @@ -1,66 +1,20 @@ -=== AUTO imports a source file as a notebook if "Databricks notebook source" is anywhere in its first 200 bytes +=== AUTO imports it as a notebook, like the legacy import-file endpoint. Overwrite can't make it a file ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/exact.sql --file exact.sql --format AUTO +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/notebook/bom.sql --file bom.sql --format AUTO ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/bom.sql --file bom.sql --format AUTO - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/second_line.py --file second_line.py --format AUTO - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/marker_past_200_bytes.py --file marker_past_200_bytes.py --format AUTO - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/doc.Rmd --file doc.Rmd --format AUTO - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto -NOTEBOOK SQL /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/bom -FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/doc.Rmd -NOTEBOOK SQL /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/exact -FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/marker_past_200_bytes.py -NOTEBOOK PYTHON /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/auto/second_line - -=== R_MARKDOWN imports R Markdown as an R notebook at the given path - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/rmd/doc --file doc.Rmd --format R_MARKDOWN - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/rmd -NOTEBOOK R /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/rmd/doc - -=== Overwrite replaces an object of the same type - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/exact.sql --file exact.sql --format AUTO - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/exact.sql --file exact.sql --format AUTO --overwrite - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/plain.sql --file plain.sql --format RAW - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/plain.sql --file plain.sql --format RAW --overwrite - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same -NOTEBOOK SQL /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/exact -FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/same/plain.sql - -=== Overwrite cannot turn a file into a notebook - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook/x.sql --file plain.sql --format RAW - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook/x.sql --file exact.sql --format AUTO --overwrite -Error: Node named 'x.sql' already exists - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook/x --file exact.sql --format SOURCE --language SQL --overwrite -Error: Node named 'x.sql' already exists - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook -FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_notebook/x.sql +>>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/notebook/bom.sql --file bom.sql --format RAW --overwrite +Error: Requested node type [FILE] is different from the existing node type [NOTEBOOK] -=== Overwrite cannot turn a notebook into a file +=== RAW imports it as a file, like this CLI. Overwrite can't make it a notebook ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x --file exact.sql --format SOURCE --language SQL +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/file/bom.sql --file bom.sql --format RAW ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x.sql --file plain.sql --format RAW --overwrite -Error: Requested node type [FILE] is different from the existing node type [NOTEBOOK] +>>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/file/bom.sql --file bom.sql --format AUTO --overwrite +Error: Node named 'bom.sql' already exists ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x.py --file plain.sql --format RAW --overwrite +>>> [CLI] workspace list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/notebook -o json +NOTEBOOK /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/notebook/bom ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file -NOTEBOOK SQL /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x -FILE - /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/to_file/x.py +>>> [CLI] workspace list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/file -o json +FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/file/bom.sql diff --git a/acceptance/cmd/workspace/import-notebook-detection/plain.sql b/acceptance/cmd/workspace/import-notebook-detection/plain.sql deleted file mode 100644 index 2e3761f7a2c..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/plain.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 diff --git a/acceptance/cmd/workspace/import-notebook-detection/script b/acceptance/cmd/workspace/import-notebook-detection/script index 6c26a8c9cf8..be11e352577 100644 --- a/acceptance/cmd/workspace/import-notebook-detection/script +++ b/acceptance/cmd/workspace/import-notebook-detection/script @@ -1,54 +1,21 @@ -# Documents how POST /api/2.0/workspace/import classifies source notebooks, and that -# overwrite never changes an existing object's type. +# Documents POST /api/2.0/workspace/import for bom.sql, a SQL notebook whose first line +# starts with a UTF-8 byte order mark (see bundle/sync-notebook-detection). DIR="/Workspace/Users/$CURRENT_USER_NAME/$UNIQUE_NAME" -$CLI workspace mkdirs "$DIR" +$CLI workspace mkdirs "$DIR/notebook" +$CLI workspace mkdirs "$DIR/file" cleanup() { $CLI workspace delete "$DIR" --recursive } trap cleanup EXIT -list() { - $CLI workspace list "$1" -o json | jq -r 'sort_by(.path)[] | "\(.object_type) \(.language // "-") \(.path)"' -} - -# bom.sql starts with a UTF-8 byte order mark. marker_past_200_bytes.py has the marker -# on its first line, after 200 bytes of "#". - -title "AUTO imports a source file as a notebook if \"Databricks notebook source\" is anywhere in its first 200 bytes\n" -# The legacy import-file endpoint only checks the first line, so second_line.py was a file -# there. AUTO imports R Markdown as a file; import-file imported it as an R notebook. -$CLI workspace mkdirs "$DIR/auto" -for f in exact.sql bom.sql second_line.py marker_past_200_bytes.py doc.Rmd; do - trace $CLI workspace import "$DIR/auto/$f" --file $f --format AUTO -done -trace list "$DIR/auto" - -title "R_MARKDOWN imports R Markdown as an R notebook at the given path\n" -$CLI workspace mkdirs "$DIR/rmd" -trace $CLI workspace import "$DIR/rmd/doc" --file doc.Rmd --format R_MARKDOWN -trace list "$DIR/rmd" - -title "Overwrite replaces an object of the same type\n" -$CLI workspace mkdirs "$DIR/same" -trace $CLI workspace import "$DIR/same/exact.sql" --file exact.sql --format AUTO -trace $CLI workspace import "$DIR/same/exact.sql" --file exact.sql --format AUTO --overwrite -trace $CLI workspace import "$DIR/same/plain.sql" --file plain.sql --format RAW -trace $CLI workspace import "$DIR/same/plain.sql" --file plain.sql --format RAW --overwrite -trace list "$DIR/same" +title "AUTO imports it as a notebook, like the legacy import-file endpoint. Overwrite can't make it a file\n" +trace $CLI workspace import "$DIR/notebook/bom.sql" --file bom.sql --format AUTO +trace musterr $CLI workspace import "$DIR/notebook/bom.sql" --file bom.sql --format RAW --overwrite -title "Overwrite cannot turn a file into a notebook\n" -# Notebook "x" would take the name x.sql, which the file already holds. -$CLI workspace mkdirs "$DIR/to_notebook" -trace $CLI workspace import "$DIR/to_notebook/x.sql" --file plain.sql --format RAW -trace musterr $CLI workspace import "$DIR/to_notebook/x.sql" --file exact.sql --format AUTO --overwrite -trace musterr $CLI workspace import "$DIR/to_notebook/x" --file exact.sql --format SOURCE --language SQL --overwrite -trace list "$DIR/to_notebook" +title "RAW imports it as a file, like this CLI. Overwrite can't make it a notebook\n" +trace $CLI workspace import "$DIR/file/bom.sql" --file bom.sql --format RAW +trace musterr $CLI workspace import "$DIR/file/bom.sql" --file bom.sql --format AUTO --overwrite -title "Overwrite cannot turn a notebook into a file\n" -# SQL notebook "x" holds the name x.sql, but not x.py. -$CLI workspace mkdirs "$DIR/to_file" -trace $CLI workspace import "$DIR/to_file/x" --file exact.sql --format SOURCE --language SQL -trace musterr $CLI workspace import "$DIR/to_file/x.sql" --file plain.sql --format RAW --overwrite -trace $CLI workspace import "$DIR/to_file/x.py" --file plain.sql --format RAW --overwrite -trace list "$DIR/to_file" +trace $CLI workspace list "$DIR/notebook" -o json | jq -r '.[] | "\(.object_type) \(.path)"' +trace $CLI workspace list "$DIR/file" -o json | jq -r '.[] | "\(.object_type) \(.path)"' diff --git a/acceptance/cmd/workspace/import-notebook-detection/second_line.py b/acceptance/cmd/workspace/import-notebook-detection/second_line.py deleted file mode 100644 index cbcf42a24f9..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/second_line.py +++ /dev/null @@ -1,3 +0,0 @@ - -# Databricks notebook source -print(1) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 856f5856fd6..abfe4fdf855 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -996,19 +996,15 @@ type notebookDetector func(filePath string, body []byte) (workspace.Language, bo // detectLegacyNotebook mirrors the legacy import-file endpoint, which looks for // the marker on the first line only (NotebookUtils.checkSourceFileNotebook in -// universe) and always imports R Markdown as an R notebook. +// universe). func detectLegacyNotebook(filePath string, body []byte) (workspace.Language, bool) { - // The endpoint matches this extension case-sensitively. - if path.Ext(filePath) == ".Rmd" { - return workspace.LanguageR, true - } language, ok := sourceNotebookLanguages[strings.ToLower(path.Ext(filePath))] firstLine, _, _ := bytes.Cut(notebookDetectionPrefix(body), []byte("\n")) return language, ok && bytes.Contains(firstLine, []byte(notebookSourceMarker)) } // detectAutoNotebook mirrors /workspace/import with format=AUTO, which finds the -// marker on any line of the leading bytes and imports R Markdown as a file. +// marker on any line of the leading bytes. func detectAutoNotebook(filePath string, body []byte) (workspace.Language, bool) { language, ok := sourceNotebookLanguages[strings.ToLower(path.Ext(filePath))] return language, ok && bytes.Contains(notebookDetectionPrefix(body), []byte(notebookSourceMarker)) diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a0245e1fc13..50ff904a573 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -81,8 +81,6 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { resp = w.WorkspaceImportRawFile(form.Path, form.Content, form.Overwrite) case string(workspace.ImportFormatSource): resp = w.WorkspaceImportNotebook(form.Path, form.Content, workspace.Language(form.Language), form.Overwrite) - case string(workspace.ImportFormatRMarkdown): - resp = w.WorkspaceImportNotebook(form.Path, form.Content, workspace.LanguageR, form.Overwrite) case string(workspace.ImportFormatJupyter): // The real endpoint reads the language from the notebook metadata. _, language, _ := notebook.DetectBytes(form.Path+notebook.ExtensionJupyter, form.Content) @@ -97,7 +95,7 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { resp = w.WorkspaceImportAutoFile(form.Path, form.Content, form.Overwrite) default: return Response{ - Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW, SOURCE, JUPYTER and R_MARKDOWN formats, got %q.", form.Format), + Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW, SOURCE and JUPYTER formats, got %q.", form.Format), StatusCode: http.StatusInternalServerError, } } diff --git a/ruff.toml b/ruff.toml index 6108b0b51fc..db6ffa98604 100644 --- a/ruff.toml +++ b/ruff.toml @@ -16,10 +16,6 @@ extend-exclude = [ # traversal and shields output/ from the acceptance/**/output prune above. "acceptance/bundle/templates/default-sql", "**/testdata", - # Byte-exact notebook headers (leading blank line, tab, invalid Python) that - # formatting would rewrite. - "acceptance/bundle/sync-notebook-detection", - "acceptance/cmd/workspace/import-notebook-detection", ] [lint] From 044db69fc2488c67540e9ed63e56622c73f26d38 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 13:10:35 +0000 Subject: [PATCH 03/16] acceptance: run the #7216 bundle test on cloud too Use a unique bundle name and clean up the workspace folder so the test can run against a real workspace, and drop generic comments from test.toml. Both tests pass on aws-cli with the recorded outputs. Co-authored-by: Isaac --- .../{databricks.yml => databricks.yml.tmpl} | 3 ++- acceptance/bundle/sync-notebook-detection/out.test.toml | 2 +- acceptance/bundle/sync-notebook-detection/output.txt | 4 ++-- acceptance/bundle/sync-notebook-detection/script | 6 ++++++ acceptance/bundle/sync-notebook-detection/test.toml | 2 +- .../cmd/workspace/import-notebook-detection/test.toml | 1 - 6 files changed, 12 insertions(+), 6 deletions(-) rename acceptance/bundle/sync-notebook-detection/{databricks.yml => databricks.yml.tmpl} (66%) diff --git a/acceptance/bundle/sync-notebook-detection/databricks.yml b/acceptance/bundle/sync-notebook-detection/databricks.yml.tmpl similarity index 66% rename from acceptance/bundle/sync-notebook-detection/databricks.yml rename to acceptance/bundle/sync-notebook-detection/databricks.yml.tmpl index 1aac7e9796b..32d766e544b 100644 --- a/acceptance/bundle/sync-notebook-detection/databricks.yml +++ b/acceptance/bundle/sync-notebook-detection/databricks.yml.tmpl @@ -1,10 +1,11 @@ bundle: - name: sync-notebook-detection + name: sync-notebook-detection-$UNIQUE_NAME # Only sync bom.sql. sync: exclude: - databricks.yml + - databricks.yml.tmpl - script - test.toml - output.txt diff --git a/acceptance/bundle/sync-notebook-detection/out.test.toml b/acceptance/bundle/sync-notebook-detection/out.test.toml index a927a5fbc06..0f167effef1 100644 --- a/acceptance/bundle/sync-notebook-detection/out.test.toml +++ b/acceptance/bundle/sync-notebook-detection/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-notebook-detection/output.txt b/acceptance/bundle/sync-notebook-detection/output.txt index 141ac1ca448..29db53c50c0 100644 --- a/acceptance/bundle/sync-notebook-detection/output.txt +++ b/acceptance/bundle/sync-notebook-detection/output.txt @@ -9,7 +9,7 @@ NOTEBOOK bom >>> musterr [CLI] bundle sync Action: PUT: bom.sql -Error: file already exists: /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection/default/files/bom.sql +Error: file already exists: /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection-[UNIQUE_NAME]/default/files/bom.sql === Downgrade: v1.20 uploads over the file this CLI created @@ -22,4 +22,4 @@ Initial Sync Complete FILE bom.sql >>> musterr legacy_import -Error: Request failed for POST /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection/default/files/bom.sql: RESOURCE_ALREADY_EXISTS: Node named 'bom.sql' already exists +Error: Request failed for POST /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection-[UNIQUE_NAME]/default/files/bom.sql: RESOURCE_ALREADY_EXISTS: Node named 'bom.sql' already exists diff --git a/acceptance/bundle/sync-notebook-detection/script b/acceptance/bundle/sync-notebook-detection/script index 514f2d36725..d4fa199c730 100644 --- a/acceptance/bundle/sync-notebook-detection/script +++ b/acceptance/bundle/sync-notebook-detection/script @@ -2,8 +2,14 @@ # import-file endpoint, used up to v1.20, imports it as notebook "bom" because its first line # contains "Databricks notebook source". This CLI requires the exact header, so it uploads # file "bom.sql". Overwrite can't turn one into the other. +envsubst < databricks.yml.tmpl > databricks.yml remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) +cleanup() { + $CLI workspace delete "$($CLI bundle summary -o json | jq -r .workspace.root_path)" --recursive +} +trap cleanup EXIT + list() { $CLI workspace list -o json "$remote" | jq -r '.[] | "\(.object_type) \(.path | sub(".*/"; ""))"' } diff --git a/acceptance/bundle/sync-notebook-detection/test.toml b/acceptance/bundle/sync-notebook-detection/test.toml index 90bee878f2e..037617015ea 100644 --- a/acceptance/bundle/sync-notebook-detection/test.toml +++ b/acceptance/bundle/sync-notebook-detection/test.toml @@ -1,6 +1,6 @@ +Cloud = true Badness = "A notebook whose first line isn't exactly the header (here: a byte order mark) is uploaded as a file, while v1.20 and earlier uploaded it as a notebook. Deploying the same bundle with both fails in either order (#7216)." -# Workspace paths start with "/"; keep Git Bash on Windows from rewriting them. Env.MSYS_NO_PATHCONV = "1" Ignore = [".databricks"] diff --git a/acceptance/cmd/workspace/import-notebook-detection/test.toml b/acceptance/cmd/workspace/import-notebook-detection/test.toml index 6b74aa538f0..bb47f764e26 100644 --- a/acceptance/cmd/workspace/import-notebook-detection/test.toml +++ b/acceptance/cmd/workspace/import-notebook-detection/test.toml @@ -1,4 +1,3 @@ Cloud = true -# Workspace paths start with "/"; keep Git Bash on Windows from rewriting them. Env.MSYS_NO_PATHCONV = "1" From 56505fad74392d0dce69a62056b75304b5f4761f Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 14:12:35 +0000 Subject: [PATCH 04/16] Revert "filer: import files the way the legacy import-file endpoint did (#7034)" This reverts commit ad9ba9864af2662e0b53c4b01e5d01873acf2cc5. --- acceptance/air/run-submit-deps/output.txt | 4 +- .../air/run-submit-overrides/output.txt | 6 +- .../bundle/apps/app_yaml/out.app.yml.txt | 2 +- .../outside_of_bundle_root/output.txt | 2 +- .../bundle/sync-upload-edge-cases/ipyNb.ipynb | 35 -------- .../bundle/sync-upload-edge-cases/output.txt | 84 +++--------------- .../query.lwquery.ipynb | 35 -------- .../bundle/sync-upload-edge-cases/script | 38 ++------ .../bundle/sync-upload-edge-cases/test.toml | 3 - .../simple/out.requests.deploy.json | 12 +-- .../simple/out.requests.destroy.json | 2 +- .../validate/sync_patterns/out.sync.txt | 2 +- .../cmd/workspace/import-zip-auto/archive.zip | Bin 408 -> 0 bytes .../workspace/import-zip-auto/out.test.toml | 1 - .../cmd/workspace/import-zip-auto/output.txt | 40 --------- .../cmd/workspace/import-zip-auto/script | 34 ------- .../import-zip-auto/space.geniespace.json | 1 - .../cmd/workspace/import-zip-auto/test.toml | 4 - libs/filer/workspace_files_client.go | 63 ++----------- libs/notebook/detect.go | 43 +++------ libs/notebook/detect_jupyter.go | 7 +- libs/notebook/detect_test.go | 27 ------ 22 files changed, 56 insertions(+), 389 deletions(-) delete mode 100644 acceptance/bundle/sync-upload-edge-cases/ipyNb.ipynb delete mode 100644 acceptance/bundle/sync-upload-edge-cases/query.lwquery.ipynb delete mode 100644 acceptance/cmd/workspace/import-zip-auto/archive.zip delete mode 100644 acceptance/cmd/workspace/import-zip-auto/out.test.toml delete mode 100644 acceptance/cmd/workspace/import-zip-auto/output.txt delete mode 100644 acceptance/cmd/workspace/import-zip-auto/script delete mode 100644 acceptance/cmd/workspace/import-zip-auto/space.geniespace.json delete mode 100644 acceptance/cmd/workspace/import-zip-auto/test.toml diff --git a/acceptance/air/run-submit-deps/output.txt b/acceptance/air/run-submit-deps/output.txt index 44f1ae35d86..de4c49e1b7f 100644 --- a/acceptance/air/run-submit-deps/output.txt +++ b/acceptance/air/run-submit-deps/output.txt @@ -11,8 +11,8 @@ Stream logs after submission using: === only config + command are uploaded; no requirements.yaml >>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh"}}} === inline deps, artifact location, and capacity id reach the submit payload >>> print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/air/run-submit-overrides/output.txt b/acceptance/air/run-submit-overrides/output.txt index 5ab207a7e3b..19323ff9a50 100644 --- a/acceptance/air/run-submit-overrides/output.txt +++ b/acceptance/air/run-submit-overrides/output.txt @@ -14,6 +14,6 @@ Stream logs after submission using: === the uploaded config preserves source order and contains overrides >>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 2\nparameters:\n model:\n hidden_size: 2048\n optimizer:\n name: adamw\n learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "RAW", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 2\nparameters:\n model:\n hidden_size: 2048\n optimizer:\n name: adamw\n learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh"}}} diff --git a/acceptance/bundle/apps/app_yaml/out.app.yml.txt b/acceptance/bundle/apps/app_yaml/out.app.yml.txt index 51431f28439..e79c7a4f57c 100644 --- a/acceptance/bundle/apps/app_yaml/out.app.yml.txt +++ b/acceptance/bundle/apps/app_yaml/out.app.yml.txt @@ -4,7 +4,7 @@ "body": { "multipart_form": { "content": "command:\n - python\n - app.py\n", - "format": "RAW", + "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml" } diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt index 72411ab8dbb..976a2f7e3e6 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt +++ b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt @@ -57,7 +57,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "path": "/api/2.0/workspace/import", "body": { "multipart_form": { - "format": "RAW", + "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl" } diff --git a/acceptance/bundle/sync-upload-edge-cases/ipyNb.ipynb b/acceptance/bundle/sync-upload-edge-cases/ipyNb.ipynb deleted file mode 100644 index d2d581029fb..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/ipyNb.ipynb +++ /dev/null @@ -1,35 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "05030541-8399-4226-a499-f11b4910888c", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Hello world!" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "py_ipynb", - "notebookOrigID": 4221142147066610, - "widgets": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/sync-upload-edge-cases/output.txt b/acceptance/bundle/sync-upload-edge-cases/output.txt index 9cda650785a..8ce0c31d5dc 100644 --- a/acceptance/bundle/sync-upload-edge-cases/output.txt +++ b/acceptance/bundle/sync-upload-edge-cases/output.txt @@ -1,21 +1,16 @@ >>> [CLI] bundle sync --output text -Action: PUT: .gitignore, archive.zip, dashboard.lvdash.json, databricks.yml, empty.txt, héllo.txt, ipyNb.ipynb, job.dbjob.json, large.bin, plain-script.py, pyNb.py, query.lwquery.ipynb, scalaNb.scala, space.geniespace.json, sqlNb.sql, with spaces.txt +Action: PUT: .gitignore, dashboard.lvdash.json, databricks.yml, empty.txt, héllo.txt, large.bin, plain-script.py, pyNb.py, scalaNb.scala, sqlNb.sql, with spaces.txt Initial Sync Complete Uploaded .gitignore -Uploaded archive.zip Uploaded dashboard.lvdash.json Uploaded databricks.yml Uploaded empty.txt Uploaded héllo.txt -Uploaded ipyNb.ipynb -Uploaded job.dbjob.json Uploaded large.bin Uploaded plain-script.py Uploaded pyNb.py -Uploaded query.lwquery.ipynb Uploaded scalaNb.scala -Uploaded space.geniespace.json Uploaded sqlNb.sql Uploaded with spaces.txt @@ -26,11 +21,6 @@ Uploaded with spaces.txt "object_type": "FILE", "language": null } -{ - "name": "archive.zip", - "object_type": "FILE", - "language": null -} { "name": "dashboard.lvdash.json", "object_type": "FILE", @@ -51,16 +41,6 @@ Uploaded with spaces.txt "object_type": "FILE", "language": null } -{ - "name": "ipyNb", - "object_type": "NOTEBOOK", - "language": "PYTHON" -} -{ - "name": "job.dbjob.json", - "object_type": "FILE", - "language": null -} { "name": "large.bin", "object_type": "FILE", @@ -76,21 +56,11 @@ Uploaded with spaces.txt "object_type": "NOTEBOOK", "language": "PYTHON" } -{ - "name": "query.lwquery", - "object_type": "NOTEBOOK", - "language": "PYTHON" -} { "name": "scalaNb", "object_type": "NOTEBOOK", "language": "SCALA" } -{ - "name": "space.geniespace.json", - "object_type": "FILE", - "language": null -} { "name": "sqlNb", "object_type": "NOTEBOOK", @@ -102,16 +72,6 @@ Uploaded with spaces.txt "language": null } -=== the .zip is uploaded with the legacy import-file endpoint ->>> print_requests.py //workspace-files/import-file --del-field raw_body --keep -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/archive.zip", - "q": { - "overwrite": "true" - } -} - === uploaded paths from each multipart POST /workspace/import (sorted) >>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/.gitignore @@ -119,44 +79,26 @@ Uploaded with spaces.txt /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/databricks.yml /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/empty.txt /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/héllo.txt -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/ipyNb -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/job.dbjob.json /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/large.bin /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/plain-script.py -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/pyNb -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/query.lwquery -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/scalaNb -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/space.geniespace.json -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/sqlNb +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/pyNb.py +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/scalaNb.scala +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/sqlNb.sql /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/with spaces.txt -=== format per upload: SOURCE/JUPYTER for notebooks, AUTO for dashboards, RAW for everything else +=== every upload set format=AUTO >>> jq -rs map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/files/"))) | - map({name: (.body.multipart_form.path | sub(".*/"; "")), format: .body.multipart_form.format}) | - sort_by(.name) | .[] | "\(.format) \(.name)" + map(.body.multipart_form.format) | unique [OUTPUT_DIR]/out.requests.txt -RAW .gitignore -AUTO dashboard.lvdash.json -RAW databricks.yml -RAW empty.txt -RAW héllo.txt -JUPYTER ipyNb -RAW job.dbjob.json -RAW large.bin -RAW plain-script.py -SOURCE pyNb -JUPYTER query.lwquery -SOURCE scalaNb -RAW space.geniespace.json -SOURCE sqlNb -RAW with spaces.txt +[ + "AUTO" +] === recorded content for the small text inputs >>> jq -rs map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/(pyNb|sqlNb|scalaNb|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | + map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | sort_by(.path) | .[] [OUTPUT_DIR]/out.requests.txt @@ -177,15 +119,15 @@ RAW with spaces.txt "content": "print(\"plain script, no header\")\n" } { - "path": "pyNb", + "path": "pyNb.py", "content": "# Databricks notebook source\nprint(\"hello\")\n" } { - "path": "scalaNb", + "path": "scalaNb.scala", "content": "// Databricks notebook source\nprintln(\"hello\")\n" } { - "path": "sqlNb", + "path": "sqlNb.sql", "content": "-- Databricks notebook source\nSELECT 1\n" } { diff --git a/acceptance/bundle/sync-upload-edge-cases/query.lwquery.ipynb b/acceptance/bundle/sync-upload-edge-cases/query.lwquery.ipynb deleted file mode 100644 index d2d581029fb..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/query.lwquery.ipynb +++ /dev/null @@ -1,35 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "05030541-8399-4226-a499-f11b4910888c", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Hello world!" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "py_ipynb", - "notebookOrigID": 4221142147066610, - "widgets": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script index 77c6708d27f..bfd3abcaa03 100644 --- a/acceptance/bundle/sync-upload-edge-cases/script +++ b/acceptance/bundle/sync-upload-edge-cases/script @@ -22,8 +22,8 @@ python3 -c "open('large.bin', 'wb').write(b'\\0' * (12 * 1024 * 1024))" # Empty file (multipart encodes an empty `content` part — distinct from JSON's empty string). touch empty.txt -# Python notebook: the CLI detects the header and imports it with format=SOURCE at the -# path with .py stripped. +# Python notebook: auto-detected as NOTEBOOK by /workspace/import; testserver mirrors +# this and stores the object at the path with .py stripped. cat > pyNb.py <<'EOF' # Databricks notebook source print("hello") @@ -41,33 +41,18 @@ cat > scalaNb.scala <<'EOF' println("hello") EOF -# ipyNb.ipynb and query.lwquery.ipynb are committed Jupyter notebooks. The second name -# ends in a suffix /workspace/import reserves for another asset type; it must still -# import as a notebook. - # Plain .py without the source header — should land as FILE, not NOTEBOOK, # even though the extension is .py. cat > plain-script.py <<'EOF' print("plain script, no header") EOF -# Lakeview dashboard descriptor: uploaded with format=AUTO, which the real /workspace/import -# imports as object_type=DASHBOARD and preserves the .lvdash.json extension. The testserver -# doesn't emulate that and stores it as a generic FILE — but the upload-side request shape -# (which is what this test asserts) is identical. +# Lakeview dashboard descriptor: the real /workspace/import assigns object_type=DASHBOARD +# and preserves the .lvdash.json extension. The testserver doesn't emulate that and stores +# it as a generic FILE — but the upload-side request shape (which is what this test asserts) +# is identical. echo '{"datasets":[],"pages":[]}' > dashboard.lvdash.json -# Names format=AUTO would map to asset types /workspace/import cannot create -# ("Requested node type [UNKNOWN] is not supported"). They must still sync as files. -echo '{}' > space.geniespace.json -echo '{}' > job.dbjob.json - -# A .zip goes through the legacy import-file endpoint, which extracts it into its parent -# directory (inner.txt next to the archive). The testserver doesn't emulate that and stores -# the archive as a FILE. /workspace/import would instead create a folder at the archive's -# path that cannot be overwritten on the next sync. -python3 -c "import zipfile; zipfile.ZipFile('archive.zip', 'w').writestr('inner.txt', 'hello')" - # Non-ASCII filename. Multipart encodes filenames with RFC 5987 / quoted-string rules, # distinct from URL-encoding in the legacy import-file endpoint. echo "hello, naïve world" > "héllo.txt" @@ -85,24 +70,19 @@ remote_files=$($CLI bundle summary -o json 2>/dev/null | jq -r .workspace.file_p trace MSYS_NO_PATHCONV=1 $CLI workspace list -o json "$remote_files" | jq 'sort_by(.path) | .[] | {name: (.path | sub(".*/"; "")), object_type, language}' -title "the .zip is uploaded with the legacy import-file endpoint" -trace print_requests.py //workspace-files/import-file --del-field raw_body --keep - title "uploaded paths from each multipart POST /workspace/import (sorted)" trace print_workspace_imports | sort -title "format per upload: SOURCE/JUPYTER for notebooks, AUTO for dashboards, RAW for everything else" +title "every upload set format=AUTO" trace jq -rs ' map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/files/"))) | - map({name: (.body.multipart_form.path | sub(".*/"; "")), format: .body.multipart_form.format}) | - sort_by(.name) | .[] | "\(.format) \(.name)" + map(.body.multipart_form.format) | unique ' "$OUT_REQUESTS" title "recorded content for the small text inputs" trace jq -rs ' map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/(pyNb|sqlNb|scalaNb|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | + map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | sort_by(.path) | .[] ' "$OUT_REQUESTS" diff --git a/acceptance/bundle/sync-upload-edge-cases/test.toml b/acceptance/bundle/sync-upload-edge-cases/test.toml index db1e57486a1..c9d06a640ab 100644 --- a/acceptance/bundle/sync-upload-edge-cases/test.toml +++ b/acceptance/bundle/sync-upload-edge-cases/test.toml @@ -8,9 +8,6 @@ Ignore = [ "scalaNb.scala", "plain-script.py", "dashboard.lvdash.json", - "space.geniespace.json", - "job.dbjob.json", - "archive.zip", "héllo.txt", "with spaces.txt", ".databricks", diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.json index 43b23a566c3..7713ace6e75 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.json @@ -159,7 +159,7 @@ "body": { "multipart_form": { "content": "", - "format": "RAW", + "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py" } @@ -176,7 +176,7 @@ "body": { "multipart_form": { "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "RAW", + "format": "AUTO", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } @@ -192,7 +192,7 @@ "body": { "multipart_form": { "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "RAW", + "format": "AUTO", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } @@ -208,7 +208,7 @@ "body": { "multipart_form": { "content": "{\"version\":1,\"seq\":1,\"cli_version\":\"[CLI_VERSION]\",\"timestamp\":\"[TIMESTAMP]\",\"files\":[{\"local_path\":\"empty.py\",\"is_notebook\":false}],\"id\":\"[UUID]\"}", - "format": "RAW", + "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" } @@ -225,7 +225,7 @@ "body": { "multipart_form": { "content": "{\n \"version\": 1,\n \"config\": {\n \"bundle\": {\n \"name\": \"test-bundle\",\n \"target\": \"default\",\n \"git\": {\n \"bundle_root_path\": \".\"\n }\n },\n \"workspace\": {\n \"file_path\": \"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"\n },\n \"resources\": {},\n \"presets\": {\n \"source_linked_deployment\": false\n }\n },\n \"extra\": {}\n}", - "format": "RAW", + "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" } @@ -242,7 +242,7 @@ "body": { "multipart_form": { "content": "{\n \"state_version\": 3,\n \"cli_version\": \"[CLI_VERSION]\",\n \"lineage\": \"[UUID]\",\n \"serial\": 1,\n \"state\": {\n \"resources.schemas.foo\": {\n \"__id__\": \"mycatalog.myschema\",\n \"state\": {\n \"catalog_name\": \"mycatalog\",\n \"name\": \"myschema\"\n }\n }\n }\n}", - "format": "RAW", + "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" } diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.json index 90328199953..168944a8311 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.json @@ -141,7 +141,7 @@ "body": { "multipart_form": { "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "RAW", + "format": "AUTO", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } diff --git a/acceptance/bundle/validate/sync_patterns/out.sync.txt b/acceptance/bundle/validate/sync_patterns/out.sync.txt index bbac212feb3..fd939c9721b 100644 --- a/acceptance/bundle/validate/sync_patterns/out.sync.txt +++ b/acceptance/bundle/validate/sync_patterns/out.sync.txt @@ -4,7 +4,7 @@ "body": { "multipart_form": { "content": "", - "format": "RAW", + "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml" } diff --git a/acceptance/cmd/workspace/import-zip-auto/archive.zip b/acceptance/cmd/workspace/import-zip-auto/archive.zip deleted file mode 100644 index fabadf6fa085ac09b3c6eaf683379184e7d65168..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 408 zcmWIWW@Zs#00GZr-&imMO0WRwg6wqt02CE(@)G8=0cAj#6RILnucV?RF{dCSkqcEZ zP#z9AfGm)4#idDTMsnZg;RVTpFfWp-WRS7R8Hq(XnW;d78JR?wak~_#4+sPp{yKta zh`UjBpt~HT4g?w)m4GBj2iPC@ECRU@0zek|1DOaPp<0ISCz!ql#!@6*U>^o}vw{MX OfdvR}0qGSW4g&y>bUdp7 diff --git a/acceptance/cmd/workspace/import-zip-auto/out.test.toml b/acceptance/cmd/workspace/import-zip-auto/out.test.toml deleted file mode 100644 index c7c6f58ed6e..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/out.test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = true diff --git a/acceptance/cmd/workspace/import-zip-auto/output.txt b/acceptance/cmd/workspace/import-zip-auto/output.txt deleted file mode 100644 index 6f9f2709742..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/output.txt +++ /dev/null @@ -1,40 +0,0 @@ - -=== AUTO extracts a .zip into a folder named after the archive, without .zip - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive.zip --file archive.zip --format AUTO - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME] -DIRECTORY /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive/a.txt -DIRECTORY /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive/sub - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive/sub -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive/sub/c.txt - -=== Importing the archive again is rejected, with or without overwrite - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive.zip --file archive.zip --format AUTO --overwrite -Error: Overwrite cannot be used for AUTO format when importing a folder. - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive.zip --file archive.zip --format AUTO -Error: /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive already exists. Please pass overwrite=true to overwrite it. - -=== RAW stores the archive as a file, and overwrites it - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/raw.zip --file archive.zip --format RAW - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/raw.zip --file archive.zip --format RAW --overwrite - -=== AUTO rejects names reserved for asset types it cannot import; RAW stores them as files - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/space.geniespace.json --file space.geniespace.json --format AUTO -Error: Requested node type [UNKNOWN] is not supported - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/space.geniespace.json --file space.geniespace.json --format RAW - ->>> list /Workspace/Users/[USERNAME]/[UNIQUE_NAME] -DIRECTORY /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/archive -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/raw.zip -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/space.geniespace.json diff --git a/acceptance/cmd/workspace/import-zip-auto/script b/acceptance/cmd/workspace/import-zip-auto/script deleted file mode 100644 index 8c48e84ee04..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/script +++ /dev/null @@ -1,34 +0,0 @@ -# Documents how POST /api/2.0/workspace/import handles a .zip and names reserved for -# other asset types. archive.zip holds a single top-level folder: pkg/a.txt, pkg/sub/c.txt. -DIR="/Workspace/Users/$CURRENT_USER_NAME/$UNIQUE_NAME" -$CLI workspace mkdirs "$DIR" - -cleanup() { - $CLI workspace delete "$DIR" --recursive -} -trap cleanup EXIT - -list() { - $CLI workspace list "$1" -o json | jq -r 'sort_by(.path)[] | "\(.object_type) \(.path)"' -} - -title "AUTO extracts a .zip into a folder named after the archive, without .zip\n" -trace $CLI workspace import "$DIR/archive.zip" --file archive.zip --format AUTO -trace list "$DIR" -# The archive's single top-level folder (pkg/) is replaced by the target folder. -trace list "$DIR/archive" -trace list "$DIR/archive/sub" - -title "Importing the archive again is rejected, with or without overwrite\n" -trace musterr $CLI workspace import "$DIR/archive.zip" --file archive.zip --format AUTO --overwrite -trace musterr $CLI workspace import "$DIR/archive.zip" --file archive.zip --format AUTO - -title "RAW stores the archive as a file, and overwrites it\n" -trace $CLI workspace import "$DIR/raw.zip" --file archive.zip --format RAW -trace $CLI workspace import "$DIR/raw.zip" --file archive.zip --format RAW --overwrite - -title "AUTO rejects names reserved for asset types it cannot import; RAW stores them as files\n" -trace musterr $CLI workspace import "$DIR/space.geniespace.json" --file space.geniespace.json --format AUTO -trace $CLI workspace import "$DIR/space.geniespace.json" --file space.geniespace.json --format RAW - -trace list "$DIR" diff --git a/acceptance/cmd/workspace/import-zip-auto/space.geniespace.json b/acceptance/cmd/workspace/import-zip-auto/space.geniespace.json deleted file mode 100644 index cb608f6e9c4..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/space.geniespace.json +++ /dev/null @@ -1 +0,0 @@ -{"version": 1} diff --git a/acceptance/cmd/workspace/import-zip-auto/test.toml b/acceptance/cmd/workspace/import-zip-auto/test.toml deleted file mode 100644 index 6b74aa538f0..00000000000 --- a/acceptance/cmd/workspace/import-zip-auto/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -Cloud = true - -# Workspace paths start with "/"; keep Git Bash on Windows from rewriting them. -Env.MSYS_NO_PATHCONV = "1" diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index f6a32369f88..24188974ed4 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -9,14 +9,11 @@ import ( "io" "io/fs" "net/http" - "net/url" "path" "slices" - "strings" "time" "github.com/databricks/cli/libs/auth" - "github.com/databricks/cli/libs/notebook" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/client" @@ -111,39 +108,6 @@ type apiClient interface { visitors ...func(*http.Request) error) error } -// Extensions that format=AUTO imports as the same asset type the legacy import-file -// endpoint created: dashboards, alerts and Lakeflow Designer files. -var autoImportAssetExtensions = []string{".lvdash.json", ".dbalert.json", ".designer.py", ".designer.ipynb"} - -// importRequest returns the workspace path and /workspace/import options for a file, -// matching how the legacy import-file endpoint stored it. Other files avoid format=AUTO -// because it types them by name and rejects names reserved for other assets -// (e.g. .geniespace.json, .lwquery.ipynb). -func importRequest(absPath string, content []byte) (string, []workspace.UploadOption) { - lower := strings.ToLower(absPath) - for _, ext := range autoImportAssetExtensions { - if strings.HasSuffix(lower, ext) { - return absPath, []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatAuto)} - } - } - - isNotebook, language, _ := notebook.DetectBytes(absPath, content) - if !isNotebook { - return absPath, []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatRaw)} - } - - // Notebooks are imported without their extension and with an explicit format. - ext := path.Ext(absPath) - notebookPath := strings.TrimSuffix(absPath, ext) - if strings.EqualFold(ext, notebook.ExtensionJupyter) { - return notebookPath, []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatJupyter)} - } - return notebookPath, []workspace.UploadOption{ - workspace.UploadFormat(workspace.ImportFormatSource), - workspace.UploadLanguage(language), - } -} - // WorkspaceFilesClient implements the files-in-workspace API. // NOTE: This API is available for files under /Repos if a workspace has files-in-repos enabled. @@ -194,27 +158,16 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - overwrite := slices.Contains(mode, OverwriteIfExists) - if strings.HasSuffix(absPath, ".zip") { - // The legacy import-file endpoint extracts a .zip into its parent directory and - // overwrites existing entries. /workspace/import imports it as a new folder that - // cannot be overwritten, so zips stay on import-file until it supports that. - urlPath := fmt.Sprintf( - "/api/2.0/workspace-files/import-file/%s?overwrite=%t", - url.PathEscape(strings.TrimLeft(absPath, "/")), - overwrite, - ) - err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) - } else { - // Upload with the SDK's multipart Workspace.Upload. The JSON-body workspace.Import - // caps content at 10 MB, the multipart form at the 500 MB workspace file limit. - uploadPath, opts := importRequest(absPath, body) - if overwrite { - opts = append(opts, workspace.UploadOverwrite()) - } - err = w.workspaceClient.Workspace.Upload(ctx, uploadPath, bytes.NewReader(body), opts...) + // Upload with the SDK's multipart Workspace.Upload; format=AUTO lets the server + // classify each payload as file or notebook. The JSON-body workspace.Import caps + // content at 10 MB, the multipart form at the 500 MB workspace file limit. + opts := []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatAuto)} + if slices.Contains(mode, OverwriteIfExists) { + opts = append(opts, workspace.UploadOverwrite()) } + err = w.workspaceClient.Workspace.Upload(ctx, absPath, bytes.NewReader(body), opts...) + // Return early on success. if err == nil { return nil diff --git a/libs/notebook/detect.go b/libs/notebook/detect.go index d9be548a5c6..579cc1de34c 100644 --- a/libs/notebook/detect.go +++ b/libs/notebook/detect.go @@ -44,10 +44,10 @@ func (f file) close() error { return f.f.Close() } -func readHeader(r io.Reader) (string, error) { +func (f file) readHeader() (string, error) { // Scan header line with some padding. buf := make([]byte, headerLength) - n, err := r.Read(buf) + n, err := f.f.Read(buf) if err != nil && err != io.EOF { return "", err } @@ -80,6 +80,8 @@ func (f file) getObjectInfo() (oi workspace.ObjectInfo, ok bool, err error) { // Detect returns whether the file at path is a Databricks notebook. // If it is, it returns the notebook language. func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.Language, err error) { + header := "" + f, err := openFile(fsys, name) if err != nil { return false, "", err @@ -96,39 +98,12 @@ func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.La return oi.ObjectType == workspace.ObjectTypeNotebook, oi.Language, nil } - if strings.ToLower(filepath.Ext(name)) == ExtensionJupyter { - return DetectJupyterWithFS(fsys, name) - } - // Read the first line of the file. - fileHeader, err := readHeader(f.f) - if err != nil { - return false, "", err - } - - notebook, language = detectSource(name, fileHeader) - return notebook, language, nil -} - -// DetectBytes is like [DetectWithFS] for file content that is already in memory. -func DetectBytes(name string, content []byte) (notebook bool, language workspace.Language, err error) { - if strings.ToLower(filepath.Ext(name)) == ExtensionJupyter { - return detectJupyter(name, bytes.NewReader(content)) - } - - fileHeader, err := readHeader(bytes.NewReader(content)) + fileHeader, err := f.readHeader() if err != nil { return false, "", err } - notebook, language = detectSource(name, fileHeader) - return notebook, language, nil -} - -// detectSource reports whether a source file with the given name and first line is a notebook. -func detectSource(name, fileHeader string) (notebook bool, language workspace.Language) { - header := "" - // Determine which header to expect based on filename extension. ext := strings.ToLower(filepath.Ext(name)) switch ext { @@ -144,15 +119,17 @@ func detectSource(name, fileHeader string) (notebook bool, language workspace.La case ExtensionSql: header = "-- Databricks notebook source" language = workspace.LanguageSql + case ExtensionJupyter: + return DetectJupyterWithFS(fsys, name) default: - return false, "" + return false, "", nil } if fileHeader != header { - return false, "" + return false, "", nil } - return true, language + return true, language, nil } // Detect calls DetectWithFS with the local filesystem. diff --git a/libs/notebook/detect_jupyter.go b/libs/notebook/detect_jupyter.go index 45233414007..f631b58127b 100644 --- a/libs/notebook/detect_jupyter.go +++ b/libs/notebook/detect_jupyter.go @@ -3,7 +3,6 @@ package notebook import ( "encoding/json" "fmt" - "io" "io/fs" "os" "path/filepath" @@ -67,12 +66,8 @@ func DetectJupyterWithFS(fsys fs.FS, name string) (notebook bool, language works defer f.Close() - return detectJupyter(name, f) -} - -func detectJupyter(name string, r io.Reader) (notebook bool, language workspace.Language, err error) { var nb jupyter - dec := json.NewDecoder(r) + dec := json.NewDecoder(f) err = dec.Decode(&nb) if err != nil { return false, "", fmt.Errorf("%s: error loading Jupyter notebook file: %w", name, err) diff --git a/libs/notebook/detect_test.go b/libs/notebook/detect_test.go index c774612c9bf..49a67d2d3ed 100644 --- a/libs/notebook/detect_test.go +++ b/libs/notebook/detect_test.go @@ -127,30 +127,3 @@ func TestDetectWithObjectInfo(t *testing.T) { assert.True(t, nb) assert.Equal(t, workspace.LanguagePython, lang) } - -func TestDetectBytes(t *testing.T) { - ipynb, err := os.ReadFile("testdata/py_ipynb.ipynb") - require.NoError(t, err) - - tests := []struct { - name string - content []byte - notebook bool - language workspace.Language - }{ - {"nb.py", []byte("# Databricks notebook source\nprint(1)\n"), true, workspace.LanguagePython}, - {"nb.sql", []byte("-- Databricks notebook source\nSELECT 1\n"), true, workspace.LanguageSql}, - {"nb.ipynb", ipynb, true, workspace.LanguagePython}, - {"plain.py", []byte("print(1)\n"), false, ""}, - {"space.geniespace.json", []byte("{}"), false, ""}, - {"empty.py", nil, false, ""}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - nb, lang, err := DetectBytes(tc.name, tc.content) - require.NoError(t, err) - assert.Equal(t, tc.notebook, nb) - assert.Equal(t, tc.language, lang) - }) - } -} From 76f8cd760b8a20bdc1ccb22131d7c2cfbf0bcb69 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 14:25:19 +0000 Subject: [PATCH 05/16] Revert "filer: switch workspace upload from import-file to /workspace/import (#6149)" This reverts commit b4f7a43ca. /workspace/import classifies files differently from import-file and can't overwrite some of them, which broke deployments (#7216). Forward-fixes for changes that landed on main after #6149: - upload-504 tests (#7014) and upload/internal_server_error inject their faults on the import-file URL again; --body-contains is gone. - The multipart request recorder in prepare_server.go stays, since snapshot uploads use it. - The testserver keeps its /workspace/import modelling for `workspace import`, minus the JUPYTER format no test uses anymore. - sync-upload-edge-cases, added by #6149, is removed. Co-authored-by: Isaac --- acceptance/air/run-submit-deps/output.txt | 6 +- acceptance/air/run-submit-deps/script | 5 +- .../air/run-submit-overrides/output.txt | 8 +- acceptance/air/run-submit-overrides/script | 2 +- acceptance/bin/fault.py | 16 +- .../bundle/apps/app_yaml/out.app.yml.txt | 14 +- acceptance/bundle/apps/app_yaml/output.txt | 2 +- acceptance/bundle/apps/app_yaml/script | 2 +- .../ai_runtime_code_source/output.txt | 4 +- .../artifacts/ai_runtime_code_source/script | 2 +- .../artifact_upload_for_workspace/output.txt | 6 +- .../artifact_upload_for_workspace/script | 2 +- .../output.txt | 6 +- .../script | 2 +- .../upload_multiple_libraries/output.txt | 18 +- .../upload_multiple_libraries/script | 2 +- .../artifacts/whl_change_version/output.txt | 60 +++--- .../artifacts/whl_change_version/script | 6 +- .../bundle/artifacts/whl_dbfs/output.txt | 2 +- acceptance/bundle/artifacts/whl_dbfs/script | 2 +- .../bundle/artifacts/whl_dynamic/output.txt | 18 +- .../bundle/artifacts/whl_dynamic/script | 4 +- .../bundle/artifacts/whl_explicit/output.txt | 6 +- .../bundle/artifacts/whl_explicit/script | 2 +- .../bundle/artifacts/whl_implicit/output.txt | 6 +- .../bundle/artifacts/whl_implicit/script | 2 +- .../whl_implicit_custom_path/output.txt | 6 +- .../artifacts/whl_implicit_custom_path/script | 2 +- .../whl_implicit_notebook/output.txt | 6 +- .../artifacts/whl_implicit_notebook/script | 2 +- .../bundle/artifacts/whl_multiple/output.txt | 10 +- .../bundle/artifacts/whl_multiple/script | 2 +- .../artifacts/whl_no_cleanup/output.txt | 6 +- .../bundle/artifacts/whl_no_cleanup/script | 2 +- .../whl_prebuilt_multiple/output.txt | 10 +- .../artifacts/whl_prebuilt_multiple/script | 2 +- .../artifacts/whl_prebuilt_outside/output.txt | 6 +- .../artifacts/whl_prebuilt_outside/script | 2 +- .../whl_prebuilt_outside_dynamic/output.txt | 6 +- .../whl_prebuilt_outside_dynamic/script | 2 +- .../whl_via_environment_key/output.txt | 6 +- .../artifacts/whl_via_environment_key/script | 2 +- .../whl_via_environment_key_extras/output.txt | 4 +- .../whl_via_environment_key_extras/script | 2 +- .../bundle/deploy/files/upload-504/output.txt | 6 +- .../bundle/deploy/files/upload-504/script | 4 +- .../partial-summary-on-push-fail/output.txt | 2 +- .../partial-summary-on-push-fail/script | 2 +- .../partial-summary-on-push-fail/test.toml | 2 +- .../deploy/readplan/whl-upload/output.txt | 20 +- .../bundle/deploy/readplan/whl-upload/script | 11 +- .../bind/pipelines/recreate/output.txt | 4 +- .../deployment/bind/pipelines/recreate/script | 6 +- .../bind/pipelines/update/output.txt | 2 +- .../deployment/bind/pipelines/update/script | 6 +- .../destroy/force-lock-node-limit/script | 4 +- .../outside_of_bundle_root/output.txt | 10 +- .../libraries/outside_of_bundle_root/script | 2 +- .../migrate/auto/empty-tfstate/output.txt | 16 +- .../bundle/migrate/auto/empty-tfstate/script | 2 +- .../migrate/auto/push-failure/output.txt | 2 +- .../bundle/migrate/auto/push-failure/script | 2 +- .../resource_deps/remote_app_url/output.txt | 6 +- .../resource_deps/remote_app_url/script | 6 +- .../resources/jobs/request-summary/output.txt | 22 +- .../change_assets_dir/output.txt | 2 +- .../quality_monitors/change_assets_dir/script | 2 +- .../change_output_schema_name/output.txt | 2 +- .../change_output_schema_name/script | 2 +- .../change_table_name/output.txt | 2 +- .../quality_monitors/change_table_name/script | 2 +- .../quality_monitors/create/output.txt | 2 +- .../resources/quality_monitors/create/script | 2 +- .../delete_scope/out.deploy.requests.txt | 2 +- .../secret_scopes/delete_scope/script | 2 +- .../sync-upload-edge-cases/databricks.yml | 2 - .../sync-upload-edge-cases/out.test.toml | 2 - .../bundle/sync-upload-edge-cases/output.txt | 136 ------------ .../bundle/sync-upload-edge-cases/script | 92 --------- .../bundle/sync-upload-edge-cases/test.toml | 14 -- acceptance/bundle/sync/upload-504/output.txt | 6 +- acceptance/bundle/sync/upload-504/script | 4 +- .../deploy-failed-engine-unknown/output.txt | 6 +- .../deploy-failed-engine-unknown/test.toml | 2 +- .../templates/default-python/classic/script | 4 +- .../default-python/serverless/script | 4 +- .../upload/internal_server_error/output.txt | 8 +- .../upload/internal_server_error/script | 2 +- acceptance/bundle/upload/timeout/output.txt | 4 +- acceptance/bundle/upload/timeout/test.toml | 2 +- acceptance/bundle/user_agent/output.txt | 14 +- .../simple/out.requests.deploy.json | 122 +++++++---- .../simple/out.requests.destroy.json | 22 +- .../validate/sync_patterns/out.sync.txt | 11 +- .../bundle/validate/sync_patterns/script | 2 +- .../cmd/sync/dryrun-missing-remote/output.txt | 4 +- acceptance/cmd/sync/upload-504/output.txt | 6 +- acceptance/cmd/sync/upload-504/script | 4 +- acceptance/script.prepare | 6 - cmd/air/runsubmit_test.go | 30 ++- cmd/air/telemetry_test.go | 11 +- libs/filer/workspace_files_client.go | 69 ++++--- libs/filer/workspace_files_client_test.go | 195 ------------------ libs/sync/watchdog.go | 5 +- libs/testserver/fault.go | 31 +-- libs/testserver/fault_test.go | 48 ++--- libs/testserver/handlers.go | 7 +- libs/testserver/server.go | 2 +- libs/testserver/workspace_test.go | 18 -- 109 files changed, 426 insertions(+), 909 deletions(-) delete mode 100644 acceptance/bundle/sync-upload-edge-cases/databricks.yml delete mode 100644 acceptance/bundle/sync-upload-edge-cases/out.test.toml delete mode 100644 acceptance/bundle/sync-upload-edge-cases/output.txt delete mode 100644 acceptance/bundle/sync-upload-edge-cases/script delete mode 100644 acceptance/bundle/sync-upload-edge-cases/test.toml diff --git a/acceptance/air/run-submit-deps/output.txt b/acceptance/air/run-submit-deps/output.txt index de4c49e1b7f..3346fcd06d7 100644 --- a/acceptance/air/run-submit-deps/output.txt +++ b/acceptance/air/run-submit-deps/output.txt @@ -10,9 +10,9 @@ Stream logs after submission using: databricks air logs 555 === only config + command are uploaded; no requirements.yaml ->>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh"}}} +>>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} === inline deps, artifact location, and capacity id reach the submit payload >>> print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/air/run-submit-deps/script b/acceptance/air/run-submit-deps/script index 0e8951cae0f..422980c99b3 100644 --- a/acceptance/air/run-submit-deps/script +++ b/acceptance/air/run-submit-deps/script @@ -2,10 +2,7 @@ title "submit with inline dependencies" trace $CLI air run -f run.yaml title "only config + command are uploaded; no requirements.yaml" -# Uploads go to POST /workspace/import, which carries the target filename in the -# multipart body rather than the URL, so each uploaded path shows up under -# body.multipart_form.path instead of in the request path. -trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep +trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep title "inline deps, artifact location, and capacity id reach the submit payload" trace print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/air/run-submit-overrides/output.txt b/acceptance/air/run-submit-overrides/output.txt index 19323ff9a50..0be49262d1b 100644 --- a/acceptance/air/run-submit-overrides/output.txt +++ b/acceptance/air/run-submit-overrides/output.txt @@ -13,7 +13,7 @@ Stream logs after submission using: databricks air logs 555 === the uploaded config preserves source order and contains overrides ->>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 2\nparameters:\n model:\n hidden_size: 2048\n optimizer:\n name: adamw\n learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml"}}} -{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh"}}} +>>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml", "q": {"overwrite": "true"}, "raw_body": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: overrides-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 2\nparameters:\n model:\n hidden_size: 2048\n optimizer:\n name: adamw\n learning_rate: 0.001\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} diff --git a/acceptance/air/run-submit-overrides/script b/acceptance/air/run-submit-overrides/script index 59234b5e874..263c1920e6c 100644 --- a/acceptance/air/run-submit-overrides/script +++ b/acceptance/air/run-submit-overrides/script @@ -5,4 +5,4 @@ trace $CLI air run -f run.yaml \ --override parameters.optimizer.learning_rate=0.001 title "the uploaded config preserves source order and contains overrides" -trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique +trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique diff --git a/acceptance/bin/fault.py b/acceptance/bin/fault.py index 5b75c0a5907..0aed44d58d1 100755 --- a/acceptance/bin/fault.py +++ b/acceptance/bin/fault.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 """Set up a fault rule on the testserver for the current test token. -Usage: fault.py [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] +Usage: fault.py [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] --after-handler run the handler first; keep its effect and replace only the response - --body-contains only fire when the request body contains SUBSTR, e.g. to target one - file's /workspace/import upload (all uploads share that path) PATTERN HTTP method and path, supports trailing * wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*" STATUS_CODE HTTP status code to return, e.g. 504 @@ -35,17 +33,8 @@ if after_handler: args.remove("--after-handler") -body_contains = "" -if "--body-contains" in args: - i = args.index("--body-contains") - body_contains = args[i + 1] - del args[i : i + 2] - if len(args) not in (4, 5): - print( - f"usage: {sys.argv[0]} [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", - file=sys.stderr, - ) + print(f"usage: {sys.argv[0]} [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr) sys.exit(1) pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3]) @@ -60,7 +49,6 @@ "offset": offset, "times": times, "after_handler": after_handler, - "body_contains": body_contains, } ).encode() diff --git a/acceptance/bundle/apps/app_yaml/out.app.yml.txt b/acceptance/bundle/apps/app_yaml/out.app.yml.txt index e79c7a4f57c..29cb7da70fa 100644 --- a/acceptance/bundle/apps/app_yaml/out.app.yml.txt +++ b/acceptance/bundle/apps/app_yaml/out.app.yml.txt @@ -1,12 +1,8 @@ { "method": "POST", - "path": "/api/2.0/workspace/import", - "body": { - "multipart_form": { - "content": "command:\n - python\n - app.py\n", - "format": "AUTO", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml" - } - } + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml", + "q": { + "overwrite": "true" + }, + "raw_body": "command:\n - python\n - app.py\n" } diff --git a/acceptance/bundle/apps/app_yaml/output.txt b/acceptance/bundle/apps/app_yaml/output.txt index 50baf9b4559..3554b68f44e 100644 --- a/acceptance/bundle/apps/app_yaml/output.txt +++ b/acceptance/bundle/apps/app_yaml/output.txt @@ -19,7 +19,7 @@ Created apps.myapp Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> jq select(.body.multipart_form.path | strings | test("app.yml")) [OUTPUT_DIR]/out.requests.txt +>>> jq select(.path | test("app.yml")) [OUTPUT_DIR]/out.requests.txt >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/apps/app_yaml/script b/acceptance/bundle/apps/app_yaml/script index 3267d66c598..3db6c9e8969 100644 --- a/acceptance/bundle/apps/app_yaml/script +++ b/acceptance/bundle/apps/app_yaml/script @@ -1,7 +1,7 @@ trace $CLI bundle validate trace $CLI bundle plan trace $CLI bundle deploy -trace jq 'select(.body.multipart_form.path | strings | test("app.yml"))' "$OUT_REQUESTS" | sed 's/\\r//g' > out.app.yml.txt +trace jq 'select(.path | test("app.yml"))' "$OUT_REQUESTS" | sed 's/\\r//g' > out.app.yml.txt #trace print_requests.py //apps # currently fails due to TF inserting description="" rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt index 1a3ffc22b37..70bbb6eb108 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt @@ -16,5 +16,5 @@ } === Expecting the code_source tarball to be uploaded ->>> print_workspace_imports -/Workspace/foo/bar/artifacts/.internal/code.tgz +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz" diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/script b/acceptance/bundle/artifacts/ai_runtime_code_source/script index 32460827ec5..396f907033e 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/script +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/script @@ -6,6 +6,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' "$OUT_REQUESTS" title "Expecting the code_source tarball to be uploaded" -trace print_workspace_imports | grep code.tgz | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep code.tgz | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt index 9997818f806..2a10679b430 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt @@ -48,9 +48,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl -/Workspace/foo/bar/artifacts/.internal/source.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script index b3c226bc752..70c9dffac83 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script @@ -9,7 +9,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt index ca39c787992..2419cc5afbc 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt @@ -4,9 +4,9 @@ Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Expecting wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl -/Workspace/foo/bar/artifacts/.internal/source.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script index 0d665cfbc5f..7ac87760449 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script @@ -6,7 +6,7 @@ $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) title "Expecting wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt index e500ebd0342..ee40963cbaa 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt @@ -38,15 +38,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 4 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl -/Workspace/foo/bar/artifacts/.internal/source1.whl -/Workspace/foo/bar/artifacts/.internal/source2.whl -/Workspace/foo/bar/artifacts/.internal/source3.whl -/Workspace/foo/bar/artifacts/.internal/source4.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source1.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source2.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source3.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl" === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/script b/acceptance/bundle/artifacts/upload_multiple_libraries/script index 89d0ba55291..f65f8154f2a 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/script +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/script @@ -12,7 +12,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 4 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 30134175168..34b272c0f08 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -29,19 +29,19 @@ dist/my_test_code-0.1.0-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME +>>> jq .path [OUTPUT_DIR]/out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" >>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0 @@ -62,14 +62,14 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME +>>> jq .path [OUTPUT_DIR]/out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" === Restore config to target old wheel >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl @@ -92,12 +92,12 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME +>>> jq .path [OUTPUT_DIR]/out.requests.txt +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" diff --git a/acceptance/bundle/artifacts/whl_change_version/script b/acceptance/bundle/artifacts/whl_change_version/script index 4b97cb54259..49d50f0b0b5 100644 --- a/acceptance/bundle/artifacts/whl_change_version/script +++ b/acceptance/bundle/artifacts/whl_change_version/script @@ -6,7 +6,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | sort +trace jq .path "$OUT_REQUESTS" | grep import | sort rm "$OUT_REQUESTS" @@ -20,7 +20,7 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | sort +trace jq .path "$OUT_REQUESTS" | grep import | sort rm "$OUT_REQUESTS" @@ -34,6 +34,6 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | sort +trace jq .path "$OUT_REQUESTS" | grep import | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index 04b3a0f58fa..b4974a6f5ca 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -25,6 +25,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting no wheels to be uploaded ->>> print_workspace_imports +>>> errcode sh -c jq .path < "$OUT_REQUESTS" | grep import | grep whl Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script index bd7b88b5806..944ecf709f9 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/script +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting no wheels to be uploaded" -trace print_workspace_imports | errcode grep whl +trace errcode sh -c 'jq .path < "$OUT_REQUESTS" | grep import | grep whl' rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index 3461014160d..51be24e7e43 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -48,11 +48,11 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 patched wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl" === Updating the local wheel and deploying again @@ -74,7 +74,7 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice) ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index d031d02715c..b73202a65ec 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -22,7 +22,7 @@ title "Expecting 2 patched wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 2 patched wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" @@ -40,4 +40,4 @@ title "Expecting 2 patched wheels in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 14358c0ab32..55aa3ff229a 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -21,9 +21,9 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index a1227cd125d..f55d7da467a 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -8,7 +8,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index c610058f32f..d589ebc5d74 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -21,6 +21,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index 8124726f4ff..a42ce2844db 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -8,6 +8,6 @@ title "Expecting 1 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 1a93fb8c9bd..04683b58ffc 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -39,6 +39,6 @@ package/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl -/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 4d5fd31d6e7..410957d8233 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -7,6 +7,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index 6a49d1e1889..4172ddf8f51 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -11,6 +11,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index 41bae0e4237..72d0ce55360 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index 96edc8e5c85..8bf5bef423d 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -25,8 +25,8 @@ my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index 8c06cd32454..df8ad98b3c4 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -8,6 +8,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm -fr "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt index b80a0145ad1..a61388c76f9 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt +++ b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt @@ -3,9 +3,9 @@ dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" === No calls to delete internal folder expected >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/script b/acceptance/bundle/artifacts/whl_no_cleanup/script index 6a4a10bc75e..f3ef86ecf70 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/script +++ b/acceptance/bundle/artifacts/whl_no_cleanup/script @@ -5,7 +5,7 @@ cat LOG.deploy | contains.py "my_test_code-0.0.1-py3-none-any.whl" > /dev/null trace find.py --expect 1 whl title "Expecting 1 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index c18e0b2d6da..e43ce01f38c 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -36,8 +36,8 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index 8034c3171cf..2b9240de98a 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -10,6 +10,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt index a034c80a088..9611c9541f2 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt @@ -32,6 +32,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/script b/acceptance/bundle/artifacts/whl_prebuilt_outside/script index 23a2e0fc1ef..ecfd7bea799 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/script @@ -11,6 +11,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt index b26bae880f5..324b0a1b18a 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt @@ -56,6 +56,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script index f972cf2504e..b53c1dc55b2 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script @@ -14,6 +14,6 @@ trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_RE trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index f6ba380bc11..eb0d23431f3 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -41,6 +41,6 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index 09f9ed82c47..d7657fa4b4f 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in environments section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt index e99a248c94b..39e7ead5144 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt @@ -41,5 +41,5 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded under its bare filename (no extras) ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index 157472b5f55..dadef34e181 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -8,6 +8,6 @@ title "Expecting the environments dependency to keep its [train] extras suffix" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" -trace print_workspace_imports | grep whl | sort +trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deploy/files/upload-504/output.txt b/acceptance/bundle/deploy/files/upload-504/output.txt index 52ff524b127..7e6db166c5c 100644 --- a/acceptance/bundle/deploy/files/upload-504/output.txt +++ b/acceptance/bundle/deploy/files/upload-504/output.txt @@ -1,6 +1,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace/import: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml: Fault injected by test. +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml diff --git a/acceptance/bundle/deploy/files/upload-504/script b/acceptance/bundle/deploy/files/upload-504/script index 78740b9becc..4e48044acc5 100644 --- a/acceptance/bundle/deploy/files/upload-504/script +++ b/acceptance/bundle/deploy/files/upload-504/script @@ -1,8 +1,8 @@ # Let the two deploy lock requests through, then inject a single 504 on the next import request, which is a file upload. # Uploads overwrite, so they are retried and the deploy succeeds. -fault.py "POST /api/2.0/workspace/import" 504 2 1 +fault.py "POST /api/2.0/workspace-files/import-file/*" 504 2 1 $CLI bundle deploy # databricks.yml is the only file. Two requests for it should appear: the initial 504 and the successful retry. -print_requests.py //api/2.0/workspace/import | jq -r .body.multipart_form.path | grep '/files/' +print_requests.py //api/2.0/workspace-files/import-file | jq -r .path | grep '/files/' diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt index 4610f302b33..a97e8642981 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt @@ -3,7 +3,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/script b/acceptance/bundle/deploy/partial-summary-on-push-fail/script index d2fef85a1d7..bc36c59cacc 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/script +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/script @@ -9,7 +9,7 @@ trap cleanup EXIT # Inject a 403 on the resource-state upload (PushResourcesState). The files are # uploaded by then, so "Files:" still reports them; "Resources:" must NOT print, # because this fails before planning and there are no resource counts to report. -fault.py --body-contains "state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 # -q: the state-push 403 races resource apply, so whether "Created jobs.my_job" is # emitted before the failure is platform-dependent. This test asserts Files is reported diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml b/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml index 64cb4896dce..01dc2713a99 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml @@ -1,5 +1,5 @@ # The resource summary must not print when pushing resource state fails, so the -# fault is injected on the state upload (POST /workspace/import). The path is fixed, so +# fault is injected on the state upload (POST import-file). The path is fixed, so # the bundle name must be stable (test-bundle, not $UNIQUE_NAME). # # The injected path is the direct-engine state file (state/resources.json), so diff --git a/acceptance/bundle/deploy/readplan/whl-upload/output.txt b/acceptance/bundle/deploy/readplan/whl-upload/output.txt index 6b21d59c6b0..48212c57e37 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-upload/output.txt @@ -7,8 +7,14 @@ Created jobs.my_job Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +>>> print_requests.py //artifacts/.internal --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "q": { + "overwrite": "true" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -26,8 +32,14 @@ Created jobs.my_job Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +>>> print_requests.py //artifacts/.internal --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "q": { + "overwrite": "true" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/readplan/whl-upload/script b/acceptance/bundle/deploy/readplan/whl-upload/script index 15b0c9aa6c3..c06707c5dd1 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/script +++ b/acceptance/bundle/deploy/readplan/whl-upload/script @@ -5,8 +5,9 @@ title "bundle deploy uploads the wheel" trace $CLI bundle deploy -# The wheel was uploaded to its .../artifacts/.internal/ workspace path. -trace print_workspace_imports | grep artifacts/.internal +# An import-file request uploaded the wheel to its .../artifacts/.internal/ workspace path. +# raw_body is the wheel's binary payload; drop it to keep the golden readable. +trace print_requests.py //artifacts/.internal --del-field raw_body trace $CLI bundle destroy --auto-approve # Discard destroy's requests so the saved-plan deploy below starts from a clean recording. @@ -18,9 +19,9 @@ title "bundle deploy --plan also uploads the wheel" $CLI bundle plan -o json > tmp.plan.json $CLI bundle plan -o json | nostamp > out.plan.json trace $CLI bundle deploy --plan tmp.plan.json -# The wheel is uploaded to the same .../artifacts/.internal/ path the job task points at, -# matching the plain `bundle deploy` above. -trace print_workspace_imports | grep artifacts/.internal +# The same import-file request uploads the wheel to the .../artifacts/.internal/ path the +# job task points at, matching the plain `bundle deploy` above. +trace print_requests.py //artifacts/.internal --del-field raw_body trace $CLI bundle destroy --auto-approve rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt index f114763c887..d641b59fc93 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//workspace/ +>>> print_requests.py ^//import-file/ ^//workspace/ >>> [CLI] bundle summary -o json @@ -34,4 +34,4 @@ Recreated pipelines.foo Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged ->>> print_requests.py ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/script b/acceptance/bundle/deployment/bind/pipelines/recreate/script index 8d02246da3b..9fd964fee14 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/script +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/script @@ -6,11 +6,11 @@ add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f "$OUT_REQUESTS" trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.txt -print_requests.py '^//workspace/' +print_requests.py '^//import-file/' '^//workspace/' rm -f "$OUT_REQUESTS" trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt -trace print_requests.py '^//workspace/' +trace print_requests.py '^//import-file/' '^//workspace/' trace $CLI bundle summary -o json | jq .resources > out.summary.json trace $CLI bundle plan @@ -19,4 +19,4 @@ trace musterr $CLI bundle deploy rm -f "$OUT_REQUESTS" trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/deployment/bind/pipelines/update/output.txt b/acceptance/bundle/deployment/bind/pipelines/update/output.txt index 732d4b8795c..5e1a9257fc1 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/update/output.txt @@ -19,4 +19,4 @@ Updated pipelines.foo Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py ^//workspace/import ^//workspace/delete ^//telemetry-ext +>>> print_requests.py ^//import-file/ ^//workspace/delete ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/update/script b/acceptance/bundle/deployment/bind/pipelines/update/script index 36c80f00f5d..68ed3eca1f2 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/script +++ b/acceptance/bundle/deployment/bind/pipelines/update/script @@ -3,15 +3,15 @@ add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f "$OUT_REQUESTS" trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.txt -print_requests.py '^//workspace/import' '^//workspace/delete' +print_requests.py '^//import-file/' '^//workspace/delete' rm -f "$OUT_REQUESTS" trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt -print_requests.py '^//workspace/import' '^//workspace/delete' +print_requests.py '^//import-file/' '^//workspace/delete' trace $CLI bundle summary -o json > out.summary.json trace $CLI bundle plan rm -f "$OUT_REQUESTS" trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//workspace/import' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//import-file/' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/destroy/force-lock-node-limit/script b/acceptance/bundle/destroy/force-lock-node-limit/script index 616b626f2ef..c7fffbc16a2 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/script +++ b/acceptance/bundle/destroy/force-lock-node-limit/script @@ -7,9 +7,9 @@ title "Deploy (creates the job and the deployment)" trace $CLI bundle deploy title "destroy without --force-lock (lock write hits the node limit: aborts)" -fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace errcode $CLI bundle destroy --auto-approve title "destroy with --force-lock (tolerates the node limit: proceeds and completes)" -fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace $CLI bundle destroy --force-lock --auto-approve diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt index 976a2f7e3e6..43e860f5d8e 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt +++ b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt @@ -54,12 +54,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> cat [OUTPUT_DIR]/out.requests.txt { "method": "POST", - "path": "/api/2.0/workspace/import", - "body": { - "multipart_form": { - "format": "AUTO", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl" - } + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl", + "q": { + "overwrite": "true" } } diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/script b/acceptance/bundle/libraries/outside_of_bundle_root/script index 7c4616890cc..df54672eee7 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/script +++ b/acceptance/bundle/libraries/outside_of_bundle_root/script @@ -11,6 +11,6 @@ title "Check that the job libraries are uploaded and the path is correct in the trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.tasks[0].libraries' trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.environments[0].spec.dependencies' trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.0/pipelines" and .method == "POST")' | jq '.body.environment.dependencies' -trace cat "$OUT_REQUESTS" | jq 'select(.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // "" | test("/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl")))' | jq 'del(.body.multipart_form.content)' +trace cat "$OUT_REQUESTS" | jq 'select(.path | test("/api/2.0/workspace-files/import-file/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl"))' rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/output.txt b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt index 2569d95ccfb..54447ea8ecc 100644 --- a/acceptance/bundle/migrate/auto/empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt @@ -20,16 +20,16 @@ direct_migrated_via_config true === The remote had no terraform state, so nothing terraform-related is backed up remotely ->>> print_requests.py //workspace/import //workspace/delete --sort +>>> print_requests.py //workspace-files/import-file //workspace/delete --sort +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" "POST /api/2.0/workspace/delete" "POST /api/2.0/workspace/delete" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" === debug states confirms no terraform.tfstate anywhere; only the local .backup remains diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/script b/acceptance/bundle/migrate/auto/empty-tfstate/script index c0df01e54ec..225092954a3 100644 --- a/acceptance/bundle/migrate/auto/empty-tfstate/script +++ b/acceptance/bundle/migrate/auto/empty-tfstate/script @@ -19,7 +19,7 @@ title "Migration is recorded in telemetry as via-config\n" trace print_migration_telemetry title "The remote had no terraform state, so nothing terraform-related is backed up remotely\n" -trace print_requests.py //workspace/import //workspace/delete --sort | jq '.method + " " + (.body.multipart_form.path // .path)' | contains.py '!terraform.tfstate' +trace print_requests.py //workspace-files/import-file //workspace/delete --sort | jq '.method + " " + .path' | contains.py '!terraform.tfstate' title "debug states confirms no terraform.tfstate anywhere; only the local .backup remains\n" trace $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/migrate/auto/push-failure/output.txt b/acceptance/bundle/migrate/auto/push-failure/output.txt index bd638d6e829..6e9a1c61b78 100644 --- a/acceptance/bundle/migrate/auto/push-failure/output.txt +++ b/acceptance/bundle/migrate/auto/push-failure/output.txt @@ -8,7 +8,7 @@ Notice: migrating your bundle to direct deployment engine (https://github.com/da Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/migrate/auto/push-failure/script b/acceptance/bundle/migrate/auto/push-failure/script index 4231617175e..da3ffaad3d9 100644 --- a/acceptance/bundle/migrate/auto/push-failure/script +++ b/acceptance/bundle/migrate/auto/push-failure/script @@ -10,7 +10,7 @@ title "Opt in via env var; inject 403 on the resources.json push so the deploy f # push fails the deploy - unlike a declined deploy, the migration is not discarded, so the # local direct state remains and a retry finishes the push. (A push failure is a hard error on # any direct deploy, not just a migrating one.) -fault.py --body-contains "Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 503055861e5..23808c5a896 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -18,7 +18,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py --nostamp ^//workspace/import ^//api/2.0/bundle +>>> print_requests.py --nostamp ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -34,7 +34,7 @@ Created pipelines.mypipeline Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --nostamp ^//workspace/import ^//api/2.0/bundle +>>> print_requests.py --nostamp ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -102,7 +102,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted ->>> print_requests.py --nostamp --sort ^//workspace/import ^//api/2.0/bundle +>>> print_requests.py --nostamp --sort ^//import-file/ ^//api/2.0/bundle { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index ccd83d2f06d..6d1e4fd5839 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py --nostamp '^//workspace/import' '^//api/2.0/bundle' +trace print_requests.py --nostamp '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle deploy -trace print_requests.py --nostamp '^//workspace/import' '^//api/2.0/bundle' +trace print_requests.py --nostamp '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle destroy --auto-approve -trace print_requests.py --nostamp --sort '^//workspace/import' '^//api/2.0/bundle' +trace print_requests.py --nostamp --sort '^//import-file/' '^//api/2.0/bundle' diff --git a/acceptance/bundle/resources/jobs/request-summary/output.txt b/acceptance/bundle/resources/jobs/request-summary/output.txt index 0f0514c2e5c..a05f10b342f 100644 --- a/acceptance/bundle/resources/jobs/request-summary/output.txt +++ b/acceptance/bundle/resources/jobs/request-summary/output.txt @@ -28,14 +28,14 @@ GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/reque GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/files/databricks.yml overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock overwrite=false +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock overwrite=false +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deployment.json overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/metadata.json overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/resources.json overwrite=true POST /api/2.0/workspace/delete POST /api/2.0/workspace/delete -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import POST /api/2.0/workspace/mkdirs POST /api/2.0/workspace/mkdirs POST /api/2.0/workspace/mkdirs @@ -57,12 +57,12 @@ GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/reque GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.2/jobs/get job_id=[FOO_ID] +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock overwrite=false +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deployment.json overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/metadata.json overwrite=true +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/resources.json overwrite=true POST /api/2.0/workspace/delete POST /api/2.0/workspace/delete -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import -POST /api/2.0/workspace/import POST /api/2.0/workspace/mkdirs >>> [CLI] bundle destroy --auto-approve @@ -81,7 +81,7 @@ GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/reque GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.0/workspace/get-status path=/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock return_export_info=true GET /api/2.2/jobs/get job_id=[FOO_ID] +POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/request-summary/default/state/deploy.lock overwrite=false POST /api/2.0/workspace/delete POST /api/2.0/workspace/delete -POST /api/2.0/workspace/import POST /api/2.2/jobs/delete diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 9795d322b3e..b53609dd402 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 70fe4992c16..5ea603ee6fe 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -31,5 +31,5 @@ nostamp < .databricks/tmp.plan.json > out.plan.json rm "$OUT_REQUESTS" trace errcode $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json trace errcode $CLI bundle plan &> out.plan_after_deploy.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt index a23a2445c8b..e82dc779ba2 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -36,7 +36,7 @@ Updated quality_monitors.monitor1 Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index a4104e4d387..fbd62ade8df 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -27,6 +27,6 @@ trace $CLI bundle plan -o json | nostamp > out.plan.json rm "$OUT_REQUESTS" trace $CLI bundle deploy # dashboard_id is output only field that terraform adds -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt index beba3487430..77656aa1913 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index a17d8b114d4..ca49568b38a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -31,7 +31,7 @@ nostamp < .databricks/tmp.plan.json > out.plan.json rm "$OUT_REQUESTS" trace errcode $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json trace errcode $CLI bundle plan &> out.plan_after_deploy.txt trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.json diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt index 5d1b3497525..fbe444e1eb7 100644 --- a/acceptance/bundle/resources/quality_monitors/create/output.txt +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -16,7 +16,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle plan -o json ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 1a3a60b1381..b45b8226f4e 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -21,7 +21,7 @@ trace $CLI bundle plan -o json | nostamp > out.plan_create.json rm "$OUT_REQUESTS" trace $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json # store state to ensure we have table_name there print_state.py | grep name > out.state.txt diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt index 1943374afeb..e21f54fcf1f 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//api/2.0/bundle ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext { "method": "POST", "path": "/api/2.0/secrets/scopes/delete", diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/script b/acceptance/bundle/resources/secret_scopes/delete_scope/script index 8d04d13a33f..6cd30fffdae 100755 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/script +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/script @@ -15,4 +15,4 @@ trace $CLI bundle plan &> out.plan.txt rm "$OUT_REQUESTS" trace $CLI bundle deploy -qq -trace print_requests.py '^//api/2.0/bundle' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt diff --git a/acceptance/bundle/sync-upload-edge-cases/databricks.yml b/acceptance/bundle/sync-upload-edge-cases/databricks.yml deleted file mode 100644 index 2261ff177c7..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/databricks.yml +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: upload-edge-cases diff --git a/acceptance/bundle/sync-upload-edge-cases/out.test.toml b/acceptance/bundle/sync-upload-edge-cases/out.test.toml deleted file mode 100644 index a927a5fbc06..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/sync-upload-edge-cases/output.txt b/acceptance/bundle/sync-upload-edge-cases/output.txt deleted file mode 100644 index 8ce0c31d5dc..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/output.txt +++ /dev/null @@ -1,136 +0,0 @@ - ->>> [CLI] bundle sync --output text -Action: PUT: .gitignore, dashboard.lvdash.json, databricks.yml, empty.txt, héllo.txt, large.bin, plain-script.py, pyNb.py, scalaNb.scala, sqlNb.sql, with spaces.txt -Initial Sync Complete -Uploaded .gitignore -Uploaded dashboard.lvdash.json -Uploaded databricks.yml -Uploaded empty.txt -Uploaded héllo.txt -Uploaded large.bin -Uploaded plain-script.py -Uploaded pyNb.py -Uploaded scalaNb.scala -Uploaded sqlNb.sql -Uploaded with spaces.txt - -=== workspace state after sync — assert object_type and language for each upload ->>> MSYS_NO_PATHCONV=1 [CLI] workspace list -o json /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files -{ - "name": ".gitignore", - "object_type": "FILE", - "language": null -} -{ - "name": "dashboard.lvdash.json", - "object_type": "FILE", - "language": null -} -{ - "name": "databricks.yml", - "object_type": "FILE", - "language": null -} -{ - "name": "empty.txt", - "object_type": "FILE", - "language": null -} -{ - "name": "héllo.txt", - "object_type": "FILE", - "language": null -} -{ - "name": "large.bin", - "object_type": "FILE", - "language": null -} -{ - "name": "plain-script.py", - "object_type": "FILE", - "language": null -} -{ - "name": "pyNb", - "object_type": "NOTEBOOK", - "language": "PYTHON" -} -{ - "name": "scalaNb", - "object_type": "NOTEBOOK", - "language": "SCALA" -} -{ - "name": "sqlNb", - "object_type": "NOTEBOOK", - "language": "SQL" -} -{ - "name": "with spaces.txt", - "object_type": "FILE", - "language": null -} - -=== uploaded paths from each multipart POST /workspace/import (sorted) ->>> print_workspace_imports -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/.gitignore -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/dashboard.lvdash.json -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/empty.txt -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/héllo.txt -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/large.bin -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/plain-script.py -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/pyNb.py -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/scalaNb.scala -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/sqlNb.sql -/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/with spaces.txt - -=== every upload set format=AUTO ->>> jq -rs - map(select(.path == "/api/2.0/workspace/import")) | - map(.body.multipart_form.format) | unique - [OUTPUT_DIR]/out.requests.txt -[ - "AUTO" -] - -=== recorded content for the small text inputs ->>> jq -rs - map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | - map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | - sort_by(.path) | .[] - [OUTPUT_DIR]/out.requests.txt -{ - "path": "dashboard.lvdash.json", - "content": "{\"datasets\":[],\"pages\":[]}\n" -} -{ - "path": "empty.txt", - "content": "" -} -{ - "path": "héllo.txt", - "content": "hello, naïve world\n" -} -{ - "path": "plain-script.py", - "content": "print(\"plain script, no header\")\n" -} -{ - "path": "pyNb.py", - "content": "# Databricks notebook source\nprint(\"hello\")\n" -} -{ - "path": "scalaNb.scala", - "content": "// Databricks notebook source\nprintln(\"hello\")\n" -} -{ - "path": "sqlNb.sql", - "content": "-- Databricks notebook source\nSELECT 1\n" -} -{ - "path": "with spaces.txt", - "content": "hello, world\n" -} diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script deleted file mode 100644 index bfd3abcaa03..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/script +++ /dev/null @@ -1,92 +0,0 @@ -# Confidence checks that /workspace/import is a drop-in for the legacy import-file: -# verify the CLI's multipart upload pipeline handles the cases that differ between the -# two endpoints — large content (>10 MB), empty content, varied extensions/object types, -# and non-ASCII / spaced filenames in the multipart `path` field. - -# Exclude framework-generated and script-internal files from the sync. We only want the -# inputs we explicitly generate below to show up in the recorded uploads. -cat > .gitignore <<'EOF' -script -output.txt -out.requests.txt -repls.json -test.toml -.databricks -EOF - -# 12 MiB binary file, over the 10 MB cap on the JSON-body variant's base64 `content` -# field. The multipart variant this test exercises has no such cap. Generated -# dynamically to keep the repo small. -python3 -c "open('large.bin', 'wb').write(b'\\0' * (12 * 1024 * 1024))" - -# Empty file (multipart encodes an empty `content` part — distinct from JSON's empty string). -touch empty.txt - -# Python notebook: auto-detected as NOTEBOOK by /workspace/import; testserver mirrors -# this and stores the object at the path with .py stripped. -cat > pyNb.py <<'EOF' -# Databricks notebook source -print("hello") -EOF - -# SQL notebook: -- comment header, .sql extension, Language=SQL on the workspace side. -cat > sqlNb.sql <<'EOF' --- Databricks notebook source -SELECT 1 -EOF - -# Scala notebook: // comment header, .scala extension, Language=SCALA. -cat > scalaNb.scala <<'EOF' -// Databricks notebook source -println("hello") -EOF - -# Plain .py without the source header — should land as FILE, not NOTEBOOK, -# even though the extension is .py. -cat > plain-script.py <<'EOF' -print("plain script, no header") -EOF - -# Lakeview dashboard descriptor: the real /workspace/import assigns object_type=DASHBOARD -# and preserves the .lvdash.json extension. The testserver doesn't emulate that and stores -# it as a generic FILE — but the upload-side request shape (which is what this test asserts) -# is identical. -echo '{"datasets":[],"pages":[]}' > dashboard.lvdash.json - -# Non-ASCII filename. Multipart encodes filenames with RFC 5987 / quoted-string rules, -# distinct from URL-encoding in the legacy import-file endpoint. -echo "hello, naïve world" > "héllo.txt" - -# Filename with a space. -echo "hello, world" > "with spaces.txt" - -trace $CLI bundle sync --output text 2>&1 | sort - -title "workspace state after sync — assert object_type and language for each upload" -remote_files=$($CLI bundle summary -o json 2>/dev/null | jq -r .workspace.file_path) -# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-slash -# workspace path into a C:/Program Files/... filesystem path before it -# reaches the CLI. -trace MSYS_NO_PATHCONV=1 $CLI workspace list -o json "$remote_files" | - jq 'sort_by(.path) | .[] | {name: (.path | sub(".*/"; "")), object_type, language}' - -title "uploaded paths from each multipart POST /workspace/import (sorted)" -trace print_workspace_imports | sort - -title "every upload set format=AUTO" -trace jq -rs ' - map(select(.path == "/api/2.0/workspace/import")) | - map(.body.multipart_form.format) | unique -' "$OUT_REQUESTS" - -title "recorded content for the small text inputs" -trace jq -rs ' - map(select(.path == "/api/2.0/workspace/import")) | - map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | - map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | - sort_by(.path) | .[] -' "$OUT_REQUESTS" - -# Drop the recorded requests; their workspace-prefixed paths are noisy and the -# assertions above already pin the relevant fields. -rm "$OUT_REQUESTS" .gitignore diff --git a/acceptance/bundle/sync-upload-edge-cases/test.toml b/acceptance/bundle/sync-upload-edge-cases/test.toml deleted file mode 100644 index c9d06a640ab..00000000000 --- a/acceptance/bundle/sync-upload-edge-cases/test.toml +++ /dev/null @@ -1,14 +0,0 @@ -RecordRequests = true - -Ignore = [ - "large.bin", - "empty.txt", - "pyNb.py", - "sqlNb.sql", - "scalaNb.scala", - "plain-script.py", - "dashboard.lvdash.json", - "héllo.txt", - "with spaces.txt", - ".databricks", -] diff --git a/acceptance/bundle/sync/upload-504/output.txt b/acceptance/bundle/sync/upload-504/output.txt index af7934522d1..d05ed02a23e 100644 --- a/acceptance/bundle/sync/upload-504/output.txt +++ b/acceptance/bundle/sync/upload-504/output.txt @@ -1,6 +1,6 @@ Action: PUT: databricks.yml -Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace/import: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml: Fault injected by test. +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml Uploaded databricks.yml Initial Sync Complete -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml diff --git a/acceptance/bundle/sync/upload-504/script b/acceptance/bundle/sync/upload-504/script index d4e9e5d1b51..c77a44296cd 100644 --- a/acceptance/bundle/sync/upload-504/script +++ b/acceptance/bundle/sync/upload-504/script @@ -1,7 +1,7 @@ # Inject a single 504 on the first file upload. Uploads overwrite, so they are retried and the sync succeeds. -fault.py "POST /api/2.0/workspace/import" 504 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/*" 504 0 1 $CLI bundle sync # databricks.yml is the only file. Two requests for it should appear: the initial 504 and the successful retry. -print_requests.py //api/2.0/workspace/import | jq -r .body.multipart_form.path +print_requests.py //api/2.0/workspace-files/import-file | jq -r .path diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt index df5252aa3d7..17eb89280ce 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt @@ -1,9 +1,9 @@ >>> [CLI] bundle deploy -Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal error -Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal error (500 INTERNAL_ERROR) +Error: Failed to acquire deployment lock: Internal error +Error: Internal error (500 INTERNAL_ERROR) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fdeploy.lock?overwrite=false HTTP Status: 500 Internal Server Error API error_code: INTERNAL_ERROR API message: Internal error diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml index 1e685cca603..86d366ba8c2 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml @@ -12,6 +12,6 @@ Cloud = false [[Server]] -Pattern = "POST /api/2.0/workspace/import" +Pattern = "POST /api/2.0/workspace-files/import-file/{path...}" Response.StatusCode = 500 Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 2f91efecbe1..3029c012d05 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -13,13 +13,13 @@ $CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.json rm "$OUT_REQUESTS" # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' -print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.txt +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.txt read_id.py -t dev sample_job > /dev/null trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' -print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.txt +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.txt read_id.py -t prod sample_job > /dev/null # Extracted with python, not jq: the id exceeds 2^53 and jq 1.6 rounds it. root_dir_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/my_default_python/prod" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index a5d2a395976..93e5048afec 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -13,13 +13,13 @@ $CLI bundle plan -o json -t prod | nostamp > ../../out.plan_prod.json rm "$OUT_REQUESTS" trace $CLI bundle deploy -t dev replace_ids.py -t dev -print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.txt +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.txt trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json | nostamp > ../../out.plan_after_deploy_dev.json trace $CLI bundle deploy -t prod replace_ids.py -t prod -print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.txt +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.txt trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json | nostamp > ../../out.plan_after_deploy_prod.json diff --git a/acceptance/bundle/upload/internal_server_error/output.txt b/acceptance/bundle/upload/internal_server_error/output.txt index 2ae3634f94b..d75ee2a71b1 100644 --- a/acceptance/bundle/upload/internal_server_error/output.txt +++ b/acceptance/bundle/upload/internal_server_error/output.txt @@ -1,9 +1,9 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Warn: retrying after 500 Internal Server Error from POST /api/2.0/workspace/import: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt: Fault injected by test. -Warn: retrying after 500 Internal Server Error from POST /api/2.0/workspace/import: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt: Fault injected by test. -Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt: Fault injected by test. (500 INJECTED) +Warn: retrying after 500 Internal Server Error from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt +Warn: retrying after 500 Internal Server Error from POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt +Error: Fault injected by test. (500 INJECTED) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true HTTP Status: 500 Internal Server Error API error_code: INJECTED API message: Fault injected by test. diff --git a/acceptance/bundle/upload/internal_server_error/script b/acceptance/bundle/upload/internal_server_error/script index 78c57ef8a94..e8a831e20af 100644 --- a/acceptance/bundle/upload/internal_server_error/script +++ b/acceptance/bundle/upload/internal_server_error/script @@ -1,2 +1,2 @@ -fault.py --body-contains "Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/files/file_to_upload.txt" "POST /api/2.0/workspace/import" 500 0 1000 +fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/files/file_to_upload.txt" 500 0 1000 musterr $CLI bundle deploy diff --git a/acceptance/bundle/upload/timeout/output.txt b/acceptance/bundle/upload/timeout/output.txt index 0c779d73dc2..c314a247707 100644 --- a/acceptance/bundle/upload/timeout/output.txt +++ b/acceptance/bundle/upload/timeout/output.txt @@ -1,3 +1,3 @@ -Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity -Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity diff --git a/acceptance/bundle/upload/timeout/test.toml b/acceptance/bundle/upload/timeout/test.toml index f0abd08ecbc..d6564ef9659 100644 --- a/acceptance/bundle/upload/timeout/test.toml +++ b/acceptance/bundle/upload/timeout/test.toml @@ -3,5 +3,5 @@ Env.DATABRICKS_BUNDLE_HTTP_TIMEOUT_SECONDS = "5" [[Server]] # CLI aborts after a single attempt when the HTTP timeout fires. Delay = "30s" -Pattern = "POST /api/2.0/workspace/import" +Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" Response.StatusCode = 200 diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 4f84546f174..458fcaa9638 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -8,14 +8,14 @@ OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct OK deploy /api/2.0/workspace/get-status engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json engine/direct +OK deploy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json engine/direct OK deploy /api/2.0/workspace/delete engine/direct OK deploy /api/2.0/workspace/delete engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct -OK deploy /api/2.0/workspace/import engine/direct OK deploy /api/2.0/workspace/mkdirs engine/direct OK deploy /api/2.0/workspace/mkdirs engine/direct OK deploy /api/2.1/unity-catalog/schemas engine/direct @@ -29,9 +29,9 @@ OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.0/workspace/get-status engine/direct OK destroy /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct +OK destroy /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct OK destroy /api/2.0/workspace/delete engine/direct OK destroy /api/2.0/workspace/delete engine/direct -OK destroy /api/2.0/workspace/import engine/direct MISS destroy /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS plan /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.json index 7713ace6e75..31bc7b0724a 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.json @@ -130,10 +130,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", - "recursive": true + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py", + "q": { + "overwrite": "true" } } { @@ -143,9 +142,15 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", + "q": { + "overwrite": "false" + }, "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" } } { @@ -155,14 +160,15 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", + "q": { + "overwrite": "false" + }, "body": { - "multipart_form": { - "content": "", - "format": "AUTO", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py" - } + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" } } { @@ -172,13 +178,22 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json", + "q": { + "overwrite": "true" + }, "body": { - "multipart_form": { - "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "AUTO", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" - } + "version": 1, + "seq": 1, + "cli_version": "[CLI_VERSION]", + "timestamp": "[TIMESTAMP]", + "files": [ + { + "local_path": "empty.py", + "is_notebook": false + } + ], + "id": "[UUID]" } } { @@ -188,13 +203,29 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json", + "q": { + "overwrite": "true" + }, "body": { - "multipart_form": { - "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "AUTO", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" - } + "version": 1, + "config": { + "bundle": { + "name": "test-bundle", + "target": "default", + "git": { + "bundle_root_path": "." + } + }, + "workspace": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" + }, + "resources": {}, + "presets": { + "source_linked_deployment": false + } + }, + "extra": {} } } { @@ -204,13 +235,23 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json", + "q": { + "overwrite": "true" + }, "body": { - "multipart_form": { - "content": "{\"version\":1,\"seq\":1,\"cli_version\":\"[CLI_VERSION]\",\"timestamp\":\"[TIMESTAMP]\",\"files\":[{\"local_path\":\"empty.py\",\"is_notebook\":false}],\"id\":\"[UUID]\"}", - "format": "AUTO", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "state": { + "resources.schemas.foo": { + "__id__": "mycatalog.myschema", + "state": { + "catalog_name": "mycatalog", + "name": "myschema" + } + } } } } @@ -221,14 +262,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace/delete", "body": { - "multipart_form": { - "content": "{\n \"version\": 1,\n \"config\": {\n \"bundle\": {\n \"name\": \"test-bundle\",\n \"target\": \"default\",\n \"git\": {\n \"bundle_root_path\": \".\"\n }\n },\n \"workspace\": {\n \"file_path\": \"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"\n },\n \"resources\": {},\n \"presets\": {\n \"source_linked_deployment\": false\n }\n },\n \"extra\": {}\n}", - "format": "AUTO", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - } + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", + "recursive": true } } { @@ -238,14 +275,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace/delete", "body": { - "multipart_form": { - "content": "{\n \"state_version\": 3,\n \"cli_version\": \"[CLI_VERSION]\",\n \"lineage\": \"[UUID]\",\n \"serial\": 1,\n \"state\": {\n \"resources.schemas.foo\": {\n \"__id__\": \"mycatalog.myschema\",\n \"state\": {\n \"catalog_name\": \"mycatalog\",\n \"name\": \"myschema\"\n }\n }\n }\n}", - "format": "AUTO", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" - } + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.json index 168944a8311..87006c49f30 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.json @@ -112,9 +112,15 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", + "q": { + "overwrite": "false" + }, "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" } } { @@ -126,8 +132,7 @@ "method": "POST", "path": "/api/2.0/workspace/delete", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "recursive": true + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" } } { @@ -137,13 +142,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/import", + "path": "/api/2.0/workspace/delete", "body": { - "multipart_form": { - "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", - "format": "AUTO", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" - } + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "recursive": true } } { diff --git a/acceptance/bundle/validate/sync_patterns/out.sync.txt b/acceptance/bundle/validate/sync_patterns/out.sync.txt index fd939c9721b..ac5aba9ab29 100644 --- a/acceptance/bundle/validate/sync_patterns/out.sync.txt +++ b/acceptance/bundle/validate/sync_patterns/out.sync.txt @@ -1,12 +1,7 @@ { "method": "POST", - "path": "/api/2.0/workspace/import", - "body": { - "multipart_form": { - "content": "", - "format": "AUTO", - "overwrite": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml" - } + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml", + "q": { + "overwrite": "true" } } diff --git a/acceptance/bundle/validate/sync_patterns/script b/acceptance/bundle/validate/sync_patterns/script index f44ee310433..d4cf7169b1b 100644 --- a/acceptance/bundle/validate/sync_patterns/script +++ b/acceptance/bundle/validate/sync_patterns/script @@ -1,5 +1,5 @@ trace $CLI bundle validate trace $CLI bundle validate -o json | jq '.sync' trace $CLI bundle deploy -jq 'select(.body.multipart_form.path | strings | test("dir/test.yml"))' "$OUT_REQUESTS" > out.sync.txt +jq 'select(.path | test("dir/test.yml"))' "$OUT_REQUESTS" > out.sync.txt rm "$OUT_REQUESTS" diff --git a/acceptance/cmd/sync/dryrun-missing-remote/output.txt b/acceptance/cmd/sync/dryrun-missing-remote/output.txt index b7453994b58..4848aae1c2d 100644 --- a/acceptance/cmd/sync/dryrun-missing-remote/output.txt +++ b/acceptance/cmd/sync/dryrun-missing-remote/output.txt @@ -15,11 +15,11 @@ Uploaded hello.py >>> print_requests.py --sort //api/ { "method": "POST", - "path": "/api/2.0/workspace/import" + "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/.gitignore" } { "method": "POST", - "path": "/api/2.0/workspace/import" + "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/hello.py" } { "method": "POST", diff --git a/acceptance/cmd/sync/upload-504/output.txt b/acceptance/cmd/sync/upload-504/output.txt index cb9b61d2b0c..669f986c419 100644 --- a/acceptance/cmd/sync/upload-504/output.txt +++ b/acceptance/cmd/sync/upload-504/output.txt @@ -1,6 +1,6 @@ Action: PUT: hello.py -Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace/import: failed to upload /Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py: Fault injected by test. +Warn: retrying after 504 Gateway Timeout from POST /api/2.0/workspace-files/import-file/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py Uploaded hello.py Initial Sync Complete -/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py -/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py +/api/2.0/workspace-files/import-file/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py +/api/2.0/workspace-files/import-file/Users/[USERNAME]/sync-504-[UNIQUE_NAME]/hello.py diff --git a/acceptance/cmd/sync/upload-504/script b/acceptance/cmd/sync/upload-504/script index 3be71acff67..6e1ea8b7479 100644 --- a/acceptance/cmd/sync/upload-504/script +++ b/acceptance/cmd/sync/upload-504/script @@ -15,10 +15,10 @@ cleanup() { trap cleanup EXIT # Inject a single 504 on the first file upload. Uploads overwrite, so they are retried and the sync succeeds. -fault.py "POST /api/2.0/workspace/import" 504 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/*" 504 0 1 # MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /Users/... to C:/Program Files/Git/Users/... MSYS_NO_PATHCONV=1 $CLI sync . "$remote_dir" # Two requests for hello.py should appear: the initial 504 and the successful retry. -print_requests.py //api/2.0/workspace/import | jq -r .body.multipart_form.path +print_requests.py //api/2.0/workspace-files/import-file | jq -r .path diff --git a/acceptance/script.prepare b/acceptance/script.prepare index 66080eb1fcb..a53627b231d 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -128,12 +128,6 @@ print_telemetry_bool_values() { jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental.bool_values // []) ) | map("\(.key) \(.value)") | .[]' "$OUT_REQUESTS" | grep -Ev '^(engine_terraform_|deployment_history_enabled )' | sort } -# Workspace paths uploaded via /workspace/import, one per line. The path is a -# multipart form field, not part of the URL, so print_requests.py can't filter on it. -print_workspace_imports() { - jq -r 'select(.path? == "/api/2.0/workspace/import") | .body.multipart_form.path | strings' "$OUT_REQUESTS" -} - sethome() { local home="$1" mkdir -p "$home" diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index 5c5a2e7f23a..b24740317af 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -411,11 +411,9 @@ func TestSubmitWorkloadValidationOutcome(t *testing.T) { t.Cleanup(server.Close) var uploads, submissions atomic.Int32 server.Handle("POST", validateConfigPath, func(req testserver.Request) any { return tt.response }) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { uploads.Add(1) - form, err := testserver.ParseImportForm(req) - assert.NoError(t, err) - return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, true) + return req.Workspace.WorkspaceFilesImportFile(req.Vars["path"], req.Body, true) }) server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { submissions.Add(1) @@ -732,19 +730,18 @@ func TestSubmitWorkloadPlainTarContentAddressed(t *testing.T) { server.Handle("POST", "/api/2.2/jobs/runs/submit", func(req testserver.Request) any { return jobs.SubmitRunResponse{RunId: 555} }) - // Count snapshot import calls, preserving fake-workspace persistence so the + // Count snapshot import-file calls, preserving fake-workspace persistence so the // second submit's existence Stat sees the first upload. A count (not a set keyed by // path) is what proves the skip: both submits resolve to the same content-addressed // name, so a set could not tell a skipped second submit from one that re-uploaded to // that same path. snapshotUploads := 0 - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - form, err := testserver.ParseImportForm(req) - assert.NoError(t, err) - if strings.Contains(form.Path, "/.air/repo_snapshots/") { + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { snapshotUploads++ } - return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) stubValidateConfig(server) testserver.AddDefaultHandlers(server) @@ -781,7 +778,7 @@ code_source: assert.Regexp(t, `^src_[0-9a-f]{16}\.tar\.gz$`, base) // Same unchanged tree → identical remote path, and the second submit moved no bytes: - // zero new import calls (a real skip), not a re-upload to the same name. + // zero new import-file calls (a real skip), not a re-upload to the same name. assert.Equal(t, first.CodeSourcePath, second.CodeSourcePath) assert.Equal(t, afterFirst, snapshotUploads, "unchanged plain_tar should skip the second upload") require.NotNil(t, first.SizeBytes) @@ -811,13 +808,12 @@ func TestSubmitWorkloadGitArchiveCaching(t *testing.T) { // Dedupe by path: the filer mkdirs-and-retries the import on a missing // parent dir, so one logical upload can hit this route more than once. uploaded := map[string]bool{} - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - form, err := testserver.ParseImportForm(req) - assert.NoError(t, err) - if strings.Contains(form.Path, "/.air/repo_snapshots/") { - uploaded[form.Path] = true + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.Contains(p, "/.air/repo_snapshots/") { + uploaded[p] = true } - return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) stubValidateConfig(server) testserver.AddDefaultHandlers(server) diff --git a/cmd/air/telemetry_test.go b/cmd/air/telemetry_test.go index 49760d24ad9..35a0cad984d 100644 --- a/cmd/air/telemetry_test.go +++ b/cmd/air/telemetry_test.go @@ -61,11 +61,10 @@ func TestAirRunTelemetry(t *testing.T) { return jobs.SubmitRunResponse{RunId: 555} }) var uploadedSize int64 - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - form, err := testserver.ParseImportForm(req) - assert.NoError(t, err) - if strings.HasSuffix(form.Path, ".tar.gz") { - uploadedSize = int64(len(form.Content)) + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(req testserver.Request) any { + p := req.Vars["path"] + if strings.HasSuffix(p, ".tar.gz") { + uploadedSize = int64(len(req.Body)) time.Sleep(20 * time.Millisecond) if tc.failUpload { return testserver.Response{StatusCode: http.StatusForbidden, Body: map[string]string{ @@ -73,7 +72,7 @@ func TestAirRunTelemetry(t *testing.T) { }} } } - return req.Workspace.WorkspaceFilesImportFile(form.Path, form.Content, form.Overwrite) + return req.Workspace.WorkspaceFilesImportFile(p, req.Body, req.URL.Query().Get("overwrite") == "true") }) stubValidateConfig(server) w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 24188974ed4..e1b62481243 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -9,8 +9,10 @@ import ( "io" "io/fs" "net/http" + "net/url" "path" "slices" + "strings" "time" "github.com/databricks/cli/libs/auth" @@ -152,29 +154,35 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } + // Remove leading "/" so we can use it in the URL. + overwrite := slices.Contains(mode, OverwriteIfExists) + urlPath := fmt.Sprintf( + "/api/2.0/workspace-files/import-file/%s?overwrite=%t", + url.PathEscape(strings.TrimLeft(absPath, "/")), + overwrite, + ) + // Buffer the file contents because we may need to retry below and we cannot read twice. body, err := io.ReadAll(reader) if err != nil { return err } - // Upload with the SDK's multipart Workspace.Upload; format=AUTO lets the server - // classify each payload as file or notebook. The JSON-body workspace.Import caps - // content at 10 MB, the multipart form at the 500 MB workspace file limit. - opts := []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatAuto)} - if slices.Contains(mode, OverwriteIfExists) { - opts = append(opts, workspace.UploadOverwrite()) - } - - err = w.workspaceClient.Workspace.Upload(ctx, absPath, bytes.NewReader(body), opts...) + err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) // Return early on success. if err == nil { return nil } - // Parent directory does not exist. - if errors.Is(err, apierr.ErrNotFound) { + // Special handling of this error only if it is an API error. + aerr, ok := errors.AsType[*apierr.APIError](err) + if !ok { + return err + } + + // This API returns a 404 if the parent directory does not exist. + if aerr.StatusCode == http.StatusNotFound { if !slices.Contains(mode, CreateParentDirectories) { return noSuchDirectoryError{path.Dir(absPath)} } @@ -182,8 +190,8 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // Create parent directory. err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath)) if err != nil { - if errors.Is(err, apierr.ErrPermissionDenied) { - return permissionError{absPath, err} + if mkdirErr, ok := errors.AsType[*apierr.APIError](err); ok && mkdirErr.StatusCode == http.StatusForbidden { + return permissionError{absPath, mkdirErr} } return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err) } @@ -192,32 +200,27 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return w.Write(ctx, name, bytes.NewReader(body), sliceWithout(mode, CreateParentDirectories)...) } - // Path already taken. ErrResourceConflict covers every 409 (including the - // bare 409 with only a message); ErrResourceAlreadyExists covers the 400 - // RESOURCE_ALREADY_EXISTS, which no status maps to a conflict. - if errors.Is(err, apierr.ErrResourceConflict) || errors.Is(err, apierr.ErrResourceAlreadyExists) { + // This API returns 409 if the file already exists, when the object type is file + if aerr.StatusCode == http.StatusConflict { return fileAlreadyExistsError{absPath} } - // Overwrite rejected because the existing object's node type differs from - // the upload. The collision carries an AIP-193 ErrorInfo with a stable - // reason, so branch on it rather than the message text. - if errors.Is(err, apierr.ErrInvalidParameterValue) { - if aerr, ok := errors.AsType[*apierr.APIError](err); ok { - if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == "WORKSPACE_OBJECT_TYPE_MISMATCH" { - return fileAlreadyExistsError{absPath} - } - } + + // This API returns 400 if the file already exists when the object type is notebook. + // Both the historical "Path () already exists." format and the newer + // "RESOURCE_ALREADY_EXISTS: already exists. ..." format end with the same + // "already exists." marker; the JSON error_code is empty in both. The new format + // might not have been rolled out to all workspaces yet, so we anchor on the shared + // marker and return absPath rather than parsing the message. + if aerr.StatusCode == http.StatusBadRequest && strings.Contains(aerr.Message, "already exists.") { + return fileAlreadyExistsError{absPath} } - // Caller has read access but no write access. - if errors.Is(err, apierr.ErrPermissionDenied) { - return permissionError{absPath, err} + // This API returns StatusForbidden when you have read access but don't have write access to a file + if aerr.StatusCode == http.StatusForbidden { + return permissionError{absPath, aerr} } - // Any other failure (e.g. a server-side error with an empty message) is - // surfaced with the target path so the user can tell which upload failed; - // %w keeps the original error inspectable by callers. - return fmt.Errorf("failed to upload %s: %w", absPath, err) + return err } func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 81832694d15..2915af60436 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -1,11 +1,8 @@ package filer import ( - "bytes" "encoding/json" "io/fs" - "net/http" - "strings" "testing" "time" @@ -18,8 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -// Confirms directory entries derived from workspace object infos report the -// name, type, and size of each object type the workspace can return. func TestWorkspaceFilesDirEntry(t *testing.T) { entries := wsfsDirEntriesFromObjectInfos([]workspace.ObjectInfo{ { @@ -66,8 +61,6 @@ func TestWorkspaceFilesDirEntry(t *testing.T) { assert.True(t, i2.IsDir()) } -// Confirms the workspace ID routing header is set only when the client config -// carries a workspace ID, and that a nil workspace client yields no headers. func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { tests := []struct { name string @@ -104,188 +97,6 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } -// Confirms each error /workspace/import returns is translated to the filer's -// own error type, and that unrecognized errors pass through unchanged. The -// endpoint signals an occupied path with three different status/error_code -// combinations (see the comment on Write), so all three are covered here to -// pin the mapping the rest of the CLI relies on. -func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { - tests := []struct { - name string - mode []WriteMode - apiErr *apierr.APIError - expectErrTarget any - }{ - { - name: "404 without create-parent maps to noSuchDirectoryError", - apiErr: &apierr.APIError{StatusCode: http.StatusNotFound, Message: "not found"}, - expectErrTarget: noSuchDirectoryError{}, - }, - { - name: "400 RESOURCE_ALREADY_EXISTS maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusBadRequest, - ErrorCode: "RESOURCE_ALREADY_EXISTS", - Message: "/dir/file.txt already exists. Please pass overwrite=true to overwrite it.", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, - { - name: "409 ALREADY_EXISTS (concurrent contention) maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusConflict, - ErrorCode: "ALREADY_EXISTS", - Message: "Node with name /dir/file.txt already exists. Please pass overwrite=true to update it.", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, - { - // A 409 carrying no error_code unwraps to ErrResourceConflict only, - // not to ErrAlreadyExists; the testserver returns exactly this shape. - name: "409 without error_code maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusConflict, - Message: "Node with name /dir/file.txt already exists.", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, - { - name: "400 INVALID_PARAMETER_VALUE other message passes through", - apiErr: &apierr.APIError{ - StatusCode: http.StatusBadRequest, - ErrorCode: "INVALID_PARAMETER_VALUE", - Message: "some other validation failure", - }, - expectErrTarget: nil, - }, - { - name: "403 maps to permissionError", - apiErr: &apierr.APIError{StatusCode: http.StatusForbidden, Message: "denied"}, - expectErrTarget: permissionError{}, - }, - { - name: "500 passes through", - apiErr: &apierr.APIError{StatusCode: http.StatusInternalServerError, Message: "boom"}, - expectErrTarget: nil, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - server := testserver.New(t) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - return testserver.Response{ - StatusCode: tc.apiErr.StatusCode, - Body: map[string]string{ - "error_code": tc.apiErr.ErrorCode, - "message": tc.apiErr.Message, - }, - } - }) - testserver.AddDefaultHandlers(server) - - client, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "testtoken", - }) - require.NoError(t, err) - - f, err := NewWorkspaceFilesClient(client, "/dir") - require.NoError(t, err) - - err = f.Write(t.Context(), "file.txt", bytes.NewReader([]byte("data")), tc.mode...) - require.Error(t, err) - switch target := tc.expectErrTarget.(type) { - case noSuchDirectoryError: - assert.ErrorAs(t, err, &target) - case fileAlreadyExistsError: - assert.ErrorAs(t, err, &target) - case permissionError: - assert.ErrorAs(t, err, &target) - case nil: - // passthrough — the underlying APIError stays inspectable, - // and the message names the file that failed to upload. - var aerr *apierr.APIError - require.ErrorAs(t, err, &aerr) - assert.Equal(t, tc.apiErr.StatusCode, aerr.StatusCode) - assert.Contains(t, err.Error(), "/dir/file.txt") - } - }) - } -} - -// writeWithImportError exercises Write through a real HTTP roundtrip so the -// SDK parses AIP-193 error details from the response body (the errorDetails -// field on apierr.APIError is unexported and only populated during response -// parsing, so it cannot be set on a directly constructed APIError). -func writeWithImportError(t *testing.T, body map[string]any) error { - t.Helper() - - server := testserver.New(t) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - return testserver.Response{ - StatusCode: http.StatusBadRequest, - Body: body, - } - }) - testserver.AddDefaultHandlers(server) - - client, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "testtoken", - }) - require.NoError(t, err) - - f, err := NewWorkspaceFilesClient(client, "/dir") - require.NoError(t, err) - - err = f.Write(t.Context(), "file.txt", strings.NewReader("data"), OverwriteIfExists) - require.Error(t, err) - return err -} - -// Confirms a type-mismatch collision is recognized from the structured -// AIP-193 ErrorInfo reason, independent of the error message wording. -func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { - // The message is deliberately unrelated to any collision wording, to prove - // the branch fires on the structured reason alone. - err := writeWithImportError(t, map[string]any{ - "error_code": "INVALID_PARAMETER_VALUE", - "message": "some future wording for the same condition", - "details": []map[string]any{ - { - "@type": "type.googleapis.com/google.rpc.ErrorInfo", - "reason": "WORKSPACE_OBJECT_TYPE_MISMATCH", - "domain": "workspace.databricks.com", - "metadata": map[string]string{"existing_type": "NOTEBOOK"}, - }, - }, - }) - var target fileAlreadyExistsError - assert.ErrorAs(t, err, &target) -} - -// Confirms an ErrorInfo carrying some other reason is not mistaken for a path -// collision, so the reason check above cannot swallow unrelated failures. -func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { - err := writeWithImportError(t, map[string]any{ - "error_code": "INVALID_PARAMETER_VALUE", - "message": "some other validation failure", - "details": []map[string]any{ - { - "@type": "type.googleapis.com/google.rpc.ErrorInfo", - "reason": "SOME_OTHER_REASON", - "domain": "workspace.databricks.com", - }, - }, - }) - var aerr *apierr.APIError - require.ErrorAs(t, err, &aerr) - assert.Equal(t, http.StatusBadRequest, aerr.StatusCode) -} - -// Confirms a workspace get-status payload unmarshals into wsfsFileInfo and that -// the fs.FileInfo methods derived from it report the expected values. func TestWorkspaceFilesClient_wsfsUnmarshal(t *testing.T) { payload := ` { @@ -352,8 +163,6 @@ func statWithError(t *testing.T, statusCode int, errorCode string) error { return err } -// Confirms a 403 from get-status surfaces as an APIError rather than being -// remapped, so callers can distinguish it from a missing file. func TestWorkspaceFilesClientStatForbidden(t *testing.T) { err := statWithError(t, 403, "PERMISSION_DENIED") var apiErr *apierr.APIError @@ -361,8 +170,6 @@ func TestWorkspaceFilesClientStatForbidden(t *testing.T) { assert.Equal(t, 403, apiErr.StatusCode) } -// Confirms a 500 from get-status surfaces as an APIError and is not treated as -// a missing file. func TestWorkspaceFilesClientStatInternalError(t *testing.T) { err := statWithError(t, 500, "INTERNAL_ERROR") var apiErr *apierr.APIError @@ -370,8 +177,6 @@ func TestWorkspaceFilesClientStatInternalError(t *testing.T) { assert.Equal(t, 500, apiErr.StatusCode) } -// Confirms RESOURCE_DOES_NOT_EXIST maps to fs.ErrNotExist so the filer plugs -// into the io/fs error conventions callers check against. func TestWorkspaceFilesClientStatNotFound(t *testing.T) { err := statWithError(t, 404, "RESOURCE_DOES_NOT_EXIST") assert.ErrorIs(t, err, fs.ErrNotExist) diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go index fa7ff0baceb..d3bb57662d9 100644 --- a/libs/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -10,10 +10,7 @@ import ( "golang.org/x/sync/errgroup" ) -// Maximum number of concurrent requests during sync. Chosen against the -// per-endpoint rate limits documented at -// https://docs.databricks.com/aws/en/resources/limits ("API rate limits" - -// Workspace API), which sync stays within for the endpoints it calls. +// Maximum number of concurrent requests during sync. const MaxRequestsInFlight = 20 // Delete the specified path. diff --git a/libs/testserver/fault.go b/libs/testserver/fault.go index 43d46f3f28c..75d3092bb70 100644 --- a/libs/testserver/fault.go +++ b/libs/testserver/fault.go @@ -1,7 +1,6 @@ package testserver import ( - "bytes" "encoding/json" "strings" "sync" @@ -20,12 +19,6 @@ type FaultRule struct { AfterHandler bool offset int times int - - // bodyContains, if non-empty, additionally requires the request body to - // contain this substring for the rule to fire. /workspace/import routes - // every upload through the same method+path, so the only way to target a - // single file's upload is by matching its multipart "path" form field. - bodyContains string } // FaultRules holds the active fault injection rules for a test server. @@ -40,16 +33,16 @@ func NewFaultRules() *FaultRules { } // Set registers or replaces a fault rule for the given token and pattern. -func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { - fr.set(token, pattern, statusCode, body, offset, times, false, bodyContains) +func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int) { + fr.set(token, pattern, statusCode, body, offset, times, false) } // SetAfterHandler is like Set, but the handler runs first so its effect is kept. -func (fr *FaultRules) SetAfterHandler(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { - fr.set(token, pattern, statusCode, body, offset, times, true, bodyContains) +func (fr *FaultRules) SetAfterHandler(token, pattern string, statusCode int, body string, offset, times int) { + fr.set(token, pattern, statusCode, body, offset, times, true) } -func (fr *FaultRules) set(token, pattern string, statusCode int, body string, offset, times int, afterHandler bool, bodyContains string) { +func (fr *FaultRules) set(token, pattern string, statusCode int, body string, offset, times int, afterHandler bool) { fr.mu.Lock() defer fr.mu.Unlock() fr.rules[faultRuleKey{token: token, pattern: pattern}] = &FaultRule{ @@ -58,14 +51,12 @@ func (fr *FaultRules) set(token, pattern string, statusCode int, body string, of AfterHandler: afterHandler, offset: offset, times: times, - bodyContains: bodyContains, } } // Check returns a matching fault rule and advances its counters, or nil if no rule matches. // Pattern supports a trailing "*" wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*". -// A rule with a non-empty bodyContains only matches when body contains that substring. -func (fr *FaultRules) Check(method, path, token string, body []byte) *FaultRule { +func (fr *FaultRules) Check(method, path, token string) *FaultRule { requestPattern := method + " " + path fr.mu.Lock() @@ -85,11 +76,6 @@ func (fr *FaultRules) Check(method, path, token string, body []byte) *FaultRule if !matched { continue } - // A non-matching body must not consume the rule's offset/times budget, - // so this check precedes the counter updates below. - if rule.bodyContains != "" && !bytes.Contains(body, []byte(rule.bodyContains)) { - continue - } if rule.offset > 0 { rule.offset-- return nil @@ -119,15 +105,14 @@ func faultEndpointHandler(fr *FaultRules) HandlerFunc { Offset int `json:"offset"` Times int `json:"times"` AfterHandler bool `json:"after_handler"` - BodyContains string `json:"body_contains"` } if err := json.Unmarshal(req.Body, &body); err != nil { return Response{StatusCode: 400, Body: map[string]string{"error": err.Error()}} } if body.AfterHandler { - fr.SetAfterHandler(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) + fr.SetAfterHandler(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) } else { - fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) + fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) } return Response{StatusCode: 200} } diff --git a/libs/testserver/fault_test.go b/libs/testserver/fault_test.go index ae27b954f93..db41b74d3ba 100644 --- a/libs/testserver/fault_test.go +++ b/libs/testserver/fault_test.go @@ -14,18 +14,18 @@ import ( func TestFaultRulesNoMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 1, "") + fr.Set("tok", "GET /foo", 504, "body", 0, 1) - assert.Nil(t, fr.Check("POST", "/foo", "tok", nil)) - assert.Nil(t, fr.Check("GET", "/bar", "tok", nil)) - assert.Nil(t, fr.Check("GET", "/foo", "other", nil)) + assert.Nil(t, fr.Check("POST", "/foo", "tok")) + assert.Nil(t, fr.Check("GET", "/bar", "tok")) + assert.Nil(t, fr.Check("GET", "/foo", "other")) } func TestFaultRulesExactMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1, "") + fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1) - rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok", nil) + rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok") require.NotNil(t, rule) assert.Equal(t, 504, rule.StatusCode) assert.Equal(t, "body", rule.Body) @@ -33,43 +33,31 @@ func TestFaultRulesExactMatch(t *testing.T) { func TestFaultRulesWildcardMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2, "") + fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2) - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok", nil)) - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) - assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) // exhausted + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok")) + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) + assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) // exhausted } func TestFaultRulesOffset(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 2, 1, "") + fr.Set("tok", "GET /foo", 504, "body", 2, 1) - assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 2→1 - assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 1→0 - assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) // fires - assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 2→1 + assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 1→0 + assert.NotNil(t, fr.Check("GET", "/foo", "tok")) // fires + assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted } func TestFaultRulesTimes(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 3, "") + fr.Set("tok", "GET /foo", 504, "body", 0, 3) for range 3 { - assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) + assert.NotNil(t, fr.Check("GET", "/foo", "tok")) } - assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted -} - -func TestFaultRulesBodyContains(t *testing.T) { - fr := testserver.NewFaultRules() - fr.Set("tok", "POST /api/2.0/workspace/import", 403, "body", 0, 1, "state/resources.json") - - // Body without the substring must not match and must not consume the budget. - assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/deploy.lock"))) - // Body with the substring fires. - assert.NotNil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) - // Budget was only consumed by the matching request. - assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) + assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted } func TestServerFaultAfterHandlerKeepsTheHandlersEffect(t *testing.T) { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 50ff904a573..044f02952f3 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -14,7 +14,6 @@ import ( "strconv" "strings" - "github.com/databricks/cli/libs/notebook" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -81,10 +80,6 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { resp = w.WorkspaceImportRawFile(form.Path, form.Content, form.Overwrite) case string(workspace.ImportFormatSource): resp = w.WorkspaceImportNotebook(form.Path, form.Content, workspace.Language(form.Language), form.Overwrite) - case string(workspace.ImportFormatJupyter): - // The real endpoint reads the language from the notebook metadata. - _, language, _ := notebook.DetectBytes(form.Path+notebook.ExtensionJupyter, form.Content) - resp = w.WorkspaceImportNotebook(form.Path, form.Content, language, form.Overwrite) case string(workspace.ImportFormatAuto): if strings.HasSuffix(strings.ToLower(form.Path), ".zip") { return w.WorkspaceImportZipFolder(form.Path, form.Content, form.Overwrite) @@ -95,7 +90,7 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { resp = w.WorkspaceImportAutoFile(form.Path, form.Content, form.Overwrite) default: return Response{ - Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW, SOURCE and JUPYTER formats, got %q.", form.Format), + Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW and SOURCE formats, got %q.", form.Format), StatusCode: http.StatusInternalServerError, } } diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 5170b318818..1f56717c50a 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -372,7 +372,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, handler HandlerFu var resp EncodedResponse - fault := s.faults.Check(r.Method, r.URL.Path, token, request.Body) + fault := s.faults.Check(r.Method, r.URL.Path, token) if fault == nil || fault.AfterHandler { if bytes.Contains(request.Body, []byte("INJECT_ERROR")) { diff --git a/libs/testserver/workspace_test.go b/libs/testserver/workspace_test.go index 401758c6648..cde1c9741c6 100644 --- a/libs/testserver/workspace_test.go +++ b/libs/testserver/workspace_test.go @@ -88,24 +88,6 @@ func TestWorkspaceDeleteNonRecursiveRequiresEmptyDirectory(t *testing.T) { assert.Equal(t, 404, getStatus(t, server.URL, "/a/b")) } -// format=AUTO matches the extension case-insensitively, so ".R" — the -// conventional spelling for R sources — is detected as a notebook just like -// ".r". Notebooks are stored with the extension stripped. -func TestWorkspaceImportDetectsNotebookExtensionCaseInsensitively(t *testing.T) { - server := testserver.New(t) - testserver.AddDefaultHandlers(server) - - mkdirs(t, server.URL, "/test-dir") - - for _, name := range []string{"lower.r", "upper.R"} { - require.Equal(t, 200, importFile(t, server.URL, "/test-dir/"+name, "# Databricks notebook source\n"), name) - } - - // Both land at the extension-stripped path, which only happens for notebooks. - assert.Equal(t, 200, getStatus(t, server.URL, "/test-dir/lower")) - assert.Equal(t, 200, getStatus(t, server.URL, "/test-dir/upper")) -} - // mkdirs creates all intermediate directories, matching "mkdir -p". func TestWorkspaceMkdirsRecursive(t *testing.T) { server := testserver.New(t) From 5dad8aabf021b6ecd58d12e34603ed9b417bad2d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 14:25:33 +0000 Subject: [PATCH 06/16] filer: replace a remote node whose type differs from the upload Overwrite can't change a node's type: import-file returns 400 when a file holds the name of a notebook upload, and 409 when a notebook holds the name of a file upload. Deployments made with v1.21 and v1.22 have such files, because those versions uploaded some notebooks as plain files. When an overwrite fails this way, delete the node of the other type and upload once more, so the remote matches the local file. Replace the #7216 Badness tests with bundle/sync/replace-node-type (Cloud), which syncs over both kinds of mismatch. Co-authored-by: Isaac --- .../bundles/revert-workspace-import.md | 1 + .../bundle/sync-notebook-detection/output.txt | 25 --------- .../bundle/sync-notebook-detection/script | 33 ------------ .../bundle/sync-notebook-detection/test.toml | 6 --- .../replace-node-type}/bom.sql | 0 .../replace-node-type}/databricks.yml.tmpl | 4 +- .../replace-node-type}/out.test.toml | 0 .../bundle/sync/replace-node-type/output.txt | 22 ++++++++ .../bundle/sync/replace-node-type/plain.sql | 1 + .../bundle/sync/replace-node-type/script | 25 +++++++++ .../sync/replace-node-type}/test.toml | 2 +- .../import-notebook-detection/bom.sql | 2 - .../import-notebook-detection/out.test.toml | 1 - .../import-notebook-detection/output.txt | 20 ------- .../import-notebook-detection/script | 21 -------- libs/filer/workspace_files_client.go | 52 +++++++++++++++++++ 16 files changed, 104 insertions(+), 111 deletions(-) create mode 100644 .nextchanges/bundles/revert-workspace-import.md delete mode 100644 acceptance/bundle/sync-notebook-detection/output.txt delete mode 100644 acceptance/bundle/sync-notebook-detection/script delete mode 100644 acceptance/bundle/sync-notebook-detection/test.toml rename acceptance/bundle/{sync-notebook-detection => sync/replace-node-type}/bom.sql (100%) rename acceptance/bundle/{sync-notebook-detection => sync/replace-node-type}/databricks.yml.tmpl (66%) rename acceptance/bundle/{sync-notebook-detection => sync/replace-node-type}/out.test.toml (100%) create mode 100644 acceptance/bundle/sync/replace-node-type/output.txt create mode 100644 acceptance/bundle/sync/replace-node-type/plain.sql create mode 100644 acceptance/bundle/sync/replace-node-type/script rename acceptance/{cmd/workspace/import-notebook-detection => bundle/sync/replace-node-type}/test.toml (61%) delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/bom.sql delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/out.test.toml delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/output.txt delete mode 100644 acceptance/cmd/workspace/import-notebook-detection/script diff --git a/.nextchanges/bundles/revert-workspace-import.md b/.nextchanges/bundles/revert-workspace-import.md new file mode 100644 index 00000000000..1b6377ba8cc --- /dev/null +++ b/.nextchanges/bundles/revert-workspace-import.md @@ -0,0 +1 @@ +* Upload bundle files through the legacy `import-file` endpoint again, because `/workspace/import` in v1.21.0 imported some notebooks as plain files; a remote file or notebook whose type differs from the upload is now replaced. ([#7218](https://github.com/databricks/cli/pull/7218)) diff --git a/acceptance/bundle/sync-notebook-detection/output.txt b/acceptance/bundle/sync-notebook-detection/output.txt deleted file mode 100644 index 29db53c50c0..00000000000 --- a/acceptance/bundle/sync-notebook-detection/output.txt +++ /dev/null @@ -1,25 +0,0 @@ - -=== Upgrade: this CLI syncs over the notebook v1.20 created - ->>> legacy_import -null - ->>> list -NOTEBOOK bom - ->>> musterr [CLI] bundle sync -Action: PUT: bom.sql -Error: file already exists: /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection-[UNIQUE_NAME]/default/files/bom.sql - -=== Downgrade: v1.20 uploads over the file this CLI created - ->>> [CLI] bundle sync --full -Action: PUT: bom.sql -Uploaded bom.sql -Initial Sync Complete - ->>> list -FILE bom.sql - ->>> musterr legacy_import -Error: Request failed for POST /Workspace/Users/[USERNAME]/.bundle/sync-notebook-detection-[UNIQUE_NAME]/default/files/bom.sql: RESOURCE_ALREADY_EXISTS: Node named 'bom.sql' already exists diff --git a/acceptance/bundle/sync-notebook-detection/script b/acceptance/bundle/sync-notebook-detection/script deleted file mode 100644 index d4fa199c730..00000000000 --- a/acceptance/bundle/sync-notebook-detection/script +++ /dev/null @@ -1,33 +0,0 @@ -# bom.sql is a SQL notebook whose first line starts with a UTF-8 byte order mark. The legacy -# import-file endpoint, used up to v1.20, imports it as notebook "bom" because its first line -# contains "Databricks notebook source". This CLI requires the exact header, so it uploads -# file "bom.sql". Overwrite can't turn one into the other. -envsubst < databricks.yml.tmpl > databricks.yml -remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) - -cleanup() { - $CLI workspace delete "$($CLI bundle summary -o json | jq -r .workspace.root_path)" --recursive -} -trap cleanup EXIT - -list() { - $CLI workspace list -o json "$remote" | jq -r '.[] | "\(.object_type) \(.path | sub(".*/"; ""))"' -} - -# Uploads bom.sql the way v1.20 did. `api post` only sends JSON; a JSON string whose -# first line contains the marker is imported as a notebook, just like bom.sql. -legacy_import() { - $CLI api post "/api/2.0/workspace-files/import-file$remote/bom.sql?overwrite=true" --json '"-- Databricks notebook source"' -} - -title "Upgrade: this CLI syncs over the notebook v1.20 created\n" -$CLI workspace mkdirs "$remote" -trace legacy_import -trace list -trace musterr $CLI bundle sync - -title "Downgrade: v1.20 uploads over the file this CLI created\n" -$CLI workspace delete "$remote" --recursive -trace $CLI bundle sync --full -trace list -trace musterr legacy_import diff --git a/acceptance/bundle/sync-notebook-detection/test.toml b/acceptance/bundle/sync-notebook-detection/test.toml deleted file mode 100644 index 037617015ea..00000000000 --- a/acceptance/bundle/sync-notebook-detection/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -Cloud = true -Badness = "A notebook whose first line isn't exactly the header (here: a byte order mark) is uploaded as a file, while v1.20 and earlier uploaded it as a notebook. Deploying the same bundle with both fails in either order (#7216)." - -Env.MSYS_NO_PATHCONV = "1" - -Ignore = [".databricks"] diff --git a/acceptance/bundle/sync-notebook-detection/bom.sql b/acceptance/bundle/sync/replace-node-type/bom.sql similarity index 100% rename from acceptance/bundle/sync-notebook-detection/bom.sql rename to acceptance/bundle/sync/replace-node-type/bom.sql diff --git a/acceptance/bundle/sync-notebook-detection/databricks.yml.tmpl b/acceptance/bundle/sync/replace-node-type/databricks.yml.tmpl similarity index 66% rename from acceptance/bundle/sync-notebook-detection/databricks.yml.tmpl rename to acceptance/bundle/sync/replace-node-type/databricks.yml.tmpl index 32d766e544b..8c9201dd352 100644 --- a/acceptance/bundle/sync-notebook-detection/databricks.yml.tmpl +++ b/acceptance/bundle/sync/replace-node-type/databricks.yml.tmpl @@ -1,7 +1,7 @@ bundle: - name: sync-notebook-detection-$UNIQUE_NAME + name: replace-node-type-$UNIQUE_NAME -# Only sync bom.sql. +# Only sync bom.sql and plain.sql. sync: exclude: - databricks.yml diff --git a/acceptance/bundle/sync-notebook-detection/out.test.toml b/acceptance/bundle/sync/replace-node-type/out.test.toml similarity index 100% rename from acceptance/bundle/sync-notebook-detection/out.test.toml rename to acceptance/bundle/sync/replace-node-type/out.test.toml diff --git a/acceptance/bundle/sync/replace-node-type/output.txt b/acceptance/bundle/sync/replace-node-type/output.txt new file mode 100644 index 00000000000..9e482f6a28e --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/output.txt @@ -0,0 +1,22 @@ + +=== A file holds the name of notebook bom.sql, a notebook holds the name of file plain.sql + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/replace-node-type-[UNIQUE_NAME]/default/files/bom.sql --file bom.sql --format RAW + +>>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/replace-node-type-[UNIQUE_NAME]/default/files/plain --file plain.sql --format SOURCE --language SQL + +>>> list +FILE bom.sql +NOTEBOOK plain + +=== Sync replaces both + +>>> [CLI] bundle sync --output text +Action: PUT: bom.sql, plain.sql +Initial Sync Complete +Uploaded bom.sql +Uploaded plain.sql + +>>> list +NOTEBOOK bom +FILE plain.sql diff --git a/acceptance/bundle/sync/replace-node-type/plain.sql b/acceptance/bundle/sync/replace-node-type/plain.sql new file mode 100644 index 00000000000..2e3761f7a2c --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/plain.sql @@ -0,0 +1 @@ +SELECT 1 diff --git a/acceptance/bundle/sync/replace-node-type/script b/acceptance/bundle/sync/replace-node-type/script new file mode 100644 index 00000000000..32dd2c3cc53 --- /dev/null +++ b/acceptance/bundle/sync/replace-node-type/script @@ -0,0 +1,25 @@ +# Overwrite can't change a node's type, so when the node at a path has the other type, +# sync deletes it and uploads again. bom.sql is a notebook: its first line starts with a +# byte order mark, and the first line only needs to contain "Databricks notebook source". +# CLI v1.21 and v1.22 uploaded it as a file. +envsubst < databricks.yml.tmpl > databricks.yml +remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) + +cleanup() { + $CLI workspace delete "$($CLI bundle summary -o json | jq -r .workspace.root_path)" --recursive +} +trap cleanup EXIT + +list() { + $CLI workspace list -o json "$remote" | jq -r 'sort_by(.path)[] | "\(.object_type) \(.path | sub(".*/"; ""))"' +} + +title "A file holds the name of notebook bom.sql, a notebook holds the name of file plain.sql\n" +$CLI workspace mkdirs "$remote" +trace $CLI workspace import "$remote/bom.sql" --file bom.sql --format RAW +trace $CLI workspace import "$remote/plain" --file plain.sql --format SOURCE --language SQL +trace list + +title "Sync replaces both\n" +trace $CLI bundle sync --output text 2>&1 | sort +trace list diff --git a/acceptance/cmd/workspace/import-notebook-detection/test.toml b/acceptance/bundle/sync/replace-node-type/test.toml similarity index 61% rename from acceptance/cmd/workspace/import-notebook-detection/test.toml rename to acceptance/bundle/sync/replace-node-type/test.toml index bb47f764e26..aab5e85edef 100644 --- a/acceptance/cmd/workspace/import-notebook-detection/test.toml +++ b/acceptance/bundle/sync/replace-node-type/test.toml @@ -1,3 +1,3 @@ Cloud = true - Env.MSYS_NO_PATHCONV = "1" +Ignore = [".databricks"] diff --git a/acceptance/cmd/workspace/import-notebook-detection/bom.sql b/acceptance/cmd/workspace/import-notebook-detection/bom.sql deleted file mode 100644 index 4aeddae0fd3..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/bom.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Databricks notebook source -SELECT 1 diff --git a/acceptance/cmd/workspace/import-notebook-detection/out.test.toml b/acceptance/cmd/workspace/import-notebook-detection/out.test.toml deleted file mode 100644 index c7c6f58ed6e..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/out.test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = true diff --git a/acceptance/cmd/workspace/import-notebook-detection/output.txt b/acceptance/cmd/workspace/import-notebook-detection/output.txt deleted file mode 100644 index 9b59bf41029..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/output.txt +++ /dev/null @@ -1,20 +0,0 @@ - -=== AUTO imports it as a notebook, like the legacy import-file endpoint. Overwrite can't make it a file - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/notebook/bom.sql --file bom.sql --format AUTO - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/notebook/bom.sql --file bom.sql --format RAW --overwrite -Error: Requested node type [FILE] is different from the existing node type [NOTEBOOK] - -=== RAW imports it as a file, like this CLI. Overwrite can't make it a notebook - ->>> [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/file/bom.sql --file bom.sql --format RAW - ->>> musterr [CLI] workspace import /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/file/bom.sql --file bom.sql --format AUTO --overwrite -Error: Node named 'bom.sql' already exists - ->>> [CLI] workspace list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/notebook -o json -NOTEBOOK /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/notebook/bom - ->>> [CLI] workspace list /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/file -o json -FILE /Workspace/Users/[USERNAME]/[UNIQUE_NAME]/file/bom.sql diff --git a/acceptance/cmd/workspace/import-notebook-detection/script b/acceptance/cmd/workspace/import-notebook-detection/script deleted file mode 100644 index be11e352577..00000000000 --- a/acceptance/cmd/workspace/import-notebook-detection/script +++ /dev/null @@ -1,21 +0,0 @@ -# Documents POST /api/2.0/workspace/import for bom.sql, a SQL notebook whose first line -# starts with a UTF-8 byte order mark (see bundle/sync-notebook-detection). -DIR="/Workspace/Users/$CURRENT_USER_NAME/$UNIQUE_NAME" -$CLI workspace mkdirs "$DIR/notebook" -$CLI workspace mkdirs "$DIR/file" - -cleanup() { - $CLI workspace delete "$DIR" --recursive -} -trap cleanup EXIT - -title "AUTO imports it as a notebook, like the legacy import-file endpoint. Overwrite can't make it a file\n" -trace $CLI workspace import "$DIR/notebook/bom.sql" --file bom.sql --format AUTO -trace musterr $CLI workspace import "$DIR/notebook/bom.sql" --file bom.sql --format RAW --overwrite - -title "RAW imports it as a file, like this CLI. Overwrite can't make it a notebook\n" -trace $CLI workspace import "$DIR/file/bom.sql" --file bom.sql --format RAW -trace musterr $CLI workspace import "$DIR/file/bom.sql" --file bom.sql --format AUTO --overwrite - -trace $CLI workspace list "$DIR/notebook" -o json | jq -r '.[] | "\(.object_type) \(.path)"' -trace $CLI workspace list "$DIR/file" -o json | jq -r '.[] | "\(.object_type) \(.path)"' diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index e1b62481243..4623bdb6d30 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -170,6 +170,18 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) + // Overwrite can't change a node's type. If a node of the other type holds the + // path, delete it and upload again so the remote matches the local file. + if err != nil && overwrite { + deleted, derr := w.deleteNodeOfOtherType(ctx, absPath, err) + if derr != nil { + return derr + } + if deleted { + err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) + } + } + // Return early on success. if err == nil { return nil @@ -223,6 +235,46 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } +// deleteNodeOfOtherType deletes the node that blocks an overwrite of absPath +// because its type differs from the upload's, and reports whether it did. +// import-file returns 400 when a file holds the name of a notebook upload, and +// 409 when a notebook holds the name of a file upload (SQL notebook "x" holds +// "x.sql"). CLI v1.21 and v1.22 uploaded some notebooks as files this way. +func (w *WorkspaceFilesClient) deleteNodeOfOtherType(ctx context.Context, absPath string, uploadErr error) (bool, error) { + aerr, ok := errors.AsType[*apierr.APIError](uploadErr) + if !ok { + return false, nil + } + + var nodePath string + var nodeType workspace.ObjectType + switch aerr.StatusCode { + case http.StatusBadRequest: + nodePath, nodeType = absPath, workspace.ObjectTypeFile + case http.StatusConflict: + nodePath, nodeType = strings.TrimSuffix(absPath, path.Ext(absPath)), workspace.ObjectTypeNotebook + default: + return false, nil + } + + info, err := w.workspaceClient.Workspace.GetStatusByPath(ctx, nodePath) + if errors.Is(err, apierr.ErrNotFound) { + return false, nil + } + if err != nil { + return false, err + } + if info.ObjectType != nodeType { + return false, nil + } + + err = w.workspaceClient.Workspace.Delete(ctx, workspace.Delete{Path: nodePath}) + if err != nil { + return false, fmt.Errorf("failed to delete %s to replace it with %s: %w", nodePath, absPath, err) + } + return true, nil +} + func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { absPath, err := w.root.Join(name) if err != nil { From f746b25d45ad9278940b3fd5e521d2a173647122 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 14:26:04 +0000 Subject: [PATCH 07/16] notebook: use errors.Is for io.EOF in the restored readHeader The revert of #7034 restored a direct comparison that errorlint now flags. Co-authored-by: Isaac --- libs/notebook/detect.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/notebook/detect.go b/libs/notebook/detect.go index 579cc1de34c..f6458cf7108 100644 --- a/libs/notebook/detect.go +++ b/libs/notebook/detect.go @@ -3,6 +3,7 @@ package notebook import ( "bufio" "bytes" + "errors" "io" "io/fs" "os" @@ -48,7 +49,7 @@ func (f file) readHeader() (string, error) { // Scan header line with some padding. buf := make([]byte, headerLength) n, err := f.f.Read(buf) - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { return "", err } From 839187782d5e8fb2499f0c89ccd6ef8bcdc8cc88 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 15:00:30 +0000 Subject: [PATCH 08/16] testserver: restore fault --body-contains, the .R detection test and JUPYTER imports The revert of #6149 and #7034 removed these, but they model the real API and stay useful independently of which endpoint the CLI uploads with. Co-authored-by: Isaac --- acceptance/bin/fault.py | 16 +++++++++-- libs/testserver/fault.go | 31 ++++++++++++++------ libs/testserver/fault_test.go | 48 +++++++++++++++++++------------ libs/testserver/handlers.go | 8 +++++- libs/testserver/server.go | 2 +- libs/testserver/workspace_test.go | 18 ++++++++++++ 6 files changed, 93 insertions(+), 30 deletions(-) diff --git a/acceptance/bin/fault.py b/acceptance/bin/fault.py index 0aed44d58d1..5b75c0a5907 100755 --- a/acceptance/bin/fault.py +++ b/acceptance/bin/fault.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 """Set up a fault rule on the testserver for the current test token. -Usage: fault.py [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] +Usage: fault.py [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] --after-handler run the handler first; keep its effect and replace only the response + --body-contains only fire when the request body contains SUBSTR, e.g. to target one + file's /workspace/import upload (all uploads share that path) PATTERN HTTP method and path, supports trailing * wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*" STATUS_CODE HTTP status code to return, e.g. 504 @@ -33,8 +35,17 @@ if after_handler: args.remove("--after-handler") +body_contains = "" +if "--body-contains" in args: + i = args.index("--body-contains") + body_contains = args[i + 1] + del args[i : i + 2] + if len(args) not in (4, 5): - print(f"usage: {sys.argv[0]} [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr) + print( + f"usage: {sys.argv[0]} [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", + file=sys.stderr, + ) sys.exit(1) pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3]) @@ -49,6 +60,7 @@ "offset": offset, "times": times, "after_handler": after_handler, + "body_contains": body_contains, } ).encode() diff --git a/libs/testserver/fault.go b/libs/testserver/fault.go index 75d3092bb70..43d46f3f28c 100644 --- a/libs/testserver/fault.go +++ b/libs/testserver/fault.go @@ -1,6 +1,7 @@ package testserver import ( + "bytes" "encoding/json" "strings" "sync" @@ -19,6 +20,12 @@ type FaultRule struct { AfterHandler bool offset int times int + + // bodyContains, if non-empty, additionally requires the request body to + // contain this substring for the rule to fire. /workspace/import routes + // every upload through the same method+path, so the only way to target a + // single file's upload is by matching its multipart "path" form field. + bodyContains string } // FaultRules holds the active fault injection rules for a test server. @@ -33,16 +40,16 @@ func NewFaultRules() *FaultRules { } // Set registers or replaces a fault rule for the given token and pattern. -func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int) { - fr.set(token, pattern, statusCode, body, offset, times, false) +func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { + fr.set(token, pattern, statusCode, body, offset, times, false, bodyContains) } // SetAfterHandler is like Set, but the handler runs first so its effect is kept. -func (fr *FaultRules) SetAfterHandler(token, pattern string, statusCode int, body string, offset, times int) { - fr.set(token, pattern, statusCode, body, offset, times, true) +func (fr *FaultRules) SetAfterHandler(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { + fr.set(token, pattern, statusCode, body, offset, times, true, bodyContains) } -func (fr *FaultRules) set(token, pattern string, statusCode int, body string, offset, times int, afterHandler bool) { +func (fr *FaultRules) set(token, pattern string, statusCode int, body string, offset, times int, afterHandler bool, bodyContains string) { fr.mu.Lock() defer fr.mu.Unlock() fr.rules[faultRuleKey{token: token, pattern: pattern}] = &FaultRule{ @@ -51,12 +58,14 @@ func (fr *FaultRules) set(token, pattern string, statusCode int, body string, of AfterHandler: afterHandler, offset: offset, times: times, + bodyContains: bodyContains, } } // Check returns a matching fault rule and advances its counters, or nil if no rule matches. // Pattern supports a trailing "*" wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*". -func (fr *FaultRules) Check(method, path, token string) *FaultRule { +// A rule with a non-empty bodyContains only matches when body contains that substring. +func (fr *FaultRules) Check(method, path, token string, body []byte) *FaultRule { requestPattern := method + " " + path fr.mu.Lock() @@ -76,6 +85,11 @@ func (fr *FaultRules) Check(method, path, token string) *FaultRule { if !matched { continue } + // A non-matching body must not consume the rule's offset/times budget, + // so this check precedes the counter updates below. + if rule.bodyContains != "" && !bytes.Contains(body, []byte(rule.bodyContains)) { + continue + } if rule.offset > 0 { rule.offset-- return nil @@ -105,14 +119,15 @@ func faultEndpointHandler(fr *FaultRules) HandlerFunc { Offset int `json:"offset"` Times int `json:"times"` AfterHandler bool `json:"after_handler"` + BodyContains string `json:"body_contains"` } if err := json.Unmarshal(req.Body, &body); err != nil { return Response{StatusCode: 400, Body: map[string]string{"error": err.Error()}} } if body.AfterHandler { - fr.SetAfterHandler(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) + fr.SetAfterHandler(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) } else { - fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) + fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) } return Response{StatusCode: 200} } diff --git a/libs/testserver/fault_test.go b/libs/testserver/fault_test.go index db41b74d3ba..ae27b954f93 100644 --- a/libs/testserver/fault_test.go +++ b/libs/testserver/fault_test.go @@ -14,18 +14,18 @@ import ( func TestFaultRulesNoMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 1) + fr.Set("tok", "GET /foo", 504, "body", 0, 1, "") - assert.Nil(t, fr.Check("POST", "/foo", "tok")) - assert.Nil(t, fr.Check("GET", "/bar", "tok")) - assert.Nil(t, fr.Check("GET", "/foo", "other")) + assert.Nil(t, fr.Check("POST", "/foo", "tok", nil)) + assert.Nil(t, fr.Check("GET", "/bar", "tok", nil)) + assert.Nil(t, fr.Check("GET", "/foo", "other", nil)) } func TestFaultRulesExactMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1) + fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1, "") - rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok") + rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok", nil) require.NotNil(t, rule) assert.Equal(t, 504, rule.StatusCode) assert.Equal(t, "body", rule.Body) @@ -33,31 +33,43 @@ func TestFaultRulesExactMatch(t *testing.T) { func TestFaultRulesWildcardMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2) + fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2, "") - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok")) - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) - assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) // exhausted + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok", nil)) + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) + assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) // exhausted } func TestFaultRulesOffset(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 2, 1) + fr.Set("tok", "GET /foo", 504, "body", 2, 1, "") - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 2→1 - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 1→0 - assert.NotNil(t, fr.Check("GET", "/foo", "tok")) // fires - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 2→1 + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 1→0 + assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) // fires + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted } func TestFaultRulesTimes(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 3) + fr.Set("tok", "GET /foo", 504, "body", 0, 3, "") for range 3 { - assert.NotNil(t, fr.Check("GET", "/foo", "tok")) + assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) } - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted +} + +func TestFaultRulesBodyContains(t *testing.T) { + fr := testserver.NewFaultRules() + fr.Set("tok", "POST /api/2.0/workspace/import", 403, "body", 0, 1, "state/resources.json") + + // Body without the substring must not match and must not consume the budget. + assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/deploy.lock"))) + // Body with the substring fires. + assert.NotNil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) + // Budget was only consumed by the matching request. + assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) } func TestServerFaultAfterHandlerKeepsTheHandlersEffect(t *testing.T) { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 044f02952f3..ad3615cd323 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -13,7 +13,9 @@ import ( "path" "strconv" "strings" + "testing/fstest" + "github.com/databricks/cli/libs/notebook" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -80,6 +82,10 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { resp = w.WorkspaceImportRawFile(form.Path, form.Content, form.Overwrite) case string(workspace.ImportFormatSource): resp = w.WorkspaceImportNotebook(form.Path, form.Content, workspace.Language(form.Language), form.Overwrite) + case string(workspace.ImportFormatJupyter): + // The real endpoint reads the language from the notebook metadata. + _, language, _ := notebook.DetectJupyterWithFS(fstest.MapFS{"nb.ipynb": {Data: form.Content}}, "nb.ipynb") + resp = w.WorkspaceImportNotebook(form.Path, form.Content, language, form.Overwrite) case string(workspace.ImportFormatAuto): if strings.HasSuffix(strings.ToLower(form.Path), ".zip") { return w.WorkspaceImportZipFolder(form.Path, form.Content, form.Overwrite) @@ -90,7 +96,7 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { resp = w.WorkspaceImportAutoFile(form.Path, form.Content, form.Overwrite) default: return Response{ - Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW and SOURCE formats, got %q.", form.Format), + Body: fmt.Sprintf("internal error: The test server only supports AUTO, RAW, SOURCE and JUPYTER formats, got %q.", form.Format), StatusCode: http.StatusInternalServerError, } } diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 1f56717c50a..5170b318818 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -372,7 +372,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, handler HandlerFu var resp EncodedResponse - fault := s.faults.Check(r.Method, r.URL.Path, token) + fault := s.faults.Check(r.Method, r.URL.Path, token, request.Body) if fault == nil || fault.AfterHandler { if bytes.Contains(request.Body, []byte("INJECT_ERROR")) { diff --git a/libs/testserver/workspace_test.go b/libs/testserver/workspace_test.go index cde1c9741c6..401758c6648 100644 --- a/libs/testserver/workspace_test.go +++ b/libs/testserver/workspace_test.go @@ -88,6 +88,24 @@ func TestWorkspaceDeleteNonRecursiveRequiresEmptyDirectory(t *testing.T) { assert.Equal(t, 404, getStatus(t, server.URL, "/a/b")) } +// format=AUTO matches the extension case-insensitively, so ".R" — the +// conventional spelling for R sources — is detected as a notebook just like +// ".r". Notebooks are stored with the extension stripped. +func TestWorkspaceImportDetectsNotebookExtensionCaseInsensitively(t *testing.T) { + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + + mkdirs(t, server.URL, "/test-dir") + + for _, name := range []string{"lower.r", "upper.R"} { + require.Equal(t, 200, importFile(t, server.URL, "/test-dir/"+name, "# Databricks notebook source\n"), name) + } + + // Both land at the extension-stripped path, which only happens for notebooks. + assert.Equal(t, 200, getStatus(t, server.URL, "/test-dir/lower")) + assert.Equal(t, 200, getStatus(t, server.URL, "/test-dir/upper")) +} + // mkdirs creates all intermediate directories, matching "mkdir -p". func TestWorkspaceMkdirsRecursive(t *testing.T) { server := testserver.New(t) From 323056e7b75af171018a8db09545196a9b55fa8a Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 15:00:42 +0000 Subject: [PATCH 09/16] testserver: extract a .zip uploaded through import-file The real import-file endpoint imports each entry of a .zip into the archive's parent folder instead of storing the archive (importZipFile in universe). The fake stored it as a file, so nothing tracked that behavior. Add bundle/sync/zip-extract (Cloud), and have the ssh tests upload valid placeholder archives, which the real endpoint requires too. Co-authored-by: Isaac --- acceptance/bundle/sync/output.txt | 12 ++++- .../bundle/sync/zip-extract/archive.zip | Bin 0 -> 299 bytes .../bundle/sync/zip-extract/archive_v2.zip | Bin 0 -> 299 bytes .../sync/zip-extract/databricks.yml.tmpl | 14 +++++ .../bundle/sync/zip-extract/out.test.toml | 2 + acceptance/bundle/sync/zip-extract/output.txt | 24 +++++++++ acceptance/bundle/sync/zip-extract/script | 24 +++++++++ acceptance/bundle/sync/zip-extract/test.toml | 3 ++ acceptance/ssh/bulk-transfer/script | 6 +-- acceptance/ssh/connect-serverless-cpu/script | 6 +-- .../output.txt | 2 +- .../script | 8 +-- acceptance/ssh/connect-serverless-gpu/script | 6 +-- acceptance/ssh/connection/script | 6 +-- acceptance/ssh/reconnect-reused-name/script | 6 +-- acceptance/ssh/script.prepare | 5 ++ libs/testserver/fake_workspace.go | 48 ++++++++++++++++++ 17 files changed, 151 insertions(+), 21 deletions(-) create mode 100644 acceptance/bundle/sync/zip-extract/archive.zip create mode 100644 acceptance/bundle/sync/zip-extract/archive_v2.zip create mode 100644 acceptance/bundle/sync/zip-extract/databricks.yml.tmpl create mode 100644 acceptance/bundle/sync/zip-extract/out.test.toml create mode 100644 acceptance/bundle/sync/zip-extract/output.txt create mode 100644 acceptance/bundle/sync/zip-extract/script create mode 100644 acceptance/bundle/sync/zip-extract/test.toml create mode 100644 acceptance/ssh/script.prepare diff --git a/acceptance/bundle/sync/output.txt b/acceptance/bundle/sync/output.txt index ca0b5b7258d..276972e0d5d 100644 --- a/acceptance/bundle/sync/output.txt +++ b/acceptance/bundle/sync/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle sync --output text -Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py, upload-504/databricks.yml, upload-504/out.test.toml +Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py, replace-node-type/bom.sql, replace-node-type/databricks.yml.tmpl, replace-node-type/out.test.toml, replace-node-type/plain.sql, upload-504/databricks.yml, upload-504/out.test.toml, zip-extract/archive.zip, zip-extract/archive_v2.zip, zip-extract/databricks.yml.tmpl, zip-extract/out.test.toml Initial Sync Complete Uploaded .gitignore Uploaded databricks.yml @@ -10,6 +10,16 @@ Uploaded dryrun/out.test.toml Uploaded ignored-folder/folder1 Uploaded ignored-folder/folder1/script.py Uploaded ignored-folder/script.py +Uploaded replace-node-type +Uploaded replace-node-type/bom.sql +Uploaded replace-node-type/databricks.yml.tmpl +Uploaded replace-node-type/out.test.toml +Uploaded replace-node-type/plain.sql Uploaded upload-504 Uploaded upload-504/databricks.yml Uploaded upload-504/out.test.toml +Uploaded zip-extract +Uploaded zip-extract/archive.zip +Uploaded zip-extract/archive_v2.zip +Uploaded zip-extract/databricks.yml.tmpl +Uploaded zip-extract/out.test.toml diff --git a/acceptance/bundle/sync/zip-extract/archive.zip b/acceptance/bundle/sync/zip-extract/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..0452f3ee8bf0a603bb1f02c2e9ce4444bff57c06 GIT binary patch literal 299 zcmWIWW@Zs#0D*?GnAjNQt)0w3HVAV9ab{j#YLQ+^MM;?H4TD z*cZOK%LJ4GVUSrM6^S6T5`jiBGKnzbwic)w3>p|g6x>> [CLI] bundle sync +Action: PUT: archive.zip +Uploaded archive.zip +Initial Sync Complete + +>>> list /Workspace/Users/[USERNAME]/.bundle/zip-extract-[UNIQUE_NAME]/default/files +FILE /inner.txt +DIRECTORY /pkg + +>>> list /Workspace/Users/[USERNAME]/.bundle/zip-extract-[UNIQUE_NAME]/default/files/pkg +FILE /pkg/a.txt + +=== Syncing a changed archive overwrites the extracted files + +>>> [CLI] bundle sync +Action: PUT: archive.zip +Uploaded archive.zip +Initial Sync Complete + +>>> [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/zip-extract-[UNIQUE_NAME]/default/files/inner.txt --format AUTO +v2 diff --git a/acceptance/bundle/sync/zip-extract/script b/acceptance/bundle/sync/zip-extract/script new file mode 100644 index 00000000000..16f6d6df107 --- /dev/null +++ b/acceptance/bundle/sync/zip-extract/script @@ -0,0 +1,24 @@ +# The import-file endpoint extracts a .zip into its parent folder instead of storing the +# archive, and overwrites the extracted files on the next upload. archive.zip holds +# inner.txt ("v1") and pkg/a.txt; archive_v2.zip is the same with inner.txt "v2". +envsubst < databricks.yml.tmpl > databricks.yml +remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) + +cleanup() { + $CLI workspace delete "$($CLI bundle summary -o json | jq -r .workspace.root_path)" --recursive +} +trap cleanup EXIT + +list() { + $CLI workspace list -o json "$1" | jq -r 'sort_by(.path)[] | "\(.object_type) \(.path | sub(".*/files"; ""))"' +} + +title "Sync extracts the archive into the bundle's file path\n" +trace $CLI bundle sync +trace list "$remote" +trace list "$remote/pkg" + +title "Syncing a changed archive overwrites the extracted files\n" +cp archive_v2.zip archive.zip +trace $CLI bundle sync +trace $CLI workspace export "$remote/inner.txt" --format AUTO diff --git a/acceptance/bundle/sync/zip-extract/test.toml b/acceptance/bundle/sync/zip-extract/test.toml new file mode 100644 index 00000000000..aab5e85edef --- /dev/null +++ b/acceptance/bundle/sync/zip-extract/test.toml @@ -0,0 +1,3 @@ +Cloud = true +Env.MSYS_NO_PATHCONV = "1" +Ignore = [".databricks"] diff --git a/acceptance/ssh/bulk-transfer/script b/acceptance/ssh/bulk-transfer/script index 7e043dedf28..7272d7d56f1 100644 --- a/acceptance/ssh/bulk-transfer/script +++ b/acceptance/ssh/bulk-transfer/script @@ -30,10 +30,10 @@ else exit 0 fi - # No release artifacts locally and they're never executed; stand in empty archives. + # No release artifacts locally and they're never executed; stand in placeholder archives. mkdir -p releases - : >releases/databricks_cli_linux_amd64.zip - : >releases/databricks_cli_linux_arm64.zip + fake_release releases/databricks_cli_linux_amd64.zip + fake_release releases/databricks_cli_linux_arm64.zip CONNECT_FLAGS="--releases-dir=$PWD/releases --known-hosts-dir=known-hosts" fi diff --git a/acceptance/ssh/connect-serverless-cpu/script b/acceptance/ssh/connect-serverless-cpu/script index 02b0068f3da..14c787e58ea 100644 --- a/acceptance/ssh/connect-serverless-cpu/script +++ b/acceptance/ssh/connect-serverless-cpu/script @@ -15,10 +15,10 @@ else exit 0 fi - # No release artifacts locally and they're never executed; stand in empty archives. + # No release artifacts locally and they're never executed; stand in placeholder archives. mkdir -p releases - touch releases/databricks_cli_linux_amd64.zip - touch releases/databricks_cli_linux_arm64.zip + fake_release releases/databricks_cli_linux_amd64.zip + fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" title "CLI ssh connect (local sshd)\n" diff --git a/acceptance/ssh/connect-serverless-feature-disabled/output.txt b/acceptance/ssh/connect-serverless-feature-disabled/output.txt index 9957a1c7e18..226d0030a71 100644 --- a/acceptance/ssh/connect-serverless-feature-disabled/output.txt +++ b/acceptance/ssh/connect-serverless-feature-disabled/output.txt @@ -10,7 +10,7 @@ DRIVER_PROXY_FORBIDDEN === Each release archive is uploaded once, and the submitted run is cancelled ->>> print_requests.py //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel +>>> print_requests.py --del-field raw_body //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel { "method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.databricks/ssh-tunnel/[CLI_VERSION]/databricks_cli_linux_amd64/databricks.zip", diff --git a/acceptance/ssh/connect-serverless-feature-disabled/script b/acceptance/ssh/connect-serverless-feature-disabled/script index 9f8fe5c1b92..67901dc76ef 100644 --- a/acceptance/ssh/connect-serverless-feature-disabled/script +++ b/acceptance/ssh/connect-serverless-feature-disabled/script @@ -6,10 +6,10 @@ # The body repeats the name: with only {"value": false} in it, the API sees an empty message. trace $CLI workspace-settings-v2 patch-public-workspace-setting port_fwd_serverless_enabled --json '{"name": "port_fwd_serverless_enabled", "boolean_val": {"value": false}}' >/dev/null -# No release artifacts locally and they're never executed; stand in empty archives. +# No release artifacts locally and they're never executed; stand in placeholder archives. mkdir -p releases -: >releases/databricks_cli_linux_amd64.zip -: >releases/databricks_cli_linux_arm64.zip +fake_release releases/databricks_cli_linux_amd64.zip +fake_release releases/databricks_cli_linux_arm64.zip title "CLI ssh connect\n" musterr $CLI ssh connect --name feature-disabled --releases-dir="$PWD/releases" -- "echo 'Connection successful'" >out.connect.txt 2>&1 @@ -19,4 +19,4 @@ title "Reported error category\n" trace jq -r 'select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.ssh_tunnel_event.error_category' "$OUT_REQUESTS" title "Each release archive is uploaded once, and the submitted run is cancelled\n" -trace print_requests.py //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel +trace print_requests.py --del-field raw_body //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel diff --git a/acceptance/ssh/connect-serverless-gpu/script b/acceptance/ssh/connect-serverless-gpu/script index b2e5e3d40fa..5aab2af0ac0 100644 --- a/acceptance/ssh/connect-serverless-gpu/script +++ b/acceptance/ssh/connect-serverless-gpu/script @@ -1,9 +1,9 @@ if [ -z "${CLOUD_ENV:-}" ]; then # Local runs have no release artifacts and never execute the uploaded - # binaries, so stand in empty archives (dev name omits the version). + # binaries, so stand in placeholder archives (dev name omits the version). mkdir -p releases - : >releases/databricks_cli_linux_amd64.zip - : >releases/databricks_cli_linux_arm64.zip + fake_release releases/databricks_cli_linux_amd64.zip + fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" fi diff --git a/acceptance/ssh/connection/script b/acceptance/ssh/connection/script index 3858cd1b6ba..25a63feca78 100644 --- a/acceptance/ssh/connection/script +++ b/acceptance/ssh/connection/script @@ -1,9 +1,9 @@ if [ -z "${CLOUD_ENV:-}" ]; then # Local runs have no release artifacts and never execute the uploaded - # binaries, so stand in empty archives (dev name omits the version). + # binaries, so stand in placeholder archives (dev name omits the version). mkdir -p releases - : >releases/databricks_cli_linux_amd64.zip - : >releases/databricks_cli_linux_arm64.zip + fake_release releases/databricks_cli_linux_amd64.zip + fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" fi diff --git a/acceptance/ssh/reconnect-reused-name/script b/acceptance/ssh/reconnect-reused-name/script index b07e4304746..a0ac016f93f 100644 --- a/acceptance/ssh/reconnect-reused-name/script +++ b/acceptance/ssh/reconnect-reused-name/script @@ -11,10 +11,10 @@ if [ -z "$(command -v sshd || ls /usr/sbin/sshd /usr/local/sbin/sshd /sbin/sshd exit 0 fi -# No release artifacts locally and they're never executed; stand in empty archives. +# No release artifacts locally and they're never executed; stand in placeholder archives. mkdir -p releases -: >releases/databricks_cli_linux_amd64.zip -: >releases/databricks_cli_linux_arm64.zip +fake_release releases/databricks_cli_linux_amd64.zip +fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" # The state a user reconnecting with a reused name is in: an entry for this connection diff --git a/acceptance/ssh/script.prepare b/acceptance/ssh/script.prepare new file mode 100644 index 00000000000..f8b26a3cd2a --- /dev/null +++ b/acceptance/ssh/script.prepare @@ -0,0 +1,5 @@ +# Writes a stand-in release archive. The import-file endpoint unzips uploaded .zip files, +# so it has to be a valid archive holding the `databricks` binary. +fake_release() { + python3 -c "import sys, zipfile; zipfile.ZipFile(sys.argv[1], 'w').writestr('databricks', '')" "$1" +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index abfe4fdf855..dd5050f6f49 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -1093,9 +1093,57 @@ func importFileNotebookTaken(filePath string) Response { // WorkspaceFilesImportFile mirrors the legacy import-file endpoint. func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, overwrite bool) Response { + // The endpoint matches this extension case-sensitively. + if path.Ext(filePath) == ".zip" { + return s.importFileZip(filePath, body, overwrite) + } return s.importFile(filePath, body, overwrite, detectLegacyNotebook, importFileConflicts) } +// importFileZip mirrors import-file for a .zip (importZipFile in universe): each +// entry is imported into the archive's parent folder, the archive itself is not +// stored, and directories must have their own entry. +func (s *FakeWorkspace) importFileZip(zipPath string, body []byte, overwrite bool) Response { + if !strings.HasPrefix(zipPath, "/") { + zipPath = "/" + zipPath + } + + zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{"message": "Failed to unzip with error: " + err.Error()}, + } + } + + parent := path.Dir(zipPath) + for _, entry := range zr.File { + // macOS archives carry an extra __MACOSX directory. + if strings.HasPrefix(entry.Name, "__MACOSX") { + continue + } + entryPath := path.Join(parent, entry.Name) + if entry.FileInfo().IsDir() { + s.WorkspaceMkdirs(workspace.Mkdirs{Path: entryPath}) + continue + } + if strings.HasSuffix(entry.Name, ".zip") { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{"message": fmt.Sprintf("Request failed for POST %s: don't support importing nested zip files", zipPath)}, + } + } + data, err := readZipEntry(entry) + if err != nil { + return Response{StatusCode: http.StatusBadRequest, Body: map[string]string{"message": err.Error()}} + } + if resp := s.importFile(entryPath, data, overwrite, detectLegacyNotebook, importFileConflicts); resp.StatusCode != 0 { + return resp + } + } + return Response{} +} + // WorkspaceImportAutoFile mirrors /workspace/import with format=AUTO for a // file that is not an archive or another asset type. func (s *FakeWorkspace) WorkspaceImportAutoFile(filePath string, body []byte, overwrite bool) Response { From 08c6c0359f0f0a70b30902d3b607cc611d76677d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 15:31:22 +0000 Subject: [PATCH 10/16] ssh: upload valid zips in TestUploadReleasesSendsEachArchiveOnce The testserver now unzips archives uploaded through import-file, like the real endpoint, so the test needs a valid archive instead of a plain string. Co-authored-by: Isaac --- .../ssh/internal/client/releases_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/experimental/ssh/internal/client/releases_test.go b/experimental/ssh/internal/client/releases_test.go index e0e2c384d9d..51b6018b614 100644 --- a/experimental/ssh/internal/client/releases_test.go +++ b/experimental/ssh/internal/client/releases_test.go @@ -1,6 +1,8 @@ package client import ( + "archive/zip" + "bytes" "context" "errors" "fmt" @@ -146,6 +148,17 @@ func TestUploadReleasesWithExistingBinary(t *testing.T) { } } +// releaseArchive returns a zip holding the databricks binary. The import-file +// endpoint unzips uploaded archives, so it has to be valid. +func releaseArchive(t *testing.T) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + _, err := zw.Create("databricks") + require.NoError(t, err) + require.NoError(t, zw.Close()) + return buf.Bytes() +} + func TestUploadReleasesSendsEachArchiveOnce(t *testing.T) { tests := []struct { name string @@ -197,8 +210,9 @@ func TestUploadReleasesSendsEachArchiveOnce(t *testing.T) { workspaceFiler, err := filer.NewWorkspaceFilesClient(client, "/Workspace/Users/tester@databricks.test/.databricks/ssh-tunnel/"+tt.version) require.NoError(t, err) + archive := releaseArchive(t) getRelease := func(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { - return io.NopCloser(strings.NewReader("archive")), nil + return io.NopCloser(bytes.NewReader(archive)), nil } for range tt.uploads { err = uploadReleases(t.Context(), workspaceFiler, getRelease, tt.version, "") From 443e49763ddf6b00ff8e70ea9f580a297c099f6c Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 15:52:28 +0000 Subject: [PATCH 11/16] filer: pick the blocking node type from the error message The status code alone is ambiguous (any 400 matched). Match the two collision messages import-file returns instead; any other error skips the cleanup, as in v1.20. Co-authored-by: Isaac --- libs/filer/workspace_files_client.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 4623bdb6d30..c452bdb66fd 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "path" + "regexp" "slices" "strings" "time" @@ -235,11 +236,19 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } +// Messages import-file returns when an overwrite would change a node's type. The +// endpoint sets no error code for them. Matching is best-effort: if the wording +// changes, the original error is returned as before. +var ( + // A file holds the name of a notebook upload. + fileHoldsNotebookName = regexp.MustCompile(`Node named '.*' already exists`) + // A notebook holds the name of a file upload (SQL notebook "x" holds "x.sql"). + notebookHoldsFileName = regexp.MustCompile(`Creating file failed\. An item with path .* already exists`) +) + // deleteNodeOfOtherType deletes the node that blocks an overwrite of absPath // because its type differs from the upload's, and reports whether it did. -// import-file returns 400 when a file holds the name of a notebook upload, and -// 409 when a notebook holds the name of a file upload (SQL notebook "x" holds -// "x.sql"). CLI v1.21 and v1.22 uploaded some notebooks as files this way. +// CLI v1.21 and v1.22 uploaded some notebooks as files this way. func (w *WorkspaceFilesClient) deleteNodeOfOtherType(ctx context.Context, absPath string, uploadErr error) (bool, error) { aerr, ok := errors.AsType[*apierr.APIError](uploadErr) if !ok { @@ -248,15 +257,17 @@ func (w *WorkspaceFilesClient) deleteNodeOfOtherType(ctx context.Context, absPat var nodePath string var nodeType workspace.ObjectType - switch aerr.StatusCode { - case http.StatusBadRequest: + switch { + case fileHoldsNotebookName.MatchString(aerr.Message): nodePath, nodeType = absPath, workspace.ObjectTypeFile - case http.StatusConflict: + case notebookHoldsFileName.MatchString(aerr.Message): nodePath, nodeType = strings.TrimSuffix(absPath, path.Ext(absPath)), workspace.ObjectTypeNotebook default: return false, nil } + // Concurrent creates of the same path return the second message too, so + // only delete a node of the expected type. info, err := w.workspaceClient.Workspace.GetStatusByPath(ctx, nodePath) if errors.Is(err, apierr.ErrNotFound) { return false, nil From 8b6c6975d8a68883f0c145f85b7488a8655b5feb Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 16:03:57 +0000 Subject: [PATCH 12/16] notebook: keep the detect.go refactor from #7034 The revert restored the pre-#7034 shape of DetectWithFS, which behaves the same for files on disk. Keep the main version instead so the PR does not touch libs/notebook, and let the testserver use DetectBytes for JUPYTER imports again. Co-authored-by: Isaac --- libs/notebook/detect.go | 46 ++++++++++++++++++++++++--------- libs/notebook/detect_jupyter.go | 7 ++++- libs/notebook/detect_test.go | 27 +++++++++++++++++++ libs/testserver/handlers.go | 3 +-- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/libs/notebook/detect.go b/libs/notebook/detect.go index f6458cf7108..d9be548a5c6 100644 --- a/libs/notebook/detect.go +++ b/libs/notebook/detect.go @@ -3,7 +3,6 @@ package notebook import ( "bufio" "bytes" - "errors" "io" "io/fs" "os" @@ -45,11 +44,11 @@ func (f file) close() error { return f.f.Close() } -func (f file) readHeader() (string, error) { +func readHeader(r io.Reader) (string, error) { // Scan header line with some padding. buf := make([]byte, headerLength) - n, err := f.f.Read(buf) - if err != nil && !errors.Is(err, io.EOF) { + n, err := r.Read(buf) + if err != nil && err != io.EOF { return "", err } @@ -81,8 +80,6 @@ func (f file) getObjectInfo() (oi workspace.ObjectInfo, ok bool, err error) { // Detect returns whether the file at path is a Databricks notebook. // If it is, it returns the notebook language. func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.Language, err error) { - header := "" - f, err := openFile(fsys, name) if err != nil { return false, "", err @@ -99,12 +96,39 @@ func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.La return oi.ObjectType == workspace.ObjectTypeNotebook, oi.Language, nil } + if strings.ToLower(filepath.Ext(name)) == ExtensionJupyter { + return DetectJupyterWithFS(fsys, name) + } + // Read the first line of the file. - fileHeader, err := f.readHeader() + fileHeader, err := readHeader(f.f) + if err != nil { + return false, "", err + } + + notebook, language = detectSource(name, fileHeader) + return notebook, language, nil +} + +// DetectBytes is like [DetectWithFS] for file content that is already in memory. +func DetectBytes(name string, content []byte) (notebook bool, language workspace.Language, err error) { + if strings.ToLower(filepath.Ext(name)) == ExtensionJupyter { + return detectJupyter(name, bytes.NewReader(content)) + } + + fileHeader, err := readHeader(bytes.NewReader(content)) if err != nil { return false, "", err } + notebook, language = detectSource(name, fileHeader) + return notebook, language, nil +} + +// detectSource reports whether a source file with the given name and first line is a notebook. +func detectSource(name, fileHeader string) (notebook bool, language workspace.Language) { + header := "" + // Determine which header to expect based on filename extension. ext := strings.ToLower(filepath.Ext(name)) switch ext { @@ -120,17 +144,15 @@ func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.La case ExtensionSql: header = "-- Databricks notebook source" language = workspace.LanguageSql - case ExtensionJupyter: - return DetectJupyterWithFS(fsys, name) default: - return false, "", nil + return false, "" } if fileHeader != header { - return false, "", nil + return false, "" } - return true, language, nil + return true, language } // Detect calls DetectWithFS with the local filesystem. diff --git a/libs/notebook/detect_jupyter.go b/libs/notebook/detect_jupyter.go index f631b58127b..45233414007 100644 --- a/libs/notebook/detect_jupyter.go +++ b/libs/notebook/detect_jupyter.go @@ -3,6 +3,7 @@ package notebook import ( "encoding/json" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -66,8 +67,12 @@ func DetectJupyterWithFS(fsys fs.FS, name string) (notebook bool, language works defer f.Close() + return detectJupyter(name, f) +} + +func detectJupyter(name string, r io.Reader) (notebook bool, language workspace.Language, err error) { var nb jupyter - dec := json.NewDecoder(f) + dec := json.NewDecoder(r) err = dec.Decode(&nb) if err != nil { return false, "", fmt.Errorf("%s: error loading Jupyter notebook file: %w", name, err) diff --git a/libs/notebook/detect_test.go b/libs/notebook/detect_test.go index 49a67d2d3ed..c774612c9bf 100644 --- a/libs/notebook/detect_test.go +++ b/libs/notebook/detect_test.go @@ -127,3 +127,30 @@ func TestDetectWithObjectInfo(t *testing.T) { assert.True(t, nb) assert.Equal(t, workspace.LanguagePython, lang) } + +func TestDetectBytes(t *testing.T) { + ipynb, err := os.ReadFile("testdata/py_ipynb.ipynb") + require.NoError(t, err) + + tests := []struct { + name string + content []byte + notebook bool + language workspace.Language + }{ + {"nb.py", []byte("# Databricks notebook source\nprint(1)\n"), true, workspace.LanguagePython}, + {"nb.sql", []byte("-- Databricks notebook source\nSELECT 1\n"), true, workspace.LanguageSql}, + {"nb.ipynb", ipynb, true, workspace.LanguagePython}, + {"plain.py", []byte("print(1)\n"), false, ""}, + {"space.geniespace.json", []byte("{}"), false, ""}, + {"empty.py", nil, false, ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + nb, lang, err := DetectBytes(tc.name, tc.content) + require.NoError(t, err) + assert.Equal(t, tc.notebook, nb) + assert.Equal(t, tc.language, lang) + }) + } +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index ad3615cd323..50ff904a573 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -13,7 +13,6 @@ import ( "path" "strconv" "strings" - "testing/fstest" "github.com/databricks/cli/libs/notebook" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -84,7 +83,7 @@ func workspaceImport(w *FakeWorkspace, form ImportForm) Response { resp = w.WorkspaceImportNotebook(form.Path, form.Content, workspace.Language(form.Language), form.Overwrite) case string(workspace.ImportFormatJupyter): // The real endpoint reads the language from the notebook metadata. - _, language, _ := notebook.DetectJupyterWithFS(fstest.MapFS{"nb.ipynb": {Data: form.Content}}, "nb.ipynb") + _, language, _ := notebook.DetectBytes(form.Path+notebook.ExtensionJupyter, form.Content) resp = w.WorkspaceImportNotebook(form.Path, form.Content, language, form.Overwrite) case string(workspace.ImportFormatAuto): if strings.HasSuffix(strings.ToLower(form.Path), ".zip") { From 77c5e225a4ff1b0bd0c80dcdf207bb133212f659 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 16:15:52 +0000 Subject: [PATCH 13/16] changelog: describe the revert, its cause and impact Co-authored-by: Isaac --- .nextchanges/bundles/revert-workspace-import.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/revert-workspace-import.md b/.nextchanges/bundles/revert-workspace-import.md index 1b6377ba8cc..01784b58052 100644 --- a/.nextchanges/bundles/revert-workspace-import.md +++ b/.nextchanges/bundles/revert-workspace-import.md @@ -1 +1 @@ -* Upload bundle files through the legacy `import-file` endpoint again, because `/workspace/import` in v1.21.0 imported some notebooks as plain files; a remote file or notebook whose type differs from the upload is now replaced. ([#7218](https://github.com/databricks/cli/pull/7218)) +* Revert bundle file uploads to the legacy `/workspace-files/import-file` endpoint, rolling back [#6149](https://github.com/databricks/cli/pull/6149) and [#7034](https://github.com/databricks/cli/pull/7034) released in [v1.21.0](https://github.com/databricks/cli/releases/tag/v1.21.0). This fixes notebooks being uploaded as plain files and deployments failing (for example SQL notebooks in [#7216](https://github.com/databricks/cli/issues/7216)). A remote file or notebook whose type differs from the local one is now deleted and replaced instead of blocking the deploy. ([#7218](https://github.com/databricks/cli/pull/7218)) From 24c35e2b7a5bd289afea0ce56b62c99e0b716b50 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 22:46:39 +0000 Subject: [PATCH 14/16] filer: only delete the node the collision message names Require the name in the message to match the upload, and only delete a notebook whose language matches the upload's extension. A folder holding x.sql also returns the 409 message, and the old check then deleted an unrelated Python notebook x. Co-authored-by: Isaac --- libs/filer/workspace_files_client.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index c452bdb66fd..30ca51f03d9 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -17,6 +17,7 @@ import ( "time" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/notebook" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/client" @@ -240,12 +241,19 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // endpoint sets no error code for them. Matching is best-effort: if the wording // changes, the original error is returned as before. var ( - // A file holds the name of a notebook upload. - fileHoldsNotebookName = regexp.MustCompile(`Node named '.*' already exists`) + // A file holds the name of a notebook upload. Captures the file's name. + fileHoldsNotebookName = regexp.MustCompile(`Node named '(.+)' already exists`) // A notebook holds the name of a file upload (SQL notebook "x" holds "x.sql"). - notebookHoldsFileName = regexp.MustCompile(`Creating file failed\. An item with path .* already exists`) + // Captures the upload's path. + notebookHoldsFileName = regexp.MustCompile(`Creating file failed\. An item with path (.+) already exists`) ) +// messageNames reports whether msg matches re and names name. +func messageNames(re *regexp.Regexp, msg, name string) bool { + m := re.FindStringSubmatch(msg) + return m != nil && m[1] == name +} + // deleteNodeOfOtherType deletes the node that blocks an overwrite of absPath // because its type differs from the upload's, and reports whether it did. // CLI v1.21 and v1.22 uploaded some notebooks as files this way. @@ -258,16 +266,18 @@ func (w *WorkspaceFilesClient) deleteNodeOfOtherType(ctx context.Context, absPat var nodePath string var nodeType workspace.ObjectType switch { - case fileHoldsNotebookName.MatchString(aerr.Message): + case messageNames(fileHoldsNotebookName, aerr.Message, path.Base(absPath)): nodePath, nodeType = absPath, workspace.ObjectTypeFile - case notebookHoldsFileName.MatchString(aerr.Message): + case messageNames(notebookHoldsFileName, aerr.Message, absPath): nodePath, nodeType = strings.TrimSuffix(absPath, path.Ext(absPath)), workspace.ObjectTypeNotebook default: return false, nil } - // Concurrent creates of the same path return the second message too, so - // only delete a node of the expected type. + // The second message is also returned when a directory holds the path or + // when two writers create it at once, so only delete a node of the expected + // type. A notebook only holds the name with its language's extension: SQL + // notebook "x" holds "x.sql", not "x.py". info, err := w.workspaceClient.Workspace.GetStatusByPath(ctx, nodePath) if errors.Is(err, apierr.ErrNotFound) { return false, nil @@ -278,6 +288,9 @@ func (w *WorkspaceFilesClient) deleteNodeOfOtherType(ctx context.Context, absPat if info.ObjectType != nodeType { return false, nil } + if nodeType == workspace.ObjectTypeNotebook && info.Language != notebook.ExtensionToLanguage[strings.ToLower(path.Ext(absPath))] { + return false, nil + } err = w.workspaceClient.Workspace.Delete(ctx, workspace.Delete{Path: nodePath}) if err != nil { From aa0b9d6e18aee60a25c8be3a82b56dbb42426d8f Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 28 Oct 2026 23:22:15 +0000 Subject: [PATCH 15/16] acceptance: use ** for import-file fault patterns Since #7241, * in a fault pattern matches within one path segment and ** across segments. Upload paths span many segments, so the faults stopped firing. Co-authored-by: Isaac --- acceptance/bundle/deploy/files/upload-504/script | 2 +- acceptance/bundle/destroy/force-lock-node-limit/script | 4 ++-- acceptance/bundle/sync/upload-504/script | 2 +- acceptance/cmd/sync/upload-504/script | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/deploy/files/upload-504/script b/acceptance/bundle/deploy/files/upload-504/script index 4e48044acc5..a6bfa10a541 100644 --- a/acceptance/bundle/deploy/files/upload-504/script +++ b/acceptance/bundle/deploy/files/upload-504/script @@ -1,6 +1,6 @@ # Let the two deploy lock requests through, then inject a single 504 on the next import request, which is a file upload. # Uploads overwrite, so they are retried and the deploy succeeds. -fault.py "POST /api/2.0/workspace-files/import-file/*" 504 2 1 +fault.py "POST /api/2.0/workspace-files/import-file/**" 504 2 1 $CLI bundle deploy diff --git a/acceptance/bundle/destroy/force-lock-node-limit/script b/acceptance/bundle/destroy/force-lock-node-limit/script index c7fffbc16a2..ccb6ae3d143 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/script +++ b/acceptance/bundle/destroy/force-lock-node-limit/script @@ -7,9 +7,9 @@ title "Deploy (creates the job and the deployment)" trace $CLI bundle deploy title "destroy without --force-lock (lock write hits the node limit: aborts)" -fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py "POST /api/2.0/workspace-files/import-file/**" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace errcode $CLI bundle destroy --auto-approve title "destroy with --force-lock (tolerates the node limit: proceeds and completes)" -fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py "POST /api/2.0/workspace-files/import-file/**" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace $CLI bundle destroy --force-lock --auto-approve diff --git a/acceptance/bundle/sync/upload-504/script b/acceptance/bundle/sync/upload-504/script index c77a44296cd..f3ca9aff682 100644 --- a/acceptance/bundle/sync/upload-504/script +++ b/acceptance/bundle/sync/upload-504/script @@ -1,5 +1,5 @@ # Inject a single 504 on the first file upload. Uploads overwrite, so they are retried and the sync succeeds. -fault.py "POST /api/2.0/workspace-files/import-file/*" 504 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/**" 504 0 1 $CLI bundle sync diff --git a/acceptance/cmd/sync/upload-504/script b/acceptance/cmd/sync/upload-504/script index 6e1ea8b7479..b7000ed9385 100644 --- a/acceptance/cmd/sync/upload-504/script +++ b/acceptance/cmd/sync/upload-504/script @@ -15,7 +15,7 @@ cleanup() { trap cleanup EXIT # Inject a single 504 on the first file upload. Uploads overwrite, so they are retried and the sync succeeds. -fault.py "POST /api/2.0/workspace-files/import-file/*" 504 0 1 +fault.py "POST /api/2.0/workspace-files/import-file/**" 504 0 1 # MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /Users/... to C:/Program Files/Git/Users/... MSYS_NO_PATHCONV=1 $CLI sync . "$remote_dir" From 640e02f4c9475850ab9bb1d5b607adf0cdcbac11 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 29 Oct 2026 07:33:49 +0000 Subject: [PATCH 16/16] Retrigger CI Co-authored-by: Isaac