From 82b0c54698ad7cd548259f2a0c82e62507c3305f Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 3 Aug 2026 14:02:18 +0000 Subject: [PATCH 01/37] initial filesystem changes --- acceptance/internal/prepare_server.go | 58 +++++ libs/filer/workspace_files_client.go | 117 +++++++--- libs/filer/workspace_files_client_test.go | 249 ++++++++++++++++++++++ libs/testserver/fake_workspace.go | 37 +++- libs/testserver/handlers.go | 75 +++++++ 5 files changed, 497 insertions(+), 39 deletions(-) diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 422044effd6..d2e34cf6cdd 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -1,8 +1,12 @@ package internal import ( + "bytes" "encoding/json" "fmt" + "io" + "mime" + "mime/multipart" "net/http" "os" "path/filepath" @@ -326,6 +330,11 @@ func getLoggedRequest(req *testserver.Request, includedHeaders []string) LoggedR if json.Valid(req.Body) { result.Body = json.RawMessage(req.Body) + } else if normalized, ok := normalizeMultipartBody(req); ok { + // Multipart bodies contain a randomly generated boundary string and binary + // content; record a normalized form (sorted form-field names with sizes for + // file parts) so recorded requests stay deterministic and reviewable. + result.Body = normalized } else { result.RawBody = string(req.Body) } @@ -333,6 +342,55 @@ func getLoggedRequest(req *testserver.Request, includedHeaders []string) LoggedR return result } +// normalizeMultipartBody returns a deterministic representation of a multipart +// form body if the request's Content-Type is multipart/*. The second return +// value is false if the body is not multipart or cannot be parsed. +// +// Text parts are recorded as their literal content so reviewers can read what +// was uploaded; existing global UNIX_TIME / UUID / USERNAME replacements +// normalize the timestamp / id / email fields embedded in deploy.lock and +// similar payloads. Non-UTF-8 (binary) parts and parts whose serialized +// content would bloat the recording past multipartContentLimit are +// summarized as "[binary content N bytes]". +func normalizeMultipartBody(req *testserver.Request) (any, bool) { + contentType := req.Headers.Get("Content-Type") + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil || !strings.HasPrefix(mediaType, "multipart/") { + return nil, false + } + boundary := params["boundary"] + if boundary == "" { + return nil, false + } + mr := multipart.NewReader(bytes.NewReader(req.Body), boundary) + parts := map[string]any{} + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false + } + name := part.FormName() + if !utf8.Valid(data) || len(data) > multipartContentLimit { + parts[name] = fmt.Sprintf("[binary content %d bytes]", len(data)) + continue + } + parts[name] = string(data) + } + return map[string]any{"multipart_form": parts}, true +} + +// Multipart parts larger than this limit are summarized as a size placeholder +// to keep recorded fixtures small. Reviewers care about *what* was uploaded +// (path, format, overwrite flag), not the bytes themselves, for large blobs. +const multipartContentLimit = 4096 + func filterHeaders(h http.Header, includedHeaders []string) http.Header { headers := make(http.Header) for k, v := range h { diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index e1b62481243..5b903ab3678 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -9,7 +9,6 @@ import ( "io" "io/fs" "net/http" - "net/url" "path" "slices" "strings" @@ -23,6 +22,11 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) +// workspaceObjectTypeMismatchReason is the AIP-193 ErrorInfo reason attached +// by /workspace/import when overwrite=true targets a path whose existing +// object's node type differs from the upload (FILE vs NOTEBOOK). +const workspaceObjectTypeMismatchReason = "WORKSPACE_OBJECT_TYPE_MISMATCH" + // Type that implements fs.DirEntry for WSFS. type wsfsDirEntry struct { wsfsFileInfo @@ -154,35 +158,42 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - // Remove leading "/" so we can use it in the URL. - overwrite := slices.Contains(mode, OverwriteIfExists) - urlPath := fmt.Sprintf( - "/api/2.0/workspace-files/import-file/%s?overwrite=%t", - url.PathEscape(strings.TrimLeft(absPath, "/")), - overwrite, - ) - // Buffer the file contents because we may need to retry below and we cannot read twice. body, err := io.ReadAll(reader) if err != nil { return err } - err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) + // Use Workspace.Upload (multipart /api/2.0/workspace/import) instead of the + // JSON-body variant of the same endpoint, which caps payloads at 10 MiB for + // AUTO format (databricks.webapp.autoExportFormatLimitBytes). The multipart + // variant has been verified against a real workspace at 450 MB for regular + // files — strictly better than the previous /workspace-files/import-file + // endpoint, which has a 200 MiB body cap + // (databricks.workspaceFilesystem.maxImportSizeBytes) plus a 305s server-side + // request timeout that cuts off uploads above ~400 MB at typical bandwidth. + // + // Notebook content (any payload with a `# Databricks notebook source` header + // detected by format=AUTO) hits a separate 10 MiB cap on the server + // (databricks.notebook.maxNotebookSizeBytes). Both /workspace-files/import-file + // and /workspace/import enforce this same cap, so switching from the former + // to the latter does not regress maximum notebook upload size. + overwrite := slices.Contains(mode, OverwriteIfExists) + uploadOpts := []func(*workspace.Import){ + workspace.UploadFormat(workspace.ImportFormatAuto), + } + if overwrite { + uploadOpts = append(uploadOpts, workspace.UploadOverwrite()) + } + err = w.workspaceClient.Workspace.Upload(ctx, absPath, bytes.NewReader(body), uploadOpts...) // Return early on success. if err == nil { return nil } - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return err - } - - // This API returns a 404 if the parent directory does not exist. - if aerr.StatusCode == http.StatusNotFound { + // Parent directory does not exist. + if errors.Is(err, apierr.ErrNotFound) { if !slices.Contains(mode, CreateParentDirectories) { return noSuchDirectoryError{path.Dir(absPath)} } @@ -190,8 +201,8 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // Create parent directory. err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath)) if err != nil { - if mkdirErr, ok := errors.AsType[*apierr.APIError](err); ok && mkdirErr.StatusCode == http.StatusForbidden { - return permissionError{absPath, mkdirErr} + if errors.Is(err, apierr.ErrPermissionDenied) { + return permissionError{absPath, err} } return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err) } @@ -200,24 +211,62 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return w.Write(ctx, name, bytes.NewReader(body), sliceWithout(mode, CreateParentDirectories)...) } - // This API returns 409 if the file already exists, when the object type is file - if aerr.StatusCode == http.StatusConflict { + // Path already taken. /workspace/import returns this in three shapes, + // verified against a real workspace: + // + // - 400 RESOURCE_ALREADY_EXISTS — sequential conflict, no overwrite flag. + // Example: "/Users/me/foo.txt already exists. Please pass overwrite=true + // to overwrite it." + // + // - 409 ALREADY_EXISTS — concurrent contention (observed in TestLock when + // five lockers race to write deploy.lock). + // Example: "Node with name /Users/me/.bundle/.../deploy.lock already + // exists. Please pass overwrite=true to update it." + // + // - 400 INVALID_PARAMETER_VALUE — overwrite=true on a path where the + // existing object's node type differs from the upload. Two distinct + // messages, both observed against aws-prod-ucws: + // + // (a) "Cannot overwrite the asset at /Users/me/foo due to type mismatch + // (asked: FILE, actual: NOTEBOOK)" — fires when the upload path is + // the same as an existing NOTEBOOK and the new content has no + // notebook header (so AUTO would store it as FILE), or the mirror + // case with FILE/NOTEBOOK swapped. + // + // (b) "Requested node type [FILE] is different from the existing node + // type [NOTEBOOK]" — fires when /foo is already a NOTEBOOK (from a + // prior /foo.py upload) and an overwrite-upload of regular content + // targets /foo.py: AUTO would store the new content as FILE at + // /foo.py, but the workspace treats /foo.py as the source view of + // the existing /foo NOTEBOOK and rejects the type change. + // + // The server refuses the overwrite even though the caller asked for + // it; from the caller's perspective the path is occupied, so we + // surface this as already-exists. + if errors.Is(err, apierr.ErrResourceAlreadyExists) || errors.Is(err, apierr.ErrAlreadyExists) { return fileAlreadyExistsError{absPath} } - - // This API returns 400 if the file already exists when the object type is notebook. - // Both the historical "Path () already exists." format and the newer - // "RESOURCE_ALREADY_EXISTS: already exists. ..." format end with the same - // "already exists." marker; the JSON error_code is empty in both. The new format - // might not have been rolled out to all workspaces yet, so we anchor on the shared - // marker and return absPath rather than parsing the message. - if aerr.StatusCode == http.StatusBadRequest && strings.Contains(aerr.Message, "already exists.") { - return fileAlreadyExistsError{absPath} + if errors.Is(err, apierr.ErrInvalidParameterValue) { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + // WCS attaches AIP-193 ErrorInfo with a stable reason to import + // path collisions (universe PR #2019174, WP-6031), so prefer + // branching on it over parsing the message. + if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == workspaceObjectTypeMismatchReason { + return fileAlreadyExistsError{absPath} + } + // Fallback for workspaces where the ErrorInfo change has not + // rolled out: as of 2026-06-12 aws-prod-ucws still returns these + // errors without details, so match the two observed messages. + // Remove once the rollout is confirmed everywhere. + if strings.Contains(aerr.Message, "type mismatch") || strings.Contains(aerr.Message, "node type") { + return fileAlreadyExistsError{absPath} + } + } } - // This API returns StatusForbidden when you have read access but don't have write access to a file - if aerr.StatusCode == http.StatusForbidden { - return permissionError{absPath, aerr} + // Caller has read access but no write access. + if errors.Is(err, apierr.ErrPermissionDenied) { + return permissionError{absPath, err} } return err diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 2915af60436..a607b116263 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -1,8 +1,13 @@ package filer import ( + "bytes" + "context" "encoding/json" + "io" "io/fs" + "net/http" + "strings" "testing" "time" @@ -10,8 +15,10 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -97,6 +104,248 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } +func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { + tests := []struct { + name string + modes []WriteMode + expectOverride bool + }{ + { + name: "no overwrite", + modes: nil, + expectOverride: false, + }, + { + name: "overwrite", + modes: []WriteMode{OverwriteIfExists}, + expectOverride: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mw := mocks.NewMockWorkspaceClient(t) + workspaceApi := mw.GetMockWorkspaceAPI() + + workspaceApi.EXPECT().Upload( + mock.Anything, + "/dir/file.txt", + mock.Anything, + mock.Anything, + mock.Anything, + ).RunAndReturn(func(_ context.Context, _ string, r io.Reader, opts ...func(*workspace.Import)) error { + body, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, "hello", string(body)) + + i := &workspace.Import{} + for _, opt := range opts { + opt(i) + } + assert.Equal(t, workspace.ImportFormatAuto, i.Format) + assert.Equal(t, tc.expectOverride, i.Overwrite) + return nil + }).Once() + + c := WorkspaceFilesClient{ + workspaceClient: mw.WorkspaceClient, + root: NewWorkspaceRootPath("/dir"), + } + err := c.Write(t.Context(), "file.txt", strings.NewReader("hello"), tc.modes...) + require.NoError(t, err) + }) + } +} + +func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { + tests := []struct { + name string + mode []WriteMode + apiErr *apierr.APIError + expectErrTarget any + }{ + { + name: "404 without create-parent maps to noSuchDirectoryError", + apiErr: &apierr.APIError{StatusCode: http.StatusNotFound, Message: "not found"}, + expectErrTarget: noSuchDirectoryError{}, + }, + { + name: "400 RESOURCE_ALREADY_EXISTS maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "RESOURCE_ALREADY_EXISTS", + Message: "/dir/file.txt already exists. Please pass overwrite=true to overwrite it.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "409 ALREADY_EXISTS (concurrent contention) maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusConflict, + ErrorCode: "ALREADY_EXISTS", + Message: "Node with name /dir/file.txt already exists. Please pass overwrite=true to update it.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "400 INVALID_PARAMETER_VALUE 'type mismatch' (overwrite=true) maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "Cannot overwrite the asset at /dir/foo due to type mismatch (asked: FILE, actual: NOTEBOOK).", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "400 INVALID_PARAMETER_VALUE 'Requested node type' (overwrite=true) maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "Requested node type [FILE] is different from the existing node type [NOTEBOOK]", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "400 INVALID_PARAMETER_VALUE other message passes through", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "some other validation failure", + }, + expectErrTarget: nil, + }, + { + name: "403 maps to permissionError", + apiErr: &apierr.APIError{StatusCode: http.StatusForbidden, Message: "denied"}, + expectErrTarget: permissionError{}, + }, + { + name: "500 passes through", + apiErr: &apierr.APIError{StatusCode: http.StatusInternalServerError, Message: "boom"}, + expectErrTarget: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mw := mocks.NewMockWorkspaceClient(t) + workspaceApi := mw.GetMockWorkspaceAPI() + workspaceApi.EXPECT().Upload( + mock.Anything, "/dir/file.txt", mock.Anything, mock.Anything, mock.Anything, + ).Return(tc.apiErr).Once() + + c := WorkspaceFilesClient{ + workspaceClient: mw.WorkspaceClient, + root: NewWorkspaceRootPath("/dir"), + } + err := c.Write(t.Context(), "file.txt", bytes.NewReader([]byte("data")), tc.mode...) + require.Error(t, err) + switch target := tc.expectErrTarget.(type) { + case noSuchDirectoryError: + assert.ErrorAs(t, err, &target) + case fileAlreadyExistsError: + assert.ErrorAs(t, err, &target) + case permissionError: + assert.ErrorAs(t, err, &target) + case nil: + // passthrough — same APIError pointer + var aerr *apierr.APIError + require.ErrorAs(t, err, &aerr) + assert.Equal(t, tc.apiErr.StatusCode, aerr.StatusCode) + } + }) + } +} + +// writeWithImportError exercises Write through a real HTTP roundtrip so the +// SDK parses AIP-193 error details from the response body (the errorDetails +// field on apierr.APIError is unexported and only populated during response +// parsing, so it cannot be set on a directly constructed APIError). +func writeWithImportError(t *testing.T, body map[string]any) error { + t.Helper() + + server := testserver.New(t) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + return testserver.Response{ + StatusCode: http.StatusBadRequest, + Body: body, + } + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewWorkspaceFilesClient(client, "/dir") + require.NoError(t, err) + + err = f.Write(t.Context(), "file.txt", strings.NewReader("data"), OverwriteIfExists) + require.Error(t, err) + return err +} + +func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { + // The message is deliberately one the fallback string match does not + // recognize, to prove the branch fires on the structured reason alone. + err := writeWithImportError(t, map[string]any{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "some future wording for the same condition", + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": workspaceObjectTypeMismatchReason, + "domain": "workspace.databricks.com", + "metadata": map[string]string{"existing_type": "NOTEBOOK"}, + }, + }, + }) + var target fileAlreadyExistsError + assert.ErrorAs(t, err, &target) +} + +func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { + err := writeWithImportError(t, map[string]any{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "some other validation failure", + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "SOME_OTHER_REASON", + "domain": "workspace.databricks.com", + }, + }, + }) + var aerr *apierr.APIError + require.ErrorAs(t, err, &aerr) + assert.Equal(t, http.StatusBadRequest, aerr.StatusCode) +} + +func TestWorkspaceFilesClientWriteCreatesParentDirectories(t *testing.T) { + mw := mocks.NewMockWorkspaceClient(t) + workspaceApi := mw.GetMockWorkspaceAPI() + + // First Upload returns 404, second returns success after MkdirsByPath. + workspaceApi.EXPECT().Upload( + mock.Anything, "/dir/sub/file.txt", mock.Anything, mock.Anything, mock.Anything, + ).Return(&apierr.APIError{StatusCode: http.StatusNotFound, Message: "not found"}).Once() + + workspaceApi.EXPECT().MkdirsByPath(mock.Anything, "/dir/sub").Return(nil).Once() + + workspaceApi.EXPECT().Upload( + mock.Anything, "/dir/sub/file.txt", mock.Anything, mock.Anything, mock.Anything, + ).Return(nil).Once() + + c := WorkspaceFilesClient{ + workspaceClient: mw.WorkspaceClient, + root: NewWorkspaceRootPath("/dir"), + } + err := c.Write(t.Context(), "sub/file.txt", strings.NewReader("data"), CreateParentDirectories) + require.NoError(t, err) +} + func TestWorkspaceFilesClient_wsfsUnmarshal(t *testing.T) { payload := ` { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 2f077b1b631..ad457115c37 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -836,6 +836,32 @@ func (s *FakeWorkspace) hasChildren(dirPath string) bool { return false } +// detectNotebookLanguage mirrors the real /workspace/import format=AUTO logic: +// a file whose extension is .py / .sql / .scala / .r and whose content starts +// with the language-appropriate "Databricks notebook source" line-comment is +// stored as a NOTEBOOK with the corresponding language; otherwise it's a FILE. +func detectNotebookLanguage(extension string, body []byte) (workspace.Language, bool) { + switch extension { + case ".py": + if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { + return workspace.LanguagePython, true + } + case ".sql": + if bytes.HasPrefix(body, []byte("-- Databricks notebook source")) { + return workspace.LanguageSql, true + } + case ".scala": + if bytes.HasPrefix(body, []byte("// Databricks notebook source")) { + return workspace.LanguageScala, true + } + case ".r": + if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { + return workspace.LanguageR, true + } + } + return "", false +} + func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, overwrite bool) Response { if !strings.HasPrefix(filePath, "/") { filePath = "/" + filePath @@ -858,18 +884,19 @@ func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, o } } - // Note: Files with .py, .scala, .r or .sql extension can - // be notebooks if they contain a magical "Databricks notebook source" - // header comment. We omit support non-python extensions for now for simplicity. + // Files with .py / .sql / .scala / .r extension can be notebooks if they + // carry the "Databricks notebook source" header comment in the language's + // line-comment syntax. Mirror the real workspace's auto-detection here so + // acceptance tests can assert the resulting object_type via /workspace/list. extension := filepath.Ext(filePath) - if extension == ".py" && strings.HasPrefix(string(body), "# Databricks notebook source") { + if lang, isNotebook := detectNotebookLanguage(extension, body); isNotebook { // Notebooks are stripped of their extension by the workspace import API. workspacePath = strings.TrimSuffix(filePath, extension) s.files[workspacePath] = FileEntry{ Info: workspace.ObjectInfo{ ObjectType: "NOTEBOOK", Path: workspacePath, - Language: "PYTHON", + Language: lang, ObjectId: nextID(), }, Data: body, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8c44d97eae7..9d77f350b10 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -149,6 +149,81 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("POST", "/api/2.0/workspace/import", func(req Request) any { + // /workspace/import accepts both a JSON body (matching workspace.Import) and a + // multipart form body. The multipart variant is what databricks-sdk-go's + // Workspace.Upload uses; the JSON variant is kept for back-compat with anything + // that still hits Workspace.Import directly. + contentType := req.Headers.Get("Content-Type") + mediaType, params, _ := mime.ParseMediaType(contentType) + if strings.HasPrefix(mediaType, "multipart/") { + mr := multipart.NewReader(bytes.NewReader(req.Body), params["boundary"]) + var ( + filePath string + content []byte + format string + overwrite bool + ) + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + data, err := io.ReadAll(part) + if err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + switch part.FormName() { + case "path": + filePath = string(data) + case "content": + content = data + case "format": + format = string(data) + case "overwrite": + overwrite = string(data) == "true" + } + } + + if format != "" && format != string(workspace.ImportFormatAuto) { + return Response{ + Body: "internal error: The test server only supports auto format.", + StatusCode: http.StatusInternalServerError, + } + } + + // Translate any 409 from the shared fake into the 400 + errorCode + // RESOURCE_ALREADY_EXISTS shape returned by the real /workspace/import + // endpoint, including the AIP-193 ErrorInfo detail it attaches to path + // collisions (universe PR #2019174, WP-6031). + resp := req.Workspace.WorkspaceFilesImportFile(filePath, content, overwrite) + if resp.StatusCode == http.StatusConflict { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]any{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Path (%s) already exists.", filePath), + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "RESOURCE_ALREADY_EXISTS", + "domain": "workspace.databricks.com", + }, + }, + }, + } + } + return resp + } + var request workspace.Import err := json.Unmarshal(req.Body, &request) if err != nil { From ed23668ae6362bd74caaba76b14c11a19bed72ab Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 3 Aug 2026 15:25:34 +0000 Subject: [PATCH 02/37] filer: switch workspace upload from import-file to /workspace/import Replace POST /api/2.0/workspace-files/import-file/{path} with the multipart variant of POST /api/2.0/workspace/import (via the SDK's Workspace.Upload + format=AUTO). The previous endpoint is deprecated; the new one has a higher rate limit (30 vs 20 rps/workspace) and is ~1.5-2x faster for typical bundle deployments. Error handling is rewritten to branch on SDK sentinels (errors.Is against ErrNotFound / ErrResourceAlreadyExists / ErrAlreadyExists / ErrInvalidParameterValue / ErrPermissionDenied) and the AIP-193 ErrorInfo reason WORKSPACE_OBJECT_TYPE_MISMATCH, with a message-substring fallback for workspaces where WP-6031 has not rolled out. The testserver now decodes the multipart /workspace/import body and mirrors the real endpoint's format=AUTO notebook detection. A new bodyContains option on the fault mechanism lets a test target a single file's upload, which is no longer possible by URL since every upload shares the /workspace/import path. Co-authored-by: Isaac --- .../bundles/workspace-import-migration.md | 1 + acceptance/bin/fault.py | 35 ++-- .../bundle/apps/app_yaml/out.app.yml.txt | 14 +- acceptance/bundle/apps/app_yaml/output.txt | 2 +- acceptance/bundle/apps/app_yaml/script | 2 +- .../ai_runtime_code_source/output.txt | 4 +- .../artifacts/ai_runtime_code_source/script | 2 +- .../artifact_upload_for_workspace/output.txt | 6 +- .../artifact_upload_for_workspace/script | 2 +- .../output.txt | 6 +- .../script | 2 +- .../upload_multiple_libraries/output.txt | 18 +-- .../upload_multiple_libraries/script | 2 +- .../artifacts/whl_change_version/output.txt | 95 +++++------ .../artifacts/whl_change_version/script | 6 +- .../bundle/artifacts/whl_dbfs/output.txt | 2 +- acceptance/bundle/artifacts/whl_dbfs/script | 2 +- .../bundle/artifacts/whl_dynamic/output.txt | 18 +-- .../bundle/artifacts/whl_dynamic/script | 4 +- .../bundle/artifacts/whl_explicit/output.txt | 6 +- .../bundle/artifacts/whl_explicit/script | 2 +- .../bundle/artifacts/whl_implicit/output.txt | 6 +- .../bundle/artifacts/whl_implicit/script | 2 +- .../whl_implicit_custom_path/output.txt | 6 +- .../artifacts/whl_implicit_custom_path/script | 2 +- .../whl_implicit_notebook/output.txt | 6 +- .../artifacts/whl_implicit_notebook/script | 2 +- .../bundle/artifacts/whl_multiple/output.txt | 10 +- .../bundle/artifacts/whl_multiple/script | 2 +- .../artifacts/whl_no_cleanup/output.txt | 6 +- .../bundle/artifacts/whl_no_cleanup/script | 2 +- .../whl_prebuilt_multiple/output.txt | 10 +- .../artifacts/whl_prebuilt_multiple/script | 2 +- .../artifacts/whl_prebuilt_outside/output.txt | 6 +- .../artifacts/whl_prebuilt_outside/script | 2 +- .../whl_prebuilt_outside_dynamic/output.txt | 6 +- .../whl_prebuilt_outside_dynamic/script | 2 +- .../whl_via_environment_key/output.txt | 6 +- .../artifacts/whl_via_environment_key/script | 2 +- .../bind/pipelines/recreate/output.txt | 4 +- .../deployment/bind/pipelines/recreate/script | 6 +- .../bind/pipelines/update/output.txt | 2 +- .../deployment/bind/pipelines/update/script | 6 +- .../destroy/force-lock-node-limit/script | 4 +- .../outside_of_bundle_root/output.txt | 10 +- .../libraries/outside_of_bundle_root/script | 2 +- .../auto-migrate-empty-tfstate/output.txt | 36 +++-- .../migrate/auto-migrate-empty-tfstate/script | 2 +- .../migrate/auto-migrate-push-failure/script | 17 +- .../resource_deps/remote_app_url/output.txt | 6 +- .../resource_deps/remote_app_url/script | 6 +- .../change_assets_dir/output.txt | 2 +- .../quality_monitors/change_assets_dir/script | 2 +- .../change_output_schema_name/output.txt | 2 +- .../change_output_schema_name/script | 2 +- .../change_table_name/output.txt | 2 +- .../quality_monitors/change_table_name/script | 2 +- .../quality_monitors/create/output.txt | 2 +- .../resources/quality_monitors/create/script | 2 +- .../delete_scope/out.deploy.requests.txt | 2 +- .../secret_scopes/delete_scope/script | 2 +- .../sync-upload-edge-cases/databricks.yml | 2 + .../sync-upload-edge-cases/out.test.toml | 3 + .../bundle/sync-upload-edge-cases/output.txt | 140 ++++++++++++++++ .../bundle/sync-upload-edge-cases/script | 95 +++++++++++ .../bundle/sync-upload-edge-cases/test.toml | 18 +++ .../templates/default-python/classic/script | 4 +- .../default-python/serverless/script | 4 +- .../upload/internal_server_error/output.txt | 4 +- .../upload/internal_server_error/test.toml | 2 +- acceptance/bundle/upload/timeout/output.txt | 4 +- acceptance/bundle/upload/timeout/test.toml | 2 +- acceptance/bundle/user_agent/output.txt | 46 +++--- .../simple/out.requests.deploy.direct.json | 122 ++++++-------- .../simple/out.requests.deploy.terraform.json | 150 ++++++------------ .../simple/out.requests.destroy.direct.json | 36 ++--- .../out.requests.destroy.terraform.json | 36 ++--- .../validate/sync_patterns/out.sync.txt | 11 +- .../bundle/validate/sync_patterns/script | 2 +- .../cmd/sync/dryrun-missing-remote/output.txt | 4 +- libs/testserver/fault.go | 31 +++- libs/testserver/fault_test.go | 48 +++--- libs/testserver/server.go | 2 +- 83 files changed, 701 insertions(+), 497 deletions(-) create mode 100644 .nextchanges/bundles/workspace-import-migration.md create mode 100644 acceptance/bundle/sync-upload-edge-cases/databricks.yml create mode 100644 acceptance/bundle/sync-upload-edge-cases/out.test.toml create mode 100644 acceptance/bundle/sync-upload-edge-cases/output.txt create mode 100644 acceptance/bundle/sync-upload-edge-cases/script create mode 100644 acceptance/bundle/sync-upload-edge-cases/test.toml diff --git a/.nextchanges/bundles/workspace-import-migration.md b/.nextchanges/bundles/workspace-import-migration.md new file mode 100644 index 00000000000..351faafb7cd --- /dev/null +++ b/.nextchanges/bundles/workspace-import-migration.md @@ -0,0 +1 @@ +Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a higher rate limit (30 vs 20 requests/sec per workspace) and is ~1.5–2× faster for typical bundle deployments. diff --git a/acceptance/bin/fault.py b/acceptance/bin/fault.py index 0aed44d58d1..261c5570862 100755 --- a/acceptance/bin/fault.py +++ b/acceptance/bin/fault.py @@ -1,16 +1,22 @@ #!/usr/bin/env python3 """Set up a fault rule on the testserver for the current test token. -Usage: fault.py [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] +Usage: fault.py [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] - --after-handler run the handler first; keep its effect and replace only the response - PATTERN HTTP method and path, supports trailing * wildcard, - e.g. "PUT /api/2.0/permissions/pipelines/*" - STATUS_CODE HTTP status code to return, e.g. 504 - OFFSET number of requests to let through before fault starts - TIMES number of times to return the fault response - ERROR_CODE optional error_code for the response body, e.g. - MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED) + --after-handler + run the handler first; keep its effect and replace only the response + PATTERN HTTP method and path, supports trailing * wildcard, + e.g. "PUT /api/2.0/permissions/pipelines/*" + STATUS_CODE HTTP status code to return, e.g. 504 + OFFSET number of requests to let through before fault starts + TIMES number of times to return the fault response + ERROR_CODE optional error_code for the response body, e.g. + MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED) + --body-contains SUBSTR + only fire when the request body contains SUBSTR. Needed to + target a single file's /workspace/import upload, since every + upload shares the same method+path and only differs by the + multipart "path" form field. The rule is scoped to the current DATABRICKS_TOKEN so it only affects the test that registers it, even when tests share a server. @@ -33,8 +39,16 @@ if after_handler: args.remove("--after-handler") +body_contains = "" +if len(args) >= 2 and args[0] == "--body-contains": + body_contains = args[1] + args = args[2:] + if len(args) not in (4, 5): - print(f"usage: {sys.argv[0]} [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr) + print( + f"usage: {sys.argv[0]} [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", + file=sys.stderr, + ) sys.exit(1) pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3]) @@ -49,6 +63,7 @@ "offset": offset, "times": times, "after_handler": after_handler, + "body_contains": body_contains, } ).encode() diff --git a/acceptance/bundle/apps/app_yaml/out.app.yml.txt b/acceptance/bundle/apps/app_yaml/out.app.yml.txt index 29cb7da70fa..e79c7a4f57c 100644 --- a/acceptance/bundle/apps/app_yaml/out.app.yml.txt +++ b/acceptance/bundle/apps/app_yaml/out.app.yml.txt @@ -1,8 +1,12 @@ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml", - "q": { - "overwrite": "true" - }, - "raw_body": "command:\n - python\n - app.py\n" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "command:\n - python\n - app.py\n", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml" + } + } } diff --git a/acceptance/bundle/apps/app_yaml/output.txt b/acceptance/bundle/apps/app_yaml/output.txt index 53429bbbb4c..37358125f39 100644 --- a/acceptance/bundle/apps/app_yaml/output.txt +++ b/acceptance/bundle/apps/app_yaml/output.txt @@ -19,7 +19,7 @@ Created apps.myapp Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> jq select(.path | test("app.yml")) out.requests.txt +>>> jq select(.body.multipart_form.path | strings | test("app.yml")) out.requests.txt >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/apps/app_yaml/script b/acceptance/bundle/apps/app_yaml/script index d0dd8d5f800..adec87dc210 100644 --- a/acceptance/bundle/apps/app_yaml/script +++ b/acceptance/bundle/apps/app_yaml/script @@ -1,7 +1,7 @@ trace $CLI bundle validate trace $CLI bundle plan trace $CLI bundle deploy -trace jq 'select(.path | test("app.yml"))' out.requests.txt | sed 's/\\r//g' > out.app.yml.txt +trace jq 'select(.body.multipart_form.path | strings | test("app.yml"))' out.requests.txt | sed 's/\\r//g' > out.app.yml.txt #trace print_requests.py //apps # currently fails due to TF inserting description="" rm out.requests.txt diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt index d384cc5190d..5fa719f5d94 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt @@ -24,5 +24,5 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Expecting the code_source tarball to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz" +>>> jq -r .body.multipart_form.path | strings +/Workspace/foo/bar/artifacts/.internal/code.tgz diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/script b/acceptance/bundle/artifacts/ai_runtime_code_source/script index 2b71cf524aa..8c7dcf45181 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/script +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/script @@ -4,6 +4,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' out.requests.txt title "Expecting the code_source tarball to be uploaded" -trace jq .path < out.requests.txt | grep import | grep code.tgz | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep code.tgz | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt index 452a913bffa..8c410005cac 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt @@ -50,9 +50,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl +/Workspace/foo/bar/artifacts/.internal/source.whl === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script index e6b1437627c..cd28cc070f5 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script @@ -8,7 +8,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt index 1072fffd3bd..98f290c0faf 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt @@ -6,9 +6,9 @@ Files: 6 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Expecting wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl +/Workspace/foo/bar/artifacts/.internal/source.whl === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script index 883601185c9..b5ee485cbd0 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script @@ -5,7 +5,7 @@ echo "test wheel content" > whl/source.whl trace $CLI bundle deploy title "Expecting wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt index 90c4730c1ab..c6b941096d4 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt @@ -40,15 +40,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 4 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source1.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source2.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source3.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl +/Workspace/foo/bar/artifacts/.internal/source1.whl +/Workspace/foo/bar/artifacts/.internal/source2.whl +/Workspace/foo/bar/artifacts/.internal/source3.whl +/Workspace/foo/bar/artifacts/.internal/source4.whl === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/script b/acceptance/bundle/artifacts/upload_multiple_libraries/script index f615147f8c4..7625d8947f1 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/script +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/script @@ -11,7 +11,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 4 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 5c76f9ae00e..d7d1271b7d5 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -3,9 +3,9 @@ Building python_artifact... Uploading dist/my_test_code-0.1.0-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.test_job -Files: 10 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Deploying resources... +Updating deployment state... +Deployment complete! >>> find.py --expect 1 whl dist/my_test_code-0.1.0-py3-none-any.whl @@ -29,23 +29,24 @@ dist/my_test_code-0.1.0-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/test.toml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> jq -r .body.multipart_form.path | strings out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/test.toml +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME >>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0 @@ -53,9 +54,9 @@ dist/my_test_code-0.1.0-py3-none-any.whl Building python_artifact... Uploading dist/my_test_code-0.2.0-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Updated jobs.test_job -Files: 4 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +Deploying resources... +Updating deployment state... +Deployment complete! >>> find.py --expect 2 whl dist/my_test_code-0.1.0-py3-none-any.whl @@ -66,16 +67,16 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> jq .path out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> jq -r .body.multipart_form.path | strings out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME === Restore config to target old wheel >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl @@ -85,9 +86,9 @@ Building python_artifact... Uploading dist/my_test_code-0.1.0-py3-none-any.whl... Uploading dist/my_test_code-0.2.0-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Updated jobs.test_job -Files: 4 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +Deploying resources... +Updating deployment state... +Deployment complete! >>> find.py --expect 2 whl dist/my_test_code-0.1.0-py3-none-any.whl @@ -98,14 +99,14 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> jq .path out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> jq -r .body.multipart_form.path | strings out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME diff --git a/acceptance/bundle/artifacts/whl_change_version/script b/acceptance/bundle/artifacts/whl_change_version/script index 9f6f480f0db..68f33c95633 100644 --- a/acceptance/bundle/artifacts/whl_change_version/script +++ b/acceptance/bundle/artifacts/whl_change_version/script @@ -6,7 +6,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path out.requests.txt | grep import | sort +trace jq -r '.body.multipart_form.path | strings' out.requests.txt | sort rm out.requests.txt @@ -20,7 +20,7 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace jq .path out.requests.txt | grep import | sort +trace jq -r '.body.multipart_form.path | strings' out.requests.txt | sort rm out.requests.txt @@ -34,6 +34,6 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace jq .path out.requests.txt | grep import | sort +trace jq -r '.body.multipart_form.path | strings' out.requests.txt | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index 3ebe5c59872..824f8f6f778 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -27,6 +27,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting no wheels to be uploaded ->>> errcode sh -c jq .path < out.requests.txt | grep import | grep whl +>>> errcode sh -c jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script index d7c93d8f38f..3f15b859126 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/script +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -7,6 +7,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting no wheels to be uploaded" -trace errcode sh -c 'jq .path < out.requests.txt | grep import | grep whl' +trace errcode sh -c "jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl" rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index eae8cd91f5f..460cc23205f 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -58,11 +58,11 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 patched wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl === Updating the local wheel and deploying again Building my_test_code... @@ -94,7 +94,7 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice) ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index 068db16caf8..1e02902b5e4 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -19,7 +19,7 @@ title "Expecting 2 patched wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt | gron.py | grep -w libraries title "Expecting 2 patched wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt @@ -35,4 +35,4 @@ title "Expecting 2 patched wheels in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 239ffa6f82b..9342f88adcd 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -29,9 +29,9 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index 4988edd8155..db664434f00 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -6,7 +6,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index b70acc109e6..59c14f08acc 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -29,6 +29,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index da61fb91f83..948e73d11d9 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -6,6 +6,6 @@ title "Expecting 1 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 835ab7c3413..eac7b9bb367 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -41,6 +41,6 @@ package/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl +/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index fdc0723f594..13f1ba1b6e9 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -6,6 +6,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index bab8fe2a99d..aab31e909d7 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -19,6 +19,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index 30196556f1e..8d68a61e6da 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -6,6 +6,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index a6eeab1b952..062d8ae68c1 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -35,8 +35,8 @@ my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index 380c6c60f8a..dfaa1bbb94a 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -6,6 +6,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm -fr out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt index e9573230b06..405884a4724 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt +++ b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt @@ -11,9 +11,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl >>> [CLI] bundle deploy Building python_artifact... diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/script b/acceptance/bundle/artifacts/whl_no_cleanup/script index 4536f929468..21e0d84ace8 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/script +++ b/acceptance/bundle/artifacts/whl_no_cleanup/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort trace $CLI bundle deploy title "No calls to delete internal folder expected" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index 560a1b1ad43..494180f75d8 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -38,8 +38,8 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index 9df7ea3d987..977c9431eac 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -9,6 +9,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt index b5cf6bf9ad3..e47166ba836 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt @@ -32,6 +32,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/script b/acceptance/bundle/artifacts/whl_prebuilt_outside/script index 939847f6c12..7459288248e 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/script @@ -11,6 +11,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt index cdd2b4432d8..1e82e6762c7 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt @@ -56,6 +56,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script index af4e84b8b3f..7306df94e9c 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script @@ -14,6 +14,6 @@ trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requ trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt title "Expecting 2 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index e0cb8bcb22b..47c6d977440 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -49,6 +49,6 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index 3a0dd929c09..0a4fbfcff24 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -6,6 +6,6 @@ title "Expecting 1 wheel in environments section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt index 32f89f53a1f..bcf56f4016b 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//import-file/ ^//workspace/ +>>> print_requests.py ^//workspace/ >>> [CLI] bundle summary -o json @@ -34,4 +34,4 @@ Recreated pipelines.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/script b/acceptance/bundle/deployment/bind/pipelines/recreate/script index 749e8ff916d..e270be4d849 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/script +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/script @@ -6,11 +6,11 @@ add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f out.requests.txt trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py '^//import-file/' '^//workspace/' +print_requests.py '^//workspace/' rm -f out.requests.txt trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt -trace print_requests.py '^//import-file/' '^//workspace/' +trace print_requests.py '^//workspace/' trace $CLI bundle summary -o json | jq .resources > out.summary.json trace $CLI bundle plan @@ -19,4 +19,4 @@ trace musterr $CLI bundle deploy rm -f out.requests.txt trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/deployment/bind/pipelines/update/output.txt b/acceptance/bundle/deployment/bind/pipelines/update/output.txt index 72de9c9187a..50f88b9b7ac 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/update/output.txt @@ -19,4 +19,4 @@ Updated pipelines.foo Files: 10 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/delete ^//telemetry-ext +>>> print_requests.py ^//workspace/import ^//workspace/delete ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/update/script b/acceptance/bundle/deployment/bind/pipelines/update/script index d3df0a767ed..b45f673d202 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/script +++ b/acceptance/bundle/deployment/bind/pipelines/update/script @@ -3,15 +3,15 @@ add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f out.requests.txt trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py '^//import-file/' '^//workspace/delete' +print_requests.py '^//workspace/import' '^//workspace/delete' rm -f out.requests.txt trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt -print_requests.py '^//import-file/' '^//workspace/delete' +print_requests.py '^//workspace/import' '^//workspace/delete' trace $CLI bundle summary -o json > out.summary.json trace $CLI bundle plan rm -f out.requests.txt trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//import-file/' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//workspace/import' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/destroy/force-lock-node-limit/script b/acceptance/bundle/destroy/force-lock-node-limit/script index c7fffbc16a2..616b626f2ef 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/script +++ b/acceptance/bundle/destroy/force-lock-node-limit/script @@ -7,9 +7,9 @@ title "Deploy (creates the job and the deployment)" trace $CLI bundle deploy title "destroy without --force-lock (lock write hits the node limit: aborts)" -fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace errcode $CLI bundle destroy --auto-approve title "destroy with --force-lock (tolerates the node limit: proceeds and completes)" -fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace $CLI bundle destroy --force-lock --auto-approve diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt index 5dc4456761e..2951ad7bfa3 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt +++ b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt @@ -54,8 +54,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> cat out.requests.txt { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl" + } } } diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/script b/acceptance/bundle/libraries/outside_of_bundle_root/script index ff72be1b690..74f4f15cf87 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/script +++ b/acceptance/bundle/libraries/outside_of_bundle_root/script @@ -11,6 +11,6 @@ title "Check that the job libraries are uploaded and the path is correct in the trace cat out.requests.txt | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.tasks[0].libraries' trace cat out.requests.txt | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.environments[0].spec.dependencies' trace cat out.requests.txt | jq 'select(.path == "/api/2.0/pipelines" and .method == "POST")' | jq '.body.environment.dependencies' -trace cat out.requests.txt | jq 'select(.path | test("/api/2.0/workspace-files/import-file/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl"))' +trace cat out.requests.txt | jq 'select(.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // "" | test("/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl")))' | jq 'del(.body.multipart_form.content)' rm out.requests.txt diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt index 792639d8819..e19d0f5a81d 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt @@ -1,11 +1,12 @@ === Deploy with bundle.engine: direct and an empty terraform state — sweep, don't migrate >>> [CLI] bundle deploy -Warn: Direct engine selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 4 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 0 unchanged -Removing empty terraform state; direct engine will be used on the next deploy (selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... +Deploying resources... +Updating deployment state... +Deployment complete! +Removing empty terraform state; direct engine will be used on the next deploy (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... === Terraform state is renamed to .backup; no resources.json (empty state, nothing to persist) @@ -21,20 +22,21 @@ direct_migrated_via_config true === Sweep also affected the workspace: .backup uploaded, terraform.tfstate deleted ->>> print_requests.py //workspace-files/import-file //workspace/delete --sort -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup" +>>> print_requests.py //workspace/import //workspace/delete --sort "POST /api/2.0/workspace/delete" "POST /api/2.0/workspace/delete" "POST /api/2.0/workspace/delete" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" === debug states confirms no terraform.tfstate anywhere; only .backup remains @@ -44,7 +46,7 @@ direct_migrated_via_config true === Next deploy uses direct engine by default (no more mismatch warning) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 2 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 0 unchanged +Deploying resources... +Deployment complete! >>> print_migration_telemetry diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script index 984bbf2fdce..afc0a262151 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script @@ -19,7 +19,7 @@ title "Sweep is recorded in telemetry as via-config\n" trace print_migration_telemetry title "Sweep also affected the workspace: .backup uploaded, terraform.tfstate deleted\n" -trace print_requests.py //workspace-files/import-file //workspace/delete --sort | jq '.method + " " + .path' | contains.py 'terraform.tfstate.backup' 'workspace/delete' +trace print_requests.py //workspace/import //workspace/delete --sort | jq '.method + " " + (.body.multipart_form.path // .path)' | contains.py 'terraform.tfstate.backup' 'workspace/delete' title "debug states confirms no terraform.tfstate anywhere; only .backup remains\n" trace $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/script b/acceptance/bundle/migrate/auto-migrate-push-failure/script index b1ccb759da5..50df7951c7d 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/script +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/script @@ -1,5 +1,18 @@ export DATABRICKS_BUNDLE_ENGINE= +# Keep harness files out of the sync: their content (this script included) contains +# the literal "state/resources.json" that the fault below matches on, and a synced +# file carrying that string would trip the fault during the files phase instead of +# on the migration commit we mean to target. +cat > .gitignore <<'EOF' +script +output.txt +out.requests.txt +repls.json +test.toml +.databricks +EOF + title "Initial deploy on terraform" trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy trace print_migration_telemetry @@ -9,7 +22,7 @@ title "Opt in via env var; inject 403 on the resources.json push so commit fails # The auto-migration's commit path uploads resources.json to the workspace # before renaming any local files. A 403 (non-retryable) fails commitMigration → # the deploy prints a warning and leaves the local state as terraform. -fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 +fault.py --body-contains "state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f out.requests.txt @@ -27,4 +40,6 @@ title "Local state is now direct\n" trace find .databricks/bundle -name "resources.json" -type f trace find .databricks/bundle -name "terraform.tfstate*" -type f +rm -f .gitignore + rm -f out.requests.txt diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 9301329af12..394a9ff9d80 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -14,7 +14,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//workspace/import { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -30,7 +30,7 @@ Created pipelines.mypipeline Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//workspace/import { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -98,7 +98,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted ->>> print_requests.py --sort ^//import-file/ +>>> print_requests.py --sort ^//workspace/import { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index d38692366b3..51ce6274ed6 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py '^//import-file/' +trace print_requests.py '^//workspace/import' trace $CLI bundle deploy -trace print_requests.py '^//import-file/' +trace print_requests.py '^//workspace/import' trace $CLI bundle destroy --auto-approve -trace print_requests.py --sort '^//import-file/' +trace print_requests.py --sort '^//workspace/import' diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 1731a30ca11..f7aabcc16e9 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 6caf49a7f4e..2bdc84ee12f 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -26,5 +26,5 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt index 16289c061a8..dffb81776bf 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -36,7 +36,7 @@ Updated quality_monitors.monitor1 Files: 4 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index 72c57e08401..e7e800e526f 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -27,6 +27,6 @@ trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy # dashboard_id is output only field that terraform adds -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt index a6d3f78f881..3bed93aaa0f 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -23,7 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 891aece1c11..0bec176f6a4 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -26,7 +26,7 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt index d1a8e3b14ff..96efef36679 100644 --- a/acceptance/bundle/resources/quality_monitors/create/output.txt +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -16,7 +16,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle plan -o json ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 97f6ce6a843..cabfc78d236 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -21,7 +21,7 @@ trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt index 2d469e4abce..b67ab287777 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext { "method": "POST", "path": "/api/2.0/secrets/scopes/delete", diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/script b/acceptance/bundle/resources/secret_scopes/delete_scope/script index 5e42e2af4be..4ba046b0585 100755 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/script +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/script @@ -15,4 +15,4 @@ trace $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt trace $CLI bundle deploy -qq -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt +trace print_requests.py '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt diff --git a/acceptance/bundle/sync-upload-edge-cases/databricks.yml b/acceptance/bundle/sync-upload-edge-cases/databricks.yml new file mode 100644 index 00000000000..2261ff177c7 --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: upload-edge-cases diff --git a/acceptance/bundle/sync-upload-edge-cases/out.test.toml b/acceptance/bundle/sync-upload-edge-cases/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/sync-upload-edge-cases/output.txt b/acceptance/bundle/sync-upload-edge-cases/output.txt new file mode 100644 index 00000000000..13e7506f2d6 --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/output.txt @@ -0,0 +1,140 @@ + +>>> [CLI] bundle sync --output text +Action: PUT: .gitignore, dashboard.lvdash.json, databricks.yml, empty.txt, héllo.txt, large.bin, plain-script.py, pyNb.py, scalaNb.scala, sqlNb.sql, with spaces.txt +Initial Sync Complete +Uploaded .gitignore +Uploaded dashboard.lvdash.json +Uploaded databricks.yml +Uploaded empty.txt +Uploaded héllo.txt +Uploaded large.bin +Uploaded plain-script.py +Uploaded pyNb.py +Uploaded scalaNb.scala +Uploaded sqlNb.sql +Uploaded with spaces.txt + +=== workspace state after sync — assert object_type and language for each upload +>>> MSYS_NO_PATHCONV=1 [CLI] workspace list -o json /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files +{ + "name": ".gitignore", + "object_type": "FILE", + "language": null +} +{ + "name": "dashboard.lvdash.json", + "object_type": "FILE", + "language": null +} +{ + "name": "databricks.yml", + "object_type": "FILE", + "language": null +} +{ + "name": "empty.txt", + "object_type": "FILE", + "language": null +} +{ + "name": "héllo.txt", + "object_type": "FILE", + "language": null +} +{ + "name": "large.bin", + "object_type": "FILE", + "language": null +} +{ + "name": "plain-script.py", + "object_type": "FILE", + "language": null +} +{ + "name": "pyNb", + "object_type": "NOTEBOOK", + "language": "PYTHON" +} +{ + "name": "scalaNb", + "object_type": "NOTEBOOK", + "language": "SCALA" +} +{ + "name": "sqlNb", + "object_type": "NOTEBOOK", + "language": "SQL" +} +{ + "name": "with spaces.txt", + "object_type": "FILE", + "language": null +} + +=== uploaded paths from each multipart POST /workspace/import (sorted) +>>> jq -rs + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.path) | + sort | .[] + out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/.gitignore +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/dashboard.lvdash.json +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/empty.txt +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/héllo.txt +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/large.bin +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/plain-script.py +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/pyNb.py +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/scalaNb.scala +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/sqlNb.sql +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/with spaces.txt + +=== every upload set format=AUTO +>>> jq -rs + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.format) | unique + out.requests.txt +[ + "AUTO" +] + +=== recorded content for the small text inputs +>>> jq -rs + map(select(.path == "/api/2.0/workspace/import")) | + map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | + map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | + sort_by(.path) | .[] + out.requests.txt +{ + "path": "dashboard.lvdash.json", + "content": "{\"datasets\":[],\"pages\":[]}\n" +} +{ + "path": "empty.txt", + "content": "" +} +{ + "path": "héllo.txt", + "content": "hello, naïve world\n" +} +{ + "path": "plain-script.py", + "content": "print(\"plain script, no header\")\n" +} +{ + "path": "pyNb.py", + "content": "# Databricks notebook source\nprint(\"hello\")\n" +} +{ + "path": "scalaNb.scala", + "content": "// Databricks notebook source\nprintln(\"hello\")\n" +} +{ + "path": "sqlNb.sql", + "content": "-- Databricks notebook source\nSELECT 1\n" +} +{ + "path": "with spaces.txt", + "content": "hello, world\n" +} diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script new file mode 100644 index 00000000000..b0402a2b003 --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/script @@ -0,0 +1,95 @@ +# Confidence checks that /workspace/import is a drop-in for the legacy import-file: +# verify the CLI's multipart upload pipeline handles the cases that differ between the +# two endpoints — large content (>10 MiB), empty content, varied extensions/object types, +# and non-ASCII / spaced filenames in the multipart `path` field. + +# Exclude framework-generated and script-internal files from the sync. We only want the +# inputs we explicitly generate below to show up in the recorded uploads. +cat > .gitignore <<'EOF' +script +output.txt +out.requests.txt +repls.json +test.toml +.databricks +EOF + +# 12 MiB binary file. The legacy /workspace/import JSON-body cap is 10 MiB; multipart has +# no such cap. Generated dynamically to keep the repo small. +python3 -c "open('large.bin', 'wb').write(b'\\0' * (12 * 1024 * 1024))" + +# Empty file (multipart encodes an empty `content` part — distinct from JSON's empty string). +touch empty.txt + +# Python notebook: auto-detected as NOTEBOOK by /workspace/import; testserver mirrors +# this and stores the object at the path with .py stripped. +cat > pyNb.py <<'EOF' +# Databricks notebook source +print("hello") +EOF + +# SQL notebook: -- comment header, .sql extension, Language=SQL on the workspace side. +cat > sqlNb.sql <<'EOF' +-- Databricks notebook source +SELECT 1 +EOF + +# Scala notebook: // comment header, .scala extension, Language=SCALA. +cat > scalaNb.scala <<'EOF' +// Databricks notebook source +println("hello") +EOF + +# Plain .py without the source header — should land as FILE, not NOTEBOOK, +# even though the extension is .py. +cat > plain-script.py <<'EOF' +print("plain script, no header") +EOF + +# Lakeview dashboard descriptor: the real /workspace/import assigns object_type=DASHBOARD +# and preserves the .lvdash.json extension. The testserver doesn't emulate that and stores +# it as a generic FILE — but the upload-side request shape (which is what this test asserts) +# is identical. +echo '{"datasets":[],"pages":[]}' > dashboard.lvdash.json + +# Non-ASCII filename. Multipart encodes filenames with RFC 5987 / quoted-string rules, +# distinct from URL-encoding in the legacy import-file endpoint. +echo "hello, naïve world" > "héllo.txt" + +# Filename with a space. +echo "hello, world" > "with spaces.txt" + +trace $CLI bundle sync --output text 2>&1 | sort + +title "workspace state after sync — assert object_type and language for each upload" +remote_files=$($CLI bundle summary -o json 2>/dev/null | jq -r .workspace.file_path) +# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-slash +# workspace path into a C:/Program Files/... filesystem path before it +# reaches the CLI. +trace MSYS_NO_PATHCONV=1 $CLI workspace list -o json "$remote_files" | + jq 'sort_by(.path) | .[] | {name: (.path | sub(".*/"; "")), object_type, language}' + +title "uploaded paths from each multipart POST /workspace/import (sorted)" +trace jq -rs ' + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.path) | + sort | .[] +' out.requests.txt + +title "every upload set format=AUTO" +trace jq -rs ' + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.format) | unique +' out.requests.txt + +title "recorded content for the small text inputs" +trace jq -rs ' + map(select(.path == "/api/2.0/workspace/import")) | + map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | + map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | + sort_by(.path) | .[] +' out.requests.txt + +# Drop the recorded requests; their workspace-prefixed paths are noisy and the +# assertions above already pin the relevant fields. +rm out.requests.txt .gitignore diff --git a/acceptance/bundle/sync-upload-edge-cases/test.toml b/acceptance/bundle/sync-upload-edge-cases/test.toml new file mode 100644 index 00000000000..8bfde26201e --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/test.toml @@ -0,0 +1,18 @@ +RecordRequests = true + +# All sync uploads go through the same /workspace/import multipart code path; running +# both engine variants is unnecessary here. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "large.bin", + "empty.txt", + "pyNb.py", + "sqlNb.sql", + "scalaNb.scala", + "plain-script.py", + "dashboard.lvdash.json", + "héllo.txt", + "with spaces.txt", + ".databricks", +] diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 6b79c6f2251..1cf42e037e4 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -13,12 +13,12 @@ $CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE rm ../../out.requests.txt # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index 86272b4a266..12d51337bdf 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -12,12 +12,12 @@ $CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE rm ../../out.requests.txt trace $CLI bundle deploy -t dev -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/upload/internal_server_error/output.txt b/acceptance/bundle/upload/internal_server_error/output.txt index 5cde6aad650..24d8b2cc4a5 100644 --- a/acceptance/bundle/upload/internal_server_error/output.txt +++ b/acceptance/bundle/upload/internal_server_error/output.txt @@ -1,7 +1,7 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: Failed to acquire deployment lock: Internal Server Error Error: Internal Server Error (500) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import HTTP Status: 500 Internal Server Error API error_code: API message: Internal Server Error diff --git a/acceptance/bundle/upload/internal_server_error/test.toml b/acceptance/bundle/upload/internal_server_error/test.toml index 8ff8501b6de..b1d2b7dc49f 100644 --- a/acceptance/bundle/upload/internal_server_error/test.toml +++ b/acceptance/bundle/upload/internal_server_error/test.toml @@ -1,3 +1,3 @@ [[Server]] -Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" +Pattern = "POST /api/2.0/workspace/import" Response.StatusCode = 500 diff --git a/acceptance/bundle/upload/timeout/output.txt b/acceptance/bundle/upload/timeout/output.txt index c314a247707..3b6eb72f28b 100644 --- a/acceptance/bundle/upload/timeout/output.txt +++ b/acceptance/bundle/upload/timeout/output.txt @@ -1,3 +1,3 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity +Error: Failed to acquire deployment lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity +Error: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity diff --git a/acceptance/bundle/upload/timeout/test.toml b/acceptance/bundle/upload/timeout/test.toml index d6564ef9659..f0abd08ecbc 100644 --- a/acceptance/bundle/upload/timeout/test.toml +++ b/acceptance/bundle/upload/timeout/test.toml @@ -3,5 +3,5 @@ Env.DATABRICKS_BUNDLE_HTTP_TIMEOUT_SECONDS = "5" [[Server]] # CLI aborts after a single attempt when the HTTP timeout fires. Delay = "30s" -Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" +Pattern = "POST /api/2.0/workspace/import" Response.StatusCode = 200 diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 318ccc3ff3c..4e0e3329054 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -8,14 +8,14 @@ OK deploy.direct /api/2.0/workspace/get-status engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json engine/direct OK deploy.direct /api/2.0/workspace/delete engine/direct OK deploy.direct /api/2.0/workspace/delete engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct OK deploy.direct /api/2.0/workspace/mkdirs engine/direct OK deploy.direct /api/2.0/workspace/mkdirs engine/direct OK deploy.direct /api/2.1/unity-catalog/schemas engine/direct @@ -30,14 +30,14 @@ OK deploy.terraform /api/2.0/workspace/get-status engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate engine/terraform OK deploy.terraform /api/2.0/workspace/delete engine/terraform OK deploy.terraform /api/2.0/workspace/delete engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform MISS deploy.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' @@ -48,7 +48,7 @@ MISS deploy.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF MISS deploy.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS deploy.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS destroy.direct /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' -MISS destroy.direct /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' +MISS destroy.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct @@ -57,21 +57,19 @@ OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct -OK destroy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct -OK destroy.direct /api/2.0/workspace/delete engine/direct OK destroy.direct /api/2.0/workspace/delete engine/direct +OK destroy.direct /api/2.0/workspace/import engine/direct MISS destroy.direct /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' -MISS destroy.terraform /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' +MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' OK destroy.terraform /api/2.0/workspace/export engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform -OK destroy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/terraform -OK destroy.terraform /api/2.0/workspace/delete engine/terraform OK destroy.terraform /api/2.0/workspace/delete engine/terraform +OK destroy.terraform /api/2.0/workspace/import engine/terraform MISS destroy.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS destroy.terraform /api/2.1/unity-catalog/current-metastore-assignment 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' @@ -95,30 +93,30 @@ MISS plan.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks MISS plan.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS plan.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' -MISS plan2.direct /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' +MISS plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan2.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan2.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' -OK plan2.direct /api/2.0/workspace/export engine/direct +OK plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct OK plan2.direct /api/2.0/workspace/get-status engine/direct OK plan2.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct MISS plan2.direct /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS plan2.terraform /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' -MISS plan2.terraform /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' +MISS plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' -OK plan2.terraform /api/2.0/workspace/export engine/terraform +OK plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/terraform OK plan2.terraform /api/2.0/workspace/get-status engine/terraform MISS plan2.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS plan2.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS plan2.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS plan2.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS run.direct /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' -MISS run.direct /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' +MISS run.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.direct /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS run.terraform /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' -MISS run.terraform /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' +MISS run.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index 5326f079064..c706828fa18 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -130,9 +130,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", + "recursive": true } } { @@ -142,15 +143,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/delete", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } { @@ -160,15 +155,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/import", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "multipart_form": { + "content": "", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py" + } } } { @@ -178,22 +172,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "seq": 1, - "cli_version": "[CLI_VERSION]", - "timestamp": "[TIMESTAMP]", - "files": [ - { - "local_path": "empty.py", - "is_notebook": false - } - ], - "id": "[UUID]" + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -203,29 +188,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "config": { - "bundle": { - "name": "test-bundle", - "target": "default", - "git": { - "bundle_root_path": "." - } - }, - "workspace": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" - }, - "resources": {}, - "presets": { - "source_linked_deployment": false - } - }, - "extra": {} + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -235,23 +204,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "state": { - "resources.schemas.foo": { - "__id__": "mycatalog.myschema", - "state": { - "catalog_name": "mycatalog", - "name": "myschema" - } - } + "multipart_form": { + "content": "{\"version\":1,\"seq\":1,\"cli_version\":\"[CLI_VERSION]\",\"timestamp\":\"[TIMESTAMP]\",\"files\":[{\"local_path\":\"empty.py\",\"is_notebook\":false}],\"id\":\"[UUID]\"}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" } } } @@ -262,10 +221,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", - "recursive": true + "multipart_form": { + "content": "{\n \"version\": 1,\n \"config\": {\n \"bundle\": {\n \"name\": \"test-bundle\",\n \"target\": \"default\",\n \"git\": {\n \"bundle_root_path\": \".\"\n }\n },\n \"workspace\": {\n \"file_path\": \"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"\n },\n \"resources\": {},\n \"presets\": {\n \"source_linked_deployment\": false\n }\n },\n \"extra\": {}\n}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + } } } { @@ -275,9 +238,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + "multipart_form": { + "content": "{\n \"state_version\": 2,\n \"cli_version\": \"[CLI_VERSION]\",\n \"lineage\": \"[UUID]\",\n \"serial\": 1,\n \"state\": {\n \"resources.schemas.foo\": {\n \"__id__\": \"mycatalog.myschema\",\n \"state\": {\n \"catalog_name\": \"mycatalog\",\n \"name\": \"myschema\"\n }\n }\n }\n}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" + } } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json index cb67aafe1b4..0dbf3e54b1e 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json @@ -130,9 +130,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", + "recursive": true } } { @@ -142,15 +143,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/delete", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } { @@ -160,15 +155,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/import", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "multipart_form": { + "content": "", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py" + } } } { @@ -178,22 +172,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "seq": 1, - "cli_version": "[CLI_VERSION]", - "timestamp": "[TIMESTAMP]", - "files": [ - { - "local_path": "empty.py", - "is_notebook": false - } - ], - "id": "[UUID]" + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -203,29 +188,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "config": { - "bundle": { - "name": "test-bundle", - "target": "default", - "git": { - "bundle_root_path": "." - } - }, - "workspace": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" - }, - "resources": {}, - "presets": { - "source_linked_deployment": false - } - }, - "extra": {} + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -235,50 +204,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 4, - "terraform_version": "1.5.5", - "serial": 1, - "lineage": "[UUID]", - "outputs": {}, - "resources": [ - { - "mode": "managed", - "type": "databricks_schema", - "name": "foo", - "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "catalog_name": "mycatalog", - "comment": null, - "enable_predictive_optimization": "INHERIT", - "force_destroy": true, - "id": "mycatalog.myschema", - "metastore_id": "[UUID]", - "name": "myschema", - "owner": "[USERNAME]", - "properties": null, - "provider_config": [ - { - "workspace_id": "[NUMID]" - } - ], - "schema_id": "[UUID]", - "storage_root": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ], - "check_results": null + "multipart_form": { + "content": "{\"version\":1,\"seq\":1,\"cli_version\":\"[CLI_VERSION]\",\"timestamp\":\"[TIMESTAMP]\",\"files\":[{\"local_path\":\"empty.py\",\"is_notebook\":false}],\"id\":\"[UUID]\"}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" + } } } { @@ -288,10 +221,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", - "recursive": true + "multipart_form": { + "content": "{\n \"version\": 1,\n \"config\": {\n \"bundle\": {\n \"name\": \"test-bundle\",\n \"target\": \"default\",\n \"git\": {\n \"bundle_root_path\": \".\"\n }\n },\n \"workspace\": {\n \"file_path\": \"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"\n },\n \"resources\": {},\n \"presets\": {\n \"source_linked_deployment\": false\n }\n },\n \"extra\": {}\n}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + } } } { @@ -301,9 +238,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + "multipart_form": { + "content": "{\n \"version\": 4,\n \"terraform_version\": \"1.5.5\",\n \"serial\": 1,\n \"lineage\": \"[UUID]\",\n \"outputs\": {},\n \"resources\": [\n {\n \"mode\": \"managed\",\n \"type\": \"databricks_schema\",\n \"name\": \"foo\",\n \"provider\": \"provider[\\\"registry.terraform.io/databricks/databricks\\\"]\",\n \"instances\": [\n {\n \"schema_version\": 0,\n \"attributes\": {\n \"catalog_name\": \"mycatalog\",\n \"comment\": null,\n \"enable_predictive_optimization\": \"INHERIT\",\n \"force_destroy\": true,\n \"id\": \"mycatalog.myschema\",\n \"metastore_id\": \"[UUID]\",\n \"name\": \"myschema\",\n \"owner\": \"[USERNAME]\",\n \"properties\": null,\n \"provider_config\": [\n {\n \"workspace_id\": \"[NUMID]\"\n }\n ],\n \"schema_id\": \"[UUID]\",\n \"storage_root\": null\n },\n \"sensitive_attributes\": [],\n \"private\": \"bnVsbA==\"\n }\n ]\n }\n ],\n \"check_results\": null\n}\n", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" + } } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json index 43de64cf21f..e22956fe86d 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json @@ -14,11 +14,7 @@ ] }, "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" - } + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" } { "headers": { @@ -118,24 +114,6 @@ "method": "GET", "path": "/api/2.1/unity-catalog/schemas/mycatalog.myschema" } -{ - "headers": { - "User-Agent": [ - "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" - ] - }, - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, - "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" - } -} { "headers": { "User-Agent": [ @@ -145,7 +123,8 @@ "method": "POST", "path": "/api/2.0/workspace/delete", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "recursive": true } } { @@ -155,10 +134,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "recursive": true + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json index 619c1335fb3..e5127a03754 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json @@ -14,11 +14,7 @@ ] }, "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" - } + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" } { "headers": { @@ -97,24 +93,6 @@ "return_export_info": "true" } } -{ - "headers": { - "User-Agent": [ - "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none engine/terraform auth/pat" - ] - }, - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, - "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" - } -} { "headers": { "User-Agent": [ @@ -124,7 +102,8 @@ "method": "POST", "path": "/api/2.0/workspace/delete", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "recursive": true } } { @@ -134,10 +113,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "recursive": true + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { diff --git a/acceptance/bundle/validate/sync_patterns/out.sync.txt b/acceptance/bundle/validate/sync_patterns/out.sync.txt index ac5aba9ab29..fd939c9721b 100644 --- a/acceptance/bundle/validate/sync_patterns/out.sync.txt +++ b/acceptance/bundle/validate/sync_patterns/out.sync.txt @@ -1,7 +1,12 @@ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml" + } } } diff --git a/acceptance/bundle/validate/sync_patterns/script b/acceptance/bundle/validate/sync_patterns/script index d2aae85444a..ce8e5cbb1b6 100644 --- a/acceptance/bundle/validate/sync_patterns/script +++ b/acceptance/bundle/validate/sync_patterns/script @@ -1,5 +1,5 @@ trace $CLI bundle validate trace $CLI bundle validate -o json | jq '.sync' trace $CLI bundle deploy -jq 'select(.path | test("dir/test.yml"))' out.requests.txt > out.sync.txt +jq 'select(.body.multipart_form.path | strings | test("dir/test.yml"))' out.requests.txt > out.sync.txt rm out.requests.txt diff --git a/acceptance/cmd/sync/dryrun-missing-remote/output.txt b/acceptance/cmd/sync/dryrun-missing-remote/output.txt index 4848aae1c2d..b7453994b58 100644 --- a/acceptance/cmd/sync/dryrun-missing-remote/output.txt +++ b/acceptance/cmd/sync/dryrun-missing-remote/output.txt @@ -15,11 +15,11 @@ Uploaded hello.py >>> print_requests.py --sort //api/ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/.gitignore" + "path": "/api/2.0/workspace/import" } { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/hello.py" + "path": "/api/2.0/workspace/import" } { "method": "POST", diff --git a/libs/testserver/fault.go b/libs/testserver/fault.go index 75d3092bb70..43d46f3f28c 100644 --- a/libs/testserver/fault.go +++ b/libs/testserver/fault.go @@ -1,6 +1,7 @@ package testserver import ( + "bytes" "encoding/json" "strings" "sync" @@ -19,6 +20,12 @@ type FaultRule struct { AfterHandler bool offset int times int + + // bodyContains, if non-empty, additionally requires the request body to + // contain this substring for the rule to fire. /workspace/import routes + // every upload through the same method+path, so the only way to target a + // single file's upload is by matching its multipart "path" form field. + bodyContains string } // FaultRules holds the active fault injection rules for a test server. @@ -33,16 +40,16 @@ func NewFaultRules() *FaultRules { } // Set registers or replaces a fault rule for the given token and pattern. -func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int) { - fr.set(token, pattern, statusCode, body, offset, times, false) +func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { + fr.set(token, pattern, statusCode, body, offset, times, false, bodyContains) } // SetAfterHandler is like Set, but the handler runs first so its effect is kept. -func (fr *FaultRules) SetAfterHandler(token, pattern string, statusCode int, body string, offset, times int) { - fr.set(token, pattern, statusCode, body, offset, times, true) +func (fr *FaultRules) SetAfterHandler(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { + fr.set(token, pattern, statusCode, body, offset, times, true, bodyContains) } -func (fr *FaultRules) set(token, pattern string, statusCode int, body string, offset, times int, afterHandler bool) { +func (fr *FaultRules) set(token, pattern string, statusCode int, body string, offset, times int, afterHandler bool, bodyContains string) { fr.mu.Lock() defer fr.mu.Unlock() fr.rules[faultRuleKey{token: token, pattern: pattern}] = &FaultRule{ @@ -51,12 +58,14 @@ func (fr *FaultRules) set(token, pattern string, statusCode int, body string, of AfterHandler: afterHandler, offset: offset, times: times, + bodyContains: bodyContains, } } // Check returns a matching fault rule and advances its counters, or nil if no rule matches. // Pattern supports a trailing "*" wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*". -func (fr *FaultRules) Check(method, path, token string) *FaultRule { +// A rule with a non-empty bodyContains only matches when body contains that substring. +func (fr *FaultRules) Check(method, path, token string, body []byte) *FaultRule { requestPattern := method + " " + path fr.mu.Lock() @@ -76,6 +85,11 @@ func (fr *FaultRules) Check(method, path, token string) *FaultRule { if !matched { continue } + // A non-matching body must not consume the rule's offset/times budget, + // so this check precedes the counter updates below. + if rule.bodyContains != "" && !bytes.Contains(body, []byte(rule.bodyContains)) { + continue + } if rule.offset > 0 { rule.offset-- return nil @@ -105,14 +119,15 @@ func faultEndpointHandler(fr *FaultRules) HandlerFunc { Offset int `json:"offset"` Times int `json:"times"` AfterHandler bool `json:"after_handler"` + BodyContains string `json:"body_contains"` } if err := json.Unmarshal(req.Body, &body); err != nil { return Response{StatusCode: 400, Body: map[string]string{"error": err.Error()}} } if body.AfterHandler { - fr.SetAfterHandler(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) + fr.SetAfterHandler(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) } else { - fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) + fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) } return Response{StatusCode: 200} } diff --git a/libs/testserver/fault_test.go b/libs/testserver/fault_test.go index db41b74d3ba..ae27b954f93 100644 --- a/libs/testserver/fault_test.go +++ b/libs/testserver/fault_test.go @@ -14,18 +14,18 @@ import ( func TestFaultRulesNoMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 1) + fr.Set("tok", "GET /foo", 504, "body", 0, 1, "") - assert.Nil(t, fr.Check("POST", "/foo", "tok")) - assert.Nil(t, fr.Check("GET", "/bar", "tok")) - assert.Nil(t, fr.Check("GET", "/foo", "other")) + assert.Nil(t, fr.Check("POST", "/foo", "tok", nil)) + assert.Nil(t, fr.Check("GET", "/bar", "tok", nil)) + assert.Nil(t, fr.Check("GET", "/foo", "other", nil)) } func TestFaultRulesExactMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1) + fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1, "") - rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok") + rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok", nil) require.NotNil(t, rule) assert.Equal(t, 504, rule.StatusCode) assert.Equal(t, "body", rule.Body) @@ -33,31 +33,43 @@ func TestFaultRulesExactMatch(t *testing.T) { func TestFaultRulesWildcardMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2) + fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2, "") - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok")) - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) - assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) // exhausted + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok", nil)) + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) + assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) // exhausted } func TestFaultRulesOffset(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 2, 1) + fr.Set("tok", "GET /foo", 504, "body", 2, 1, "") - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 2→1 - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 1→0 - assert.NotNil(t, fr.Check("GET", "/foo", "tok")) // fires - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 2→1 + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 1→0 + assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) // fires + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted } func TestFaultRulesTimes(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 3) + fr.Set("tok", "GET /foo", 504, "body", 0, 3, "") for range 3 { - assert.NotNil(t, fr.Check("GET", "/foo", "tok")) + assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) } - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted +} + +func TestFaultRulesBodyContains(t *testing.T) { + fr := testserver.NewFaultRules() + fr.Set("tok", "POST /api/2.0/workspace/import", 403, "body", 0, 1, "state/resources.json") + + // Body without the substring must not match and must not consume the budget. + assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/deploy.lock"))) + // Body with the substring fires. + assert.NotNil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) + // Budget was only consumed by the matching request. + assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) } func TestServerFaultAfterHandlerKeepsTheHandlersEffect(t *testing.T) { diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 7f949ebf5cd..78cde200864 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -372,7 +372,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, handler HandlerFu var resp EncodedResponse - fault := s.faults.Check(r.Method, r.URL.Path, token) + fault := s.faults.Check(r.Method, r.URL.Path, token, request.Body) if fault == nil || fault.AfterHandler { if bytes.Contains(request.Body, []byte("INJECT_ERROR")) { From 86cf272ffe111041c50c5d25ad1eab73e983a4cc Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 4 Aug 2026 11:27:52 +0000 Subject: [PATCH 03/37] update comments --- libs/filer/workspace_files_client.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 5b903ab3678..1f6c21571e0 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -239,6 +239,9 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // targets /foo.py: AUTO would store the new content as FILE at // /foo.py, but the workspace treats /foo.py as the source view of // the existing /foo NOTEBOOK and rejects the type change. + // Unlike (a), this message comes from the legacy webapp tree path + // (webapp/.../tree/TreeBackendHelper.scala), not from WCS, so the + // WCS-only ErrorInfo work below does not cover it. // // The server refuses the overwrite even though the caller asked for // it; from the caller's perspective the path is occupied, so we @@ -254,10 +257,19 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == workspaceObjectTypeMismatchReason { return fileAlreadyExistsError{absPath} } - // Fallback for workspaces where the ErrorInfo change has not - // rolled out: as of 2026-06-12 aws-prod-ucws still returns these - // errors without details, so match the two observed messages. - // Remove once the rollout is confirmed everywhere. + // Fallback for errors that carry no ErrorInfo. Two reasons this + // is still needed, both verified against a live workspace on + // 2026-08-04: + // + // - Rollout lag: universe #2019174 merged 2026-06-03 with its + // SAFE flag defaulting to true, but workspaces on an older WCS + // build still return WCS-worded collisions without details. + // That half is temporary. + // + // - Message (b) above is thrown by webapp, which #2019174 never + // touched, so it has no ErrorInfo regardless of WCS rollout. + // Once the lag clears this can narrow to "node type" alone, + // but it cannot be dropped until webapp attaches details too. if strings.Contains(aerr.Message, "type mismatch") || strings.Contains(aerr.Message, "node type") { return fileAlreadyExistsError{absPath} } From 15d3b59ff11a46ba323955c59c1fdc7a2c3dd233 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 5 Aug 2026 13:14:57 +0000 Subject: [PATCH 04/37] Source upload size and rate limits from public docs The size limits in WorkspaceFilesClient.Write were described using internal config flag names and figures measured by hand against one workspace, neither of which a reader can verify. Replace them with the documented limits and link the sources. This corrects one figure: the notebook cap was described as a flat 10 MiB, but IPYNB notebooks are documented at 100 MB. Only source-format notebooks are capped at 10 MB, so the limit depends on how format=AUTO classifies the payload. Also restore the rate limit doc link to MaxRequestsInFlight, which the comment carried when the constant was introduced in #81. The value is unchanged. Add a comment above each test in workspace_files_client_test.go stating what it covers. Co-authored-by: Isaac --- .../bundles/workspace-import-migration.md | 2 +- .../bundle/sync-upload-edge-cases/script | 7 ++-- libs/filer/workspace_files_client.go | 38 ++++++++++++------- libs/filer/workspace_files_client_test.go | 30 +++++++++++++++ libs/sync/watchdog.go | 5 ++- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/.nextchanges/bundles/workspace-import-migration.md b/.nextchanges/bundles/workspace-import-migration.md index 351faafb7cd..4212003bdc7 100644 --- a/.nextchanges/bundles/workspace-import-migration.md +++ b/.nextchanges/bundles/workspace-import-migration.md @@ -1 +1 @@ -Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a higher rate limit (30 vs 20 requests/sec per workspace) and is ~1.5–2× faster for typical bundle deployments. +Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a documented rate limit of 30 requests per second per workspace ([API rate limits](https://docs.databricks.com/aws/en/resources/limits)), higher than the limit that applied to the previous endpoint, and is ~1.5–2× faster for typical bundle deployments. diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script index b0402a2b003..950c2157f3c 100644 --- a/acceptance/bundle/sync-upload-edge-cases/script +++ b/acceptance/bundle/sync-upload-edge-cases/script @@ -1,6 +1,6 @@ # Confidence checks that /workspace/import is a drop-in for the legacy import-file: # verify the CLI's multipart upload pipeline handles the cases that differ between the -# two endpoints — large content (>10 MiB), empty content, varied extensions/object types, +# two endpoints — large content (>10 MB), empty content, varied extensions/object types, # and non-ASCII / spaced filenames in the multipart `path` field. # Exclude framework-generated and script-internal files from the sync. We only want the @@ -14,8 +14,9 @@ test.toml .databricks EOF -# 12 MiB binary file. The legacy /workspace/import JSON-body cap is 10 MiB; multipart has -# no such cap. Generated dynamically to keep the repo small. +# 12 MiB binary file, over the 10 MB cap on the JSON-body variant's base64 `content` +# field. The multipart variant this test exercises has no such cap. Generated +# dynamically to keep the repo small. python3 -c "open('large.bin', 'wb').write(b'\\0' * (12 * 1024 * 1024))" # Empty file (multipart encodes an empty `content` part — distinct from JSON's empty string). diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 1f6c21571e0..24714339119 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -164,20 +164,32 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - // Use Workspace.Upload (multipart /api/2.0/workspace/import) instead of the - // JSON-body variant of the same endpoint, which caps payloads at 10 MiB for - // AUTO format (databricks.webapp.autoExportFormatLimitBytes). The multipart - // variant has been verified against a real workspace at 450 MB for regular - // files — strictly better than the previous /workspace-files/import-file - // endpoint, which has a 200 MiB body cap - // (databricks.workspaceFilesystem.maxImportSizeBytes) plus a 305s server-side - // request timeout that cuts off uploads above ~400 MB at typical bandwidth. + // Use Workspace.Upload (multipart /api/2.0/workspace/import) rather than + // Workspace.Import (the JSON-body variant of the same endpoint). The JSON + // variant sends content base64-encoded in a `content` field that is capped + // at 10 MB, returning MAX_NOTEBOOK_SIZE_EXCEEDED above it; the multipart + // variant posts the bytes as a file part instead and is bounded only by the + // 500 MB workspace file size limit. See the `content` field description in + // .codegen/cli.json (workspace.Import) and + // https://docs.databricks.com/aws/en/files/workspace ("Workspace file size + // is limited to 500MB"). // - // Notebook content (any payload with a `# Databricks notebook source` header - // detected by format=AUTO) hits a separate 10 MiB cap on the server - // (databricks.notebook.maxNotebookSizeBytes). Both /workspace-files/import-file - // and /workspace/import enforce this same cap, so switching from the former - // to the latter does not regress maximum notebook upload size. + // Because format=AUTO lets the server classify each payload, the applicable + // limit depends on how the content is classified, not on our request: + // + // - Regular file: 500 MB. + // - Notebook in a source format, i.e. content whose extension and + // "Databricks notebook source" header make AUTO import it as a + // notebook: 10 MB. + // - Notebook in IPYNB format: 100 MB. + // + // Notebook limits are documented at + // https://docs.databricks.com/aws/en/notebooks/notebook-limitations + // ("Import and export is supported for IPYNB notebooks up to 100 MB" and up + // to 10 MB for other formats). They are enforced by the workspace for both + // /workspace/import and the /workspace-files/import-file endpoint this + // replaced, so migrating between the two does not change the maximum + // uploadable notebook size. overwrite := slices.Contains(mode, OverwriteIfExists) uploadOpts := []func(*workspace.Import){ workspace.UploadFormat(workspace.ImportFormatAuto), diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index a607b116263..2b71eb9820d 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -22,6 +22,8 @@ import ( "github.com/stretchr/testify/require" ) +// Confirms directory entries derived from workspace object infos report the +// name, type, and size of each object type the workspace can return. func TestWorkspaceFilesDirEntry(t *testing.T) { entries := wsfsDirEntriesFromObjectInfos([]workspace.ObjectInfo{ { @@ -68,6 +70,8 @@ func TestWorkspaceFilesDirEntry(t *testing.T) { assert.True(t, i2.IsDir()) } +// Confirms the workspace ID routing header is set only when the client config +// carries a workspace ID, and that a nil workspace client yields no headers. func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { tests := []struct { name string @@ -104,6 +108,11 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } +// Confirms Write uploads through Workspace.Upload with format=AUTO, and that +// the overwrite flag is set only when OverwriteIfExists is passed. AUTO is what +// lets the server decide between storing content as a file or a notebook, so +// the format is asserted explicitly rather than left to the SDK default +// (SOURCE), which would import every file as a notebook. func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { tests := []struct { name string @@ -157,6 +166,11 @@ func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { } } +// Confirms each error /workspace/import returns is translated to the filer's +// own error type, and that unrecognized errors pass through unchanged. The +// endpoint signals an occupied path with three different status/error_code +// combinations (see the comment on Write), so all three are covered here to +// pin the mapping the rest of the CLI relies on. func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { tests := []struct { name string @@ -287,6 +301,8 @@ func writeWithImportError(t *testing.T, body map[string]any) error { return err } +// Confirms a type-mismatch collision is recognized from the structured +// AIP-193 ErrorInfo reason, independent of the error message wording. func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { // The message is deliberately one the fallback string match does not // recognize, to prove the branch fires on the structured reason alone. @@ -306,6 +322,8 @@ func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { assert.ErrorAs(t, err, &target) } +// Confirms an ErrorInfo carrying some other reason is not mistaken for a path +// collision, so the reason check above cannot swallow unrelated failures. func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { err := writeWithImportError(t, map[string]any{ "error_code": "INVALID_PARAMETER_VALUE", @@ -323,6 +341,10 @@ func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { assert.Equal(t, http.StatusBadRequest, aerr.StatusCode) } +// Confirms a 404 from a missing parent directory triggers a mkdirs call and a +// retry of the upload when CreateParentDirectories is passed. The retry re-reads +// the buffered body, so this also covers that the content survives the second +// attempt. func TestWorkspaceFilesClientWriteCreatesParentDirectories(t *testing.T) { mw := mocks.NewMockWorkspaceClient(t) workspaceApi := mw.GetMockWorkspaceAPI() @@ -346,6 +368,8 @@ func TestWorkspaceFilesClientWriteCreatesParentDirectories(t *testing.T) { require.NoError(t, err) } +// Confirms a workspace get-status payload unmarshals into wsfsFileInfo and that +// the fs.FileInfo methods derived from it report the expected values. func TestWorkspaceFilesClient_wsfsUnmarshal(t *testing.T) { payload := ` { @@ -412,6 +436,8 @@ func statWithError(t *testing.T, statusCode int, errorCode string) error { return err } +// Confirms a 403 from get-status surfaces as an APIError rather than being +// remapped, so callers can distinguish it from a missing file. func TestWorkspaceFilesClientStatForbidden(t *testing.T) { err := statWithError(t, 403, "PERMISSION_DENIED") var apiErr *apierr.APIError @@ -419,6 +445,8 @@ func TestWorkspaceFilesClientStatForbidden(t *testing.T) { assert.Equal(t, 403, apiErr.StatusCode) } +// Confirms a 500 from get-status surfaces as an APIError and is not treated as +// a missing file. func TestWorkspaceFilesClientStatInternalError(t *testing.T) { err := statWithError(t, 500, "INTERNAL_ERROR") var apiErr *apierr.APIError @@ -426,6 +454,8 @@ func TestWorkspaceFilesClientStatInternalError(t *testing.T) { assert.Equal(t, 500, apiErr.StatusCode) } +// Confirms RESOURCE_DOES_NOT_EXIST maps to fs.ErrNotExist so the filer plugs +// into the io/fs error conventions callers check against. func TestWorkspaceFilesClientStatNotFound(t *testing.T) { err := statWithError(t, 404, "RESOURCE_DOES_NOT_EXIST") assert.ErrorIs(t, err, fs.ErrNotExist) diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go index d3bb57662d9..fa7ff0baceb 100644 --- a/libs/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -10,7 +10,10 @@ import ( "golang.org/x/sync/errgroup" ) -// Maximum number of concurrent requests during sync. +// Maximum number of concurrent requests during sync. Chosen against the +// per-endpoint rate limits documented at +// https://docs.databricks.com/aws/en/resources/limits ("API rate limits" - +// Workspace API), which sync stays within for the endpoints it calls. const MaxRequestsInFlight = 20 // Delete the specified path. From 83b8bc526b52c4527d50cd63bad1aaa5ef2779c0 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 5 Aug 2026 13:33:15 +0000 Subject: [PATCH 05/37] acceptance: migrate ai_runtime_task snapshot upload assertions The local_code_source test filtered recorded uploads by URL path, which no longer matches: /workspace/import carries the target filename in the multipart body. Filter on the body instead, the same way auto-migrate-empty-tfstate does. --del-field raw_body is dropped because the tarball is binary and the request recorder already summarizes it as a size placeholder. This test was added in #6110, after the upload migration branch was cut, so it was not covered by the earlier fixture updates. Co-authored-by: Isaac --- .../local_code_source/output.txt | 37 +++++++++++++------ .../ai_runtime_task/local_code_source/script | 15 +++++--- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt index 768955bc2b2..a08103ce148 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt +++ b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt @@ -21,19 +21,29 @@ src/train.py === both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec ->>> print_requests.py --sort --del-field raw_body //.air_snapshots/ //jobs/create +>>> print_requests.py --sort //workspace/import //jobs/create { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "[binary content 189 bytes]", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz" + } } } { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "[binary content 273 bytes]", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz" + } } } { @@ -114,12 +124,17 @@ Updated jobs.train Files: 4 uploaded, 1 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ->>> print_requests.py --sort --del-field raw_body //.air_snapshots/ +>>> print_requests.py --sort //workspace/import { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "[binary content 276 bytes]", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz" + } } } diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/script b/acceptance/bundle/ai_runtime_task/local_code_source/script index fe5e707ce01..6135bad0010 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/script +++ b/acceptance/bundle/ai_runtime_task/local_code_source/script @@ -11,10 +11,14 @@ title "each task's tarball holds only synced files (both under the repo root .ai trace list_code_snapshot.py title "both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec\n" -# --del-field raw_body drops the binary tarball upload payload (kept readable). Filters -# use a leading // so Git Bash on Windows does not path-convert them. --keep is not -# passed, so print_requests.py consumes out.requests.txt. -trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' '//jobs/create' +# Uploads go to POST /workspace/import, which carries the target filename in the +# multipart body rather than the URL, so .air_snapshots is matched with jq on the +# recorded body instead of by path filter. The tarball payload is binary and is +# already summarized as "[binary content N bytes]" by the request recorder. +# Filters use a leading // so Git Bash on Windows does not path-convert them. +# --keep is not passed, so print_requests.py consumes out.requests.txt. +trace print_requests.py --sort '//workspace/import' '//jobs/create' | + jq 'select((.body.multipart_form.path // .path) | test("[.]air_snapshots/|/jobs/create"))' title "re-planning unchanged code is a no-op (no changes)\n" trace $CLI bundle plan @@ -22,7 +26,8 @@ trace $CLI bundle plan title "editing a file changes the snapshot hash (content-addressed name changes)\n" update_file.py src/train.py 'print("training")' 'print("training v2")' trace $CLI bundle deploy -trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' +trace print_requests.py --sort '//workspace/import' | + jq 'select((.body.multipart_form.path // .path) | test("[.]air_snapshots/"))' title "destroy removes the deployed bundle (including the synced snapshots)\n" trace $CLI bundle destroy --auto-approve From c25f052882d9b1ba8f0bcec80d356cb549080475 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 5 Aug 2026 15:08:00 +0000 Subject: [PATCH 06/37] filer: fix already-exists detection for errors without an error_code Two error shapes stopped mapping to fileAlreadyExistsError when Write moved to sentinel matching. A 409 carrying no error_code unwraps to ErrResourceConflict only. ErrAlreadyExists and ErrResourceAlreadyExists are siblings under it, not parents, so neither matched. The workspace returns exactly this shape, and locker relies on fs.ErrExist here to report which client holds the lock. Match the parent sentinel instead. A 400 with an empty error_code matches no sentinel but ErrBadRequest, so the notebook already-exists check added in #5106 no longer fired. Restore it; that PR notes the newer message format may not have reached all workspaces, and this change has no evidence the rollout has since completed. Both shapes now have test cases; each fails against the previous logic. Co-authored-by: Isaac --- libs/filer/workspace_files_client.go | 26 ++++++++++++++++++++++- libs/filer/workspace_files_client_test.go | 21 ++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 24714339119..1c9e19ef58e 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -258,7 +258,14 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // The server refuses the overwrite even though the caller asked for // it; from the caller's perspective the path is occupied, so we // surface this as already-exists. - if errors.Is(err, apierr.ErrResourceAlreadyExists) || errors.Is(err, apierr.ErrAlreadyExists) { + // ErrResourceConflict covers every 409 regardless of error_code, including + // the bare 409 the workspace returns with only a message. ErrAlreadyExists + // alone would miss those: it and ErrResourceConflict are siblings under the + // SDK's error mapping, so a 409 without an error_code unwraps to the parent + // only. ErrResourceAlreadyExists is listed separately because the + // RESOURCE_ALREADY_EXISTS case arrives as a 400, which no status mapping + // resolves to a conflict. + if errors.Is(err, apierr.ErrResourceConflict) || errors.Is(err, apierr.ErrResourceAlreadyExists) { return fileAlreadyExistsError{absPath} } if errors.Is(err, apierr.ErrInvalidParameterValue) { @@ -288,6 +295,23 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io } } + // This API returns 400 if the file already exists when the object type is notebook. + // Both the historical "Path () already exists." format and the newer + // "RESOURCE_ALREADY_EXISTS: already exists. ..." format end with the same + // "already exists." marker; the JSON error_code is empty in both. The new format + // might not have been rolled out to all workspaces yet, so we anchor on the shared + // marker and return absPath rather than parsing the message. + // + // An empty error_code unwraps to ErrBadRequest by status alone, so this is checked + // after the ErrInvalidParameterValue branch above rather than before it. + if errors.Is(err, apierr.ErrBadRequest) { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + if strings.Contains(aerr.Message, "already exists.") { + return fileAlreadyExistsError{absPath} + } + } + } + // Caller has read access but no write access. if errors.Is(err, apierr.ErrPermissionDenied) { return permissionError{absPath, err} diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 2b71eb9820d..a0d6446ef5c 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -201,6 +201,27 @@ func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { }, expectErrTarget: fileAlreadyExistsError{}, }, + { + // A 409 carrying no error_code unwraps to ErrResourceConflict only, + // not to ErrAlreadyExists; the testserver returns exactly this shape. + name: "409 without error_code maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusConflict, + Message: "Node with name /dir/file.txt already exists.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + // Notebook conflicts arrive as a 400 with an empty error_code, so + // they match neither ErrResourceAlreadyExists nor + // ErrInvalidParameterValue and are detected by message marker. + name: "400 without error_code and 'already exists.' maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + Message: "Path (/dir/notebook) already exists.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, { name: "400 INVALID_PARAMETER_VALUE 'type mismatch' (overwrite=true) maps to fileAlreadyExistsError", apiErr: &apierr.APIError{ From ac122b495b8397e38eb4e29bd1620cd3d6ee614a Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 08:41:49 +0000 Subject: [PATCH 07/37] filer: build the import form locally to keep workspace routing correct The SDK's Workspace.Upload sets X-Databricks-Workspace-Id from cfg.WorkspaceID behind a bare != "" check, so it forwards the CLI-only "none" sentinel as a literal workspace ID. That sentinel is written to .databrickscfg by `auth login --skip-workspace` and names no real workspace, so the upload is misrouted. Encode the same multipart body in newImportForm and post it through apiClient.Do, which routes the header through auth.WorkspaceIDHeaders and maps the sentinel to no header at all. Field names and layout match Upload; the `language` field is omitted because it only applies to format=SOURCE. Co-authored-by: Isaac --- libs/filer/workspace_files_client.go | 76 +++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 1c9e19ef58e..cce13ff456f 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "io/fs" + "mime/multipart" "net/http" "path" "slices" @@ -138,6 +139,43 @@ func (w *WorkspaceFilesClient) workspaceIDHeaders() map[string]string { return auth.WorkspaceIDHeaders(w.workspaceClient.Config) } +// newImportForm encodes the multipart body for POST /api/2.0/workspace/import +// and returns it alongside the Content-Type header carrying the generated +// boundary. Field names and layout match the SDK's Workspace.Upload; format is +// always AUTO so the server classifies each payload as a file or a notebook. +// The `language` field Upload derives for source-format uploads is omitted +// because it only applies to format=SOURCE. +func newImportForm(absPath string, content []byte, overwrite bool) ([]byte, string, error) { + var body bytes.Buffer + mw := multipart.NewWriter(&body) + + if err := mw.WriteField("path", absPath); err != nil { + return nil, "", fmt.Errorf("failed to write path: %w", err) + } + if err := mw.WriteField("format", string(workspace.ImportFormatAuto)); err != nil { + return nil, "", fmt.Errorf("failed to write format: %w", err) + } + // Upload omits this field entirely when false, and the endpoint defaults to + // no overwrite, so only write it when set. + if overwrite { + if err := mw.WriteField("overwrite", "true"); err != nil { + return nil, "", fmt.Errorf("failed to write overwrite: %w", err) + } + } + part, err := mw.CreateFormFile("content", "content") + if err != nil { + return nil, "", fmt.Errorf("failed to create content part: %w", err) + } + if _, err := part.Write(content); err != nil { + return nil, "", fmt.Errorf("failed to write content: %w", err) + } + if err := mw.Close(); err != nil { + return nil, "", fmt.Errorf("failed to finalize multipart body: %w", err) + } + + return body.Bytes(), mw.FormDataContentType(), nil +} + func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, error) { apiClient, err := client.New(w.Config) if err != nil { @@ -164,15 +202,14 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - // Use Workspace.Upload (multipart /api/2.0/workspace/import) rather than - // Workspace.Import (the JSON-body variant of the same endpoint). The JSON - // variant sends content base64-encoded in a `content` field that is capped - // at 10 MB, returning MAX_NOTEBOOK_SIZE_EXCEEDED above it; the multipart - // variant posts the bytes as a file part instead and is bounded only by the - // 500 MB workspace file size limit. See the `content` field description in - // .codegen/cli.json (workspace.Import) and - // https://docs.databricks.com/aws/en/files/workspace ("Workspace file size - // is limited to 500MB"). + // Post the multipart form of /api/2.0/workspace/import rather than its + // JSON-body variant (workspace.Import). The JSON variant sends content + // base64-encoded in a `content` field that is capped at 10 MB, returning + // MAX_NOTEBOOK_SIZE_EXCEEDED above it; the multipart form posts the bytes as + // a file part instead and is bounded only by the 500 MB workspace file size + // limit. See the `content` field description in .codegen/cli.json + // (workspace.Import) and https://docs.databricks.com/aws/en/files/workspace + // ("Workspace file size is limited to 500MB"). // // Because format=AUTO lets the server classify each payload, the applicable // limit depends on how the content is classified, not on our request: @@ -190,14 +227,25 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // /workspace/import and the /workspace-files/import-file endpoint this // replaced, so migrating between the two does not change the maximum // uploadable notebook size. + // + // The body is built here rather than via the SDK's Workspace.Upload because + // that helper derives the routing header from cfg.WorkspaceID with a bare + // != "" check, which sends the CLI-only "none" sentinel as a literal + // workspace ID. Going through apiClient.Do keeps the header under + // workspaceIDHeaders, which maps that sentinel to no header at all. overwrite := slices.Contains(mode, OverwriteIfExists) - uploadOpts := []func(*workspace.Import){ - workspace.UploadFormat(workspace.ImportFormatAuto), + requestBody, contentType, err := newImportForm(absPath, body, overwrite) + if err != nil { + return err } - if overwrite { - uploadOpts = append(uploadOpts, workspace.UploadOverwrite()) + + headers := w.workspaceIDHeaders() + if headers == nil { + headers = make(map[string]string) } - err = w.workspaceClient.Workspace.Upload(ctx, absPath, bytes.NewReader(body), uploadOpts...) + headers["Content-Type"] = contentType + + err = w.apiClient.Do(ctx, http.MethodPost, "/api/2.0/workspace/import", headers, nil, requestBody, nil) // Return early on success. if err == nil { From dc0e0c62372b6bcdf97aa29de8cb006ad11aab6a Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 08:41:57 +0000 Subject: [PATCH 08/37] filer: assert Write against a testserver instead of a mocked SDK client Write no longer calls Workspace.Upload, so the mock expectations on it asserted against a call that is gone. Drive the tests through testserver and inspect the multipart body the CLI actually sends, which pins the wire format rather than the SDK call shape: path, content, format=AUTO, and overwrite present only when requested. Adds coverage for the routing header the previous commit fixes, including the "none" sentinel case. Co-authored-by: Isaac --- libs/filer/workspace_files_client_test.go | 246 +++++++++++++++------- 1 file changed, 175 insertions(+), 71 deletions(-) diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index a0d6446ef5c..d011b2a703b 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -2,23 +2,24 @@ package filer import ( "bytes" - "context" "encoding/json" + "errors" "io" "io/fs" + "mime" + "mime/multipart" "net/http" "strings" "testing" "time" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -108,60 +109,130 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } -// Confirms Write uploads through Workspace.Upload with format=AUTO, and that -// the overwrite flag is set only when OverwriteIfExists is passed. AUTO is what -// lets the server decide between storing content as a file or a notebook, so -// the format is asserted explicitly rather than left to the SDK default -// (SOURCE), which would import every file as a notebook. +// importFormFields parses the multipart body of a recorded /workspace/import +// request into a field name -> value map. +func importFormFields(t *testing.T, contentType string, body []byte) map[string]string { + t.Helper() + + _, params, err := mime.ParseMediaType(contentType) + require.NoError(t, err) + mr := multipart.NewReader(bytes.NewReader(body), params["boundary"]) + + fields := map[string]string{} + for { + part, err := mr.NextPart() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + data, err := io.ReadAll(part) + require.NoError(t, err) + fields[part.FormName()] = string(data) + } + return fields +} + +// Confirms Write posts a multipart /workspace/import body with format=AUTO, and +// that the overwrite field is present only when OverwriteIfExists is passed. +// AUTO is what lets the server decide between storing content as a file or a +// notebook, so the format is asserted explicitly rather than left to the +// endpoint default (SOURCE), which would import every file as a notebook. func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { tests := []struct { - name string - modes []WriteMode - expectOverride bool + name string + modes []WriteMode + expectPresent bool }{ { - name: "no overwrite", - modes: nil, - expectOverride: false, + name: "no overwrite", + modes: nil, + expectPresent: false, }, { - name: "overwrite", - modes: []WriteMode{OverwriteIfExists}, - expectOverride: true, + name: "overwrite", + modes: []WriteMode{OverwriteIfExists}, + expectPresent: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - mw := mocks.NewMockWorkspaceClient(t) - workspaceApi := mw.GetMockWorkspaceAPI() - - workspaceApi.EXPECT().Upload( - mock.Anything, - "/dir/file.txt", - mock.Anything, - mock.Anything, - mock.Anything, - ).RunAndReturn(func(_ context.Context, _ string, r io.Reader, opts ...func(*workspace.Import)) error { - body, err := io.ReadAll(r) - require.NoError(t, err) - assert.Equal(t, "hello", string(body)) - - i := &workspace.Import{} - for _, opt := range opts { - opt(i) - } - assert.Equal(t, workspace.ImportFormatAuto, i.Format) - assert.Equal(t, tc.expectOverride, i.Overwrite) - return nil - }).Once() - - c := WorkspaceFilesClient{ - workspaceClient: mw.WorkspaceClient, - root: NewWorkspaceRootPath("/dir"), + var gotForm map[string]string + server := testserver.New(t) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + gotForm = importFormFields(t, req.Headers.Get("Content-Type"), req.Body) + return testserver.Response{StatusCode: http.StatusOK} + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewWorkspaceFilesClient(client, "/dir") + require.NoError(t, err) + + require.NoError(t, f.Write(t.Context(), "file.txt", strings.NewReader("hello"), tc.modes...)) + + assert.Equal(t, "/dir/file.txt", gotForm["path"]) + assert.Equal(t, "hello", gotForm["content"]) + assert.Equal(t, string(workspace.ImportFormatAuto), gotForm["format"]) + overwrite, present := gotForm["overwrite"] + assert.Equal(t, tc.expectPresent, present) + if present { + assert.Equal(t, "true", overwrite) } - err := c.Write(t.Context(), "file.txt", strings.NewReader("hello"), tc.modes...) + }) + } +} + +// Confirms the workspace routing header is sent when the config carries a real +// workspace ID, and that the CLI-only "none" sentinel is never forwarded as a +// literal routing identifier. The sentinel is written to .databrickscfg by +// `auth login --skip-workspace`; the platform has no workspace named "none", +// so sending it would misroute the upload. +func TestWorkspaceFilesClientWriteWorkspaceIDHeader(t *testing.T) { + tests := []struct { + name string + workspaceID string + expect string + }{ + { + name: "real workspace ID is forwarded", + workspaceID: "7474644166319138", + expect: "7474644166319138", + }, + { + name: "none sentinel is not forwarded", + workspaceID: auth.WorkspaceIDNone, + expect: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotHeader string + server := testserver.New(t) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + gotHeader = req.Headers.Get(auth.WorkspaceIDHeader) + return testserver.Response{StatusCode: http.StatusOK} + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + WorkspaceID: tc.workspaceID, + }) + require.NoError(t, err) + + f, err := NewWorkspaceFilesClient(client, "/dir") require.NoError(t, err) + + require.NoError(t, f.Write(t.Context(), "file.txt", strings.NewReader("hello"))) + assert.Equal(t, tc.expect, gotHeader) }) } } @@ -263,17 +334,28 @@ func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - mw := mocks.NewMockWorkspaceClient(t) - workspaceApi := mw.GetMockWorkspaceAPI() - workspaceApi.EXPECT().Upload( - mock.Anything, "/dir/file.txt", mock.Anything, mock.Anything, mock.Anything, - ).Return(tc.apiErr).Once() - - c := WorkspaceFilesClient{ - workspaceClient: mw.WorkspaceClient, - root: NewWorkspaceRootPath("/dir"), - } - err := c.Write(t.Context(), "file.txt", bytes.NewReader([]byte("data")), tc.mode...) + server := testserver.New(t) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + return testserver.Response{ + StatusCode: tc.apiErr.StatusCode, + Body: map[string]string{ + "error_code": tc.apiErr.ErrorCode, + "message": tc.apiErr.Message, + }, + } + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewWorkspaceFilesClient(client, "/dir") + require.NoError(t, err) + + err = f.Write(t.Context(), "file.txt", bytes.NewReader([]byte("data")), tc.mode...) require.Error(t, err) switch target := tc.expectErrTarget.(type) { case noSuchDirectoryError: @@ -367,26 +449,48 @@ func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { // the buffered body, so this also covers that the content survives the second // attempt. func TestWorkspaceFilesClientWriteCreatesParentDirectories(t *testing.T) { - mw := mocks.NewMockWorkspaceClient(t) - workspaceApi := mw.GetMockWorkspaceAPI() + var uploads int + var mkdirs []string + var lastContent string - // First Upload returns 404, second returns success after MkdirsByPath. - workspaceApi.EXPECT().Upload( - mock.Anything, "/dir/sub/file.txt", mock.Anything, mock.Anything, mock.Anything, - ).Return(&apierr.APIError{StatusCode: http.StatusNotFound, Message: "not found"}).Once() - - workspaceApi.EXPECT().MkdirsByPath(mock.Anything, "/dir/sub").Return(nil).Once() + server := testserver.New(t) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + uploads++ + // The first attempt 404s as it would against a missing parent; the retry + // after mkdirs succeeds. + if uploads == 1 { + return testserver.Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{"message": "The parent folder (/dir/sub) does not exist."}, + } + } + form := importFormFields(t, req.Headers.Get("Content-Type"), req.Body) + lastContent = form["content"] + return testserver.Response{StatusCode: http.StatusOK} + }) + server.Handle("POST", "/api/2.0/workspace/mkdirs", func(req testserver.Request) any { + var request workspace.Mkdirs + require.NoError(t, json.Unmarshal(req.Body, &request)) + mkdirs = append(mkdirs, request.Path) + return testserver.Response{StatusCode: http.StatusOK} + }) + testserver.AddDefaultHandlers(server) - workspaceApi.EXPECT().Upload( - mock.Anything, "/dir/sub/file.txt", mock.Anything, mock.Anything, mock.Anything, - ).Return(nil).Once() + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) - c := WorkspaceFilesClient{ - workspaceClient: mw.WorkspaceClient, - root: NewWorkspaceRootPath("/dir"), - } - err := c.Write(t.Context(), "sub/file.txt", strings.NewReader("data"), CreateParentDirectories) + f, err := NewWorkspaceFilesClient(client, "/dir") require.NoError(t, err) + + require.NoError(t, f.Write(t.Context(), "sub/file.txt", strings.NewReader("data"), CreateParentDirectories)) + + assert.Equal(t, 2, uploads) + assert.Equal(t, []string{"/dir/sub"}, mkdirs) + // The retry re-reads the buffered body, so the content must survive. + assert.Equal(t, "data", lastContent) } // Confirms a workspace get-status payload unmarshals into wsfsFileInfo and that From c97c1af5e22a5d68f0503bbae7a873125ceb3da3 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 08:49:55 +0000 Subject: [PATCH 09/37] filer: condense the upload size-limit and conflict comments Tighten the size-limit block to the classification rule and its two doc links, and reorganize the path-conflict list so each bullet names the branch that handles it, including the two shapes that carry no error_code. Also move multipartContentLimit above the function that reads it. Co-authored-by: Isaac --- acceptance/internal/prepare_server.go | 10 +++--- libs/filer/workspace_files_client.go | 48 +++++++++++---------------- 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index d2e34cf6cdd..250adfbf5bc 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -342,6 +342,11 @@ func getLoggedRequest(req *testserver.Request, includedHeaders []string) LoggedR return result } +// Multipart parts larger than this limit are summarized as a size placeholder +// to keep recorded fixtures small. Reviewers care about *what* was uploaded +// (path, format, overwrite flag), not the bytes themselves, for large blobs. +const multipartContentLimit = 4096 + // normalizeMultipartBody returns a deterministic representation of a multipart // form body if the request's Content-Type is multipart/*. The second return // value is false if the body is not multipart or cannot be parsed. @@ -386,11 +391,6 @@ func normalizeMultipartBody(req *testserver.Request) (any, bool) { return map[string]any{"multipart_form": parts}, true } -// Multipart parts larger than this limit are summarized as a size placeholder -// to keep recorded fixtures small. Reviewers care about *what* was uploaded -// (path, format, overwrite flag), not the bytes themselves, for large blobs. -const multipartContentLimit = 4096 - func filterHeaders(h http.Header, includedHeaders []string) http.Header { headers := make(http.Header) for k, v := range h { diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index cce13ff456f..8d50b5f5725 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -203,30 +203,15 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io } // Post the multipart form of /api/2.0/workspace/import rather than its - // JSON-body variant (workspace.Import). The JSON variant sends content - // base64-encoded in a `content` field that is capped at 10 MB, returning - // MAX_NOTEBOOK_SIZE_EXCEEDED above it; the multipart form posts the bytes as - // a file part instead and is bounded only by the 500 MB workspace file size - // limit. See the `content` field description in .codegen/cli.json - // (workspace.Import) and https://docs.databricks.com/aws/en/files/workspace - // ("Workspace file size is limited to 500MB"). - // - // Because format=AUTO lets the server classify each payload, the applicable - // limit depends on how the content is classified, not on our request: - // - // - Regular file: 500 MB. - // - Notebook in a source format, i.e. content whose extension and - // "Databricks notebook source" header make AUTO import it as a - // notebook: 10 MB. - // - Notebook in IPYNB format: 100 MB. - // - // Notebook limits are documented at + // JSON-body variant (workspace.Import), whose base64 `content` field is + // capped at 10 MB; the multipart form is bounded only by the 500 MB + // workspace file size limit. Because format=AUTO lets the server classify + // each payload, the applicable limit follows the classification: 500 MB for + // a regular file, 100 MB for an IPYNB notebook, 10 MB for a source-format + // notebook. Those notebook limits applied equally to the import-file + // endpoint this replaced, so the migration does not lower any ceiling. + // See https://docs.databricks.com/aws/en/files/workspace and // https://docs.databricks.com/aws/en/notebooks/notebook-limitations - // ("Import and export is supported for IPYNB notebooks up to 100 MB" and up - // to 10 MB for other formats). They are enforced by the workspace for both - // /workspace/import and the /workspace-files/import-file endpoint this - // replaced, so migrating between the two does not change the maximum - // uploadable notebook size. // // The body is built here rather than via the SDK's Workspace.Upload because // that helper derives the routing header from cfg.WorkspaceID with a bare @@ -271,17 +256,22 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return w.Write(ctx, name, bytes.NewReader(body), sliceWithout(mode, CreateParentDirectories)...) } - // Path already taken. /workspace/import returns this in three shapes, - // verified against a real workspace: + // Path already taken. /workspace/import signals this with several + // status/error_code combinations, all verified against a real workspace, and + // each of the branches below handles one group of them: // // - 400 RESOURCE_ALREADY_EXISTS — sequential conflict, no overwrite flag. // Example: "/Users/me/foo.txt already exists. Please pass overwrite=true // to overwrite it." // - // - 409 ALREADY_EXISTS — concurrent contention (observed in TestLock when - // five lockers race to write deploy.lock). - // Example: "Node with name /Users/me/.bundle/.../deploy.lock already - // exists. Please pass overwrite=true to update it." + // - 409, with ALREADY_EXISTS or with no error_code at all — concurrent + // contention (observed in TestLock when five lockers race to write + // deploy.lock). Example: "Node with name + // /Users/me/.bundle/.../deploy.lock already exists. Please pass + // overwrite=true to update it." + // + // - 400 with no error_code — notebook conflict; detected by message marker + // in the ErrBadRequest branch further below. // // - 400 INVALID_PARAMETER_VALUE — overwrite=true on a path where the // existing object's node type differs from the upload. Two distinct From 90becb3dc18fab7f11427cc70ca155d16a8969c1 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 08:50:10 +0000 Subject: [PATCH 10/37] testserver: tighten format=AUTO handling in the import fake Match the notebook extension case-insensitively, so a ".R" source is detected the way the real endpoint detects it. Also require the format field to be AUTO rather than accepting an absent value. The real endpoint defaults to SOURCE, which imports every payload as a notebook, so a caller that omits the field is a bug the fake should surface. The rejection message now reports what was received. Co-authored-by: Isaac --- libs/testserver/fake_workspace.go | 5 ++++- libs/testserver/handlers.go | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index ad457115c37..660b9317670 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -840,8 +840,11 @@ func (s *FakeWorkspace) hasChildren(dirPath string) bool { // a file whose extension is .py / .sql / .scala / .r and whose content starts // with the language-appropriate "Databricks notebook source" line-comment is // stored as a NOTEBOOK with the corresponding language; otherwise it's a FILE. +// +// The extension is lowercased because the real endpoint matches it +// case-insensitively, and ".R" is the conventional spelling for R sources. func detectNotebookLanguage(extension string, body []byte) (workspace.Language, bool) { - switch extension { + switch strings.ToLower(extension) { case ".py": if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { return workspace.LanguagePython, true diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 9d77f350b10..89b21d4f1ea 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -193,9 +193,12 @@ func AddDefaultHandlers(server *Server) { } } - if format != "" && format != string(workspace.ImportFormatAuto) { + // Require format explicitly: the real endpoint defaults to SOURCE, which + // would import every upload as a notebook, so a caller that drops the + // field is a bug rather than a request for the fake's usual behavior. + if format != string(workspace.ImportFormatAuto) { return Response{ - Body: "internal error: The test server only supports auto format.", + Body: fmt.Sprintf("internal error: The test server only supports auto format, got %q.", format), StatusCode: http.StatusInternalServerError, } } From c5a41a3deb4f13d75183edf015e8b1cd0f52eec2 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 08:50:23 +0000 Subject: [PATCH 11/37] acceptance: parse fault.py arguments with argparse Replace the hand-rolled sys.argv slicing with argparse. The manual parser had to special-case --body-contains before the positionals, so the flag only worked in that one position and the usage string was maintained in two places. Behavior is unchanged: same positionals, same optional error_code default, same payload. Co-authored-by: Isaac --- acceptance/bin/fault.py | 79 +++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/acceptance/bin/fault.py b/acceptance/bin/fault.py index 261c5570862..4d6f5a22b9f 100755 --- a/acceptance/bin/fault.py +++ b/acceptance/bin/fault.py @@ -1,32 +1,44 @@ #!/usr/bin/env python3 """Set up a fault rule on the testserver for the current test token. -Usage: fault.py [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] - - --after-handler - run the handler first; keep its effect and replace only the response - PATTERN HTTP method and path, supports trailing * wildcard, - e.g. "PUT /api/2.0/permissions/pipelines/*" - STATUS_CODE HTTP status code to return, e.g. 504 - OFFSET number of requests to let through before fault starts - TIMES number of times to return the fault response - ERROR_CODE optional error_code for the response body, e.g. - MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED) - --body-contains SUBSTR - only fire when the request body contains SUBSTR. Needed to - target a single file's /workspace/import upload, since every - upload shares the same method+path and only differs by the - multipart "path" form field. - The rule is scoped to the current DATABRICKS_TOKEN so it only affects the test that registers it, even when tests share a server. """ +import argparse import json import os import sys import urllib.request +parser = argparse.ArgumentParser(description="Set up a fault rule on the testserver for the current test token.") +parser.add_argument( + "pattern", help='HTTP method and path, supports trailing * wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*"' +) +parser.add_argument("status_code", type=int, help="HTTP status code to return, e.g. 504") +parser.add_argument("offset", type=int, help="number of requests to let through before fault starts") +parser.add_argument("times", type=int, help="number of times to return the fault response") +parser.add_argument( + "error_code", + nargs="?", + default="INJECTED", + help="error_code for the response body, e.g. MAX_CHILD_NODE_SIZE_EXCEEDED", +) +parser.add_argument( + "--after-handler", + action="store_true", + help="run the handler first; keep its effect and replace only the response", +) +parser.add_argument( + "--body-contains", + default="", + metavar="SUBSTR", + help="only fire when the request body contains SUBSTR. Needed to target a single file's " + "/workspace/import upload, since every upload shares the same method+path and only " + 'differs by the multipart "path" form field.', +) +args = parser.parse_args() + host = os.environ.get("DATABRICKS_HOST", "") token = os.environ.get("DATABRICKS_TOKEN", "") @@ -34,36 +46,17 @@ print("DATABRICKS_HOST not set", file=sys.stderr) sys.exit(1) -args = sys.argv[1:] -after_handler = "--after-handler" in args -if after_handler: - args.remove("--after-handler") - -body_contains = "" -if len(args) >= 2 and args[0] == "--body-contains": - body_contains = args[1] - args = args[2:] - -if len(args) not in (4, 5): - print( - f"usage: {sys.argv[0]} [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", - file=sys.stderr, - ) - sys.exit(1) - -pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3]) -error_code = args[4] if len(args) == 5 else "INJECTED" -body = json.dumps({"error_code": error_code, "message": "Fault injected by test."}) +body = json.dumps({"error_code": args.error_code, "message": "Fault injected by test."}) data = json.dumps( { - "pattern": pattern, - "status_code": status_code, + "pattern": args.pattern, + "status_code": args.status_code, "body": body, - "offset": offset, - "times": times, - "after_handler": after_handler, - "body_contains": body_contains, + "offset": args.offset, + "times": args.times, + "after_handler": args.after_handler, + "body_contains": args.body_contains, } ).encode() From 0f683dbac3978c5a9e9751f61147145e2d18c0d5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 08:52:09 +0000 Subject: [PATCH 12/37] testserver: cover case-insensitive notebook extension detection Guards the ToLower added to detectNotebookLanguage: the test fails against the previous case-sensitive switch, where ".R" was stored as a FILE at the unstripped path. Co-authored-by: Isaac --- libs/testserver/workspace_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libs/testserver/workspace_test.go b/libs/testserver/workspace_test.go index cde1c9741c6..401758c6648 100644 --- a/libs/testserver/workspace_test.go +++ b/libs/testserver/workspace_test.go @@ -88,6 +88,24 @@ func TestWorkspaceDeleteNonRecursiveRequiresEmptyDirectory(t *testing.T) { assert.Equal(t, 404, getStatus(t, server.URL, "/a/b")) } +// format=AUTO matches the extension case-insensitively, so ".R" — the +// conventional spelling for R sources — is detected as a notebook just like +// ".r". Notebooks are stored with the extension stripped. +func TestWorkspaceImportDetectsNotebookExtensionCaseInsensitively(t *testing.T) { + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + + mkdirs(t, server.URL, "/test-dir") + + for _, name := range []string{"lower.r", "upper.R"} { + require.Equal(t, 200, importFile(t, server.URL, "/test-dir/"+name, "# Databricks notebook source\n"), name) + } + + // Both land at the extension-stripped path, which only happens for notebooks. + assert.Equal(t, 200, getStatus(t, server.URL, "/test-dir/lower")) + assert.Equal(t, 200, getStatus(t, server.URL, "/test-dir/upper")) +} + // mkdirs creates all intermediate directories, matching "mkdir -p". func TestWorkspaceMkdirsRecursive(t *testing.T) { server := testserver.New(t) From bb2a05dfcd12ca5a68ba82311c8f1c93f42d4a08 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 09:16:02 +0000 Subject: [PATCH 13/37] acceptance: re-aim run-submit-deps upload assertions at /workspace/import The test filtered recorded uploads by URL path, which no longer matches: /workspace/import carries the target filename in the multipart body. Both assertions still show exactly two uploads (the config and the command), so the test's point -- that the requirements file is never uploaded -- is unchanged; only the recorded request shape differs. This test was added in #6166, which merged into this branch after the upload migration, so it was not covered by the earlier fixture updates. Mirrors the fix applied to ai_runtime_task/local_code_source for the same reason. Co-authored-by: Isaac --- acceptance/experimental/air/run-submit-deps/script | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/acceptance/experimental/air/run-submit-deps/script b/acceptance/experimental/air/run-submit-deps/script index 12dc884740d..b644c9177ff 100644 --- a/acceptance/experimental/air/run-submit-deps/script +++ b/acceptance/experimental/air/run-submit-deps/script @@ -2,7 +2,10 @@ title "submit with inline dependencies" trace $CLI experimental air run -f run.yaml title "only config + command are uploaded; no requirements.yaml" -trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep +# Uploads go to POST /workspace/import, which carries the target filename in the +# multipart body rather than the URL, so each uploaded path shows up under +# body.multipart_form.path instead of in the request path. +trace print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep title "declared deps ride on environments[].spec.dependencies" trace print_requests.py //api/2.2/jobs/runs/submit From 78428663b99df4ba0a9d7efe63a3da42cb54a0dc Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 10:07:07 +0000 Subject: [PATCH 14/37] acceptance: pin ai_runtime_task snapshot inputs to LF The local_code_source test asserts the uploaded size of each .air_snapshots tarball. On Windows, git checks the packaged files out with CRLF, which adds a byte per newline and shifts every recorded size, so both Windows jobs failed with 2-4 byte differences. Pin the packaged files to eol=lf, following the per-directory pattern in acceptance/experimental/air/run-submit-deps/.gitattributes. Only the files that actually land in a tarball are listed; debug.log is excluded by .gitignore and stays unpinned. The recorded sizes only became load-bearing when uploads moved to /workspace/import, which records the multipart body; the previous endpoint put the filename in the URL and never recorded the payload. Co-authored-by: Isaac --- .../ai_runtime_task/local_code_source/.gitattributes | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 acceptance/bundle/ai_runtime_task/local_code_source/.gitattributes diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/.gitattributes b/acceptance/bundle/ai_runtime_task/local_code_source/.gitattributes new file mode 100644 index 00000000000..8abc0d3b855 --- /dev/null +++ b/acceptance/bundle/ai_runtime_task/local_code_source/.gitattributes @@ -0,0 +1,9 @@ +# These files are packaged verbatim into the .air_snapshots tarballs whose +# uploaded sizes the test asserts, so their line endings must stay \n on every +# OS — a Windows \r would change every recorded payload size. +src/.gitignore text eol=lf +src/command.sh text eol=lf +src/data/model.bin text eol=lf +src/train.py text eol=lf +src2/command.sh text eol=lf +src2/train.py text eol=lf From 770cf457a34e1e7828c665cc704b29a0ad3fcf22 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 11:16:03 +0000 Subject: [PATCH 15/37] acceptance: keep update_file.py from rewriting line endings Python's text mode translates \n to \r\n on write under Windows, so editing a file through this helper replaced its LF endings with CRLF. Pinning the checked out bytes with .gitattributes does not help, because the rewrite happens after checkout: in ai_runtime_task/local_code_source only the upload recorded after the update_file.py step still reported a different size on Windows. Pass newline="" on both the read and the write so the file's existing endings survive verbatim, in either direction. Co-authored-by: Isaac --- acceptance/bin/update_file.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/acceptance/bin/update_file.py b/acceptance/bin/update_file.py index ef643239533..46ba30925af 100755 --- a/acceptance/bin/update_file.py +++ b/acceptance/bin/update_file.py @@ -20,9 +20,12 @@ # call update_file.py on that file. assert filename != "output.txt" -data = open(filename).read() +data = open(filename, newline="").read() newdata = data.replace(old, new) if newdata == data: sys.exit(f"{old=} not found in {filename=}\n{data}") -with open(filename, "w") as fobj: +# newline="" on both ends keeps the file's line endings byte-for-byte. Without it +# Python rewrites every \n as \r\n on Windows, which changes the content of files +# whose uploaded bytes or hashes a test asserts. +with open(filename, "w", newline="") as fobj: fobj.write(newdata) From 2e42fb4924063df9a5b9fcab69a7ec56b0732a6d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 7 Aug 2026 10:01:09 +0000 Subject: [PATCH 16/37] add a descriptive error on 500 upload failure --- libs/filer/workspace_files_client.go | 5 ++++- libs/filer/workspace_files_client_test.go | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 8d50b5f5725..8055a3d53de 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -355,7 +355,10 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return permissionError{absPath, err} } - return err + // Any other failure (e.g. a server-side error with an empty message) is + // surfaced with the target path so the user can tell which upload failed; + // %w keeps the original error inspectable by callers. + return fmt.Errorf("failed to upload %s: %w", absPath, err) } func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index d011b2a703b..923e64c34b9 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -365,10 +365,12 @@ func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { case permissionError: assert.ErrorAs(t, err, &target) case nil: - // passthrough — same APIError pointer + // passthrough — the underlying APIError stays inspectable, + // and the message names the file that failed to upload. var aerr *apierr.APIError require.ErrorAs(t, err, &aerr) assert.Equal(t, tc.apiErr.StatusCode, aerr.StatusCode) + assert.Contains(t, err.Error(), "/dir/file.txt") } }) } From b42de03d900e6106fcb77b291784d68164709b82 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 7 Aug 2026 11:16:59 +0000 Subject: [PATCH 17/37] acceptance: regenerate upload goldens for the descriptive error fb99c278a wrapped unmapped upload failures with "failed to upload ", which changed the message the upload/internal_server_error and upload/timeout tests capture. Regenerate their goldens to match. Co-authored-by: Isaac --- acceptance/bundle/upload/internal_server_error/output.txt | 4 ++-- acceptance/bundle/upload/timeout/output.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/upload/internal_server_error/output.txt b/acceptance/bundle/upload/internal_server_error/output.txt index 24d8b2cc4a5..d7f01415559 100644 --- a/acceptance/bundle/upload/internal_server_error/output.txt +++ b/acceptance/bundle/upload/internal_server_error/output.txt @@ -1,5 +1,5 @@ -Error: Failed to acquire deployment lock: Internal Server Error -Error: Internal Server Error (500) +Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal Server Error +Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal Server Error (500) Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import HTTP Status: 500 Internal Server Error diff --git a/acceptance/bundle/upload/timeout/output.txt b/acceptance/bundle/upload/timeout/output.txt index 3b6eb72f28b..0c779d73dc2 100644 --- a/acceptance/bundle/upload/timeout/output.txt +++ b/acceptance/bundle/upload/timeout/output.txt @@ -1,3 +1,3 @@ -Error: Failed to acquire deployment lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity -Error: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity +Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity +Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity From a99899fda154cb3927e82fb980cb07fd69f0222b Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 7 Aug 2026 11:50:17 +0000 Subject: [PATCH 18/37] acceptance: sort auto-migrate-empty-tfstate uploads deterministically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit print_requests.py --sort orders by the whole request, including the multipart body. One synced file is out.requests.txt, whose recorded body differs between runs, so its position in the list was nondeterministic — passing locally but failing in CI. Drop --sort and sort the displayed "METHOD path" lines instead, which are stable. The contains.py assertions are unaffected. Co-authored-by: Isaac --- .../auto-migrate-empty-tfstate/output.txt | 18 +++++++++--------- .../migrate/auto-migrate-empty-tfstate/script | 5 ++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt index e19d0f5a81d..ef3365a5818 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt @@ -22,21 +22,21 @@ direct_migrated_via_config true === Sweep also affected the workspace: .backup uploaded, terraform.tfstate deleted ->>> print_requests.py //workspace/import //workspace/delete --sort -"POST /api/2.0/workspace/delete" -"POST /api/2.0/workspace/delete" -"POST /api/2.0/workspace/delete" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" +>>> print_requests.py //workspace/import //workspace/delete +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup" +"POST /api/2.0/workspace/delete" +"POST /api/2.0/workspace/delete" +"POST /api/2.0/workspace/delete" === debug states confirms no terraform.tfstate anywhere; only .backup remains diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script index afc0a262151..03c39f4d84e 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script @@ -19,7 +19,10 @@ title "Sweep is recorded in telemetry as via-config\n" trace print_migration_telemetry title "Sweep also affected the workspace: .backup uploaded, terraform.tfstate deleted\n" -trace print_requests.py //workspace/import //workspace/delete --sort | jq '.method + " " + (.body.multipart_form.path // .path)' | contains.py 'terraform.tfstate.backup' 'workspace/delete' +# Sort on the displayed method+path rather than with print_requests.py --sort, +# which orders by the whole request including the multipart body — nondeterministic +# for the uploaded out.requests.txt whose body content varies between runs. +trace print_requests.py //workspace/import //workspace/delete | jq '.method + " " + (.body.multipart_form.path // .path)' | sort | contains.py 'terraform.tfstate.backup' 'workspace/delete' title "debug states confirms no terraform.tfstate anywhere; only .backup remains\n" trace $CLI bundle debug states --force-pull From 11fefde0b9cfb6a0a4730e688eac5b43586db573 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 7 Aug 2026 12:43:00 +0000 Subject: [PATCH 19/37] acceptance: regenerate sync-upload-edge-cases out.test.toml #6196 removed the Local config field and regenerated every out.test.toml snapshot, but this test was added on this branch in parallel, so its committed snapshot still carried Local = true. The post-test "no files changed" check regenerates it and fails on the diff. Regenerate to match. Co-authored-by: Isaac --- acceptance/bundle/sync-upload-edge-cases/out.test.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/bundle/sync-upload-edge-cases/out.test.toml b/acceptance/bundle/sync-upload-edge-cases/out.test.toml index e90b6d5d1ba..0938e678987 100644 --- a/acceptance/bundle/sync-upload-edge-cases/out.test.toml +++ b/acceptance/bundle/sync-upload-edge-cases/out.test.toml @@ -1,3 +1,2 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 191309388e729f3e420fb7bbddf00b3d16f56205 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 7 Aug 2026 13:09:42 +0000 Subject: [PATCH 20/37] acceptance: match update_file.py search strings across CRLF The previous newline="" fix stopped Python from rewriting \n to \r\n on write, but it also made the read preserve CRLF verbatim. Script search strings are always LF, so a multi-line replacement (e.g. duplicate_yaml_merge_key) no longer matched on Windows, where databricks.yml is checked out with CRLF, and update_file.py errored with "old=... not found". Normalize CRLF to LF on read so the LF search string matches, and keep the newline="" write so the result stays LF. Net effect on Windows: the edited file is normalized to LF, which is what every downstream upload/hash assertion expects anyway. Co-authored-by: Isaac --- acceptance/bin/update_file.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/acceptance/bin/update_file.py b/acceptance/bin/update_file.py index 46ba30925af..b78c9dda9b2 100755 --- a/acceptance/bin/update_file.py +++ b/acceptance/bin/update_file.py @@ -20,12 +20,14 @@ # call update_file.py on that file. assert filename != "output.txt" -data = open(filename, newline="").read() +# Read raw and normalize CRLF to LF so the search string (always LF in the +# script) matches on Windows, where the file is checked out with CRLF. Write +# with newline="" so Python does not turn the \n back into \r\n: the acceptance +# harness treats every file as LF, and a stray \r would change any uploaded +# bytes or content hash a test later asserts. +data = open(filename, newline="").read().replace("\r\n", "\n") newdata = data.replace(old, new) if newdata == data: sys.exit(f"{old=} not found in {filename=}\n{data}") -# newline="" on both ends keeps the file's line endings byte-for-byte. Without it -# Python rewrites every \n as \r\n on Windows, which changes the content of files -# whose uploaded bytes or hashes a test asserts. with open(filename, "w", newline="") as fobj: fobj.write(newdata) From 0552229d5d7eaec7e4cddd9ed293efb0214dc19a Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 12 Aug 2026 12:42:52 +0000 Subject: [PATCH 21/37] filer: upload via the SDK's Workspace.Upload Replace the hand-built multipart form (newImportForm + apiClient.Do) with the SDK's multipart Workspace.Upload. To keep the CLI-only "none" workspace-id sentinel from being forwarded as a literal routing id by Upload's bare cfg.WorkspaceID != "" check, normalize it to "" in NewWorkspaceFilesClient. The error mapping and multipart wire format are unchanged, so the existing Write tests still hold; add TestNewWorkspaceFilesClientNormalizesWorkspaceID to pin the sentinel normalization. Co-authored-by: Isaac --- libs/filer/workspace_files_client.go | 76 ++++------------------- libs/filer/workspace_files_client_test.go | 30 +++++++++ 2 files changed, 42 insertions(+), 64 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 8055a3d53de..509e303664f 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "io/fs" - "mime/multipart" "net/http" "path" "slices" @@ -139,44 +138,13 @@ func (w *WorkspaceFilesClient) workspaceIDHeaders() map[string]string { return auth.WorkspaceIDHeaders(w.workspaceClient.Config) } -// newImportForm encodes the multipart body for POST /api/2.0/workspace/import -// and returns it alongside the Content-Type header carrying the generated -// boundary. Field names and layout match the SDK's Workspace.Upload; format is -// always AUTO so the server classifies each payload as a file or a notebook. -// The `language` field Upload derives for source-format uploads is omitted -// because it only applies to format=SOURCE. -func newImportForm(absPath string, content []byte, overwrite bool) ([]byte, string, error) { - var body bytes.Buffer - mw := multipart.NewWriter(&body) - - if err := mw.WriteField("path", absPath); err != nil { - return nil, "", fmt.Errorf("failed to write path: %w", err) - } - if err := mw.WriteField("format", string(workspace.ImportFormatAuto)); err != nil { - return nil, "", fmt.Errorf("failed to write format: %w", err) - } - // Upload omits this field entirely when false, and the endpoint defaults to - // no overwrite, so only write it when set. - if overwrite { - if err := mw.WriteField("overwrite", "true"); err != nil { - return nil, "", fmt.Errorf("failed to write overwrite: %w", err) - } - } - part, err := mw.CreateFormFile("content", "content") - if err != nil { - return nil, "", fmt.Errorf("failed to create content part: %w", err) - } - if _, err := part.Write(content); err != nil { - return nil, "", fmt.Errorf("failed to write content: %w", err) - } - if err := mw.Close(); err != nil { - return nil, "", fmt.Errorf("failed to finalize multipart body: %w", err) +func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, error) { + // Workspace.Upload/Download forward cfg.WorkspaceID behind a bare != "" check, + // so normalize the "none" sentinel to "" to keep the SDK from routing on it. + if w.Config.WorkspaceID == auth.WorkspaceIDNone { + w.Config.WorkspaceID = "" } - return body.Bytes(), mw.FormDataContentType(), nil -} - -func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, error) { apiClient, err := client.New(w.Config) if err != nil { return nil, err @@ -202,35 +170,15 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - // Post the multipart form of /api/2.0/workspace/import rather than its - // JSON-body variant (workspace.Import), whose base64 `content` field is - // capped at 10 MB; the multipart form is bounded only by the 500 MB - // workspace file size limit. Because format=AUTO lets the server classify - // each payload, the applicable limit follows the classification: 500 MB for - // a regular file, 100 MB for an IPYNB notebook, 10 MB for a source-format - // notebook. Those notebook limits applied equally to the import-file - // endpoint this replaced, so the migration does not lower any ceiling. - // See https://docs.databricks.com/aws/en/files/workspace and - // https://docs.databricks.com/aws/en/notebooks/notebook-limitations - // - // The body is built here rather than via the SDK's Workspace.Upload because - // that helper derives the routing header from cfg.WorkspaceID with a bare - // != "" check, which sends the CLI-only "none" sentinel as a literal - // workspace ID. Going through apiClient.Do keeps the header under - // workspaceIDHeaders, which maps that sentinel to no header at all. - overwrite := slices.Contains(mode, OverwriteIfExists) - requestBody, contentType, err := newImportForm(absPath, body, overwrite) - if err != nil { - return err - } - - headers := w.workspaceIDHeaders() - if headers == nil { - headers = make(map[string]string) + // Upload with the SDK's multipart Workspace.Upload; format=AUTO lets the server + // classify each payload as file or notebook. The JSON-body workspace.Import caps + // content at 10 MB, the multipart form at the 500 MB workspace file limit. + opts := []workspace.UploadOption{workspace.UploadFormat(workspace.ImportFormatAuto)} + if slices.Contains(mode, OverwriteIfExists) { + opts = append(opts, workspace.UploadOverwrite()) } - headers["Content-Type"] = contentType - err = w.apiClient.Do(ctx, http.MethodPost, "/api/2.0/workspace/import", headers, nil, requestBody, nil) + err = w.workspaceClient.Workspace.Upload(ctx, absPath, bytes.NewReader(body), opts...) // Return early on success. if err == nil { diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 923e64c34b9..e598108bfdf 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -109,6 +109,36 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } +// Confirms NewWorkspaceFilesClient clears the CLI-only "none" sentinel so the +// SDK's Workspace.Upload/Download never route on it, while a real workspace ID +// is left untouched. +func TestNewWorkspaceFilesClientNormalizesWorkspaceID(t *testing.T) { + server := testserver.New(t) + + tests := []struct { + name string + input string + want string + }{ + {"none sentinel is cleared", auth.WorkspaceIDNone, ""}, + {"real workspace ID is kept", "7474644166319138", "7474644166319138"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + WorkspaceID: tc.input, + }) + require.NoError(t, err) + + _, err = NewWorkspaceFilesClient(client, "/dir") + require.NoError(t, err) + assert.Equal(t, tc.want, client.Config.WorkspaceID) + }) + } +} + // importFormFields parses the multipart body of a recorded /workspace/import // request into a field name -> value map. func importFormFields(t *testing.T, contentType string, body []byte) map[string]string { From 6ccfb3720ef9022865fd8ac98d9d137d7efec5da Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 14 Aug 2026 15:16:26 +0000 Subject: [PATCH 22/37] inline the constant --- libs/filer/workspace_files_client.go | 7 +------ libs/filer/workspace_files_client_test.go | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 509e303664f..1ea449ab9d7 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -22,11 +22,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) -// workspaceObjectTypeMismatchReason is the AIP-193 ErrorInfo reason attached -// by /workspace/import when overwrite=true targets a path whose existing -// object's node type differs from the upload (FILE vs NOTEBOOK). -const workspaceObjectTypeMismatchReason = "WORKSPACE_OBJECT_TYPE_MISMATCH" - // Type that implements fs.DirEntry for WSFS. type wsfsDirEntry struct { wsfsFileInfo @@ -259,7 +254,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // WCS attaches AIP-193 ErrorInfo with a stable reason to import // path collisions (universe PR #2019174, WP-6031), so prefer // branching on it over parsing the message. - if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == workspaceObjectTypeMismatchReason { + if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == "WORKSPACE_OBJECT_TYPE_MISMATCH" { return fileAlreadyExistsError{absPath} } // Fallback for errors that carry no ErrorInfo. Two reasons this diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index e598108bfdf..1b6b2f8eeeb 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -447,7 +447,7 @@ func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { "details": []map[string]any{ { "@type": "type.googleapis.com/google.rpc.ErrorInfo", - "reason": workspaceObjectTypeMismatchReason, + "reason": "WORKSPACE_OBJECT_TYPE_MISMATCH", "domain": "workspace.databricks.com", "metadata": map[string]string{"existing_type": "NOTEBOOK"}, }, From 2590a5a03f92df753e355989164fca697fb19a34 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 09:25:26 +0000 Subject: [PATCH 23/37] update comment to reflect latest change --- libs/filer/workspace_files_client.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 1ea449ab9d7..27d399b89bd 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -257,19 +257,22 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == "WORKSPACE_OBJECT_TYPE_MISMATCH" { return fileAlreadyExistsError{absPath} } - // Fallback for errors that carry no ErrorInfo. Two reasons this - // is still needed, both verified against a live workspace on - // 2026-08-04: + // Fallback for collisions that reach us without ErrorInfo. WCS does + // attach ErrorInfo to the exception (flag enableImportCollisionErrorInfo, + // default-on), but the details don't yet reach the HTTP response for two + // independent reasons: // - // - Rollout lag: universe #2019174 merged 2026-06-03 with its - // SAFE flag defaulting to true, but workspaces on an older WCS - // build still return WCS-worded collisions without details. - // That half is temporary. + // - The /workspace/import HTTP edge (WorkspaceContentWebBackend) builds + // its error body by hand as {error_code, message}, dropping details; + // they are correct only on WCS's internal RPC responses. universe + // #2429097 (WP-7085) routes the HTTP edge through the framework error + // serializer instead, but behind a default-false SAFE flag + // (enableWebBackendErrorDetailsPropagation) that still has to ramp, so + // message (a) stays detail-less until it does. // - // - Message (b) above is thrown by webapp, which #2019174 never - // touched, so it has no ErrorInfo regardless of WCS rollout. - // Once the lag clears this can narrow to "node type" alone, - // but it cannot be dropped until webapp attaches details too. + // - Message (b) is thrown bare by webapp's TreeBackendHelper with no + // ErrorInfo attached, so #2429097 has nothing to serialize for it. It + // can only be dropped once webapp attaches ErrorInfo at that throw site. if strings.Contains(aerr.Message, "type mismatch") || strings.Contains(aerr.Message, "node type") { return fileAlreadyExistsError{absPath} } From e2f004adc9be65a5ecd66b8582d6d0814eb3c9c7 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 17 Aug 2026 11:27:05 +0000 Subject: [PATCH 24/37] update comments --- libs/filer/workspace_files_client.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 27d399b89bd..499408a3b97 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -257,22 +257,22 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == "WORKSPACE_OBJECT_TYPE_MISMATCH" { return fileAlreadyExistsError{absPath} } - // Fallback for collisions that reach us without ErrorInfo. WCS does - // attach ErrorInfo to the exception (flag enableImportCollisionErrorInfo, - // default-on), but the details don't yet reach the HTTP response for two - // independent reasons: + // Fallback for collisions that reach us without ErrorInfo. WCS attaches + // ErrorInfo to the exception (flag enableImportCollisionErrorInfo, + // default-on), but it doesn't yet reach the HTTP body, and two universe + // fixes are needed before this can drop: // - // - The /workspace/import HTTP edge (WorkspaceContentWebBackend) builds - // its error body by hand as {error_code, message}, dropping details; - // they are correct only on WCS's internal RPC responses. universe - // #2429097 (WP-7085) routes the HTTP edge through the framework error - // serializer instead, but behind a default-false SAFE flag - // (enableWebBackendErrorDetailsPropagation) that still has to ramp, so - // message (a) stays detail-less until it does. + // - Both messages come over WCS's /workspace/import HTTP edge + // (WorkspaceContentWebBackend), which hand-rolled its error body as + // {error_code, message} and dropped details (they survive only on + // WCS's internal RPC responses). #2429097 (WP-7085) routes that edge + // through the framework serializer, behind a default-false SAFE flag + // (enableWebBackendErrorDetailsPropagation) that still has to ramp. // - // - Message (b) is thrown bare by webapp's TreeBackendHelper with no - // ErrorInfo attached, so #2429097 has nothing to serialize for it. It - // can only be dropped once webapp attaches ErrorInfo at that throw site. + // - Message (b) originates in webapp's TreeBackendHelper (reached from + // WCS over gRPC, which preserves the detail) and was thrown bare, so + // even with #2429097 there was nothing to serialize; #2437047 attaches + // the ErrorInfo there. So (a) needs only #2429097; (b) needs both. if strings.Contains(aerr.Message, "type mismatch") || strings.Contains(aerr.Message, "node type") { return fileAlreadyExistsError{absPath} } From 1766a0ac2a0c692a54660342d9f5d9111d5fd151 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 26 Aug 2026 11:02:55 +0000 Subject: [PATCH 25/37] update PR with latest universe changes --- libs/filer/workspace_files_client.go | 88 +++-------------------- libs/filer/workspace_files_client_test.go | 22 +----- 2 files changed, 12 insertions(+), 98 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 499408a3b97..f00b12d769c 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -199,95 +199,27 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return w.Write(ctx, name, bytes.NewReader(body), sliceWithout(mode, CreateParentDirectories)...) } - // Path already taken. /workspace/import signals this with several - // status/error_code combinations, all verified against a real workspace, and - // each of the branches below handles one group of them: - // - // - 400 RESOURCE_ALREADY_EXISTS — sequential conflict, no overwrite flag. - // Example: "/Users/me/foo.txt already exists. Please pass overwrite=true - // to overwrite it." - // - // - 409, with ALREADY_EXISTS or with no error_code at all — concurrent - // contention (observed in TestLock when five lockers race to write - // deploy.lock). Example: "Node with name - // /Users/me/.bundle/.../deploy.lock already exists. Please pass - // overwrite=true to update it." - // - // - 400 with no error_code — notebook conflict; detected by message marker - // in the ErrBadRequest branch further below. - // - // - 400 INVALID_PARAMETER_VALUE — overwrite=true on a path where the - // existing object's node type differs from the upload. Two distinct - // messages, both observed against aws-prod-ucws: - // - // (a) "Cannot overwrite the asset at /Users/me/foo due to type mismatch - // (asked: FILE, actual: NOTEBOOK)" — fires when the upload path is - // the same as an existing NOTEBOOK and the new content has no - // notebook header (so AUTO would store it as FILE), or the mirror - // case with FILE/NOTEBOOK swapped. - // - // (b) "Requested node type [FILE] is different from the existing node - // type [NOTEBOOK]" — fires when /foo is already a NOTEBOOK (from a - // prior /foo.py upload) and an overwrite-upload of regular content - // targets /foo.py: AUTO would store the new content as FILE at - // /foo.py, but the workspace treats /foo.py as the source view of - // the existing /foo NOTEBOOK and rejects the type change. - // Unlike (a), this message comes from the legacy webapp tree path - // (webapp/.../tree/TreeBackendHelper.scala), not from WCS, so the - // WCS-only ErrorInfo work below does not cover it. - // - // The server refuses the overwrite even though the caller asked for - // it; from the caller's perspective the path is occupied, so we - // surface this as already-exists. - // ErrResourceConflict covers every 409 regardless of error_code, including - // the bare 409 the workspace returns with only a message. ErrAlreadyExists - // alone would miss those: it and ErrResourceConflict are siblings under the - // SDK's error mapping, so a 409 without an error_code unwraps to the parent - // only. ErrResourceAlreadyExists is listed separately because the - // RESOURCE_ALREADY_EXISTS case arrives as a 400, which no status mapping - // resolves to a conflict. + // Path already taken. ErrResourceConflict covers every 409 (including the + // bare 409 with only a message); ErrResourceAlreadyExists covers the 400 + // RESOURCE_ALREADY_EXISTS, which no status maps to a conflict. if errors.Is(err, apierr.ErrResourceConflict) || errors.Is(err, apierr.ErrResourceAlreadyExists) { return fileAlreadyExistsError{absPath} } + // Overwrite rejected because the existing object's node type differs from + // the upload. The collision carries an AIP-193 ErrorInfo with a stable + // reason, so branch on it rather than the message text. if errors.Is(err, apierr.ErrInvalidParameterValue) { if aerr, ok := errors.AsType[*apierr.APIError](err); ok { - // WCS attaches AIP-193 ErrorInfo with a stable reason to import - // path collisions (universe PR #2019174, WP-6031), so prefer - // branching on it over parsing the message. if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == "WORKSPACE_OBJECT_TYPE_MISMATCH" { return fileAlreadyExistsError{absPath} } - // Fallback for collisions that reach us without ErrorInfo. WCS attaches - // ErrorInfo to the exception (flag enableImportCollisionErrorInfo, - // default-on), but it doesn't yet reach the HTTP body, and two universe - // fixes are needed before this can drop: - // - // - Both messages come over WCS's /workspace/import HTTP edge - // (WorkspaceContentWebBackend), which hand-rolled its error body as - // {error_code, message} and dropped details (they survive only on - // WCS's internal RPC responses). #2429097 (WP-7085) routes that edge - // through the framework serializer, behind a default-false SAFE flag - // (enableWebBackendErrorDetailsPropagation) that still has to ramp. - // - // - Message (b) originates in webapp's TreeBackendHelper (reached from - // WCS over gRPC, which preserves the detail) and was thrown bare, so - // even with #2429097 there was nothing to serialize; #2437047 attaches - // the ErrorInfo there. So (a) needs only #2429097; (b) needs both. - if strings.Contains(aerr.Message, "type mismatch") || strings.Contains(aerr.Message, "node type") { - return fileAlreadyExistsError{absPath} - } } } - // This API returns 400 if the file already exists when the object type is notebook. - // Both the historical "Path () already exists." format and the newer - // "RESOURCE_ALREADY_EXISTS: already exists. ..." format end with the same - // "already exists." marker; the JSON error_code is empty in both. The new format - // might not have been rolled out to all workspaces yet, so we anchor on the shared - // marker and return absPath rather than parsing the message. - // - // An empty error_code unwraps to ErrBadRequest by status alone, so this is checked - // after the ErrInvalidParameterValue branch above rather than before it. + // A notebook that already exists returns 400 with an empty error_code, which + // unwraps to ErrBadRequest by status alone, so this runs after the + // ErrInvalidParameterValue branch. Anchor on the shared "already exists." + // marker rather than parsing the full message. if errors.Is(err, apierr.ErrBadRequest) { if aerr, ok := errors.AsType[*apierr.APIError](err); ok { if strings.Contains(aerr.Message, "already exists.") { diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 1b6b2f8eeeb..add99bd2517 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -323,24 +323,6 @@ func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { }, expectErrTarget: fileAlreadyExistsError{}, }, - { - name: "400 INVALID_PARAMETER_VALUE 'type mismatch' (overwrite=true) maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusBadRequest, - ErrorCode: "INVALID_PARAMETER_VALUE", - Message: "Cannot overwrite the asset at /dir/foo due to type mismatch (asked: FILE, actual: NOTEBOOK).", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, - { - name: "400 INVALID_PARAMETER_VALUE 'Requested node type' (overwrite=true) maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusBadRequest, - ErrorCode: "INVALID_PARAMETER_VALUE", - Message: "Requested node type [FILE] is different from the existing node type [NOTEBOOK]", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, { name: "400 INVALID_PARAMETER_VALUE other message passes through", apiErr: &apierr.APIError{ @@ -439,8 +421,8 @@ func writeWithImportError(t *testing.T, body map[string]any) error { // Confirms a type-mismatch collision is recognized from the structured // AIP-193 ErrorInfo reason, independent of the error message wording. func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { - // The message is deliberately one the fallback string match does not - // recognize, to prove the branch fires on the structured reason alone. + // The message is deliberately unrelated to any collision wording, to prove + // the branch fires on the structured reason alone. err := writeWithImportError(t, map[string]any{ "error_code": "INVALID_PARAMETER_VALUE", "message": "some future wording for the same condition", From d4fb7c0d3187e2dce3e49ce697b7da0576ff0f78 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 26 Aug 2026 11:43:08 +0000 Subject: [PATCH 26/37] acceptance: update goldens for /workspace/import after rebase Regenerate acceptance outputs affected by the workspace upload endpoint switch, for tests that landed on main after this branch was created, and re-aim the whl_via_environment_key_extras assertion at the multipart body path since the /workspace/import URL no longer carries the wheel filename. Co-authored-by: Isaac --- .../local_code_source/output.txt | 2 +- .../artifacts/whl_change_version/output.txt | 19 +++++++++---------- .../whl_via_environment_key_extras/output.txt | 4 ++-- .../whl_via_environment_key_extras/script | 2 +- .../partial-summary-on-push-fail/output.txt | 11 ++--------- .../auto-migrate-empty-tfstate/output.txt | 14 ++++++-------- .../auto-migrate-push-failure/output.txt | 6 +++--- acceptance/bundle/user_agent/output.txt | 18 ++++++++++-------- .../simple/out.requests.destroy.direct.json | 18 +++++++++++++++++- .../out.requests.destroy.terraform.json | 18 +++++++++++++++++- .../air/run-submit-deps/output.txt | 6 +++--- 11 files changed, 71 insertions(+), 47 deletions(-) diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt index a08103ce148..df6fd7a28e9 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt +++ b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt @@ -4,7 +4,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files... Created jobs.train -Files: 13 uploaded, 0 deleted +Files: 14 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === each task's tarball holds only synced files (both under the repo root .air_snapshots) diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index d7d1271b7d5..5055166b304 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -3,9 +3,9 @@ Building python_artifact... Uploading dist/my_test_code-0.1.0-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Created jobs.test_job +Files: 10 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> find.py --expect 1 whl dist/my_test_code-0.1.0-py3-none-any.whl @@ -38,7 +38,6 @@ dist/my_test_code-0.1.0-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/test.toml @@ -54,9 +53,9 @@ dist/my_test_code-0.1.0-py3-none-any.whl Building python_artifact... Uploading dist/my_test_code-0.2.0-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Updated jobs.test_job +Files: 4 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> find.py --expect 2 whl dist/my_test_code-0.1.0-py3-none-any.whl @@ -86,9 +85,9 @@ Building python_artifact... Uploading dist/my_test_code-0.1.0-py3-none-any.whl... Uploading dist/my_test_code-0.2.0-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Updated jobs.test_job +Files: 4 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> find.py --expect 2 whl dist/my_test_code-0.1.0-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt index 2ecf86709e1..03c9ee108e3 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt @@ -49,5 +49,5 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded under its bare filename (no extras) ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index 426eb0b71c9..c2977a640d3 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -6,6 +6,6 @@ title "Expecting the environments dependency to keep its [train] extras suffix" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt index 36d57ffdbfa..dd90889f46f 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt @@ -1,16 +1,9 @@ >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true -HTTP Status: 403 Forbidden -API error_code: INJECTED -API message: Fault injected by test. - +Created jobs.my_job Files: 4 uploaded, 0 deleted - -Exit code: 1 +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt index ef3365a5818..3191e1066c8 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt @@ -1,12 +1,11 @@ === Deploy with bundle.engine: direct and an empty terraform state — sweep, don't migrate >>> [CLI] bundle deploy -Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Warn: Direct engine selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! -Removing empty terraform state; direct engine will be used on the next deploy (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... +Files: 4 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged +Removing empty terraform state; direct engine will be used on the next deploy (selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... === Terraform state is renamed to .backup; no resources.json (empty state, nothing to persist) @@ -26,7 +25,6 @@ direct_migrated_via_config true "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" "POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" @@ -46,7 +44,7 @@ direct_migrated_via_config true === Next deploy uses direct engine by default (no more mismatch warning) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Deployment complete! +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged >>> print_migration_telemetry diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt b/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt index 94d30286b16..cf125b4bd74 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt @@ -3,7 +3,7 @@ >>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.test_job -Files: 4 uploaded, 0 deleted +Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_migration_telemetry @@ -14,7 +14,7 @@ direct_drymigrate_warnings false >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 2 uploaded, 0 deleted +Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... Warn: automatic migration to direct engine failed: pushing direct state to workspace: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json @@ -33,7 +33,7 @@ direct_migrate_commit_error true >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 2 uploaded, 0 deleted +Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... Migrated 1 resource to direct deployment engine. diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 4e0e3329054..d516ece0c26 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -48,7 +48,7 @@ MISS deploy.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF MISS deploy.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS deploy.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS destroy.direct /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' -MISS destroy.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' +MISS destroy.direct /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct @@ -58,10 +58,11 @@ OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct OK destroy.direct /api/2.0/workspace/delete engine/direct +OK destroy.direct /api/2.0/workspace/delete engine/direct OK destroy.direct /api/2.0/workspace/import engine/direct MISS destroy.direct /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' -MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' +MISS destroy.terraform /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' OK destroy.terraform /api/2.0/workspace/export engine/terraform @@ -69,6 +70,7 @@ OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/delete engine/terraform +OK destroy.terraform /api/2.0/workspace/delete engine/terraform OK destroy.terraform /api/2.0/workspace/import engine/terraform MISS destroy.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' @@ -93,30 +95,30 @@ MISS plan.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks MISS plan.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS plan.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS plan2.direct /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' -MISS plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' +MISS plan2.direct /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan2.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan2.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' -OK plan2.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct +OK plan2.direct /api/2.0/workspace/export engine/direct OK plan2.direct /api/2.0/workspace/get-status engine/direct OK plan2.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct MISS plan2.direct /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS plan2.terraform /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' -MISS plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' +MISS plan2.terraform /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' MISS plan2.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none auth/pat' -OK plan2.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/terraform +OK plan2.terraform /api/2.0/workspace/export engine/terraform OK plan2.terraform /api/2.0/workspace/get-status engine/terraform MISS plan2.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS plan2.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS plan2.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS plan2.terraform /.well-known/databricks-config 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5' MISS run.direct /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' -MISS run.direct /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' +MISS run.direct /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.direct /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.direct /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS run.terraform /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' -MISS run.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' +MISS run.terraform /api/2.0/workspace/export 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.terraform /api/2.0/workspace/get-status 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_run cmd-exec-id/[UUID] interactive/none auth/pat' MISS run.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json index e22956fe86d..49ae5e2cfc9 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json @@ -14,7 +14,11 @@ ] }, "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" + } } { "headers": { @@ -114,6 +118,18 @@ "method": "GET", "path": "/api/2.1/unity-catalog/schemas/mycatalog.myschema" } +{ + "headers": { + "User-Agent": [ + "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" + } +} { "headers": { "User-Agent": [ diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json index e5127a03754..15cd96dc647 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json @@ -14,7 +14,11 @@ ] }, "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" + } } { "headers": { @@ -93,6 +97,18 @@ "return_export_info": "true" } } +{ + "headers": { + "User-Agent": [ + "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none engine/terraform auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle" + } +} { "headers": { "User-Agent": [ diff --git a/acceptance/experimental/air/run-submit-deps/output.txt b/acceptance/experimental/air/run-submit-deps/output.txt index 3c7280bc5ba..ab12d6b5360 100644 --- a/acceptance/experimental/air/run-submit-deps/output.txt +++ b/acceptance/experimental/air/run-submit-deps/output.txt @@ -8,9 +8,9 @@ View job run at: [DATABRICKS_URL]/jobs/runs/555 Tip: use --watch to stream logs until the run completes. === only config + command are uploaded; no requirements.yaml ->>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep -{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} -{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\n"} +>>> print_requests.py //api/2.0/workspace/import --oneline --sort --unique --keep +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\n", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml"}}} +{"method": "POST", "path": "/api/2.0/workspace/import", "body": {"multipart_form": {"content": "python train.py", "format": "AUTO", "overwrite": "true", "path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh"}}} === declared deps ride on environments[].spec.dependencies >>> print_requests.py //api/2.2/jobs/runs/submit From e1e59d57ceeb9118e54d600645ad9f9c1a48b7f5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 16 Sep 2026 09:19:40 +0000 Subject: [PATCH 27/37] changelog: reformat nextchanges fragment as a single bullet line main added tools/validate_nextchanges.py which requires each fragment to be a single `* `-prefixed line ending with a period; rename to 6149.md to match the PR-number convention. Co-authored-by: Isaac --- .nextchanges/bundles/6149.md | 1 + .nextchanges/bundles/workspace-import-migration.md | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .nextchanges/bundles/6149.md delete mode 100644 .nextchanges/bundles/workspace-import-migration.md diff --git a/.nextchanges/bundles/6149.md b/.nextchanges/bundles/6149.md new file mode 100644 index 00000000000..12bf478ca8d --- /dev/null +++ b/.nextchanges/bundles/6149.md @@ -0,0 +1 @@ +* Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`, raising the per-workspace upload rate limit and improving deploy throughput under concurrency. ([#6149](https://github.com/databricks/cli/pull/6149)) diff --git a/.nextchanges/bundles/workspace-import-migration.md b/.nextchanges/bundles/workspace-import-migration.md deleted file mode 100644 index 4212003bdc7..00000000000 --- a/.nextchanges/bundles/workspace-import-migration.md +++ /dev/null @@ -1 +0,0 @@ -Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a documented rate limit of 30 requests per second per workspace ([API rate limits](https://docs.databricks.com/aws/en/resources/limits)), higher than the limit that applied to the previous endpoint, and is ~1.5–2× faster for typical bundle deployments. From f8e5f5fb36d786cd9d0af6103857fe885ebd81e6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 16 Sep 2026 09:35:52 +0000 Subject: [PATCH 28/37] acceptance: restore goldens corrupted by test-update timeouts The full-suite test-update ran under load; 13 resource tests exceeded the 1m per-test timeout and the timeout error was baked into their goldens. These tests don't exercise the /workspace/import change, so restore their directories verbatim from main. Co-authored-by: Isaac --- .../out.plan_noop.terraform.json | 11 +++ .../pipelines_recreate/output.txt | 44 ++++++++++- .../out.plan_.terraform.json | 16 ++++ .../out.plan_.terraform.txt | 6 ++ .../update-and-resize-autoscale/output.txt | 60 ++++++++++++++- .../out.plan_.terraform.json | 8 ++ .../update-and-resize/out.plan_.terraform.txt | 3 + .../deploy/update-and-resize/output.txt | 39 +++++++++- .../resources/experiments/basic/output.txt | 3 +- .../jobs/update/out.state.terraform.json | 4 +- .../jobs/update/out.update.requests.json | 2 +- .../bundle/resources/jobs/update/output.txt | 63 ++++++++++++++- .../jobs/update_single_node/output.txt | 13 +++- .../bundle/resources/models/basic/output.txt | 6 +- .../update/out.plan_set_empty.terraform.json | 12 +++ .../out.requests_destroy.terraform.json | 7 ++ .../out.requests_set_empty.terraform.json | 12 +++ .../permissions/jobs/update/output.txt | 17 ++++- .../out.plan.restore.terraform.json | 3 + .../out.requests.restore.terraform.json | 23 ++++++ .../out.requests.update.terraform.json | 23 ++++++ .../update_protected/output.txt | 71 ++++++++++++++++- .../update/out.plan.restore.terraform.json | 3 + .../out.requests.restore.terraform.json | 31 ++++++++ .../update/out.requests.update.terraform.json | 31 ++++++++ .../postgres_databases/update/output.txt | 70 ++++++++++++++++- .../out.plan.restore.terraform.json | 3 + .../out.requests.restore.terraform.json | 29 +++++++ .../out.requests.update.terraform.json | 29 +++++++ .../update_autoscaling/output.txt | 71 ++++++++++++++++- .../out.plan.restore.terraform.json | 3 + .../out.requests.restore.terraform.json | 24 ++++++ .../update_display_name/output.txt | 76 ++++++++++++++++++- .../update/out.plan.restore.terraform.json | 3 + .../out.requests.restore.terraform.json | 29 +++++++ .../update/out.requests.update.terraform.json | 29 +++++++ .../postgres_roles/update/output.txt | 72 +++++++++++++++++- 37 files changed, 931 insertions(+), 18 deletions(-) create mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json create mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json create mode 100644 acceptance/bundle/resources/postgres_branches/update_protected/out.plan.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_branches/update_protected/out.requests.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_databases/update/out.plan.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_databases/update/out.requests.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_databases/update/out.requests.update.terraform.json create mode 100644 acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.update.terraform.json create mode 100644 acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_projects/update_display_name/out.requests.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_roles/update/out.plan.restore.terraform.json create mode 100644 acceptance/bundle/resources/postgres_roles/update/out.requests.restore.terraform.json diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.terraform.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.terraform.json index e69de29bb2d..ea25aa7e8e7 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.terraform.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.terraform.json @@ -0,0 +1,11 @@ +{ + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.jobs.bar": { + "action": "skip" + }, + "resources.pipelines.foo": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index 4f3d6118a10..73802cef930 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -34,8 +34,8 @@ Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline properties such as the 'catalog' or 'storage' are changed: recreate resources.pipelines.foo -Recreated pipelines.foo Updated jobs.bar +Recreated pipelines.foo Files: 3 uploaded, 0 deleted Resources: 1 created, 1 changed, 1 deleted, 0 unchanged @@ -96,4 +96,44 @@ Error: The specified pipeline [FOO_ID] was not found. >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged -Error: Test script killed due to a timeout (1m0s) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --nostamp //jobs //pipelines ^//api/2.0/bundle + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.bar + delete resources.pipelines.foo + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 2 deleted + +>>> print_requests.py --nostamp //jobs //pipelines ^//api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[FOO_ID_2]" +} + +>>> musterr [CLI] pipelines get [FOO_ID] +Error: The specified pipeline [FOO_ID] was not found. + +>>> musterr [CLI] pipelines get [FOO_ID_2] +Error: The specified pipeline [FOO_ID_2] was not found. + +>>> musterr [CLI] jobs get [BAR_ID] +Error: Job [BAR_ID] does not exist. diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json index 3cbb285079a..d3c66d89f8f 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json @@ -22,3 +22,19 @@ } } } +{ + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} +{ + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt index 546acdd553e..73b40dacde6 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt @@ -7,3 +7,9 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged update clusters.test_cluster Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt index 34716a099b1..5de12861f62 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt @@ -106,4 +106,62 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Changing autoscale should call resize API on running cluster -Error: Test script killed due to a timeout (1m0s) +>>> [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) [OUTPUT_DIR]/out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "autoscale": { + "max_workers": 6, + "min_workers": 4 + }, + "cluster_id": "[CLUSTER_ID]" + } +} + +=== Cluster should have new autoscale +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": null, + "autoscale": { + "max_workers": 6, + "min_workers": 4 + } +} + +=== Removing autoscale section should call resize API + +>>> [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) [OUTPUT_DIR]/out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "cluster_id": "[CLUSTER_ID]", + "num_workers": 3 + } +} + +=== Cluster should have num_workers +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 3, + "autoscale": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.test_cluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.json index 3cbb285079a..9dccd78ae21 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.json @@ -22,3 +22,11 @@ } } } +{ + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.txt index 546acdd553e..d51cd2abf07 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.txt @@ -7,3 +7,6 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged update clusters.test_cluster Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt index fdf9f5bb5a4..006cff9bb2f 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt @@ -69,4 +69,41 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Changing num_workers and spark_conf should call update API -Error: Test script killed due to a timeout (1m0s) +>>> [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk) [OUTPUT_DIR]/out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/edit", + "body": { + "autotermination_minutes": 60, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]", + "num_workers": 5, + "spark_conf": { + "spark.executor.memory": "4g" + }, + "spark_version": "13.3.x-snapshot-scala2.12" + } +} + +=== Cluster should have new num_workers and spark_conf +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 5, + "spark_conf": { + "spark.executor.memory": "4g" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.test_cluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/experiments/basic/output.txt b/acceptance/bundle/resources/experiments/basic/output.txt index 9af0e0e48c4..9c1cf620c81 100644 --- a/acceptance/bundle/resources/experiments/basic/output.txt +++ b/acceptance/bundle/resources/experiments/basic/output.txt @@ -178,5 +178,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/experiment-basic-[UNIQUE_NAME]/default - -Error: Test script killed due to a timeout (1m0s) +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/jobs/update/out.state.terraform.json b/acceptance/bundle/resources/jobs/update/out.state.terraform.json index a30248f01d7..750dbd3baa0 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.terraform.json +++ b/acceptance/bundle/resources/jobs/update/out.state.terraform.json @@ -44,7 +44,7 @@ "format": "MULTI_TASK", "git_source": [], "health": [], - "id": "[NUMID]", + "id": "[FOO_ID]", "job_cluster": [ { "job_cluster_key": "key", @@ -151,7 +151,7 @@ } ], "triggers": [], - "url": "[DATABRICKS_URL]/#job/[NUMID]", + "url": "[DATABRICKS_URL]/#job/[FOO_ID]", "usage_policy_id": null, "webhook_notifications": [ { diff --git a/acceptance/bundle/resources/jobs/update/out.update.requests.json b/acceptance/bundle/resources/jobs/update/out.update.requests.json index b307fedb5b5..35d04dc7b9b 100644 --- a/acceptance/bundle/resources/jobs/update/out.update.requests.json +++ b/acceptance/bundle/resources/jobs/update/out.update.requests.json @@ -2,7 +2,7 @@ "method": "POST", "path": "/api/2.2/jobs/reset", "body": { - "job_id": [NUMID], + "job_id": [FOO_ID], "new_settings": { "deployment": { "kind": "BUNDLE", diff --git a/acceptance/bundle/resources/jobs/update/output.txt b/acceptance/bundle/resources/jobs/update/output.txt index 7a1d1b9a816..bfd827f0f2e 100644 --- a/acceptance/bundle/resources/jobs/update/output.txt +++ b/acceptance/bundle/resources/jobs/update/output.txt @@ -35,5 +35,66 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_requests.py //jobs --nostamp >>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Fetch job ID and verify remote state +>>> [CLI] jobs get [FOO_ID] +{ + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "job_id": [FOO_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "key", + "new_cluster": { + "num_workers": 0, + "spark_version": "13.3.x-scala2.12" + } + } + ], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "trigger": { + "pause_status": "UNPAUSED", + "periodic": { + "interval": 1, + "unit": "HOURS" + } + }, + "webhook_notifications": {} + } +} + +=== Destroy the job and verify that it's removed from the state and from remote +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +>>> print_requests.py //jobs --nostamp +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [FOO_ID] + } +} -Error: Test script killed due to a timeout (1m0s) +>>> musterr [CLI] jobs get [FOO_ID] +Error: Job [FOO_ID] does not exist. diff --git a/acceptance/bundle/resources/jobs/update_single_node/output.txt b/acceptance/bundle/resources/jobs/update_single_node/output.txt index 05759178bfc..5841cbdd52d 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/output.txt +++ b/acceptance/bundle/resources/jobs/update_single_node/output.txt @@ -86,5 +86,16 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Destroy: 1 deleted -Error: Test script killed due to a timeout (1m0s) +>>> print_requests.py //jobs --nostamp +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [FOO_ID] + } +} + +>>> musterr [CLI] jobs get [FOO_ID] +Error: Job [FOO_ID] does not exist. diff --git a/acceptance/bundle/resources/models/basic/output.txt b/acceptance/bundle/resources/models/basic/output.txt index 2598d7e35b9..6727e0d9d9e 100644 --- a/acceptance/bundle/resources/models/basic/output.txt +++ b/acceptance/bundle/resources/models/basic/output.txt @@ -85,5 +85,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged } >>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.models.my_model -Error: Test script killed due to a timeout (1m0s) +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-models-basic-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json index e69de29bb2d..4910a177337 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json @@ -0,0 +1,12 @@ +{ + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.jobs.job_with_permissions": { + "action": "skip" + }, + "resources.jobs.job_with_permissions.permissions": { + "action": "delete", + "state_only": true + } + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json new file mode 100644 index 00000000000..73495bdc38f --- /dev/null +++ b/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json @@ -0,0 +1,7 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [JOB_WITH_PERMISSIONS_ID] + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json new file mode 100644 index 00000000000..95d354f99a4 --- /dev/null +++ b/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json @@ -0,0 +1,12 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/update/output.txt b/acceptance/bundle/resources/permissions/jobs/update/output.txt index e56b725b3f9..010de6f81c3 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/update/output.txt @@ -146,4 +146,19 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle plan -o json -Error: Test script killed due to a timeout (2m0s) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --nostamp //jobs/ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job_with_permissions + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default + +Destroy: 1 deleted + +>>> print_requests.py --nostamp //jobs/ diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.restore.terraform.json b/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.restore.terraform.json new file mode 100644 index 00000000000..375b496fef0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.plan.restore.terraform.json @@ -0,0 +1,3 @@ +{ + "action": "update" +} diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.requests.restore.terraform.json b/acceptance/bundle/resources/postgres_branches/update_protected/out.requests.restore.terraform.json new file mode 100644 index 00000000000..14a6068d6b3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.requests.restore.terraform.json @@ -0,0 +1,23 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[DEV_BRANCH_ID]" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/[DEV_BRANCH_ID]", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "[DEV_BRANCH_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "spec": { + "is_protected": false, + "no_expiry": true + } + } +} diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.requests.update.terraform.json b/acceptance/bundle/resources/postgres_branches/update_protected/out.requests.update.terraform.json index e69de29bb2d..1fbae288953 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.requests.update.terraform.json +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.requests.update.terraform.json @@ -0,0 +1,23 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[DEV_BRANCH_ID]" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/[DEV_BRANCH_ID]", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "[DEV_BRANCH_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "spec": { + "is_protected": true, + "no_expiry": true + } + } +} diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/output.txt b/acceptance/bundle/resources/postgres_branches/update_protected/output.txt index d8837871231..1a2eadd2c68 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/output.txt +++ b/acceptance/bundle/resources/postgres_branches/update_protected/output.txt @@ -101,4 +101,73 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ -Error: Test script killed due to a timeout (1m0s) +>>> [CLI] postgres get-branch [DEV_BRANCH_ID] +{ + "branch_id": "dev-branch", + "name": "[DEV_BRANCH_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "status": { + "branch_id": "dev-branch", + "current_state": "READY", + "default": false, + "is_protected": true, + "source_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "source_branch_lsn": "[LSN]", + "source_branch_time": "[TIMESTAMP]", + "state_change_time": "[TIMESTAMP]" + }, + "uid": "[BRANCH_UID]" +} + +=== Restore is_protected to false +>>> update_file.py databricks.yml is_protected: true is_protected: false + +>>> [CLI] bundle plan +update postgres_branches.dev_branch + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-branch-[UNIQUE_NAME]/default/files... +Updated postgres_branches.dev_branch +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] postgres get-branch [DEV_BRANCH_ID] +{ + "branch_id": "dev-branch", + "name": "[DEV_BRANCH_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "status": { + "branch_id": "dev-branch", + "current_state": "READY", + "default": false, + "is_protected": false, + "source_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "source_branch_lsn": "[LSN]", + "source_branch_time": "[TIMESTAMP]", + "state_change_time": "[TIMESTAMP]" + }, + "uid": "[BRANCH_UID]" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.dev_branch + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.dev_branch + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-postgres-branch-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/postgres_databases/update/out.plan.restore.terraform.json b/acceptance/bundle/resources/postgres_databases/update/out.plan.restore.terraform.json new file mode 100644 index 00000000000..375b496fef0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/update/out.plan.restore.terraform.json @@ -0,0 +1,3 @@ +{ + "action": "update" +} diff --git a/acceptance/bundle/resources/postgres_databases/update/out.requests.restore.terraform.json b/acceptance/bundle/resources/postgres_databases/update/out.requests.restore.terraform.json new file mode 100644 index 00000000000..be32218e65e --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/update/out.requests.restore.terraform.json @@ -0,0 +1,31 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_DATABASE_ID]" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/[MY_DATABASE_ID]", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "[MY_DATABASE_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "postgres_database": "initial_db_name", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + } + } +} diff --git a/acceptance/bundle/resources/postgres_databases/update/out.requests.update.terraform.json b/acceptance/bundle/resources/postgres_databases/update/out.requests.update.terraform.json new file mode 100644 index 00000000000..f3e8e7858c3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/update/out.requests.update.terraform.json @@ -0,0 +1,31 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_DATABASE_ID]" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/[MY_DATABASE_ID]", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "[MY_DATABASE_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "postgres_database": "renamed_db_name", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + } + } +} diff --git a/acceptance/bundle/resources/postgres_databases/update/output.txt b/acceptance/bundle/resources/postgres_databases/update/output.txt index 57c894bc9a9..b94e0180d7e 100644 --- a/acceptance/bundle/resources/postgres_databases/update/output.txt +++ b/acceptance/bundle/resources/postgres_databases/update/output.txt @@ -101,5 +101,73 @@ Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-database-[UNIQUE_NAME]/default/files... +Updated postgres_databases.my_database +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] postgres get-database [MY_DATABASE_ID] +{ + "database_id": "my-database", + "name": "[MY_DATABASE_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "status": { + "database_id": "my-database", + "postgres_database": "renamed_db_name", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + } +} + +=== Restore postgres_database to original value +>>> update_file.py databricks.yml postgres_database: renamed_db_name postgres_database: initial_db_name + +>>> [CLI] bundle plan +update postgres_databases.my_database + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-database-[UNIQUE_NAME]/default/files... +Updated postgres_databases.my_database +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] postgres get-database [MY_DATABASE_ID] +{ + "database_id": "my-database", + "name": "[MY_DATABASE_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "status": { + "database_id": "my-database", + "postgres_database": "initial_db_name", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.main + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.main + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.my_database + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-postgres-database-[UNIQUE_NAME]/default -Error: Test script killed due to a timeout (1m0s) +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.terraform.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.terraform.json new file mode 100644 index 00000000000..375b496fef0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.plan.restore.terraform.json @@ -0,0 +1,3 @@ +{ + "action": "update" +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.terraform.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.terraform.json new file mode 100644 index 00000000000..e551ed5eeae --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.restore.terraform.json @@ -0,0 +1,29 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "[MY_ENDPOINT_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "autoscaling_limit_max_cu": 8, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "suspend_timeout_duration": "300s" + } + } +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.update.terraform.json b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.update.terraform.json new file mode 100644 index 00000000000..26c0897d868 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.requests.update.terraform.json @@ -0,0 +1,29 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/[MY_ENDPOINT_ID]", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "[MY_ENDPOINT_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "suspend_timeout_duration": "300s" + } + } +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt index 9336f9a452a..1a613bbb5b8 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt @@ -98,5 +98,74 @@ Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/files... +Updated postgres_endpoints.my_endpoint +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] postgres get-endpoint [MY_ENDPOINT_ID] +{ + "endpoint_id": "my-endpoint", + "name": "[MY_ENDPOINT_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "uid": "[ENDPOINT_UID]", + "status": { + "endpoint_id": "my-endpoint", + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "autoscaling_limit_min_cu": 0.5, + "autoscaling_limit_max_cu": 4, + "suspend_timeout_duration": "300s" + } +} + +=== Restore endpoint autoscaling to original value +>>> update_file.py databricks.yml autoscaling_limit_max_cu: 4 autoscaling_limit_max_cu: 8 + +>>> [CLI] bundle plan +update postgres_endpoints.my_endpoint + +Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default/files... +Updated postgres_endpoints.my_endpoint +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] postgres get-endpoint [MY_ENDPOINT_ID] +{ + "endpoint_id": "my-endpoint", + "name": "[MY_ENDPOINT_ID]", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "uid": "[ENDPOINT_UID]", + "status": { + "endpoint_id": "my-endpoint", + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "autoscaling_limit_min_cu": 0.5, + "autoscaling_limit_max_cu": 8, + "suspend_timeout_duration": "300s" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.main + delete resources.postgres_endpoints.my_endpoint + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.main + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-[UNIQUE_NAME]/default -Error: Test script killed due to a timeout (1m0s) +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.terraform.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.terraform.json new file mode 100644 index 00000000000..375b496fef0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.plan.restore.terraform.json @@ -0,0 +1,3 @@ +{ + "action": "update" +} diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.requests.restore.terraform.json b/acceptance/bundle/resources/postgres_projects/update_display_name/out.requests.restore.terraform.json new file mode 100644 index 00000000000..0289e24a593 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.requests.restore.terraform.json @@ -0,0 +1,24 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/[MY_PROJECT_ID]" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/[MY_PROJECT_ID]", + "q": { + "update_mask": "initial_branch_spec,initial_endpoint_spec,spec" + }, + "body": { + "name": "[MY_PROJECT_ID]", + "spec": { + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Original Name", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt b/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt index 0a8732683a9..edb088c3a45 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt @@ -97,4 +97,78 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ -Error: Test script killed due to a timeout (1m0s) +>>> [CLI] postgres get-project [MY_PROJECT_ID] +{ + "name": "[MY_PROJECT_ID]", + "project_id": "test-pg-proj-[UNIQUE_NAME]", + "status": { + "branch_logical_size_limit_bytes": [NUMID], + "default_branch": "[MY_PROJECT_ID]/branches/production", + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Updated Name", + "enable_pg_native_login": false, + "history_retention_duration": "604800s", + "owner": "[USERNAME]", + "pg_version": 16, + "project_id": "test-pg-proj-[UNIQUE_NAME]", + "synthetic_storage_size_bytes": 0 + }, + "uid": "[UUID]" +} + +=== Restore display_name to original value +>>> update_file.py databricks.yml Updated Name Original Name + +>>> [CLI] bundle plan +update postgres_projects.my_project + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-project-[UNIQUE_NAME]/default/files... +Updated postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] postgres get-project [MY_PROJECT_ID] +{ + "name": "[MY_PROJECT_ID]", + "project_id": "test-pg-proj-[UNIQUE_NAME]", + "status": { + "branch_logical_size_limit_bytes": [NUMID], + "default_branch": "[MY_PROJECT_ID]/branches/production", + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Original Name", + "enable_pg_native_login": false, + "history_retention_duration": "604800s", + "owner": "[USERNAME]", + "pg_version": 16, + "project_id": "test-pg-proj-[UNIQUE_NAME]", + "synthetic_storage_size_bytes": 0 + }, + "uid": "[UUID]" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-postgres-project-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/postgres_roles/update/out.plan.restore.terraform.json b/acceptance/bundle/resources/postgres_roles/update/out.plan.restore.terraform.json new file mode 100644 index 00000000000..375b496fef0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/update/out.plan.restore.terraform.json @@ -0,0 +1,3 @@ +{ + "action": "update" +} diff --git a/acceptance/bundle/resources/postgres_roles/update/out.requests.restore.terraform.json b/acceptance/bundle/resources/postgres_roles/update/out.requests.restore.terraform.json new file mode 100644 index 00000000000..d62792a95a0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/update/out.requests.restore.terraform.json @@ -0,0 +1,29 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "attributes": { + "createdb": false + }, + "postgres_role": "app_role" + } + } +} diff --git a/acceptance/bundle/resources/postgres_roles/update/out.requests.update.terraform.json b/acceptance/bundle/resources/postgres_roles/update/out.requests.update.terraform.json index e69de29bb2d..3f56f596530 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.requests.update.terraform.json +++ b/acceptance/bundle/resources/postgres_roles/update/out.requests.update.terraform.json @@ -0,0 +1,29 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "attributes": { + "createdb": true + }, + "postgres_role": "app_role" + } + } +} diff --git a/acceptance/bundle/resources/postgres_roles/update/output.txt b/acceptance/bundle/resources/postgres_roles/update/output.txt index 0d378ab4c19..f95c6c0118e 100644 --- a/acceptance/bundle/resources/postgres_roles/update/output.txt +++ b/acceptance/bundle/resources/postgres_roles/update/output.txt @@ -106,4 +106,74 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ -Error: Test script killed due to a timeout (1m0s) +>>> [CLI] postgres get-role projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "role_id": "test-role", + "status": { + "attributes": { + "bypassrls": false, + "createdb": true, + "createrole": false + }, + "auth_method": "PG_PASSWORD_SCRAM_SHA_256", + "identity_type": "IDENTITY_TYPE_UNSPECIFIED", + "postgres_role": "app_role", + "role_id": "test-role" + } +} + +=== Restore attributes.createdb to original value +>>> update_file.py databricks.yml createdb: true createdb: false + +>>> [CLI] bundle plan +update postgres_roles.my_role + +Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-role-[UNIQUE_NAME]/default/files... +Updated postgres_roles.my_role +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] postgres get-role projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "role_id": "test-role", + "status": { + "attributes": { + "bypassrls": false, + "createdb": false, + "createrole": false + }, + "auth_method": "PG_PASSWORD_SCRAM_SHA_256", + "identity_type": "IDENTITY_TYPE_UNSPECIFIED", + "postgres_role": "app_role", + "role_id": "test-role" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.main + delete resources.postgres_projects.my_project + delete resources.postgres_roles.my_role + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.main + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-postgres-role-[UNIQUE_NAME]/default + +Destroy: 3 deleted From 4c7da85a090bc093e1dedf6ef9b1988e46637433 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 16 Sep 2026 10:35:07 +0000 Subject: [PATCH 29/37] acceptance: fix permissions and telemetry goldens after merge Two more casualties of the earlier full-suite test-update timeouts: - bundle/resources/permissions is a meta-test whose output.txt reflects the state of its subdirs; restore it to main now that the subdirs are back to main's content. - bundle/telemetry/deploy-failed-engine-unknown: regenerate its metadata golden so the state_size fields match the merged code. Co-authored-by: Isaac --- .../bundle/resources/permissions/output.txt | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index fa5daf2e086..85a16ad6a38 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -257,8 +257,25 @@ DIFF jobs/update/out.requests_delete_all.direct.json + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" + } +] -ERROR jobs/update/out.requests_destroy.direct.json: Missing terraform file jobs/update/out.requests_destroy.terraform.json -ERROR jobs/update/out.requests_set_empty.direct.json: Missing terraform file jobs/update/out.requests_set_empty.terraform.json +EXACT jobs/update/out.requests_destroy.direct.json +DIFF jobs/update/out.requests_set_empty.direct.json +--- jobs/update/out.requests_set_empty.direct.json ++++ jobs/update/out.requests_set_empty.terraform.json +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { ++ "access_control_list": [ ++ { ++ "permission_level": "IS_OWNER", ++ "user_name": "[USERNAME]" ++ } ++ ] ++ }, ++ "method": "PUT", ++ "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" ++ } ++] MATCH jobs/viewers/out.requests.deploy.direct.json DIFF jobs/viewers/out.requests.destroy.direct.json --- jobs/viewers/out.requests.destroy.direct.json From f38be3997eb0649f74fca4ab8601599daf093cf9 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 16 Sep 2026 10:50:44 +0000 Subject: [PATCH 30/37] acceptance: re-aim upload fault stubs at /workspace/import Two fault-injection tests stubbed the deprecated /workspace-files/import-file endpoint, which the migrated filer no longer calls, so the injected failure never fired: - deploy-failed-engine-unknown: [[Server]] stub now matches POST /workspace/import so the deploy.lock write fails at lock acquisition as intended (restores the deterministic no-state-size metadata golden). - partial-summary-on-push-fail: fault.py now matches the resources.json upload via --body-contains on POST /workspace/import (the multipart body carries the path, not the URL). Co-authored-by: Isaac --- .../partial-summary-on-push-fail/output.txt | 16 ++++++++++++---- .../deploy/partial-summary-on-push-fail/script | 2 +- .../partial-summary-on-push-fail/test.toml | 2 +- .../out.resources_metadata.direct.txt | 6 +----- .../deploy-failed-engine-unknown/output.txt | 14 ++++++++++---- .../deploy-failed-engine-unknown/test.toml | 2 +- 6 files changed, 26 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt index 9a4e20d7ab0..168113682f0 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt @@ -1,13 +1,21 @@ >>> errcode [CLI] bundle deploy -q Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 3 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Error: Failed to update, encountered possible permission error: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files +Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files as [USERNAME]. +Please make sure the current user or one of their groups is listed under the permissions of this bundle. +For assistance, contact the owners of this project. +They may need to redeploy the bundle to apply the new permissions. +Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions. + + +Exit code: 1 >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged +Created jobs.my_job +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/script b/acceptance/bundle/deploy/partial-summary-on-push-fail/script index bc36c59cacc..d2fef85a1d7 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/script +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/script @@ -9,7 +9,7 @@ trap cleanup EXIT # Inject a 403 on the resource-state upload (PushResourcesState). The files are # uploaded by then, so "Files:" still reports them; "Resources:" must NOT print, # because this fails before planning and there are no resource counts to report. -fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 +fault.py --body-contains "state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 # -q: the state-push 403 races resource apply, so whether "Created jobs.my_job" is # emitted before the failure is platform-dependent. This test asserts Files is reported diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml b/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml index 7ad1225aec5..fce577746c8 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/test.toml @@ -1,5 +1,5 @@ # The resource summary must not print when pushing resource state fails, so the -# fault is injected on the state upload (POST import-file). The path is fixed, so +# fault is injected on the state upload (POST /workspace/import). The path is fixed, so # the bundle name must be stable (test-bundle, not $UNIQUE_NAME). # # The injected path is the direct-engine state file (state/resources.json), so diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/out.resources_metadata.direct.txt b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/out.resources_metadata.direct.txt index 3eaab19c4a2..a403bee65c8 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/out.resources_metadata.direct.txt +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/out.resources_metadata.direct.txt @@ -1,13 +1,9 @@ { "state_engine": "direct", - "state_file_size_bytes": 543, "resources": [ { "resource_type": "jobs", - "count": 1, - "state_size_max_bytes": 254, - "state_size_mean_bytes": 254, - "state_size_median_bytes": 254 + "count": 1 } ] } diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt index 0eb98b7ffc4..df5252aa3d7 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/output.txt @@ -1,6 +1,12 @@ >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.job_one -Files: 1 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal error +Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +Exit code: 1 diff --git a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml index 86d366ba8c2..1e685cca603 100644 --- a/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml +++ b/acceptance/bundle/telemetry/deploy-failed-engine-unknown/test.toml @@ -12,6 +12,6 @@ Cloud = false [[Server]] -Pattern = "POST /api/2.0/workspace-files/import-file/{path...}" +Pattern = "POST /api/2.0/workspace/import" Response.StatusCode = 500 Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' From de55cacec60e146161d3dea1105baa1b0d535375 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 16 Sep 2026 20:58:31 +0000 Subject: [PATCH 31/37] filer: drop the already-exists message fallback The collision now returns error_code RESOURCE_ALREADY_EXISTS (with an ErrorInfo reason), caught by the ErrResourceAlreadyExists branch, so the empty-error_code message match is dead. enableWebBackendErrorDetailsPropagation is rolled out to dogfood and most prod. Co-authored-by: Isaac --- libs/filer/workspace_files_client.go | 13 ------------- libs/filer/workspace_files_client_test.go | 11 ----------- 2 files changed, 24 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index f00b12d769c..c25bfa0f8c2 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -11,7 +11,6 @@ import ( "net/http" "path" "slices" - "strings" "time" "github.com/databricks/cli/libs/auth" @@ -216,18 +215,6 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io } } - // A notebook that already exists returns 400 with an empty error_code, which - // unwraps to ErrBadRequest by status alone, so this runs after the - // ErrInvalidParameterValue branch. Anchor on the shared "already exists." - // marker rather than parsing the full message. - if errors.Is(err, apierr.ErrBadRequest) { - if aerr, ok := errors.AsType[*apierr.APIError](err); ok { - if strings.Contains(aerr.Message, "already exists.") { - return fileAlreadyExistsError{absPath} - } - } - } - // Caller has read access but no write access. if errors.Is(err, apierr.ErrPermissionDenied) { return permissionError{absPath, err} diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index add99bd2517..4aca3a12a4f 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -312,17 +312,6 @@ func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { }, expectErrTarget: fileAlreadyExistsError{}, }, - { - // Notebook conflicts arrive as a 400 with an empty error_code, so - // they match neither ErrResourceAlreadyExists nor - // ErrInvalidParameterValue and are detected by message marker. - name: "400 without error_code and 'already exists.' maps to fileAlreadyExistsError", - apiErr: &apierr.APIError{ - StatusCode: http.StatusBadRequest, - Message: "Path (/dir/notebook) already exists.", - }, - expectErrTarget: fileAlreadyExistsError{}, - }, { name: "400 INVALID_PARAMETER_VALUE other message passes through", apiErr: &apierr.APIError{ From 62738b7c778eba3900e11a4ab50c177f21e775d4 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 12 Oct 2026 13:42:42 +0000 Subject: [PATCH 32/37] filer: drop the workspace ID "none" normalization Upload now goes through the generated SDK client like every other DABs call, none of which strip the sentinel, so special-casing it here only mutated the shared client config. Co-authored-by: Isaac --- libs/filer/workspace_files_client.go | 6 -- libs/filer/workspace_files_client_test.go | 80 ----------------------- 2 files changed, 86 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index c25bfa0f8c2..24188974ed4 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -133,12 +133,6 @@ func (w *WorkspaceFilesClient) workspaceIDHeaders() map[string]string { } func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, error) { - // Workspace.Upload/Download forward cfg.WorkspaceID behind a bare != "" check, - // so normalize the "none" sentinel to "" to keep the SDK from routing on it. - if w.Config.WorkspaceID == auth.WorkspaceIDNone { - w.Config.WorkspaceID = "" - } - apiClient, err := client.New(w.Config) if err != nil { return nil, err diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 4aca3a12a4f..f06766cb0f6 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -13,7 +13,6 @@ import ( "testing" "time" - "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -109,36 +108,6 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } -// Confirms NewWorkspaceFilesClient clears the CLI-only "none" sentinel so the -// SDK's Workspace.Upload/Download never route on it, while a real workspace ID -// is left untouched. -func TestNewWorkspaceFilesClientNormalizesWorkspaceID(t *testing.T) { - server := testserver.New(t) - - tests := []struct { - name string - input string - want string - }{ - {"none sentinel is cleared", auth.WorkspaceIDNone, ""}, - {"real workspace ID is kept", "7474644166319138", "7474644166319138"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - client, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "testtoken", - WorkspaceID: tc.input, - }) - require.NoError(t, err) - - _, err = NewWorkspaceFilesClient(client, "/dir") - require.NoError(t, err) - assert.Equal(t, tc.want, client.Config.WorkspaceID) - }) - } -} - // importFormFields parses the multipart body of a recorded /workspace/import // request into a field name -> value map. func importFormFields(t *testing.T, contentType string, body []byte) map[string]string { @@ -218,55 +187,6 @@ func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { } } -// Confirms the workspace routing header is sent when the config carries a real -// workspace ID, and that the CLI-only "none" sentinel is never forwarded as a -// literal routing identifier. The sentinel is written to .databrickscfg by -// `auth login --skip-workspace`; the platform has no workspace named "none", -// so sending it would misroute the upload. -func TestWorkspaceFilesClientWriteWorkspaceIDHeader(t *testing.T) { - tests := []struct { - name string - workspaceID string - expect string - }{ - { - name: "real workspace ID is forwarded", - workspaceID: "7474644166319138", - expect: "7474644166319138", - }, - { - name: "none sentinel is not forwarded", - workspaceID: auth.WorkspaceIDNone, - expect: "", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var gotHeader string - server := testserver.New(t) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - gotHeader = req.Headers.Get(auth.WorkspaceIDHeader) - return testserver.Response{StatusCode: http.StatusOK} - }) - testserver.AddDefaultHandlers(server) - - client, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "testtoken", - WorkspaceID: tc.workspaceID, - }) - require.NoError(t, err) - - f, err := NewWorkspaceFilesClient(client, "/dir") - require.NoError(t, err) - - require.NoError(t, f.Write(t.Context(), "file.txt", strings.NewReader("hello"))) - assert.Equal(t, tc.expect, gotHeader) - }) - } -} - // Confirms each error /workspace/import returns is translated to the filer's // own error type, and that unrecognized errors pass through unchanged. The // endpoint signals an occupied path with three different status/error_code From eb90d8d0444695d6011b08a0fc7cdbeabdf68779 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 12 Oct 2026 13:54:56 +0000 Subject: [PATCH 33/37] acceptance: keep fault.py's hand-rolled parser, add only --body-contains Revert the unrelated argparse rewrite so the usage docstring stays intact; the migration only needs the new --body-contains flag. Co-authored-by: Isaac --- acceptance/bin/fault.py | 76 ++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/acceptance/bin/fault.py b/acceptance/bin/fault.py index 4d6f5a22b9f..5b75c0a5907 100755 --- a/acceptance/bin/fault.py +++ b/acceptance/bin/fault.py @@ -1,44 +1,28 @@ #!/usr/bin/env python3 """Set up a fault rule on the testserver for the current test token. +Usage: fault.py [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] + + --after-handler run the handler first; keep its effect and replace only the response + --body-contains only fire when the request body contains SUBSTR, e.g. to target one + file's /workspace/import upload (all uploads share that path) + PATTERN HTTP method and path, supports trailing * wildcard, + e.g. "PUT /api/2.0/permissions/pipelines/*" + STATUS_CODE HTTP status code to return, e.g. 504 + OFFSET number of requests to let through before fault starts + TIMES number of times to return the fault response + ERROR_CODE optional error_code for the response body, e.g. + MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED) + The rule is scoped to the current DATABRICKS_TOKEN so it only affects the test that registers it, even when tests share a server. """ -import argparse import json import os import sys import urllib.request -parser = argparse.ArgumentParser(description="Set up a fault rule on the testserver for the current test token.") -parser.add_argument( - "pattern", help='HTTP method and path, supports trailing * wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*"' -) -parser.add_argument("status_code", type=int, help="HTTP status code to return, e.g. 504") -parser.add_argument("offset", type=int, help="number of requests to let through before fault starts") -parser.add_argument("times", type=int, help="number of times to return the fault response") -parser.add_argument( - "error_code", - nargs="?", - default="INJECTED", - help="error_code for the response body, e.g. MAX_CHILD_NODE_SIZE_EXCEEDED", -) -parser.add_argument( - "--after-handler", - action="store_true", - help="run the handler first; keep its effect and replace only the response", -) -parser.add_argument( - "--body-contains", - default="", - metavar="SUBSTR", - help="only fire when the request body contains SUBSTR. Needed to target a single file's " - "/workspace/import upload, since every upload shares the same method+path and only " - 'differs by the multipart "path" form field.', -) -args = parser.parse_args() - host = os.environ.get("DATABRICKS_HOST", "") token = os.environ.get("DATABRICKS_TOKEN", "") @@ -46,17 +30,37 @@ print("DATABRICKS_HOST not set", file=sys.stderr) sys.exit(1) -body = json.dumps({"error_code": args.error_code, "message": "Fault injected by test."}) +args = sys.argv[1:] +after_handler = "--after-handler" in args +if after_handler: + args.remove("--after-handler") + +body_contains = "" +if "--body-contains" in args: + i = args.index("--body-contains") + body_contains = args[i + 1] + del args[i : i + 2] + +if len(args) not in (4, 5): + print( + f"usage: {sys.argv[0]} [--after-handler] [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", + file=sys.stderr, + ) + sys.exit(1) + +pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3]) +error_code = args[4] if len(args) == 5 else "INJECTED" +body = json.dumps({"error_code": error_code, "message": "Fault injected by test."}) data = json.dumps( { - "pattern": args.pattern, - "status_code": args.status_code, + "pattern": pattern, + "status_code": status_code, "body": body, - "offset": args.offset, - "times": args.times, - "after_handler": args.after_handler, - "body_contains": args.body_contains, + "offset": offset, + "times": times, + "after_handler": after_handler, + "body_contains": body_contains, } ).encode() From 9da608d3d07f494f49ecc7c77c3bf1bbe1125d44 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 12 Oct 2026 14:06:03 +0000 Subject: [PATCH 34/37] acceptance: add print_workspace_imports helper for uploaded paths Upload paths live in the multipart body of POST /workspace/import, so tests listed them with an inline jq over $OUT_REQUESTS. Name that in one helper that also pins the endpoint, and use it at every path-listing call site. Co-authored-by: Isaac --- .../bundle/artifacts/ai_runtime_code_source/output.txt | 2 +- acceptance/bundle/artifacts/ai_runtime_code_source/script | 2 +- .../artifacts/artifact_upload_for_workspace/output.txt | 2 +- .../bundle/artifacts/artifact_upload_for_workspace/script | 2 +- .../artifact_upload_with_no_library_reference/output.txt | 2 +- .../artifact_upload_with_no_library_reference/script | 2 +- .../bundle/artifacts/upload_multiple_libraries/output.txt | 2 +- .../bundle/artifacts/upload_multiple_libraries/script | 2 +- acceptance/bundle/artifacts/whl_change_version/output.txt | 6 +++--- acceptance/bundle/artifacts/whl_change_version/script | 6 +++--- acceptance/bundle/artifacts/whl_dbfs/output.txt | 2 +- acceptance/bundle/artifacts/whl_dbfs/script | 2 +- acceptance/bundle/artifacts/whl_dynamic/output.txt | 4 ++-- acceptance/bundle/artifacts/whl_dynamic/script | 4 ++-- acceptance/bundle/artifacts/whl_explicit/output.txt | 2 +- acceptance/bundle/artifacts/whl_explicit/script | 2 +- acceptance/bundle/artifacts/whl_implicit/output.txt | 2 +- acceptance/bundle/artifacts/whl_implicit/script | 2 +- .../bundle/artifacts/whl_implicit_custom_path/output.txt | 2 +- acceptance/bundle/artifacts/whl_implicit_custom_path/script | 2 +- .../bundle/artifacts/whl_implicit_notebook/output.txt | 2 +- acceptance/bundle/artifacts/whl_implicit_notebook/script | 2 +- acceptance/bundle/artifacts/whl_multiple/output.txt | 2 +- acceptance/bundle/artifacts/whl_multiple/script | 2 +- acceptance/bundle/artifacts/whl_no_cleanup/output.txt | 2 +- acceptance/bundle/artifacts/whl_no_cleanup/script | 2 +- .../bundle/artifacts/whl_prebuilt_multiple/output.txt | 2 +- acceptance/bundle/artifacts/whl_prebuilt_multiple/script | 2 +- acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt | 2 +- acceptance/bundle/artifacts/whl_prebuilt_outside/script | 2 +- .../artifacts/whl_prebuilt_outside_dynamic/output.txt | 2 +- .../bundle/artifacts/whl_prebuilt_outside_dynamic/script | 2 +- .../bundle/artifacts/whl_via_environment_key/output.txt | 2 +- acceptance/bundle/artifacts/whl_via_environment_key/script | 2 +- .../artifacts/whl_via_environment_key_extras/output.txt | 2 +- .../bundle/artifacts/whl_via_environment_key_extras/script | 2 +- acceptance/bundle/deploy/readplan/whl-upload/output.txt | 4 ++-- acceptance/bundle/deploy/readplan/whl-upload/script | 4 ++-- acceptance/bundle/sync-upload-edge-cases/output.txt | 6 +----- acceptance/bundle/sync-upload-edge-cases/script | 6 +----- acceptance/script.prepare | 6 ++++++ 41 files changed, 54 insertions(+), 56 deletions(-) diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt index 89cedc80d51..1a3ffc22b37 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt @@ -16,5 +16,5 @@ } === Expecting the code_source tarball to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/foo/bar/artifacts/.internal/code.tgz diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/script b/acceptance/bundle/artifacts/ai_runtime_code_source/script index fdcb0d38aac..32460827ec5 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/script +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/script @@ -6,6 +6,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' "$OUT_REQUESTS" title "Expecting the code_source tarball to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep code.tgz | sort +trace print_workspace_imports | grep code.tgz | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt index bf3d3b3d32d..9997818f806 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt @@ -48,7 +48,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl /Workspace/foo/bar/artifacts/.internal/source.whl diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script index 4f2493ddb8c..b3c226bc752 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script @@ -9,7 +9,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt index 2fde3d7b022..ca39c787992 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt @@ -4,7 +4,7 @@ Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Expecting wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl /Workspace/foo/bar/artifacts/.internal/source.whl diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script index a3b350f59e6..0d665cfbc5f 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script @@ -6,7 +6,7 @@ $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) title "Expecting wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt index 9993ef1698d..e500ebd0342 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt @@ -38,7 +38,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 4 wheels to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/script b/acceptance/bundle/artifacts/upload_multiple_libraries/script index ded4cb0b195..89d0ba55291 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/script +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/script @@ -12,7 +12,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 4 wheels to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 3298d531974..30134175168 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -29,7 +29,7 @@ dist/my_test_code-0.1.0-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings [OUTPUT_DIR]/out.requests.txt +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml @@ -62,7 +62,7 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings [OUTPUT_DIR]/out.requests.txt +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py @@ -92,7 +92,7 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings [OUTPUT_DIR]/out.requests.txt +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml diff --git a/acceptance/bundle/artifacts/whl_change_version/script b/acceptance/bundle/artifacts/whl_change_version/script index d7ca4f3af51..4b97cb54259 100644 --- a/acceptance/bundle/artifacts/whl_change_version/script +++ b/acceptance/bundle/artifacts/whl_change_version/script @@ -6,7 +6,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' "$OUT_REQUESTS" | sort +trace print_workspace_imports | sort rm "$OUT_REQUESTS" @@ -20,7 +20,7 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' "$OUT_REQUESTS" | sort +trace print_workspace_imports | sort rm "$OUT_REQUESTS" @@ -34,6 +34,6 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' "$OUT_REQUESTS" | sort +trace print_workspace_imports | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index f731471b36e..04b3a0f58fa 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -25,6 +25,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting no wheels to be uploaded ->>> errcode sh -c jq -r ".body.multipart_form.path | strings" < "$OUT_REQUESTS" | grep whl +>>> print_workspace_imports Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script index e55a7f85fbd..bd7b88b5806 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/script +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting no wheels to be uploaded" -trace errcode sh -c 'jq -r ".body.multipart_form.path | strings" < "$OUT_REQUESTS" | grep whl' +trace print_workspace_imports | errcode grep whl rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index f88662b4b2c..3461014160d 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -48,7 +48,7 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 patched wheels to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl @@ -74,7 +74,7 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice) ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index 163e1892da9..d031d02715c 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -22,7 +22,7 @@ title "Expecting 2 patched wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 2 patched wheels to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" @@ -40,4 +40,4 @@ title "Expecting 2 patched wheels in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' "$OUT_REQUESTS" | gron.py | grep -w libraries title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 1ee3e7c880a..14358c0ab32 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -21,7 +21,7 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index 849d7ddb112..a1227cd125d 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -8,7 +8,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index 73d2b4ef20e..c610058f32f 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -21,6 +21,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheels to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index 0539c729925..8124726f4ff 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -8,6 +8,6 @@ title "Expecting 1 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 1 wheels to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 11d3b8f33a8..1a93fb8c9bd 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -39,6 +39,6 @@ package/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl /Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 8a1d0fa8f1e..4d5fd31d6e7 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -7,6 +7,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index cd78556e973..6a49d1e1889 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -11,6 +11,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index c519521621c..41bae0e4237 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries' "$OUT_REQUESTS" title "Expecting 1 wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index 2f92eb08574..96edc8e5c85 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -25,7 +25,7 @@ my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index ec263f050e1..8c06cd32454 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -8,6 +8,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm -fr "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt index d2740e64aff..b80a0145ad1 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt +++ b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt @@ -3,7 +3,7 @@ dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/script b/acceptance/bundle/artifacts/whl_no_cleanup/script index c6c1ef84223..6a4a10bc75e 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/script +++ b/acceptance/bundle/artifacts/whl_no_cleanup/script @@ -5,7 +5,7 @@ cat LOG.deploy | contains.py "my_test_code-0.0.1-py3-none-any.whl" > /dev/null trace find.py --expect 1 whl title "Expecting 1 wheels to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index 752f81fc2a3..c18e0b2d6da 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -36,7 +36,7 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index e5d936db74f..8034c3171cf 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -10,6 +10,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt index b2635f7dbb6..a034c80a088 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt @@ -32,6 +32,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/script b/acceptance/bundle/artifacts/whl_prebuilt_outside/script index 84abfc0e85d..23a2e0fc1ef 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/script @@ -11,6 +11,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt index b5f3e9561bc..b26bae880f5 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt @@ -56,6 +56,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script index a4054fd0774..f972cf2504e 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script @@ -14,6 +14,6 @@ trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' "$OUT_RE trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' "$OUT_REQUESTS" title "Expecting 2 wheels to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index 2d45c8c3028..f6ba380bc11 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -41,6 +41,6 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index 17470657ad9..09f9ed82c47 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -8,6 +8,6 @@ title "Expecting 1 wheel in environments section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt index 110b742e084..e99a248c94b 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt @@ -41,5 +41,5 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded under its bare filename (no extras) ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index cd7f51f3e2a..157472b5f55 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -8,6 +8,6 @@ title "Expecting the environments dependency to keep its [train] extras suffix" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' "$OUT_REQUESTS" | nostamp title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep whl | sort +trace print_workspace_imports | grep whl | sort rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/deploy/readplan/whl-upload/output.txt b/acceptance/bundle/deploy/readplan/whl-upload/output.txt index 57c35386651..6b21d59c6b0 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-upload/output.txt @@ -7,7 +7,7 @@ Created jobs.my_job Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl >>> [CLI] bundle destroy --auto-approve @@ -26,7 +26,7 @@ Created jobs.my_job Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> jq -r .body.multipart_form.path | strings +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/whl-upload/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/readplan/whl-upload/script b/acceptance/bundle/deploy/readplan/whl-upload/script index 09d14f7005f..15b0c9aa6c3 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/script +++ b/acceptance/bundle/deploy/readplan/whl-upload/script @@ -6,7 +6,7 @@ title "bundle deploy uploads the wheel" trace $CLI bundle deploy # The wheel was uploaded to its .../artifacts/.internal/ workspace path. -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep artifacts/.internal +trace print_workspace_imports | grep artifacts/.internal trace $CLI bundle destroy --auto-approve # Discard destroy's requests so the saved-plan deploy below starts from a clean recording. @@ -20,7 +20,7 @@ $CLI bundle plan -o json | nostamp > out.plan.json trace $CLI bundle deploy --plan tmp.plan.json # The wheel is uploaded to the same .../artifacts/.internal/ path the job task points at, # matching the plain `bundle deploy` above. -trace jq -r '.body.multipart_form.path | strings' < "$OUT_REQUESTS" | grep artifacts/.internal +trace print_workspace_imports | grep artifacts/.internal trace $CLI bundle destroy --auto-approve rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/sync-upload-edge-cases/output.txt b/acceptance/bundle/sync-upload-edge-cases/output.txt index ad2f6d2906e..8ce0c31d5dc 100644 --- a/acceptance/bundle/sync-upload-edge-cases/output.txt +++ b/acceptance/bundle/sync-upload-edge-cases/output.txt @@ -73,11 +73,7 @@ Uploaded with spaces.txt } === uploaded paths from each multipart POST /workspace/import (sorted) ->>> jq -rs - map(select(.path == "/api/2.0/workspace/import")) | - map(.body.multipart_form.path) | - sort | .[] - [OUTPUT_DIR]/out.requests.txt +>>> print_workspace_imports /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/.gitignore /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/dashboard.lvdash.json /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/databricks.yml diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script index dfcb226315a..bfd3abcaa03 100644 --- a/acceptance/bundle/sync-upload-edge-cases/script +++ b/acceptance/bundle/sync-upload-edge-cases/script @@ -71,11 +71,7 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace list -o json "$remote_files" | jq 'sort_by(.path) | .[] | {name: (.path | sub(".*/"; "")), object_type, language}' title "uploaded paths from each multipart POST /workspace/import (sorted)" -trace jq -rs ' - map(select(.path == "/api/2.0/workspace/import")) | - map(.body.multipart_form.path) | - sort | .[] -' "$OUT_REQUESTS" +trace print_workspace_imports | sort title "every upload set format=AUTO" trace jq -rs ' diff --git a/acceptance/script.prepare b/acceptance/script.prepare index a53627b231d..66080eb1fcb 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -128,6 +128,12 @@ print_telemetry_bool_values() { jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental.bool_values // []) ) | map("\(.key) \(.value)") | .[]' "$OUT_REQUESTS" | grep -Ev '^(engine_terraform_|deployment_history_enabled )' | sort } +# Workspace paths uploaded via /workspace/import, one per line. The path is a +# multipart form field, not part of the URL, so print_requests.py can't filter on it. +print_workspace_imports() { + jq -r 'select(.path? == "/api/2.0/workspace/import") | .body.multipart_form.path | strings' "$OUT_REQUESTS" +} + sethome() { local home="$1" mkdir -p "$home" From a8129af26a33baa686c025ad28f175e47fe71b37 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 12 Oct 2026 14:27:36 +0000 Subject: [PATCH 35/37] acceptance: fault only file_to_upload.txt in upload/internal_server_error The [[Server]] stub matched every /workspace/import call, so the deploy.lock upload failed first and the test stopped covering a failed file upload. Target the file with fault.py --body-contains instead. Co-authored-by: Isaac --- acceptance/bundle/upload/internal_server_error/output.txt | 8 ++++---- acceptance/bundle/upload/internal_server_error/script | 1 + acceptance/bundle/upload/internal_server_error/test.toml | 3 --- 3 files changed, 5 insertions(+), 7 deletions(-) delete mode 100644 acceptance/bundle/upload/internal_server_error/test.toml diff --git a/acceptance/bundle/upload/internal_server_error/output.txt b/acceptance/bundle/upload/internal_server_error/output.txt index d7f01415559..794a0ed2072 100644 --- a/acceptance/bundle/upload/internal_server_error/output.txt +++ b/acceptance/bundle/upload/internal_server_error/output.txt @@ -1,8 +1,8 @@ -Error: Failed to acquire deployment lock: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal Server Error -Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock: Internal Server Error (500) +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: failed to upload /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/file_to_upload.txt: Fault injected by test. (500 INJECTED) Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import HTTP Status: 500 Internal Server Error -API error_code: -API message: Internal Server Error +API error_code: INJECTED +API message: Fault injected by test. diff --git a/acceptance/bundle/upload/internal_server_error/script b/acceptance/bundle/upload/internal_server_error/script index c1a222958e1..0fc4896ef9e 100644 --- a/acceptance/bundle/upload/internal_server_error/script +++ b/acceptance/bundle/upload/internal_server_error/script @@ -1 +1,2 @@ +fault.py --body-contains "/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/files/file_to_upload.txt" "POST /api/2.0/workspace/import" 500 0 1000 musterr $CLI bundle deploy diff --git a/acceptance/bundle/upload/internal_server_error/test.toml b/acceptance/bundle/upload/internal_server_error/test.toml deleted file mode 100644 index b1d2b7dc49f..00000000000 --- a/acceptance/bundle/upload/internal_server_error/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -[[Server]] -Pattern = "POST /api/2.0/workspace/import" -Response.StatusCode = 500 From 885339c6c593ada2c70a88f3c3136943e64d6439 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 12 Oct 2026 15:10:43 +0000 Subject: [PATCH 36/37] filer: drop Write unit tests covered by filer and acceptance tests The form-field and mkdirs-retry cases are exercised by the common filer tests and by every acceptance deploy against the testserver, which rejects uploads that are not format=AUTO and returns 404 for a missing parent. Co-authored-by: Isaac --- libs/filer/workspace_files_client_test.go | 132 ---------------------- 1 file changed, 132 deletions(-) diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index f06766cb0f6..81832694d15 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -3,11 +3,7 @@ package filer import ( "bytes" "encoding/json" - "errors" - "io" "io/fs" - "mime" - "mime/multipart" "net/http" "strings" "testing" @@ -108,85 +104,6 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } -// importFormFields parses the multipart body of a recorded /workspace/import -// request into a field name -> value map. -func importFormFields(t *testing.T, contentType string, body []byte) map[string]string { - t.Helper() - - _, params, err := mime.ParseMediaType(contentType) - require.NoError(t, err) - mr := multipart.NewReader(bytes.NewReader(body), params["boundary"]) - - fields := map[string]string{} - for { - part, err := mr.NextPart() - if errors.Is(err, io.EOF) { - break - } - require.NoError(t, err) - data, err := io.ReadAll(part) - require.NoError(t, err) - fields[part.FormName()] = string(data) - } - return fields -} - -// Confirms Write posts a multipart /workspace/import body with format=AUTO, and -// that the overwrite field is present only when OverwriteIfExists is passed. -// AUTO is what lets the server decide between storing content as a file or a -// notebook, so the format is asserted explicitly rather than left to the -// endpoint default (SOURCE), which would import every file as a notebook. -func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { - tests := []struct { - name string - modes []WriteMode - expectPresent bool - }{ - { - name: "no overwrite", - modes: nil, - expectPresent: false, - }, - { - name: "overwrite", - modes: []WriteMode{OverwriteIfExists}, - expectPresent: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var gotForm map[string]string - server := testserver.New(t) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - gotForm = importFormFields(t, req.Headers.Get("Content-Type"), req.Body) - return testserver.Response{StatusCode: http.StatusOK} - }) - testserver.AddDefaultHandlers(server) - - client, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "testtoken", - }) - require.NoError(t, err) - - f, err := NewWorkspaceFilesClient(client, "/dir") - require.NoError(t, err) - - require.NoError(t, f.Write(t.Context(), "file.txt", strings.NewReader("hello"), tc.modes...)) - - assert.Equal(t, "/dir/file.txt", gotForm["path"]) - assert.Equal(t, "hello", gotForm["content"]) - assert.Equal(t, string(workspace.ImportFormatAuto), gotForm["format"]) - overwrite, present := gotForm["overwrite"] - assert.Equal(t, tc.expectPresent, present) - if present { - assert.Equal(t, "true", overwrite) - } - }) - } -} - // Confirms each error /workspace/import returns is translated to the filer's // own error type, and that unrecognized errors pass through unchanged. The // endpoint signals an occupied path with three different status/error_code @@ -367,55 +284,6 @@ func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { assert.Equal(t, http.StatusBadRequest, aerr.StatusCode) } -// Confirms a 404 from a missing parent directory triggers a mkdirs call and a -// retry of the upload when CreateParentDirectories is passed. The retry re-reads -// the buffered body, so this also covers that the content survives the second -// attempt. -func TestWorkspaceFilesClientWriteCreatesParentDirectories(t *testing.T) { - var uploads int - var mkdirs []string - var lastContent string - - server := testserver.New(t) - server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { - uploads++ - // The first attempt 404s as it would against a missing parent; the retry - // after mkdirs succeeds. - if uploads == 1 { - return testserver.Response{ - StatusCode: http.StatusNotFound, - Body: map[string]string{"message": "The parent folder (/dir/sub) does not exist."}, - } - } - form := importFormFields(t, req.Headers.Get("Content-Type"), req.Body) - lastContent = form["content"] - return testserver.Response{StatusCode: http.StatusOK} - }) - server.Handle("POST", "/api/2.0/workspace/mkdirs", func(req testserver.Request) any { - var request workspace.Mkdirs - require.NoError(t, json.Unmarshal(req.Body, &request)) - mkdirs = append(mkdirs, request.Path) - return testserver.Response{StatusCode: http.StatusOK} - }) - testserver.AddDefaultHandlers(server) - - client, err := databricks.NewWorkspaceClient(&databricks.Config{ - Host: server.URL, - Token: "testtoken", - }) - require.NoError(t, err) - - f, err := NewWorkspaceFilesClient(client, "/dir") - require.NoError(t, err) - - require.NoError(t, f.Write(t.Context(), "sub/file.txt", strings.NewReader("data"), CreateParentDirectories)) - - assert.Equal(t, 2, uploads) - assert.Equal(t, []string{"/dir/sub"}, mkdirs) - // The retry re-reads the buffered body, so the content must survive. - assert.Equal(t, "data", lastContent) -} - // Confirms a workspace get-status payload unmarshals into wsfsFileInfo and that // the fs.FileInfo methods derived from it report the expected values. func TestWorkspaceFilesClient_wsfsUnmarshal(t *testing.T) { From 932444a0fa8a2a643286b4b84ba06ade139f3fce Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 12 Oct 2026 19:42:49 +0000 Subject: [PATCH 37/37] acceptance: avoid a leading slash in fault.py --body-contains Git Bash on Windows rewrites a leading-/ argument into a Windows path before it reaches python, so the substring never matched and the fault never fired. Co-authored-by: Isaac --- acceptance/bundle/migrate/auto/push-failure/script | 2 +- acceptance/bundle/upload/internal_server_error/script | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/migrate/auto/push-failure/script b/acceptance/bundle/migrate/auto/push-failure/script index 72e4f8ca282..4231617175e 100644 --- a/acceptance/bundle/migrate/auto/push-failure/script +++ b/acceptance/bundle/migrate/auto/push-failure/script @@ -10,7 +10,7 @@ title "Opt in via env var; inject 403 on the resources.json push so the deploy f # push fails the deploy - unlike a declined deploy, the migration is not discarded, so the # local direct state remains and a retry finishes the push. (A push failure is a hard error on # any direct deploy, not just a migrating one.) -fault.py --body-contains "/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 +fault.py --body-contains "Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/upload/internal_server_error/script b/acceptance/bundle/upload/internal_server_error/script index 0fc4896ef9e..78c57ef8a94 100644 --- a/acceptance/bundle/upload/internal_server_error/script +++ b/acceptance/bundle/upload/internal_server_error/script @@ -1,2 +1,2 @@ -fault.py --body-contains "/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/files/file_to_upload.txt" "POST /api/2.0/workspace/import" 500 0 1000 +fault.py --body-contains "Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/files/file_to_upload.txt" "POST /api/2.0/workspace/import" 500 0 1000 musterr $CLI bundle deploy