From 14724215c6e3fd1b651b00c229d8add55447dbf1 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 6 Oct 2026 16:38:32 +0200 Subject: [PATCH 1/2] Translate paths in job environment_variables files The job `environment_variables[].spec.files` field (added in the v0.185.0 SDK bump) references `.env` files by path, but relative paths were passed through untranslated. Translate them to their deployed workspace locations like other job file paths. Co-authored-by: Isaac --- .../job-environment-variables-files.md | 1 + .../databricks.yml | 5 +++ .../env/common.env | 1 + .../out.test.toml | 2 ++ .../output.txt | 17 ++++++++++ .../resources/job.yml | 23 +++++++++++++ .../job_environment_variables_files/script | 1 + .../config/mutator/paths/job_paths_visitor.go | 20 ++++++++++- .../mutator/paths/job_paths_visitor_test.go | 34 +++++++++++++++++++ 9 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 .nextchanges/bundles/job-environment-variables-files.md create mode 100644 acceptance/bundle/paths/job_environment_variables_files/databricks.yml create mode 100644 acceptance/bundle/paths/job_environment_variables_files/env/common.env create mode 100644 acceptance/bundle/paths/job_environment_variables_files/out.test.toml create mode 100644 acceptance/bundle/paths/job_environment_variables_files/output.txt create mode 100644 acceptance/bundle/paths/job_environment_variables_files/resources/job.yml create mode 100644 acceptance/bundle/paths/job_environment_variables_files/script diff --git a/.nextchanges/bundles/job-environment-variables-files.md b/.nextchanges/bundles/job-environment-variables-files.md new file mode 100644 index 00000000000..85b6aa50c91 --- /dev/null +++ b/.nextchanges/bundles/job-environment-variables-files.md @@ -0,0 +1 @@ +* Translate relative `.env` file paths in a job's `environment_variables[].spec.files` to their deployed workspace locations. ([#6985](https://github.com/databricks/cli/pull/6985)) diff --git a/acceptance/bundle/paths/job_environment_variables_files/databricks.yml b/acceptance/bundle/paths/job_environment_variables_files/databricks.yml new file mode 100644 index 00000000000..3eb7d935c5e --- /dev/null +++ b/acceptance/bundle/paths/job_environment_variables_files/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: job_environment_variables_files + +include: + - resources/*.yml diff --git a/acceptance/bundle/paths/job_environment_variables_files/env/common.env b/acceptance/bundle/paths/job_environment_variables_files/env/common.env new file mode 100644 index 00000000000..c075a74be94 --- /dev/null +++ b/acceptance/bundle/paths/job_environment_variables_files/env/common.env @@ -0,0 +1 @@ +FOO=bar diff --git a/acceptance/bundle/paths/job_environment_variables_files/out.test.toml b/acceptance/bundle/paths/job_environment_variables_files/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/paths/job_environment_variables_files/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/paths/job_environment_variables_files/output.txt b/acceptance/bundle/paths/job_environment_variables_files/output.txt new file mode 100644 index 00000000000..c2976892187 --- /dev/null +++ b/acceptance/bundle/paths/job_environment_variables_files/output.txt @@ -0,0 +1,17 @@ + +>>> [CLI] bundle validate -o json +[ + { + "environment_variables_key": "default", + "spec": { + "files": [ + "/Workspace/Users/[USERNAME]/.bundle/job_environment_variables_files/default/files/env/common.env", + "/Volumes/main/default/vol/shared.env", + "/Workspace/Shared/shared.env" + ], + "variables": { + "FOO": "bar" + } + } + } +] diff --git a/acceptance/bundle/paths/job_environment_variables_files/resources/job.yml b/acceptance/bundle/paths/job_environment_variables_files/resources/job.yml new file mode 100644 index 00000000000..54b66074fb2 --- /dev/null +++ b/acceptance/bundle/paths/job_environment_variables_files/resources/job.yml @@ -0,0 +1,23 @@ +resources: + jobs: + my_job: + name: my job + environment_variables: + - environment_variables_key: default + spec: + variables: + FOO: bar + files: + - ../env/common.env + - /Volumes/main/default/vol/shared.env + - /Workspace/Shared/shared.env + tasks: + - task_key: my_task + environment_key: default + environment_variables_key: default + spark_python_task: + python_file: /Workspace/Shared/main.py + environments: + - environment_key: default + spec: + environment_version: "5" diff --git a/acceptance/bundle/paths/job_environment_variables_files/script b/acceptance/bundle/paths/job_environment_variables_files/script new file mode 100644 index 00000000000..ec97db7e2b6 --- /dev/null +++ b/acceptance/bundle/paths/job_environment_variables_files/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq '.resources.jobs.my_job.environment_variables' diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index cb400112b7e..ccc02f72f9f 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -1,6 +1,8 @@ package paths import ( + "slices" + "github.com/databricks/cli/libs/dyn" ) @@ -69,7 +71,23 @@ func jobRewritePatterns() []jobRewritePattern { taskPatterns := jobTaskRewritePatterns(base) forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - return append(taskPatterns, forEachPatterns...) + jobPatterns := []jobRewritePattern{ + { + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environment_variables"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("files"), + dyn.AnyIndex(), + ), + TranslateModeFile, + noSkipRewrite, + }, + } + return slices.Concat(taskPatterns, forEachPatterns, jobPatterns) } // VisitJobPaths visits all paths in job resources and applies a function to each path. diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go index 7584737b1c0..23fb6408267 100644 --- a/bundle/config/mutator/paths/job_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -131,6 +131,40 @@ func TestVisitJobPaths_environments(t *testing.T) { assert.ElementsMatch(t, expected, actual) } +func TestVisitJobPaths_environmentVariables(t *testing.T) { + job0 := &resources.Job{ + JobSettings: jobs.JobSettings{ + EnvironmentVariables: []jobs.JobEnvironmentVariables{ + { + EnvironmentVariablesKey: "default", + Spec: &jobs.JobEnvironmentVariablesSpec{ + Files: []string{ + "./a.env", + "./b.env", + }, + }, + }, + }, + }, + } + + root := config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job0": job0, + }, + }, + } + + actual := collectVisitedPaths(t, root, VisitJobPaths) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.environment_variables[0].spec.files[0]"), + dyn.MustPathFromString("resources.jobs.job0.environment_variables[0].spec.files[1]"), + } + + assert.ElementsMatch(t, expected, actual) +} + func TestVisitJobPaths_foreach(t *testing.T) { task0 := jobs.Task{ ForEachTask: &jobs.ForEachTask{ From 0e7baff0007bff0f22da53ff52f49a7aba0361e2 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 6 Oct 2026 16:55:44 +0200 Subject: [PATCH 2/2] Use append for job-level path pattern Co-authored-by: Isaac --- bundle/config/mutator/paths/job_paths_visitor.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go index ccc02f72f9f..6e82e544f02 100644 --- a/bundle/config/mutator/paths/job_paths_visitor.go +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -1,8 +1,6 @@ package paths import ( - "slices" - "github.com/databricks/cli/libs/dyn" ) @@ -71,8 +69,9 @@ func jobRewritePatterns() []jobRewritePattern { taskPatterns := jobTaskRewritePatterns(base) forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - jobPatterns := []jobRewritePattern{ - { + patterns := append(taskPatterns, forEachPatterns...) + return append(patterns, + jobRewritePattern{ dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), @@ -86,8 +85,7 @@ func jobRewritePatterns() []jobRewritePattern { TranslateModeFile, noSkipRewrite, }, - } - return slices.Concat(taskPatterns, forEachPatterns, jobPatterns) + ) } // VisitJobPaths visits all paths in job resources and applies a function to each path.