From ea401df959a933858d187203503f639886b996a9 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 01:09:17 +0000 Subject: [PATCH 1/5] Replace deprecated bundle syntax and stale runtime pins in knowledge_base Co-authored-by: omnigent --- .../databricks_app/resources/hello_world.job.yml | 2 +- .../databricks_app/resources/job_manager.app.yml | 2 +- knowledge_base/job_backfill_data/pyproject.toml | 2 +- .../job_conditional_execution/pyproject.toml | 2 +- knowledge_base/job_file_arrival/pyproject.toml | 2 +- .../job_programmatic_generation/pyproject.toml | 2 +- knowledge_base/job_table_update_trigger/pyproject.toml | 2 +- knowledge_base/job_with_for_each/pyproject.toml | 2 +- knowledge_base/job_with_multiple_wheels/databricks.yml | 5 +++++ .../job_with_multiple_wheels/resources/job.yml | 4 ++-- knowledge_base/job_with_run_job_tasks/databricks.yml | 4 ++-- knowledge_base/job_with_task_values/pyproject.toml | 2 +- .../pipeline_with_schema/resources/pipeline.yml | 2 +- knowledge_base/private_wheel_packages/databricks.yml | 5 +++++ .../private_wheel_packages/resources/cluster.job.yml | 2 +- .../resources/serverless.job.yml | 2 +- knowledge_base/python_wheel_poetry/databricks.yml | 5 +++++ knowledge_base/python_wheel_poetry/resources/job.yml | 4 ++-- .../serverless_job/resources/serverless_job.yml | 2 +- .../bundle_with_shared_code/databricks.yml | 2 +- .../bundle_with_shared_configuration/databricks.yml | 2 +- knowledge_base/spark_jar_task/databricks.yml | 10 ++++++++-- .../resources/query-job.yml | 2 +- .../resources/setup-job.yml | 2 +- 24 files changed, 46 insertions(+), 25 deletions(-) diff --git a/knowledge_base/databricks_app/resources/hello_world.job.yml b/knowledge_base/databricks_app/resources/hello_world.job.yml index 189ea0c0..ea6268d4 100644 --- a/knowledge_base/databricks_app/resources/hello_world.job.yml +++ b/knowledge_base/databricks_app/resources/hello_world.job.yml @@ -11,6 +11,6 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "4" dependencies: - cowsay \ No newline at end of file diff --git a/knowledge_base/databricks_app/resources/job_manager.app.yml b/knowledge_base/databricks_app/resources/job_manager.app.yml index 125b76ee..1ed9fb54 100644 --- a/knowledge_base/databricks_app/resources/job_manager.app.yml +++ b/knowledge_base/databricks_app/resources/job_manager.app.yml @@ -1,7 +1,7 @@ resources: apps: job_manager: - name: "job_manager" + name: "job-manager" description: "App which manages job created by this bundle" # The location of the source code for the app. diff --git a/knowledge_base/job_backfill_data/pyproject.toml b/knowledge_base/job_backfill_data/pyproject.toml index ebf98ae6..b473e3d5 100644 --- a/knowledge_base/job_backfill_data/pyproject.toml +++ b/knowledge_base/job_backfill_data/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.18.0", ] [build-system] diff --git a/knowledge_base/job_conditional_execution/pyproject.toml b/knowledge_base/job_conditional_execution/pyproject.toml index c0ebf80c..95b6e3cf 100644 --- a/knowledge_base/job_conditional_execution/pyproject.toml +++ b/knowledge_base/job_conditional_execution/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.18.0", ] [build-system] diff --git a/knowledge_base/job_file_arrival/pyproject.toml b/knowledge_base/job_file_arrival/pyproject.toml index c431da83..f891ea34 100644 --- a/knowledge_base/job_file_arrival/pyproject.toml +++ b/knowledge_base/job_file_arrival/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.18.0", ] [build-system] diff --git a/knowledge_base/job_programmatic_generation/pyproject.toml b/knowledge_base/job_programmatic_generation/pyproject.toml index eb1b6b9b..c0c1a886 100644 --- a/knowledge_base/job_programmatic_generation/pyproject.toml +++ b/knowledge_base/job_programmatic_generation/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.18.0", ] [build-system] diff --git a/knowledge_base/job_table_update_trigger/pyproject.toml b/knowledge_base/job_table_update_trigger/pyproject.toml index a3962718..05ec68ca 100644 --- a/knowledge_base/job_table_update_trigger/pyproject.toml +++ b/knowledge_base/job_table_update_trigger/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.18.0", ] [build-system] diff --git a/knowledge_base/job_with_for_each/pyproject.toml b/knowledge_base/job_with_for_each/pyproject.toml index 084c4b11..ee3adaed 100644 --- a/knowledge_base/job_with_for_each/pyproject.toml +++ b/knowledge_base/job_with_for_each/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.18.0", ] [build-system] diff --git a/knowledge_base/job_with_multiple_wheels/databricks.yml b/knowledge_base/job_with_multiple_wheels/databricks.yml index 0fa31515..bf1ccb93 100644 --- a/knowledge_base/job_with_multiple_wheels/databricks.yml +++ b/knowledge_base/job_with_multiple_wheels/databricks.yml @@ -18,6 +18,11 @@ artifacts: build: poetry build path: ./my_custom_wheel2 +variables: + node_type_id: + description: "Classic cluster node type (AWS: i3.xlarge; Azure: Standard_DS3_v2)" + default: i3.xlarge + targets: dev: default: true diff --git a/knowledge_base/job_with_multiple_wheels/resources/job.yml b/knowledge_base/job_with_multiple_wheels/resources/job.yml index 0b530241..528254ba 100644 --- a/knowledge_base/job_with_multiple_wheels/resources/job.yml +++ b/knowledge_base/job_with_multiple_wheels/resources/job.yml @@ -13,9 +13,9 @@ resources: - whl: ../my_custom_wheel2/dist/*.whl new_cluster: - node_type_id: i3.xlarge + node_type_id: ${var.node_type_id} num_workers: 0 - spark_version: 14.3.x-scala2.12 + spark_version: 16.4.x-scala2.12 spark_conf: "spark.databricks.cluster.profile": "singleNode" "spark.master": "local[*, 4]" diff --git a/knowledge_base/job_with_run_job_tasks/databricks.yml b/knowledge_base/job_with_run_job_tasks/databricks.yml index 17b06bb2..8a7126b7 100644 --- a/knowledge_base/job_with_run_job_tasks/databricks.yml +++ b/knowledge_base/job_with_run_job_tasks/databricks.yml @@ -49,7 +49,7 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "4" leaf2: @@ -72,7 +72,7 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "4" targets: diff --git a/knowledge_base/job_with_task_values/pyproject.toml b/knowledge_base/job_with_task_values/pyproject.toml index 21cc3a97..50469db1 100644 --- a/knowledge_base/job_with_task_values/pyproject.toml +++ b/knowledge_base/job_with_task_values/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.18.0", ] [build-system] diff --git a/knowledge_base/pipeline_with_schema/resources/pipeline.yml b/knowledge_base/pipeline_with_schema/resources/pipeline.yml index fc20c2cc..92d6b2c6 100644 --- a/knowledge_base/pipeline_with_schema/resources/pipeline.yml +++ b/knowledge_base/pipeline_with_schema/resources/pipeline.yml @@ -4,7 +4,7 @@ resources: name: "Example of a pipeline with a schema" catalog: ${resources.schemas.example_schema.catalog_name} - target: ${resources.schemas.example_schema.name} + schema: ${resources.schemas.example_schema.name} libraries: - file: diff --git a/knowledge_base/private_wheel_packages/databricks.yml b/knowledge_base/private_wheel_packages/databricks.yml index b1d147e8..8c97e663 100644 --- a/knowledge_base/private_wheel_packages/databricks.yml +++ b/knowledge_base/private_wheel_packages/databricks.yml @@ -7,6 +7,11 @@ include: workspace: # host: https://myworkspace.cloud.databricks.com +variables: + node_type_id: + description: "Classic cluster node type (AWS: i3.xlarge; Azure: Standard_DS3_v2)" + default: i3.xlarge + targets: dev: default: true diff --git a/knowledge_base/private_wheel_packages/resources/cluster.job.yml b/knowledge_base/private_wheel_packages/resources/cluster.job.yml index b2922780..1874c77b 100644 --- a/knowledge_base/private_wheel_packages/resources/cluster.job.yml +++ b/knowledge_base/private_wheel_packages/resources/cluster.job.yml @@ -25,7 +25,7 @@ resources: - job_cluster_key: default new_cluster: spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge + node_type_id: ${var.node_type_id} data_security_mode: SINGLE_USER num_workers: 0 spark_conf: diff --git a/knowledge_base/private_wheel_packages/resources/serverless.job.yml b/knowledge_base/private_wheel_packages/resources/serverless.job.yml index b21f20cc..f25530ce 100644 --- a/knowledge_base/private_wheel_packages/resources/serverless.job.yml +++ b/knowledge_base/private_wheel_packages/resources/serverless.job.yml @@ -14,7 +14,7 @@ resources: - environment_key: default spec: - client: "1" + environment_version: "4" dependencies: # Relative local path reference to the private wheel package. # This references alone is sufficient to upload it on deployment. diff --git a/knowledge_base/python_wheel_poetry/databricks.yml b/knowledge_base/python_wheel_poetry/databricks.yml index 53ba433e..bbfa531f 100644 --- a/knowledge_base/python_wheel_poetry/databricks.yml +++ b/knowledge_base/python_wheel_poetry/databricks.yml @@ -12,6 +12,11 @@ artifacts: type: whl build: poetry build +variables: + node_type_id: + description: "Classic cluster node type (AWS: i3.xlarge; Azure: Standard_DS3_v2)" + default: i3.xlarge + targets: dev: default: true diff --git a/knowledge_base/python_wheel_poetry/resources/job.yml b/knowledge_base/python_wheel_poetry/resources/job.yml index 22e557e9..88cb4d57 100644 --- a/knowledge_base/python_wheel_poetry/resources/job.yml +++ b/knowledge_base/python_wheel_poetry/resources/job.yml @@ -13,6 +13,6 @@ resources: - whl: ../dist/*.whl new_cluster: - node_type_id: i3.xlarge + node_type_id: ${var.node_type_id} num_workers: 1 - spark_version: 13.3.x-scala2.12 + spark_version: 16.4.x-scala2.12 diff --git a/knowledge_base/serverless_job/resources/serverless_job.yml b/knowledge_base/serverless_job/resources/serverless_job.yml index 8f93eeff..d598ad14 100644 --- a/knowledge_base/serverless_job/resources/serverless_job.yml +++ b/knowledge_base/serverless_job/resources/serverless_job.yml @@ -24,6 +24,6 @@ resources: # Full documentation of this spec can be found at: # https://docs.databricks.com/api/workspace/jobs/create#environments-spec spec: - client: "1" + environment_version: "4" dependencies: - cowsay diff --git a/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml b/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml index 0fdaf0b7..f160f0d1 100644 --- a/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml +++ b/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml @@ -28,4 +28,4 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "4" \ No newline at end of file diff --git a/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml b/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml index e3bf4d83..e172ee76 100644 --- a/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml +++ b/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml @@ -28,4 +28,4 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "4" \ No newline at end of file diff --git a/knowledge_base/spark_jar_task/databricks.yml b/knowledge_base/spark_jar_task/databricks.yml index 5491cb38..f93ba15d 100644 --- a/knowledge_base/spark_jar_task/databricks.yml +++ b/knowledge_base/spark_jar_task/databricks.yml @@ -5,6 +5,12 @@ workspace: host: https://myworkspace.cloud.databricks.com artifact_path: /Volumes/my/artifact/path + +variables: + node_type_id: + description: "Classic cluster node type (AWS: i3.xlarge; Azure: Standard_DS3_v2)" + default: i3.xlarge + artifacts: my_java_code: path: ./print_args @@ -19,9 +25,9 @@ resources: tasks: - task_key: TestSparkJarTask new_cluster: - node_type_id: i3.xlarge + node_type_id: ${var.node_type_id} num_workers: 1 - spark_version: 14.3.x-scala2.12 + spark_version: 16.4.x-scala2.12 spark_jar_task: main_class_name: PrintArgs libraries: diff --git a/knowledge_base/vector_search_product_discovery/resources/query-job.yml b/knowledge_base/vector_search_product_discovery/resources/query-job.yml index 6309b0ee..99a873d7 100644 --- a/knowledge_base/vector_search_product_discovery/resources/query-job.yml +++ b/knowledge_base/vector_search_product_discovery/resources/query-job.yml @@ -20,7 +20,7 @@ resources: environments: - environment_key: serverless_env spec: - client: "3" + environment_version: "4" dependencies: - databricks-vectorsearch diff --git a/knowledge_base/vector_search_product_discovery/resources/setup-job.yml b/knowledge_base/vector_search_product_discovery/resources/setup-job.yml index 54ae9649..c322a54d 100644 --- a/knowledge_base/vector_search_product_discovery/resources/setup-job.yml +++ b/knowledge_base/vector_search_product_discovery/resources/setup-job.yml @@ -18,7 +18,7 @@ resources: environments: - environment_key: serverless_env spec: - client: "3" + environment_version: "4" dependencies: - databricks-vectorsearch From d2c4a01972927a53d5ca71bd2c9271f120c4d128 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 01:13:54 +0000 Subject: [PATCH 2/5] Revert out-of-scope node_type_id variable parameterization Co-authored-by: omnigent --- knowledge_base/job_with_multiple_wheels/databricks.yml | 5 ----- knowledge_base/job_with_multiple_wheels/resources/job.yml | 2 +- knowledge_base/private_wheel_packages/databricks.yml | 5 ----- .../private_wheel_packages/resources/cluster.job.yml | 2 +- knowledge_base/python_wheel_poetry/databricks.yml | 5 ----- knowledge_base/python_wheel_poetry/resources/job.yml | 2 +- knowledge_base/spark_jar_task/databricks.yml | 8 +------- 7 files changed, 4 insertions(+), 25 deletions(-) diff --git a/knowledge_base/job_with_multiple_wheels/databricks.yml b/knowledge_base/job_with_multiple_wheels/databricks.yml index bf1ccb93..0fa31515 100644 --- a/knowledge_base/job_with_multiple_wheels/databricks.yml +++ b/knowledge_base/job_with_multiple_wheels/databricks.yml @@ -18,11 +18,6 @@ artifacts: build: poetry build path: ./my_custom_wheel2 -variables: - node_type_id: - description: "Classic cluster node type (AWS: i3.xlarge; Azure: Standard_DS3_v2)" - default: i3.xlarge - targets: dev: default: true diff --git a/knowledge_base/job_with_multiple_wheels/resources/job.yml b/knowledge_base/job_with_multiple_wheels/resources/job.yml index 528254ba..0a0f0240 100644 --- a/knowledge_base/job_with_multiple_wheels/resources/job.yml +++ b/knowledge_base/job_with_multiple_wheels/resources/job.yml @@ -13,7 +13,7 @@ resources: - whl: ../my_custom_wheel2/dist/*.whl new_cluster: - node_type_id: ${var.node_type_id} + node_type_id: i3.xlarge num_workers: 0 spark_version: 16.4.x-scala2.12 spark_conf: diff --git a/knowledge_base/private_wheel_packages/databricks.yml b/knowledge_base/private_wheel_packages/databricks.yml index 8c97e663..b1d147e8 100644 --- a/knowledge_base/private_wheel_packages/databricks.yml +++ b/knowledge_base/private_wheel_packages/databricks.yml @@ -7,11 +7,6 @@ include: workspace: # host: https://myworkspace.cloud.databricks.com -variables: - node_type_id: - description: "Classic cluster node type (AWS: i3.xlarge; Azure: Standard_DS3_v2)" - default: i3.xlarge - targets: dev: default: true diff --git a/knowledge_base/private_wheel_packages/resources/cluster.job.yml b/knowledge_base/private_wheel_packages/resources/cluster.job.yml index 1874c77b..b2922780 100644 --- a/knowledge_base/private_wheel_packages/resources/cluster.job.yml +++ b/knowledge_base/private_wheel_packages/resources/cluster.job.yml @@ -25,7 +25,7 @@ resources: - job_cluster_key: default new_cluster: spark_version: 15.4.x-scala2.12 - node_type_id: ${var.node_type_id} + node_type_id: i3.xlarge data_security_mode: SINGLE_USER num_workers: 0 spark_conf: diff --git a/knowledge_base/python_wheel_poetry/databricks.yml b/knowledge_base/python_wheel_poetry/databricks.yml index bbfa531f..53ba433e 100644 --- a/knowledge_base/python_wheel_poetry/databricks.yml +++ b/knowledge_base/python_wheel_poetry/databricks.yml @@ -12,11 +12,6 @@ artifacts: type: whl build: poetry build -variables: - node_type_id: - description: "Classic cluster node type (AWS: i3.xlarge; Azure: Standard_DS3_v2)" - default: i3.xlarge - targets: dev: default: true diff --git a/knowledge_base/python_wheel_poetry/resources/job.yml b/knowledge_base/python_wheel_poetry/resources/job.yml index 88cb4d57..c93041ab 100644 --- a/knowledge_base/python_wheel_poetry/resources/job.yml +++ b/knowledge_base/python_wheel_poetry/resources/job.yml @@ -13,6 +13,6 @@ resources: - whl: ../dist/*.whl new_cluster: - node_type_id: ${var.node_type_id} + node_type_id: i3.xlarge num_workers: 1 spark_version: 16.4.x-scala2.12 diff --git a/knowledge_base/spark_jar_task/databricks.yml b/knowledge_base/spark_jar_task/databricks.yml index f93ba15d..010f22bb 100644 --- a/knowledge_base/spark_jar_task/databricks.yml +++ b/knowledge_base/spark_jar_task/databricks.yml @@ -5,12 +5,6 @@ workspace: host: https://myworkspace.cloud.databricks.com artifact_path: /Volumes/my/artifact/path - -variables: - node_type_id: - description: "Classic cluster node type (AWS: i3.xlarge; Azure: Standard_DS3_v2)" - default: i3.xlarge - artifacts: my_java_code: path: ./print_args @@ -25,7 +19,7 @@ resources: tasks: - task_key: TestSparkJarTask new_cluster: - node_type_id: ${var.node_type_id} + node_type_id: i3.xlarge num_workers: 1 spark_version: 16.4.x-scala2.12 spark_jar_task: From 4743ca018b077491893a27d2fdb86e20fc0d08e7 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 03:45:46 +0000 Subject: [PATCH 3/5] Bump serverless env to 6, classic DBR to 18, and job_* pins to 1.20.0. Co-authored-by: omnigent --- knowledge_base/databricks_app/resources/hello_world.job.yml | 2 +- knowledge_base/job_backfill_data/pyproject.toml | 2 +- knowledge_base/job_conditional_execution/pyproject.toml | 2 +- knowledge_base/job_file_arrival/pyproject.toml | 2 +- knowledge_base/job_programmatic_generation/pyproject.toml | 2 +- knowledge_base/job_table_update_trigger/pyproject.toml | 2 +- knowledge_base/job_with_for_each/pyproject.toml | 2 +- knowledge_base/job_with_multiple_wheels/resources/job.yml | 2 +- knowledge_base/job_with_run_job_tasks/databricks.yml | 4 ++-- knowledge_base/job_with_task_values/pyproject.toml | 2 +- .../private_wheel_packages/resources/serverless.job.yml | 2 +- knowledge_base/python_wheel_poetry/resources/job.yml | 2 +- knowledge_base/serverless_job/resources/serverless_job.yml | 2 +- .../bundle_with_shared_code/databricks.yml | 2 +- .../bundle_with_shared_configuration/databricks.yml | 2 +- knowledge_base/spark_jar_task/databricks.yml | 2 +- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/knowledge_base/databricks_app/resources/hello_world.job.yml b/knowledge_base/databricks_app/resources/hello_world.job.yml index ea6268d4..d8ec2876 100644 --- a/knowledge_base/databricks_app/resources/hello_world.job.yml +++ b/knowledge_base/databricks_app/resources/hello_world.job.yml @@ -11,6 +11,6 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6" dependencies: - cowsay \ No newline at end of file diff --git a/knowledge_base/job_backfill_data/pyproject.toml b/knowledge_base/job_backfill_data/pyproject.toml index b473e3d5..fe054efe 100644 --- a/knowledge_base/job_backfill_data/pyproject.toml +++ b/knowledge_base/job_backfill_data/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==1.18.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_conditional_execution/pyproject.toml b/knowledge_base/job_conditional_execution/pyproject.toml index 95b6e3cf..a0a7ab3e 100644 --- a/knowledge_base/job_conditional_execution/pyproject.toml +++ b/knowledge_base/job_conditional_execution/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==1.18.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_file_arrival/pyproject.toml b/knowledge_base/job_file_arrival/pyproject.toml index f891ea34..a3e7d928 100644 --- a/knowledge_base/job_file_arrival/pyproject.toml +++ b/knowledge_base/job_file_arrival/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==1.18.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_programmatic_generation/pyproject.toml b/knowledge_base/job_programmatic_generation/pyproject.toml index c0c1a886..627a05df 100644 --- a/knowledge_base/job_programmatic_generation/pyproject.toml +++ b/knowledge_base/job_programmatic_generation/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==1.18.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_table_update_trigger/pyproject.toml b/knowledge_base/job_table_update_trigger/pyproject.toml index 05ec68ca..e8bd50f2 100644 --- a/knowledge_base/job_table_update_trigger/pyproject.toml +++ b/knowledge_base/job_table_update_trigger/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==1.18.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_with_for_each/pyproject.toml b/knowledge_base/job_with_for_each/pyproject.toml index ee3adaed..7dae9b92 100644 --- a/knowledge_base/job_with_for_each/pyproject.toml +++ b/knowledge_base/job_with_for_each/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==1.18.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_with_multiple_wheels/resources/job.yml b/knowledge_base/job_with_multiple_wheels/resources/job.yml index 0a0f0240..d4293a78 100644 --- a/knowledge_base/job_with_multiple_wheels/resources/job.yml +++ b/knowledge_base/job_with_multiple_wheels/resources/job.yml @@ -15,7 +15,7 @@ resources: new_cluster: node_type_id: i3.xlarge num_workers: 0 - spark_version: 16.4.x-scala2.12 + spark_version: 18.x-scala2.13 spark_conf: "spark.databricks.cluster.profile": "singleNode" "spark.master": "local[*, 4]" diff --git a/knowledge_base/job_with_run_job_tasks/databricks.yml b/knowledge_base/job_with_run_job_tasks/databricks.yml index 8a7126b7..a682a6de 100644 --- a/knowledge_base/job_with_run_job_tasks/databricks.yml +++ b/knowledge_base/job_with_run_job_tasks/databricks.yml @@ -49,7 +49,7 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6" leaf2: @@ -72,7 +72,7 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6" targets: diff --git a/knowledge_base/job_with_task_values/pyproject.toml b/knowledge_base/job_with_task_values/pyproject.toml index 50469db1..b1ce7f8d 100644 --- a/knowledge_base/job_with_task_values/pyproject.toml +++ b/knowledge_base/job_with_task_values/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==1.18.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/private_wheel_packages/resources/serverless.job.yml b/knowledge_base/private_wheel_packages/resources/serverless.job.yml index f25530ce..4c42550f 100644 --- a/knowledge_base/private_wheel_packages/resources/serverless.job.yml +++ b/knowledge_base/private_wheel_packages/resources/serverless.job.yml @@ -14,7 +14,7 @@ resources: - environment_key: default spec: - environment_version: "4" + environment_version: "6" dependencies: # Relative local path reference to the private wheel package. # This references alone is sufficient to upload it on deployment. diff --git a/knowledge_base/python_wheel_poetry/resources/job.yml b/knowledge_base/python_wheel_poetry/resources/job.yml index c93041ab..a7fa483a 100644 --- a/knowledge_base/python_wheel_poetry/resources/job.yml +++ b/knowledge_base/python_wheel_poetry/resources/job.yml @@ -15,4 +15,4 @@ resources: new_cluster: node_type_id: i3.xlarge num_workers: 1 - spark_version: 16.4.x-scala2.12 + spark_version: 18.x-scala2.13 diff --git a/knowledge_base/serverless_job/resources/serverless_job.yml b/knowledge_base/serverless_job/resources/serverless_job.yml index d598ad14..36b2f352 100644 --- a/knowledge_base/serverless_job/resources/serverless_job.yml +++ b/knowledge_base/serverless_job/resources/serverless_job.yml @@ -24,6 +24,6 @@ resources: # Full documentation of this spec can be found at: # https://docs.databricks.com/api/workspace/jobs/create#environments-spec spec: - environment_version: "4" + environment_version: "6" dependencies: - cowsay diff --git a/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml b/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml index f160f0d1..aebac997 100644 --- a/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml +++ b/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml @@ -28,4 +28,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" \ No newline at end of file + environment_version: "6" \ No newline at end of file diff --git a/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml b/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml index e172ee76..80cb0f54 100644 --- a/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml +++ b/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml @@ -28,4 +28,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" \ No newline at end of file + environment_version: "6" \ No newline at end of file diff --git a/knowledge_base/spark_jar_task/databricks.yml b/knowledge_base/spark_jar_task/databricks.yml index 010f22bb..64248994 100644 --- a/knowledge_base/spark_jar_task/databricks.yml +++ b/knowledge_base/spark_jar_task/databricks.yml @@ -21,7 +21,7 @@ resources: new_cluster: node_type_id: i3.xlarge num_workers: 1 - spark_version: 16.4.x-scala2.12 + spark_version: 18.x-scala2.13 spark_jar_task: main_class_name: PrintArgs libraries: From 282593be4037481aa79aaf57d97655ff9c4aab78 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 03:48:21 +0000 Subject: [PATCH 4/5] Document why vector_search jobs stay on environment_version 4. Co-authored-by: omnigent --- .../vector_search_product_discovery/resources/query-job.yml | 1 + .../vector_search_product_discovery/resources/setup-job.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/knowledge_base/vector_search_product_discovery/resources/query-job.yml b/knowledge_base/vector_search_product_discovery/resources/query-job.yml index 99a873d7..e9000b9c 100644 --- a/knowledge_base/vector_search_product_discovery/resources/query-job.yml +++ b/knowledge_base/vector_search_product_discovery/resources/query-job.yml @@ -20,6 +20,7 @@ resources: environments: - environment_key: serverless_env spec: + # Stay on 4: databricks-vectorsearch needs protobuf 5.x, environment 6 ships protobuf 6. environment_version: "4" dependencies: - databricks-vectorsearch diff --git a/knowledge_base/vector_search_product_discovery/resources/setup-job.yml b/knowledge_base/vector_search_product_discovery/resources/setup-job.yml index c322a54d..0d6f3556 100644 --- a/knowledge_base/vector_search_product_discovery/resources/setup-job.yml +++ b/knowledge_base/vector_search_product_discovery/resources/setup-job.yml @@ -18,6 +18,7 @@ resources: environments: - environment_key: serverless_env spec: + # Stay on 4: databricks-vectorsearch needs protobuf 5.x, environment 6 ships protobuf 6. environment_version: "4" dependencies: - databricks-vectorsearch From da7955e353bd2da4c330ab1457c171eb3916a783 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 16:39:23 +0000 Subject: [PATCH 5/5] Bump metric_view_dbt serverless environment_version to 6. Co-authored-by: omnigent --- .../metric_view_dbt/resources/metric_view_dbt.job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knowledge_base/metric_view_dbt/resources/metric_view_dbt.job.yml b/knowledge_base/metric_view_dbt/resources/metric_view_dbt.job.yml index 970bd1b3..1703f1b1 100644 --- a/knowledge_base/metric_view_dbt/resources/metric_view_dbt.job.yml +++ b/knowledge_base/metric_view_dbt/resources/metric_view_dbt.job.yml @@ -27,7 +27,7 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6" dependencies: # The metric_view materialization landed in dbt-databricks 1.12.0 # (https://github.com/databricks/dbt-databricks/pull/1285).