diff --git a/knowledge_base/databricks_app/resources/hello_world.job.yml b/knowledge_base/databricks_app/resources/hello_world.job.yml index 189ea0c0..d8ec2876 100644 --- a/knowledge_base/databricks_app/resources/hello_world.job.yml +++ b/knowledge_base/databricks_app/resources/hello_world.job.yml @@ -11,6 +11,6 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "6" dependencies: - cowsay \ No newline at end of file diff --git a/knowledge_base/databricks_app/resources/job_manager.app.yml b/knowledge_base/databricks_app/resources/job_manager.app.yml index 125b76ee..1ed9fb54 100644 --- a/knowledge_base/databricks_app/resources/job_manager.app.yml +++ b/knowledge_base/databricks_app/resources/job_manager.app.yml @@ -1,7 +1,7 @@ resources: apps: job_manager: - name: "job_manager" + name: "job-manager" description: "App which manages job created by this bundle" # The location of the source code for the app. diff --git a/knowledge_base/job_backfill_data/pyproject.toml b/knowledge_base/job_backfill_data/pyproject.toml index ebf98ae6..fe054efe 100644 --- a/knowledge_base/job_backfill_data/pyproject.toml +++ b/knowledge_base/job_backfill_data/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_conditional_execution/pyproject.toml b/knowledge_base/job_conditional_execution/pyproject.toml index c0ebf80c..a0a7ab3e 100644 --- a/knowledge_base/job_conditional_execution/pyproject.toml +++ b/knowledge_base/job_conditional_execution/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_file_arrival/pyproject.toml b/knowledge_base/job_file_arrival/pyproject.toml index c431da83..a3e7d928 100644 --- a/knowledge_base/job_file_arrival/pyproject.toml +++ b/knowledge_base/job_file_arrival/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_programmatic_generation/pyproject.toml b/knowledge_base/job_programmatic_generation/pyproject.toml index eb1b6b9b..627a05df 100644 --- a/knowledge_base/job_programmatic_generation/pyproject.toml +++ b/knowledge_base/job_programmatic_generation/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_table_update_trigger/pyproject.toml b/knowledge_base/job_table_update_trigger/pyproject.toml index a3962718..e8bd50f2 100644 --- a/knowledge_base/job_table_update_trigger/pyproject.toml +++ b/knowledge_base/job_table_update_trigger/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_with_for_each/pyproject.toml b/knowledge_base/job_with_for_each/pyproject.toml index 084c4b11..7dae9b92 100644 --- a/knowledge_base/job_with_for_each/pyproject.toml +++ b/knowledge_base/job_with_for_each/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/job_with_multiple_wheels/resources/job.yml b/knowledge_base/job_with_multiple_wheels/resources/job.yml index 0b530241..d4293a78 100644 --- a/knowledge_base/job_with_multiple_wheels/resources/job.yml +++ b/knowledge_base/job_with_multiple_wheels/resources/job.yml @@ -15,7 +15,7 @@ resources: new_cluster: node_type_id: i3.xlarge num_workers: 0 - spark_version: 14.3.x-scala2.12 + spark_version: 18.x-scala2.13 spark_conf: "spark.databricks.cluster.profile": "singleNode" "spark.master": "local[*, 4]" diff --git a/knowledge_base/job_with_run_job_tasks/databricks.yml b/knowledge_base/job_with_run_job_tasks/databricks.yml index 17b06bb2..a682a6de 100644 --- a/knowledge_base/job_with_run_job_tasks/databricks.yml +++ b/knowledge_base/job_with_run_job_tasks/databricks.yml @@ -49,7 +49,7 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "6" leaf2: @@ -72,7 +72,7 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "6" targets: diff --git a/knowledge_base/job_with_task_values/pyproject.toml b/knowledge_base/job_with_task_values/pyproject.toml index 21cc3a97..b1ce7f8d 100644 --- a/knowledge_base/job_with_task_values/pyproject.toml +++ b/knowledge_base/job_with_task_values/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ [dependency-groups] dev = [ "pytest", - "databricks-bundles==0.275.0", + "databricks-bundles==1.20.0", ] [build-system] diff --git a/knowledge_base/metric_view_dbt/resources/metric_view_dbt.job.yml b/knowledge_base/metric_view_dbt/resources/metric_view_dbt.job.yml index 970bd1b3..1703f1b1 100644 --- a/knowledge_base/metric_view_dbt/resources/metric_view_dbt.job.yml +++ b/knowledge_base/metric_view_dbt/resources/metric_view_dbt.job.yml @@ -27,7 +27,7 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6" dependencies: # The metric_view materialization landed in dbt-databricks 1.12.0 # (https://github.com/databricks/dbt-databricks/pull/1285). diff --git a/knowledge_base/pipeline_with_schema/resources/pipeline.yml b/knowledge_base/pipeline_with_schema/resources/pipeline.yml index fc20c2cc..92d6b2c6 100644 --- a/knowledge_base/pipeline_with_schema/resources/pipeline.yml +++ b/knowledge_base/pipeline_with_schema/resources/pipeline.yml @@ -4,7 +4,7 @@ resources: name: "Example of a pipeline with a schema" catalog: ${resources.schemas.example_schema.catalog_name} - target: ${resources.schemas.example_schema.name} + schema: ${resources.schemas.example_schema.name} libraries: - file: diff --git a/knowledge_base/private_wheel_packages/resources/serverless.job.yml b/knowledge_base/private_wheel_packages/resources/serverless.job.yml index b21f20cc..4c42550f 100644 --- a/knowledge_base/private_wheel_packages/resources/serverless.job.yml +++ b/knowledge_base/private_wheel_packages/resources/serverless.job.yml @@ -14,7 +14,7 @@ resources: - environment_key: default spec: - client: "1" + environment_version: "6" dependencies: # Relative local path reference to the private wheel package. # This references alone is sufficient to upload it on deployment. diff --git a/knowledge_base/python_wheel_poetry/resources/job.yml b/knowledge_base/python_wheel_poetry/resources/job.yml index 22e557e9..a7fa483a 100644 --- a/knowledge_base/python_wheel_poetry/resources/job.yml +++ b/knowledge_base/python_wheel_poetry/resources/job.yml @@ -15,4 +15,4 @@ resources: new_cluster: node_type_id: i3.xlarge num_workers: 1 - spark_version: 13.3.x-scala2.12 + spark_version: 18.x-scala2.13 diff --git a/knowledge_base/serverless_job/resources/serverless_job.yml b/knowledge_base/serverless_job/resources/serverless_job.yml index 8f93eeff..36b2f352 100644 --- a/knowledge_base/serverless_job/resources/serverless_job.yml +++ b/knowledge_base/serverless_job/resources/serverless_job.yml @@ -24,6 +24,6 @@ resources: # Full documentation of this spec can be found at: # https://docs.databricks.com/api/workspace/jobs/create#environments-spec spec: - client: "1" + environment_version: "6" dependencies: - cowsay diff --git a/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml b/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml index 0fdaf0b7..aebac997 100644 --- a/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml +++ b/knowledge_base/share_files_across_bundles/bundle_with_shared_code/databricks.yml @@ -28,4 +28,4 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "6" \ No newline at end of file diff --git a/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml b/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml index e3bf4d83..80cb0f54 100644 --- a/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml +++ b/knowledge_base/share_files_across_bundles/bundle_with_shared_configuration/databricks.yml @@ -28,4 +28,4 @@ resources: environments: - environment_key: default spec: - client: "1" + environment_version: "6" \ No newline at end of file diff --git a/knowledge_base/spark_jar_task/databricks.yml b/knowledge_base/spark_jar_task/databricks.yml index 5491cb38..64248994 100644 --- a/knowledge_base/spark_jar_task/databricks.yml +++ b/knowledge_base/spark_jar_task/databricks.yml @@ -21,7 +21,7 @@ resources: new_cluster: node_type_id: i3.xlarge num_workers: 1 - spark_version: 14.3.x-scala2.12 + spark_version: 18.x-scala2.13 spark_jar_task: main_class_name: PrintArgs libraries: diff --git a/knowledge_base/vector_search_product_discovery/resources/query-job.yml b/knowledge_base/vector_search_product_discovery/resources/query-job.yml index 6309b0ee..e9000b9c 100644 --- a/knowledge_base/vector_search_product_discovery/resources/query-job.yml +++ b/knowledge_base/vector_search_product_discovery/resources/query-job.yml @@ -20,7 +20,8 @@ resources: environments: - environment_key: serverless_env spec: - client: "3" + # Stay on 4: databricks-vectorsearch needs protobuf 5.x, environment 6 ships protobuf 6. + environment_version: "4" dependencies: - databricks-vectorsearch diff --git a/knowledge_base/vector_search_product_discovery/resources/setup-job.yml b/knowledge_base/vector_search_product_discovery/resources/setup-job.yml index 54ae9649..0d6f3556 100644 --- a/knowledge_base/vector_search_product_discovery/resources/setup-job.yml +++ b/knowledge_base/vector_search_product_discovery/resources/setup-job.yml @@ -18,7 +18,8 @@ resources: environments: - environment_key: serverless_env spec: - client: "3" + # Stay on 4: databricks-vectorsearch needs protobuf 5.x, environment 6 ships protobuf 6. + environment_version: "4" dependencies: - databricks-vectorsearch