From e9bb09ba4fc4e98c8120844068d9ca0cc3f44534 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:34:40 +0000 Subject: [PATCH 1/3] Add permissions and lifecycle example Co-authored-by: omnigent --- .../permissions_and_lifecycle/.gitignore | 2 + .../permissions_and_lifecycle/README.md | 55 +++++++++++++++++++ .../permissions_and_lifecycle/databricks.yml | 52 ++++++++++++++++++ .../resources/permissions.yml | 34 ++++++++++++ .../permissions_and_lifecycle/src/main.py | 1 + .../permissions_and_lifecycle/src/pipeline.py | 6 ++ 6 files changed, 150 insertions(+) create mode 100644 knowledge_base/permissions_and_lifecycle/.gitignore create mode 100644 knowledge_base/permissions_and_lifecycle/README.md create mode 100644 knowledge_base/permissions_and_lifecycle/databricks.yml create mode 100644 knowledge_base/permissions_and_lifecycle/resources/permissions.yml create mode 100644 knowledge_base/permissions_and_lifecycle/src/main.py create mode 100644 knowledge_base/permissions_and_lifecycle/src/pipeline.py diff --git a/knowledge_base/permissions_and_lifecycle/.gitignore b/knowledge_base/permissions_and_lifecycle/.gitignore new file mode 100644 index 0000000..159d3d4 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/.gitignore @@ -0,0 +1,2 @@ +.databricks/ + diff --git a/knowledge_base/permissions_and_lifecycle/README.md b/knowledge_base/permissions_and_lifecycle/README.md new file mode 100644 index 0000000..da9ab2a --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/README.md @@ -0,0 +1,55 @@ +# Permissions and lifecycle + +This Declarative Automation Bundles example shows combined bundle and resource permissions, target overrides, lifecycle protection, and resource-key replacement. See the [permissions documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/permissions). + +## Prerequisites + +* Databricks CLI v1.20.0 or above. +* Workspace administrator access. +* A temporary workspace group name. +* Existing `_dev` and `_prod` Unity Catalog schemas for the pipelines. + +## Usage + +1. Create a temporary group. + + `databricks groups create --display-name bundle-permissions-test --profile dabs-examples` + +2. Validate and deploy each target. + + `databricks bundle validate -t dev --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + + `databricks bundle deploy -t dev --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + + Repeat both commands with `-t prod`. + +| Target | `users` from bundle | Temporary group on job | Temporary group on pipeline | +| --- | --- | --- | --- | +| dev | `CAN_VIEW` | `CAN_MANAGE_RUN` | `CAN_RUN` | +| prod | `CAN_VIEW` | `CAN_MANAGE` | `CAN_MANAGE` | + +3. Run the job and pipeline for each target. + + `databricks bundle run permission_job -t dev --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + + `databricks bundle run permission_pipeline -t dev --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + + Repeat both commands with `-t prod`. + +4. Get each job ID from `databricks bundle summary` and read its effective permissions. + + `databricks permissions get jobs --profile dabs-examples` + +5. Rename `permission_job` in both YAML files and deploy again to replace the job. + + Set `lifecycle.prevent_destroy` to `true` to block this replacement or cleanup. + +6. Restore `lifecycle.prevent_destroy` to `false` and destroy both targets. + + `databricks bundle destroy -t dev --auto-approve --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + + Repeat the command with `-t prod`. + +7. Delete the temporary group. + + `databricks groups delete --profile dabs-examples` diff --git a/knowledge_base/permissions_and_lifecycle/databricks.yml b/knowledge_base/permissions_and_lifecycle/databricks.yml new file mode 100644 index 0000000..cbb4185 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/databricks.yml @@ -0,0 +1,52 @@ +bundle: + name: permissions_and_lifecycle + +include: + - resources/*.yml + +variables: + temporary_group: + description: Temporary group used to test resource permissions + catalog_name: + description: Unity Catalog catalog containing the pipeline schema + schema_name: + description: Existing schema used by the pipeline + +permissions: + - level: CAN_VIEW + group_name: users + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + resources: + jobs: + permission_job: + permissions: + - level: CAN_MANAGE_RUN + group_name: ${var.temporary_group} + pipelines: + permission_pipeline: + permissions: + - level: CAN_RUN + group_name: ${var.temporary_group} + + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + resources: + jobs: + permission_job: + permissions: + - level: CAN_MANAGE + group_name: ${var.temporary_group} + pipelines: + permission_pipeline: + permissions: + - level: CAN_MANAGE + group_name: ${var.temporary_group} + diff --git a/knowledge_base/permissions_and_lifecycle/resources/permissions.yml b/knowledge_base/permissions_and_lifecycle/resources/permissions.yml new file mode 100644 index 0000000..7d2e4fb --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/resources/permissions.yml @@ -0,0 +1,34 @@ +resources: + jobs: + permission_job: + name: permissions lifecycle job + lifecycle: + prevent_destroy: false + permissions: + - level: CAN_MANAGE_RUN + group_name: ${var.temporary_group} + tasks: + - task_key: permission_check + spark_python_task: + python_file: ../src/main.py + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "4" + + pipelines: + permission_pipeline: + name: permissions lifecycle pipeline + lifecycle: + prevent_destroy: false + permissions: + - level: CAN_RUN + group_name: ${var.temporary_group} + serverless: true + catalog: ${var.catalog_name} + target: ${var.schema_name}_${bundle.target} + libraries: + - file: + path: ../src/pipeline.py + diff --git a/knowledge_base/permissions_and_lifecycle/src/main.py b/knowledge_base/permissions_and_lifecycle/src/main.py new file mode 100644 index 0000000..8e20285 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/src/main.py @@ -0,0 +1 @@ +print("Permissions and lifecycle example") diff --git a/knowledge_base/permissions_and_lifecycle/src/pipeline.py b/knowledge_base/permissions_and_lifecycle/src/pipeline.py new file mode 100644 index 0000000..3aff258 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/src/pipeline.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.materialized_view +def permission_example(): + return spark.range(1) From 6f687fe4cc68458074fb9f5f76c40c48aee801b6 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:39:33 +0000 Subject: [PATCH 2/3] Use team group terminology Co-authored-by: omnigent --- .../permissions_and_lifecycle/README.md | 22 ++++++++++--------- .../permissions_and_lifecycle/databricks.yml | 12 +++++----- .../resources/permissions.yml | 4 ++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/knowledge_base/permissions_and_lifecycle/README.md b/knowledge_base/permissions_and_lifecycle/README.md index da9ab2a..4f8c681 100644 --- a/knowledge_base/permissions_and_lifecycle/README.md +++ b/knowledge_base/permissions_and_lifecycle/README.md @@ -6,33 +6,33 @@ This Declarative Automation Bundles example shows combined bundle and resource p * Databricks CLI v1.20.0 or above. * Workspace administrator access. -* A temporary workspace group name. +* A workspace group for your team. * Existing `_dev` and `_prod` Unity Catalog schemas for the pipelines. ## Usage -1. Create a temporary group. +1. Create a workspace group for your team. - `databricks groups create --display-name bundle-permissions-test --profile dabs-examples` + `databricks groups create --display-name data-engineering-team --profile dabs-examples` 2. Validate and deploy each target. - `databricks bundle validate -t dev --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + `databricks bundle validate -t dev --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` - `databricks bundle deploy -t dev --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + `databricks bundle deploy -t dev --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` Repeat both commands with `-t prod`. -| Target | `users` from bundle | Temporary group on job | Temporary group on pipeline | +| Target | `users` from bundle | Team group on job | Team group on pipeline | | --- | --- | --- | --- | | dev | `CAN_VIEW` | `CAN_MANAGE_RUN` | `CAN_RUN` | | prod | `CAN_VIEW` | `CAN_MANAGE` | `CAN_MANAGE` | 3. Run the job and pipeline for each target. - `databricks bundle run permission_job -t dev --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + `databricks bundle run permission_job -t dev --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` - `databricks bundle run permission_pipeline -t dev --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + `databricks bundle run permission_pipeline -t dev --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` Repeat both commands with `-t prod`. @@ -40,16 +40,18 @@ This Declarative Automation Bundles example shows combined bundle and resource p `databricks permissions get jobs --profile dabs-examples` + `databricks permissions get pipelines --profile dabs-examples` + 5. Rename `permission_job` in both YAML files and deploy again to replace the job. Set `lifecycle.prevent_destroy` to `true` to block this replacement or cleanup. 6. Restore `lifecycle.prevent_destroy` to `false` and destroy both targets. - `databricks bundle destroy -t dev --auto-approve --var "temporary_group=bundle-permissions-test,catalog_name=,schema_name=" --profile dabs-examples` + `databricks bundle destroy -t dev --auto-approve --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` Repeat the command with `-t prod`. -7. Delete the temporary group. +7. Delete the example team group. `databricks groups delete --profile dabs-examples` diff --git a/knowledge_base/permissions_and_lifecycle/databricks.yml b/knowledge_base/permissions_and_lifecycle/databricks.yml index cbb4185..f6e6715 100644 --- a/knowledge_base/permissions_and_lifecycle/databricks.yml +++ b/knowledge_base/permissions_and_lifecycle/databricks.yml @@ -5,8 +5,8 @@ include: - resources/*.yml variables: - temporary_group: - description: Temporary group used to test resource permissions + team_group: + description: Workspace group for your team catalog_name: description: Unity Catalog catalog containing the pipeline schema schema_name: @@ -27,12 +27,12 @@ targets: permission_job: permissions: - level: CAN_MANAGE_RUN - group_name: ${var.temporary_group} + group_name: ${var.team_group} pipelines: permission_pipeline: permissions: - level: CAN_RUN - group_name: ${var.temporary_group} + group_name: ${var.team_group} prod: mode: production @@ -43,10 +43,10 @@ targets: permission_job: permissions: - level: CAN_MANAGE - group_name: ${var.temporary_group} + group_name: ${var.team_group} pipelines: permission_pipeline: permissions: - level: CAN_MANAGE - group_name: ${var.temporary_group} + group_name: ${var.team_group} diff --git a/knowledge_base/permissions_and_lifecycle/resources/permissions.yml b/knowledge_base/permissions_and_lifecycle/resources/permissions.yml index 7d2e4fb..e0cb560 100644 --- a/knowledge_base/permissions_and_lifecycle/resources/permissions.yml +++ b/knowledge_base/permissions_and_lifecycle/resources/permissions.yml @@ -6,7 +6,7 @@ resources: prevent_destroy: false permissions: - level: CAN_MANAGE_RUN - group_name: ${var.temporary_group} + group_name: ${var.team_group} tasks: - task_key: permission_check spark_python_task: @@ -24,7 +24,7 @@ resources: prevent_destroy: false permissions: - level: CAN_RUN - group_name: ${var.temporary_group} + group_name: ${var.team_group} serverless: true catalog: ${var.catalog_name} target: ${var.schema_name}_${bundle.target} From 584eff32d5890382bf60ff174b57fcc1dc94792b Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 02:37:50 +0000 Subject: [PATCH 3/3] Use serverless environment version 6 Move the serverless job environment from version 4 to version 6, the newest GA version. Co-authored-by: Isaac Co-authored-by: omnigent --- .../permissions_and_lifecycle/resources/permissions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knowledge_base/permissions_and_lifecycle/resources/permissions.yml b/knowledge_base/permissions_and_lifecycle/resources/permissions.yml index e0cb560..30fa9b1 100644 --- a/knowledge_base/permissions_and_lifecycle/resources/permissions.yml +++ b/knowledge_base/permissions_and_lifecycle/resources/permissions.yml @@ -15,7 +15,7 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6" pipelines: permission_pipeline: