diff --git a/knowledge_base/permissions_and_lifecycle/.gitignore b/knowledge_base/permissions_and_lifecycle/.gitignore new file mode 100644 index 0000000..159d3d4 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/.gitignore @@ -0,0 +1,2 @@ +.databricks/ + diff --git a/knowledge_base/permissions_and_lifecycle/README.md b/knowledge_base/permissions_and_lifecycle/README.md new file mode 100644 index 0000000..4f8c681 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/README.md @@ -0,0 +1,57 @@ +# Permissions and lifecycle + +This Declarative Automation Bundles example shows combined bundle and resource permissions, target overrides, lifecycle protection, and resource-key replacement. See the [permissions documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/permissions). + +## Prerequisites + +* Databricks CLI v1.20.0 or above. +* Workspace administrator access. +* A workspace group for your team. +* Existing `_dev` and `_prod` Unity Catalog schemas for the pipelines. + +## Usage + +1. Create a workspace group for your team. + + `databricks groups create --display-name data-engineering-team --profile dabs-examples` + +2. Validate and deploy each target. + + `databricks bundle validate -t dev --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` + + `databricks bundle deploy -t dev --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` + + Repeat both commands with `-t prod`. + +| Target | `users` from bundle | Team group on job | Team group on pipeline | +| --- | --- | --- | --- | +| dev | `CAN_VIEW` | `CAN_MANAGE_RUN` | `CAN_RUN` | +| prod | `CAN_VIEW` | `CAN_MANAGE` | `CAN_MANAGE` | + +3. Run the job and pipeline for each target. + + `databricks bundle run permission_job -t dev --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` + + `databricks bundle run permission_pipeline -t dev --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` + + Repeat both commands with `-t prod`. + +4. Get each job ID from `databricks bundle summary` and read its effective permissions. + + `databricks permissions get jobs --profile dabs-examples` + + `databricks permissions get pipelines --profile dabs-examples` + +5. Rename `permission_job` in both YAML files and deploy again to replace the job. + + Set `lifecycle.prevent_destroy` to `true` to block this replacement or cleanup. + +6. Restore `lifecycle.prevent_destroy` to `false` and destroy both targets. + + `databricks bundle destroy -t dev --auto-approve --var "team_group=data-engineering-team,catalog_name=,schema_name=" --profile dabs-examples` + + Repeat the command with `-t prod`. + +7. Delete the example team group. + + `databricks groups delete --profile dabs-examples` diff --git a/knowledge_base/permissions_and_lifecycle/databricks.yml b/knowledge_base/permissions_and_lifecycle/databricks.yml new file mode 100644 index 0000000..f6e6715 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/databricks.yml @@ -0,0 +1,52 @@ +bundle: + name: permissions_and_lifecycle + +include: + - resources/*.yml + +variables: + team_group: + description: Workspace group for your team + catalog_name: + description: Unity Catalog catalog containing the pipeline schema + schema_name: + description: Existing schema used by the pipeline + +permissions: + - level: CAN_VIEW + group_name: users + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + resources: + jobs: + permission_job: + permissions: + - level: CAN_MANAGE_RUN + group_name: ${var.team_group} + pipelines: + permission_pipeline: + permissions: + - level: CAN_RUN + group_name: ${var.team_group} + + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + resources: + jobs: + permission_job: + permissions: + - level: CAN_MANAGE + group_name: ${var.team_group} + pipelines: + permission_pipeline: + permissions: + - level: CAN_MANAGE + group_name: ${var.team_group} + diff --git a/knowledge_base/permissions_and_lifecycle/resources/permissions.yml b/knowledge_base/permissions_and_lifecycle/resources/permissions.yml new file mode 100644 index 0000000..30fa9b1 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/resources/permissions.yml @@ -0,0 +1,34 @@ +resources: + jobs: + permission_job: + name: permissions lifecycle job + lifecycle: + prevent_destroy: false + permissions: + - level: CAN_MANAGE_RUN + group_name: ${var.team_group} + tasks: + - task_key: permission_check + spark_python_task: + python_file: ../src/main.py + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "6" + + pipelines: + permission_pipeline: + name: permissions lifecycle pipeline + lifecycle: + prevent_destroy: false + permissions: + - level: CAN_RUN + group_name: ${var.team_group} + serverless: true + catalog: ${var.catalog_name} + target: ${var.schema_name}_${bundle.target} + libraries: + - file: + path: ../src/pipeline.py + diff --git a/knowledge_base/permissions_and_lifecycle/src/main.py b/knowledge_base/permissions_and_lifecycle/src/main.py new file mode 100644 index 0000000..8e20285 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/src/main.py @@ -0,0 +1 @@ +print("Permissions and lifecycle example") diff --git a/knowledge_base/permissions_and_lifecycle/src/pipeline.py b/knowledge_base/permissions_and_lifecycle/src/pipeline.py new file mode 100644 index 0000000..3aff258 --- /dev/null +++ b/knowledge_base/permissions_and_lifecycle/src/pipeline.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.materialized_view +def permission_example(): + return spark.range(1)