diff --git a/knowledge_base/unity_catalog_objects/.gitignore b/knowledge_base/unity_catalog_objects/.gitignore new file mode 100644 index 0000000..15bcc6d --- /dev/null +++ b/knowledge_base/unity_catalog_objects/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/knowledge_base/unity_catalog_objects/README.md b/knowledge_base/unity_catalog_objects/README.md new file mode 100644 index 0000000..dd69829 --- /dev/null +++ b/knowledge_base/unity_catalog_objects/README.md @@ -0,0 +1,31 @@ +# Unity Catalog objects + +This Declarative Automation Bundles example creates a managed Unity Catalog catalog, schema, and volume with grants. +A serverless job writes a marker file into the volume. +See the [catalog resource documentation](https://docs.databricks.com/dev-tools/bundles/resources.html#catalogs). + +## Prerequisites + +* Databricks CLI v1.20.0 or above +* A workspace with Unity Catalog and serverless jobs +* Permission to create catalogs in the metastore +* A cloud storage root URL for the catalog when the metastore has no default root + +## Usage + +1. Set the catalog storage root: `export BUNDLE_VAR_catalog_storage_root=abfss://container@storageaccount.dfs.core.windows.net/` +2. Validate the development target: `databricks bundle validate -t dev --profile ` +3. Deploy the development target: `databricks bundle deploy -t dev --profile ` +4. Run the write job: `databricks bundle run -t dev write_marker --profile ` +5. Check catalog grants: `databricks grants get catalog --profile ` +6. In `resources/data.schema.yml`, set `lifecycle.prevent_destroy` to `false`. +7. Remove the development deployment: `databricks bundle destroy -t dev --auto-approve --profile ` +8. Deploy the production target: `databricks bundle deploy -t prod --profile ` +9. Remove the production deployment after the same `prevent_destroy` steps: `databricks bundle destroy -t prod --auto-approve --profile ` + +| Resource | Purpose | +| --- | --- | +| `catalogs.uc_demo` | Managed catalog with grants | +| `schemas.data` | Schema with `lifecycle.prevent_destroy` | +| `volumes.landing` | Managed volume for the marker file | +| `jobs.write_marker` | Serverless job that proves volume access | diff --git a/knowledge_base/unity_catalog_objects/databricks.yml b/knowledge_base/unity_catalog_objects/databricks.yml new file mode 100644 index 0000000..0b32c0a --- /dev/null +++ b/knowledge_base/unity_catalog_objects/databricks.yml @@ -0,0 +1,34 @@ +# Bundle-managed Unity Catalog catalog, schema, volume, grants, and a write job. +# Catalogs require the direct deployment engine (default in CLI >= 1.20). +bundle: + name: unity_catalog_objects + engine: direct + databricks_cli_version: ">= 1.20.0" + +include: + - resources/*.yml + +variables: + grant_group: + description: Group that receives USE_CATALOG, USE_SCHEMA, and READ_VOLUME. + default: users + # Catalog names allow only alphanumeric characters and underscores. + catalog_name: + description: Unique catalog name that includes the target and deploying user. + default: uc_objects_${bundle.target}_${workspace.current_user.short_name} + # Required when the metastore has no default managed storage root. + catalog_storage_root: + description: Cloud storage root URL for managed tables in the catalog. + default: abfss://container@storageaccount.dfs.core.windows.net/ + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} diff --git a/knowledge_base/unity_catalog_objects/resources/data.schema.yml b/knowledge_base/unity_catalog_objects/resources/data.schema.yml new file mode 100644 index 0000000..b3267b7 --- /dev/null +++ b/knowledge_base/unity_catalog_objects/resources/data.schema.yml @@ -0,0 +1,13 @@ +resources: + schemas: + data: + name: data + catalog_name: ${resources.catalogs.uc_demo.name} + comment: Data-holding schema with prevent_destroy. + grants: + - principal: ${var.grant_group} + privileges: + - USE_SCHEMA + # Client-side guard: destroy fails while true. Set false, then destroy. + lifecycle: + prevent_destroy: true diff --git a/knowledge_base/unity_catalog_objects/resources/data.volume.yml b/knowledge_base/unity_catalog_objects/resources/data.volume.yml new file mode 100644 index 0000000..f279a0a --- /dev/null +++ b/knowledge_base/unity_catalog_objects/resources/data.volume.yml @@ -0,0 +1,12 @@ +resources: + volumes: + landing: + name: landing + catalog_name: ${resources.catalogs.uc_demo.name} + schema_name: ${resources.schemas.data.name} + volume_type: MANAGED + comment: Managed volume for marker files from the write job. + grants: + - principal: ${var.grant_group} + privileges: + - READ_VOLUME diff --git a/knowledge_base/unity_catalog_objects/resources/uc_demo.catalog.yml b/knowledge_base/unity_catalog_objects/resources/uc_demo.catalog.yml new file mode 100644 index 0000000..655fb23 --- /dev/null +++ b/knowledge_base/unity_catalog_objects/resources/uc_demo.catalog.yml @@ -0,0 +1,11 @@ +resources: + catalogs: + uc_demo: + # Managed catalog. Set catalog_storage_root when the metastore has no default root. + name: ${var.catalog_name} + storage_root: ${var.catalog_storage_root} + comment: Bundle-managed catalog for the unity_catalog_objects example. + grants: + - principal: ${var.grant_group} + privileges: + - USE_CATALOG diff --git a/knowledge_base/unity_catalog_objects/resources/write_marker.job.yml b/knowledge_base/unity_catalog_objects/resources/write_marker.job.yml new file mode 100644 index 0000000..cf937ab --- /dev/null +++ b/knowledge_base/unity_catalog_objects/resources/write_marker.job.yml @@ -0,0 +1,15 @@ +resources: + jobs: + write_marker: + name: write_marker + tasks: + - task_key: write + spark_python_task: + python_file: ../src/write_marker.py + parameters: + - /Volumes/${resources.catalogs.uc_demo.name}/${resources.schemas.data.name}/${resources.volumes.landing.name}/marker.txt + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "6" diff --git a/knowledge_base/unity_catalog_objects/src/write_marker.py b/knowledge_base/unity_catalog_objects/src/write_marker.py new file mode 100644 index 0000000..ba29c35 --- /dev/null +++ b/knowledge_base/unity_catalog_objects/src/write_marker.py @@ -0,0 +1,20 @@ +"""Write a marker file into the bundle-managed Unity Catalog volume.""" + +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: write_marker.py ") + path = Path(sys.argv[1]) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f"written_at={datetime.now(timezone.utc).isoformat()}\n", encoding="utf-8" + ) + print(f"wrote {path}") + + +if __name__ == "__main__": + main()