diff --git a/.nextchanges/bundles/7229.md b/.nextchanges/bundles/7229.md new file mode 100644 index 00000000000..346486808bf --- /dev/null +++ b/.nextchanges/bundles/7229.md @@ -0,0 +1 @@ +* Fix migration from Terraform dropping the principal of a permission or grant that references a resource that was never deployed. ([#7229](https://github.com/databricks/cli/pull/7229)) diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/app.py b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/app.py new file mode 100644 index 00000000000..f1a18139c84 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/databricks.yml b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/databricks.yml new file mode 100644 index 00000000000..5bb57bf4480 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/databricks.yml @@ -0,0 +1,31 @@ +bundle: + name: migrate-basic-test + +resources: + # apps: + # new_app: + # name: new-app + # source_code_path: . + + jobs: + test_job: + name: "Test Migration Job" + tasks: + - task_key: "main" + notebook_task: + notebook_path: "./notebook.py" + permissions: + # - level: CAN_MANAGE + # service_principal_name: ${resources.apps.new_app.service_principal_client_id} + - level: CAN_VIEW + user_name: viewer@databricks.com + + pipelines: + test_pipeline: + name: "Test Migration Pipeline" + libraries: + - notebook: + path: "./pipeline.py" + permissions: + - level: CAN_MANAGE + user_name: manager@databricks.com diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/notebook.py b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/notebook.py new file mode 100644 index 00000000000..250b3c9f766 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Hello from test migration job") diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/out.test.toml b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/output.txt b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/output.txt new file mode 100644 index 00000000000..79e48e24095 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/output.txt @@ -0,0 +1,239 @@ + +=== Replay a prior terraform deployment of a job and a pipeline with permissions +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +=== Add a new app and a job permission that references its service principal +=== Plan shows the new app and the added permission +>>> [CLI] bundle plan -o json +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +{ + "plan_version": [PLAN_VERSION], + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 7, + "plan": { + "resources.apps.new_app": { + "action": "create", + "new_state": { + "value": { + "description": "", + "name": "new-app", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files" + } + } + }, + "resources.jobs.test_job": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [TEST_JOB_ID], + "max_concurrent_runs": 1, + "name": "Test Migration Job", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "tasks": [ + { + "email_notifications": {}, + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/notebook", + "source": "WORKSPACE" + }, + "run_if": "ALL_SUCCESS", + "task_key": "main", + "timeout_seconds": 0 + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='main'].email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "tasks[task_key='main'].notebook_task.source": { + "action": "skip", + "reason": "backend_default", + "remote": "WORKSPACE" + }, + "tasks[task_key='main'].run_if": { + "action": "skip", + "reason": "backend_default", + "remote": "ALL_SUCCESS" + }, + "tasks[task_key='main'].timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.test_job.permissions": { + "depends_on": [ + { + "node": "resources.apps.new_app", + "label": "${resources.apps.new_app.service_principal_client_id}" + }, + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.id}" + } + ], + "action": "update", + "new_state": { + "value": { + "object_id": "/jobs/[TEST_JOB_ID]", + "__embed__": [ + { + "level": "CAN_MANAGE", + "service_principal_name": "${resources.apps.new_app.service_principal_client_id}" + }, + { + "level": "CAN_VIEW", + "user_name": "viewer@databricks.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "[0].service_principal_name": "${resources.apps.new_app.service_principal_client_id}" + } + }, + "remote_state": { + "object_id": "/jobs/[TEST_JOB_ID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@databricks.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "changes": { + "[service_principal_name='${resources.apps.new_app.service_principal_client_id}']": { + "action": "update", + "new": { + "level": "CAN_MANAGE", + "service_principal_name": "${resources.apps.new_app.service_principal_client_id}" + } + } + } + }, + "resources.pipelines.test_pipeline": { + "action": "skip", + "remote_state": { + "channel": "CURRENT", + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json" + }, + "edition": "ADVANCED", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "id": "[TEST_PIPELINE_ID]", + "last_modified": [UNIX_TIME_MILLIS][1], + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/pipeline" + } + } + ], + "name": "Test Migration Pipeline", + "pipeline_id": "[TEST_PIPELINE_ID]", + "run_as_user_name": "[USERNAME]", + "state": "IDLE", + "storage": "dbfs:[PIPELINE_TEST_PIPELINE_ID]" + }, + "changes": { + "storage": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "dbfs:[PIPELINE_TEST_PIPELINE_ID]" + } + } + }, + "resources.pipelines.test_pipeline.permissions": { + "depends_on": [ + { + "node": "resources.pipelines.test_pipeline", + "label": "${resources.pipelines.test_pipeline.id}" + } + ], + "action": "skip", + "remote_state": { + "object_id": "[PIPELINE_TEST_PIPELINE_ID]", + "__embed__": [ + { + "level": "CAN_MANAGE", + "user_name": "manager@databricks.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + } + } +} + +=== Deploy migrates to direct, creates the app and grants the permission +>>> [CLI] bundle deploy +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files... +Migrated 4 resources to direct deployment engine. +Created apps.new_app +Updated jobs.test_job.permissions +Files: 4 uploaded, 0 deleted +Resources: 1 created, 1 changed, 0 deleted, 3 unchanged + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.new_app + delete resources.jobs.test_job + delete resources.pipelines.test_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.test_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/pipeline.py b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/pipeline.py new file mode 100644 index 00000000000..cd19031e962 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/pipeline.py @@ -0,0 +1,2 @@ +# Databricks notebook source +# This is a test pipeline notebook diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/script b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/script new file mode 100644 index 00000000000..005859e731b --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/script @@ -0,0 +1,25 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Replay a prior terraform deployment of a job and a pipeline with permissions" +trace replay_tfstate.py terraform-requests.json terraform.tfstate +replace_ids.py +rm -f "$OUT_REQUESTS" + +# The job gains a permission for the service principal of a new app. Terraform never deployed +# either, so the migrated state must not hold a permission entry without a principal. +title "Add a new app and a job permission that references its service principal" +update_file.py databricks.yml '# ' '' + +title "Plan shows the new app and the added permission" +trace $CLI bundle plan -o json +rm -f "$OUT_REQUESTS" + +title "Deploy migrates to direct, creates the app and grants the permission" +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Redeploy is a no-op" +trace $CLI bundle plan + +trace $CLI bundle destroy --auto-approve +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/terraform-requests.json b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/terraform-requests.json new file mode 100644 index 00000000000..a8894d9f79a --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/terraform-requests.json @@ -0,0 +1,80 @@ +[ + { + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/files/pipeline" + } + } + ], + "name": "Test Migration Pipeline" + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Test Migration Job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "main" + } + ] + } + }, + { + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/79fe85ff-42ab-b5b8-79fe-85ff42abb5b9", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "manager@databricks.com" + }, + { + "permission_level": "IS_OWNER", + "user_name": "tester@databricks.com" + } + ] + } + }, + { + "method": "PUT", + "path": "/api/2.0/permissions/jobs/8790610854067598000", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_VIEW", + "user_name": "viewer@databricks.com" + }, + { + "permission_level": "IS_OWNER", + "user_name": "tester@databricks.com" + } + ] + } + } +] diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/terraform.tfstate b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/terraform.tfstate new file mode 100644 index 00000000000..a1561224071 --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/terraform.tfstate @@ -0,0 +1,387 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 5, + "lineage": "a0fc6106-3686-738c-7a93-06c0a226574f", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "test_job", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8790610854067598000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "Test Migration Job", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [ + { + "ai_runtime_task": [], + "alert_task": [], + "clean_rooms_notebook_task": [], + "compute": [], + "condition_task": [], + "dashboard_task": [], + "dbt_cloud_task": [], + "dbt_platform_task": [], + "dbt_task": [], + "depends_on": [], + "description": "", + "disable_auto_optimization": false, + "disabled": false, + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment_key": "", + "existing_cluster_id": "", + "for_each_task": [], + "gen_ai_compute_task": [], + "health": [], + "job_cluster_key": "", + "library": [], + "max_retries": 0, + "min_retry_interval_millis": 0, + "new_cluster": [], + "notebook_task": [ + { + "base_parameters": null, + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/files/notebook", + "source": "WORKSPACE", + "warehouse_id": "" + } + ], + "notification_settings": [], + "pipeline_task": [], + "power_bi_task": [], + "python_operator_task": [], + "python_wheel_task": [], + "retry_on_timeout": false, + "run_if": "ALL_SUCCESS", + "run_job_task": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "sql_task": [], + "task_key": "main", + "timeout_seconds": 0, + "webhook_notifications": [] + } + ], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8790610854067598000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + }, + { + "mode": "managed", + "type": "databricks_permissions", + "name": "job_test_job", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "access_control": [ + { + "group_name": "", + "permission_level": "CAN_VIEW", + "service_principal_name": "", + "user_name": "viewer@databricks.com" + }, + { + "group_name": "", + "permission_level": "IS_OWNER", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "alert_v2_id": null, + "app_name": null, + "authorization": null, + "cluster_id": null, + "cluster_policy_id": null, + "dashboard_id": null, + "database_instance_name": null, + "database_project_name": null, + "directory_id": null, + "directory_path": null, + "experiment_id": null, + "id": "/jobs/8790610854067598000", + "instance_pool_id": null, + "job_id": "8790610854067598000", + "knowledge_assistant_id": null, + "notebook_id": null, + "notebook_path": null, + "object_type": "job", + "pipeline_id": null, + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "registered_model_id": null, + "repo_id": null, + "repo_path": null, + "serving_endpoint_id": null, + "sql_alert_id": null, + "sql_dashboard_id": null, + "sql_endpoint_id": null, + "sql_query_id": null, + "supervisor_agent_id": null, + "vector_search_endpoint_id": null, + "workspace_file_id": null, + "workspace_file_path": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "databricks_job.test_job" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_permissions", + "name": "pipeline_test_pipeline", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "access_control": [ + { + "group_name": "", + "permission_level": "CAN_MANAGE", + "service_principal_name": "", + "user_name": "manager@databricks.com" + }, + { + "group_name": "", + "permission_level": "IS_OWNER", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "alert_v2_id": null, + "app_name": null, + "authorization": null, + "cluster_id": null, + "cluster_policy_id": null, + "dashboard_id": null, + "database_instance_name": null, + "database_project_name": null, + "directory_id": null, + "directory_path": null, + "experiment_id": null, + "id": "/pipelines/79fe85ff-42ab-b5b8-79fe-85ff42abb5b9", + "instance_pool_id": null, + "job_id": null, + "knowledge_assistant_id": null, + "notebook_id": null, + "notebook_path": null, + "object_type": "pipelines", + "pipeline_id": "79fe85ff-42ab-b5b8-79fe-85ff42abb5b9", + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "registered_model_id": null, + "repo_id": null, + "repo_path": null, + "serving_endpoint_id": null, + "sql_alert_id": null, + "sql_dashboard_id": null, + "sql_endpoint_id": null, + "sql_query_id": null, + "supervisor_agent_id": null, + "vector_search_endpoint_id": null, + "workspace_file_id": null, + "workspace_file_path": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "databricks_pipeline.test_pipeline" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_pipeline", + "name": "test_pipeline", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "allow_duplicate_names": false, + "budget_policy_id": null, + "catalog": null, + "cause": null, + "channel": "CURRENT", + "cluster": [], + "cluster_id": null, + "configuration": null, + "continuous": false, + "creator_user_name": "tester@databricks.com", + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/state/metadata.json", + "version_id": "" + } + ], + "development": false, + "edition": "ADVANCED", + "environment": [], + "event_log": [], + "expected_last_modified": 0, + "filters": [], + "gateway_definition": [], + "health": null, + "id": "79fe85ff-42ab-b5b8-79fe-85ff42abb5b9", + "ingestion_definition": [], + "last_modified": 1790610854055, + "latest_updates": null, + "library": [ + { + "file": [], + "glob": [], + "jar": "", + "maven": [], + "notebook": [ + { + "path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/files/pipeline" + } + ], + "whl": "" + } + ], + "name": "Test Migration Pipeline", + "notification": [], + "photon": false, + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "restart_window": [], + "root_path": null, + "run_as": [], + "run_as_user_name": "tester@databricks.com", + "schema": null, + "serverless": false, + "serverless_compute_id": null, + "state": "IDLE", + "storage": "dbfs:/pipelines/79fe85ff-42ab-b5b8-79fe-85ff42abb5b9", + "tags": null, + "target": null, + "timeouts": null, + "trigger": [], + "url": "[DATABRICKS_URL]/#joblist/pipelines/79fe85ff-42ab-b5b8-79fe-85ff42abb5b9", + "usage_policy_id": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/acceptance/bundle/migrate/auto/new-resource-ref-permissions/test.toml b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/test.toml new file mode 100644 index 00000000000..34dd8bbcbda --- /dev/null +++ b/acceptance/bundle/migrate/auto/new-resource-ref-permissions/test.toml @@ -0,0 +1,9 @@ +# Local-only: the replay fixtures are captured against the fake server, so they reproduce a +# terraform state only there, not on a real workspace (replaying them on cloud fails auth and +# hits backend validation the mock does not enforce). +Cloud = false +RecordRequests = true + +Ignore = [ + ".databricks", +] diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index be8132fd82d..855b032e8ab 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -199,6 +199,8 @@ func BuildStateFromTF( pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) } + // Entries of a permissions or grants node to drop, see below. + var droppedEntries []int for _, pending := range pendingRefs { fieldPath, err := structpath.ParsePath(pending.fieldPathStr) if err != nil { @@ -209,6 +211,13 @@ func BuildStateFromTF( // resource cannot have the field set either. Record it as unset: the first direct // plan sees the difference and applies it once the referenced resource exists. if referencesUndeployed(pending.refTemplate, tfIDs) { + // Entries of a permissions or grants node are keyed by their principal, so an + // entry without it is invalid. Drop the whole entry instead: it was not deployed. + if idx, ok := fieldPath.AsSlice()[0].Index(); ok && len(parts) == 4 { + droppedEntries = append(droppedEntries, idx) + delete(sv.Refs, pending.fieldPathStr) + continue + } current, err := structaccess.Get(sv.Value, fieldPath) if err != nil { return warningsSeen, fmt.Errorf("%s: reading field %q: %w", node, pending.fieldPathStr, err) @@ -241,6 +250,10 @@ func BuildStateFromTF( delete(sv.Refs, pending.fieldPathStr) } + if err := dropEntries(sv.Value, droppedEntries); err != nil { + return warningsSeen, fmt.Errorf("%s: dropping undeployed entries: %w", node, err) + } + if len(sv.Refs) > 0 { return warningsSeen, fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) } @@ -279,6 +292,23 @@ func BuildStateFromTF( return warningsSeen, nil } +// dropEntries removes the given entries from the slice embedded in a permissions or grants state. +func dropEntries(state any, indexes []int) error { + if len(indexes) == 0 { + return nil + } + slice := reflect.ValueOf(state).Elem().FieldByName("EmbeddedSlice") + if !slice.IsValid() { + return fmt.Errorf("%T has no embedded slice", state) + } + slices.Sort(indexes) + // Delete from the end so the remaining indexes stay valid. + for _, idx := range slices.Backward(slices.Compact(indexes)) { + slice.Set(reflect.AppendSlice(slice.Slice(0, idx), slice.Slice(idx+1, slice.Len()))) + } + return nil +} + // referencesUndeployed reports whether refTemplate references a resource that is not in the terraform state. func referencesUndeployed(refTemplate string, tfIDs map[string]string) bool { ref, ok := dynvar.NewRef(dyn.V(refTemplate)) diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index 108c0591968..28fa5321553 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -359,3 +359,65 @@ resources: ) assert.Contains(t, string(state["resources.jobs.my_job"].State), `"name": "hello"`) } + +func TestBuildStateFromTFPermissionRefToUndeployedResourceDropsEntry(t *testing.T) { + state := runBuildStateFromTF(t, ` +resources: + apps: + new_app: + name: new-app + source_code_path: ./app + jobs: + my_job: + name: "hello" + permissions: + - level: CAN_VIEW + group_name: users + - level: CAN_MANAGE + service_principal_name: "${resources.apps.new_app.service_principal_client_id}" +`, + migrate.TFStateAttrs{ + "databricks_job": {"my_job": json.RawMessage(`{"id": "1", "name": "hello"}`)}, + "databricks_permissions": {"my_job": json.RawMessage(`{"id": "/jobs/1"}`)}, + }, + map[string]string{"resources.jobs.my_job": "1", "resources.jobs.my_job.permissions": "/jobs/1"}, + ) + + var got struct { + Entries []map[string]string `json:"__embed__"` + } + require.NoError(t, json.Unmarshal(state["resources.jobs.my_job.permissions"].State, &got)) + assert.Equal(t, []map[string]string{{"level": "CAN_VIEW", "group_name": "users"}}, got.Entries) +} + +func TestBuildStateFromTFGrantRefToUndeployedResourceDropsEntry(t *testing.T) { + state := runBuildStateFromTF(t, ` +resources: + schemas: + new_schema: + catalog_name: main + name: new + my_schema: + catalog_name: main + name: existing + grants: + - principal: viewers + privileges: [USE_SCHEMA] + - principal: "${resources.schemas.new_schema.name}" + privileges: [CREATE_TABLE] +`, + migrate.TFStateAttrs{ + "databricks_schema": {"my_schema": json.RawMessage(`{"id": "main.existing", "catalog_name": "main", "name": "existing"}`)}, + }, + map[string]string{"resources.schemas.my_schema": "main.existing", "resources.schemas.my_schema.grants": "schema/main.existing"}, + ) + + var got struct { + Entries []struct { + Principal string `json:"principal"` + } `json:"__embed__"` + } + require.NoError(t, json.Unmarshal(state["resources.schemas.my_schema.grants"].State, &got)) + require.Len(t, got.Entries, 1) + assert.Equal(t, "viewers", got.Entries[0].Principal) +}