Drop permission and grant entries that reference undeployed resources on migration

.nextchanges/bundles/7229.md

@@ -0,0 +1 @@
+* Fix migration from Terraform dropping the principal of a permission or grant that references a resource that was never deployed. ([#7229](https://github.com/databricks/cli/pull/7229))

acceptance/bundle/migrate/auto/new-resource-ref-permissions/app.py

@@ -0,0 +1 @@
+print("Hello world!")

acceptance/bundle/migrate/auto/new-resource-ref-permissions/databricks.yml

@@ -0,0 +1,31 @@
+bundle:
+  name: migrate-basic-test
+
+resources:
+  # apps:
+  #   new_app:
+  #     name: new-app
+  #     source_code_path: .
+
+  jobs:
+    test_job:
+      name: "Test Migration Job"
+      tasks:
+        - task_key: "main"
+          notebook_task:
+            notebook_path: "./notebook.py"
+      permissions:
+        - level: CAN_VIEW
+          user_name: viewer@databricks.com
+        # - level: CAN_MANAGE
+        #   service_principal_name: ${resources.apps.new_app.service_principal_client_id}
+
+  pipelines:
+    test_pipeline:
+      name: "Test Migration Pipeline"
+      libraries:
+        - notebook:
+            path: "./pipeline.py"
+      permissions:
+        - level: CAN_MANAGE
+          user_name: manager@databricks.com

acceptance/bundle/migrate/auto/new-resource-ref-permissions/notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("Hello from test migration job")

acceptance/bundle/migrate/auto/new-resource-ref-permissions/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/migrate/auto/new-resource-ref-permissions/output.txt

@@ -0,0 +1,239 @@
+
+=== Replay a prior terraform deployment of a job and a pipeline with permissions
+>>> replay_tfstate.py terraform-requests.json terraform.tfstate
+
+=== Add a new app and a job permission that references its service principal
+=== Plan shows the new app and the added permission
+>>> [CLI] bundle plan -o json
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+{
+  "plan_version": [PLAN_VERSION],
+  "cli_version": "[CLI_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 7,
+  "plan": {
+    "resources.apps.new_app": {
+      "action": "create",
+      "new_state": {
+        "value": {
+          "description": "",
+          "name": "new-app",
+          "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files"
+        }
+      }
+    },
+    "resources.jobs.test_job": {
+      "action": "skip",
+      "remote_state": {
+        "created_time": [UNIX_TIME_MILLIS][0],
+        "creator_user_name": "[USERNAME]",
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
+        },
+        "edit_mode": "UI_LOCKED",
+        "email_notifications": {},
+        "format": "MULTI_TASK",
+        "job_id": [TEST_JOB_ID],
+        "max_concurrent_runs": 1,
+        "name": "Test Migration Job",
+        "queue": {
+          "enabled": true
+        },
+        "run_as_user_name": "[USERNAME]",
+        "tasks": [
+          {
+            "email_notifications": {},
+            "notebook_task": {
+              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/notebook",
+              "source": "WORKSPACE"
+            },
+            "run_if": "ALL_SUCCESS",
+            "task_key": "main",
+            "timeout_seconds": 0
+          }
+        ],
+        "timeout_seconds": 0,
+        "webhook_notifications": {}
+      },
+      "changes": {
+        "email_notifications": {
+          "action": "skip",
+          "reason": "empty",
+          "remote": {}
+        },
+        "tasks[task_key='main'].email_notifications": {
+          "action": "skip",
+          "reason": "empty",
+          "remote": {}
+        },
+        "tasks[task_key='main'].notebook_task.source": {
+          "action": "skip",
+          "reason": "backend_default",
+          "remote": "WORKSPACE"
+        },
+        "tasks[task_key='main'].run_if": {
+          "action": "skip",
+          "reason": "backend_default",
+          "remote": "ALL_SUCCESS"
+        },
+        "tasks[task_key='main'].timeout_seconds": {
+          "action": "skip",
+          "reason": "empty",
+          "remote": 0
+        },
+        "timeout_seconds": {
+          "action": "skip",
+          "reason": "empty",
+          "remote": 0
+        },
+        "webhook_notifications": {
+          "action": "skip",
+          "reason": "empty",
+          "remote": {}
+        }
+      }
+    },
+    "resources.jobs.test_job.permissions": {
+      "depends_on": [
+        {
+          "node": "resources.apps.new_app",
+          "label": "${resources.apps.new_app.service_principal_client_id}"
+        },
+        {
+          "node": "resources.jobs.test_job",
+          "label": "${resources.jobs.test_job.id}"
+        }
+      ],
+      "action": "update",
+      "new_state": {
+        "value": {
+          "object_id": "/jobs/[TEST_JOB_ID]",
+          "__embed__": [
+            {
+              "level": "CAN_VIEW",
+              "user_name": "viewer@databricks.com"
+            },
+            {
+              "level": "CAN_MANAGE",
+              "service_principal_name": "${resources.apps.new_app.service_principal_client_id}"
+            },
+            {
+              "level": "IS_OWNER",
+              "user_name": "[USERNAME]"
+            }
+          ]
+        },
+        "vars": {
+          "[1].service_principal_name": "${resources.apps.new_app.service_principal_client_id}"
+        }
+      },
+      "remote_state": {
+        "object_id": "/jobs/[TEST_JOB_ID]",
+        "__embed__": [
+          {
+            "level": "CAN_VIEW",
+            "user_name": "viewer@databricks.com"
+          },
+          {
+            "level": "IS_OWNER",
+            "user_name": "[USERNAME]"
+          }
+        ]
+      },
+      "changes": {
+        "[service_principal_name='${resources.apps.new_app.service_principal_client_id}']": {
+          "action": "update",
+          "new": {
+            "level": "CAN_MANAGE",
+            "service_principal_name": "${resources.apps.new_app.service_principal_client_id}"
+          }
+        }
+      }
+    },
+    "resources.pipelines.test_pipeline": {
+      "action": "skip",
+      "remote_state": {
+        "channel": "CURRENT",
+        "creator_user_name": "[USERNAME]",
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/state/metadata.json"
+        },
+        "edition": "ADVANCED",
+        "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE",
+        "id": "[TEST_PIPELINE_ID]",
+        "last_modified": [UNIX_TIME_MILLIS][1],
+        "libraries": [
+          {
+            "notebook": {
+              "path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files/pipeline"
+            }
+          }
+        ],
+        "name": "Test Migration Pipeline",
+        "pipeline_id": "[TEST_PIPELINE_ID]",
+        "run_as_user_name": "[USERNAME]",
+        "state": "IDLE",
+        "storage": "dbfs:[PIPELINE_TEST_PIPELINE_ID]"
+      },
+      "changes": {
+        "storage": {
+          "action": "skip",
+          "reason": "immutable_backend_value",
+          "remote": "dbfs:[PIPELINE_TEST_PIPELINE_ID]"
+        }
+      }
+    },
+    "resources.pipelines.test_pipeline.permissions": {
+      "depends_on": [
+        {
+          "node": "resources.pipelines.test_pipeline",
+          "label": "${resources.pipelines.test_pipeline.id}"
+        }
+      ],
+      "action": "skip",
+      "remote_state": {
+        "object_id": "[PIPELINE_TEST_PIPELINE_ID]",
+        "__embed__": [
+          {
+            "level": "CAN_MANAGE",
+            "user_name": "manager@databricks.com"
+          },
+          {
+            "level": "IS_OWNER",
+            "user_name": "[USERNAME]"
+          }
+        ]
+      }
+    }
+  }
+}
+
+=== Deploy migrates to direct, creates the app and grants the permission
+>>> [CLI] bundle deploy
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default/files...
+Migrated 4 resources to direct deployment engine.
+Created apps.new_app
+Updated jobs.test_job.permissions
+Files: 4 uploaded, 0 deleted
+Resources: 1 created, 1 changed, 0 deleted, 3 unchanged
+
+=== Redeploy is a no-op
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.apps.new_app
+  delete resources.jobs.test_job
+  delete resources.pipelines.test_pipeline
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.test_pipeline
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/default
+
+Destroy: 3 deleted

acceptance/bundle/migrate/auto/new-resource-ref-permissions/pipeline.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+# This is a test pipeline notebook

acceptance/bundle/migrate/auto/new-resource-ref-permissions/script

@@ -0,0 +1,25 @@
+export DATABRICKS_BUNDLE_ENGINE=
+
+title "Replay a prior terraform deployment of a job and a pipeline with permissions"
+trace replay_tfstate.py terraform-requests.json terraform.tfstate
+replace_ids.py
+rm -f "$OUT_REQUESTS"
+
+# The job gains a permission for the service principal of a new app. Terraform never deployed
+# either, so the migrated state must not hold a permission entry without a principal.
+title "Add a new app and a job permission that references its service principal"
+update_file.py databricks.yml '# ' ''
+
+title "Plan shows the new app and the added permission"
+trace $CLI bundle plan -o json
+rm -f "$OUT_REQUESTS"
+
+title "Deploy migrates to direct, creates the app and grants the permission"
+trace $CLI bundle deploy
+rm -f "$OUT_REQUESTS"
+
+title "Redeploy is a no-op"
+trace $CLI bundle plan
+
+trace $CLI bundle destroy --auto-approve
+rm -f "$OUT_REQUESTS"

acceptance/bundle/migrate/auto/new-resource-ref-permissions/terraform-requests.json

@@ -0,0 +1,80 @@
+[
+  {
+    "method": "POST",
+    "path": "/api/2.0/pipelines",
+    "body": {
+      "channel": "CURRENT",
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/state/metadata.json"
+      },
+      "edition": "ADVANCED",
+      "libraries": [
+        {
+          "notebook": {
+            "path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/files/pipeline"
+          }
+        }
+      ],
+      "name": "Test Migration Pipeline"
+    }
+  },
+  {
+    "method": "POST",
+    "path": "/api/2.2/jobs/create",
+    "body": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "Test Migration Job",
+      "queue": {
+        "enabled": true
+      },
+      "tasks": [
+        {
+          "notebook_task": {
+            "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-basic-test/default/files/notebook",
+            "source": "WORKSPACE"
+          },
+          "task_key": "main"
+        }
+      ]
+    }
+  },
+  {
+    "method": "PUT",
+    "path": "/api/2.0/permissions/pipelines/79fe85ff-42ab-b5b8-79fe-85ff42abb5b9",
+    "body": {
+      "access_control_list": [
+        {
+          "permission_level": "CAN_MANAGE",
+          "user_name": "manager@databricks.com"
+        },
+        {
+          "permission_level": "IS_OWNER",
+          "user_name": "tester@databricks.com"
+        }
+      ]
+    }
+  },
+  {
+    "method": "PUT",
+    "path": "/api/2.0/permissions/jobs/8790610854067598000",
+    "body": {
+      "access_control_list": [
+        {
+          "permission_level": "CAN_VIEW",
+          "user_name": "viewer@databricks.com"
+        },
+        {
+          "permission_level": "IS_OWNER",
+          "user_name": "tester@databricks.com"
+        }
+      ]
+    }
+  }
+]