acc: cover spurious recreate/update when referencing a recreated resource

acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl

@@ -0,0 +1,48 @@
+bundle:
+  name: recreate-ref-update-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+# When a resource is recreated, the direct deploy engine cannot resolve a reference to its
+# output fields at plan time (a recreate does not preserve the id), so the reference stays the
+# literal "${...}" placeholder. A dependent that uses it then diffs its saved value against the
+# placeholder and plans a phantom change -- even when the referenced value is actually
+# unchanged. This test covers the mutable-field case (a spurious UPDATE); the immutable case
+# (a spurious recreate) is postgres_projects/recreate_ref_both_engines.
+#
+# Changing postgres_role recreates the role in the direct engine (the backend rejects this
+# field in the PATCH update_mask, so it is delete + create -- see postgres_roles/
+# recreate-postgres-role). role_id is unchanged, so my_role.name (projects/.../roles/app-owner)
+# is stable. The direct engine cannot resolve ${my_role.name} while the role is recreated, so
+# my_database.role diffs the saved name against the "${...}" placeholder and plans an
+# unnecessary update.
+#
+# This test runs the direct engine only. For context, the terraform engine (now being removed)
+# left my_database alone: against the lenient testserver it updates the role in place, so the
+# name reference resolved at plan and never diffed.
+resources:
+  postgres_projects:
+    my_project:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: My project
+      pg_version: 16
+
+  postgres_branches:
+    my_branch:
+      parent: ${resources.postgres_projects.my_project.name}
+      branch_id: main
+      no_expiry: true
+
+  postgres_roles:
+    my_role:
+      parent: ${resources.postgres_branches.my_branch.name}
+      role_id: app-owner
+      postgres_role: app_role
+
+  postgres_databases:
+    my_database:
+      database_id: my-db
+      parent: ${resources.postgres_branches.my_branch.name}
+      postgres_database: mydb
+      role: ${resources.postgres_roles.my_role.name}

acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.json

@@ -0,0 +1,6 @@
+{
+  "action": "update",
+  "old": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner",
+  "new": "${resources.postgres_roles.my_role.name}",
+  "remote": ""
+}

acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.txt

@@ -0,0 +1,29 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner"
+}
+{
+  "method": "PATCH",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases/my-db",
+  "q": {
+    "update_mask": "spec.role"
+  },
+  "body": {
+    "spec": {
+      "postgres_database": "mydb",
+      "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner"
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles",
+  "q": {
+    "role_id": "app-owner"
+  },
+  "body": {
+    "spec": {
+      "postgres_role": "new_role"
+    }
+  }
+}

acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml

@@ -0,0 +1,4 @@
+Cloud = false
+CloudEnvs.azure = false
+CloudEnvs.gcp = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt

@@ -0,0 +1,48 @@
+
+=== Initial deployment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-ref-update-[UNIQUE_NAME]/default/files...
+Created postgres_branches.my_branch
+Created postgres_databases.my_database
+Created postgres_projects.my_project
+Created postgres_roles.my_role
+Files: 0 uploaded, 0 deleted
+Resources: 4 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Recreate the role; my_database.role references the role's (unchanged) name
+>>> update_file.py databricks.yml postgres_role: app_role postgres_role: new_role
+
+=== Plan
+>>> [CLI] bundle plan
+update postgres_databases.my_database
+recreate postgres_roles.my_role
+
+Plan: 1 to add, 1 to change, 1 to delete, 2 unchanged
+
+>>> [CLI] bundle plan -o json
+
+=== Deploy and record the requests (see out.requests.txt)
+>>> print_requests.py --sort //postgres
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.postgres_branches.my_branch
+  delete resources.postgres_databases.my_database
+  delete resources.postgres_projects.my_project
+  delete resources.postgres_roles.my_role
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase branches.
+All data stored in them will be permanently lost:
+  delete resources.postgres_branches.my_branch
+
+This action will result in the deletion of the following Lakebase databases.
+All data stored in them will be permanently lost:
+  delete resources.postgres_databases.my_database
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-ref-update-[UNIQUE_NAME]/default
+
+Destroy: 4 deleted

acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script

@@ -0,0 +1,32 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f "$OUT_REQUESTS"
+}
+trap cleanup EXIT
+
+title "Initial deployment"
+trace $CLI bundle deploy
+rm -f "$OUT_REQUESTS"   # drop create requests; capture only the edit below
+
+# Changing postgres_role recreates the role in the direct engine (the backend rejects this
+# field in the PATCH update_mask, so it is delete + create -- see recreate-postgres-role).
+# role_id is unchanged, so my_role.name (projects/.../roles/app-owner) is stable.
+title "Recreate the role; my_database.role references the role's (unchanged) name"
+trace update_file.py databricks.yml "postgres_role: app_role" "postgres_role: new_role"
+
+# PLANNED: the direct engine recreates the role, so it cannot resolve ${my_role.name} (a
+# recreate does not preserve the id) and diffs the saved name against the literal "${...}"
+# placeholder -> spurious "update postgres_databases.my_database".
+title "Plan"
+trace $CLI bundle plan
+# Field-level proof: old is the real name, new is the still-unresolved reference.
+trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database".changes.role' > out.database_role.json
+
+# WHAT ACTUALLY HAPPENED: deploy and record the requests (deploy stdout -> LOG.deploy; parallel
+# completion order is nondeterministic). out.requests.txt shows the direct engine issues
+# a PATCH against my_database -- a spurious update writing back the unchanged role.
+title "Deploy and record the requests (see out.requests.txt)"
+$CLI bundle deploy --auto-approve &> LOG.deploy
+trace print_requests.py --sort '//postgres' > out.requests.txt

acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml

@@ -0,0 +1,3 @@
+# Direct-engine demonstration of a plan-time reference-resolution limitation (a reference to a
+# resource being recreated reads as a phantom change). Not Cloud: shown against the testserver.
+Cloud = false

acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl

@@ -0,0 +1,34 @@
+bundle:
+  name: recreate-ref-both-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+# When a resource is recreated, the direct deploy engine cannot resolve a reference to its
+# output fields at plan time (a recreate does not preserve the id), so the reference stays the
+# literal "${...}" placeholder. A dependent that uses it then diffs its saved value against the
+# placeholder and plans a phantom change -- even when the referenced value is actually
+# unchanged. The dependent's action depends on the field: an immutable/id field -> recreate, a
+# mutable field -> update. This test covers the immutable case (a spurious RECREATE).
+#
+# pg_version is spec:immutable, so a bump recreates the project. project_id is unchanged, so the
+# project name (projects/{project_id}) is stable, and my_catalog.branch -- built from
+# ${my_project.name} -- resolves to the same value: the catalog does not actually change, yet
+# the direct engine plans and executes its recreate.
+#
+# This test runs the direct engine only. For context, the terraform engine (now being removed)
+# behaved the same here -- it also recreated the catalog, because it cannot cancel a replace
+# once the plan commits it to separate destroy/create graph nodes.
+resources:
+  postgres_projects:
+    my_project:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: My project
+      pg_version: 16
+
+  postgres_catalogs:
+    my_catalog:
+      catalog_id: mycat
+      branch: ${resources.postgres_projects.my_project.name}/branches/production
+      postgres_database: appdb
+      create_database_if_missing: true

acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.txt

@@ -0,0 +1,35 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/catalogs/mycat"
+}
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/catalogs",
+  "q": {
+    "catalog_id": "mycat"
+  },
+  "body": {
+    "spec": {
+      "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production",
+      "create_database_if_missing": true,
+      "postgres_database": "appdb"
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects",
+  "q": {
+    "project_id": "test-pg-proj-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "display_name": "My project",
+      "pg_version": 17
+    }
+  }
+}

acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml

@@ -0,0 +1,4 @@
+Cloud = false
+CloudEnvs.azure = false
+CloudEnvs.gcp = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt

@@ -0,0 +1,34 @@
+
+=== Initial deployment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-ref-both-[UNIQUE_NAME]/default/files...
+Created postgres_catalogs.my_catalog
+Created postgres_projects.my_project
+Files: 0 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Bump pg_version: recreates the project; project_id and name are unchanged
+>>> update_file.py databricks.yml pg_version: 16 pg_version: 17
+
+=== Plan
+>>> [CLI] bundle plan
+recreate postgres_catalogs.my_catalog
+recreate postgres_projects.my_project
+
+Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged
+
+=== Deploy the recreate and record the requests (see out.requests.txt)
+>>> print_requests.py --sort //postgres
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.postgres_catalogs.my_catalog
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.my_project
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-ref-both-[UNIQUE_NAME]/default
+
+Destroy: 2 deleted

acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script

@@ -0,0 +1,33 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f "$OUT_REQUESTS"
+}
+trap cleanup EXIT
+
+title "Initial deployment"
+trace $CLI bundle deploy
+rm -f "$OUT_REQUESTS"   # drop create requests; capture only the recreate below
+
+# pg_version is spec:immutable, so a bump recreates the project. project_id (hence the project
+# name "projects/{project_id}") is unchanged, so my_catalog.branch -- built from
+# ${my_project.name} -- resolves to the same value: the catalog does not change.
+title "Bump pg_version: recreates the project; project_id and name are unchanged"
+trace update_file.py databricks.yml "pg_version: 16" "pg_version: 17"
+
+# PLANNED: the direct engine cannot resolve ${my_project.name} for a project being recreated
+# (a recreate does not preserve the id), so it diffs the saved branch against the literal
+# "${...}" placeholder and plans a spurious recreate of the catalog alongside the project. The
+# catalog's branch value never actually changes.
+title "Plan"
+trace $CLI bundle plan
+
+# WHAT ACTUALLY HAPPENED: the direct engine performs a real delete + create of the catalog.
+# Deploy stdout goes to LOG.deploy (parallel completion order is nondeterministic, so it is not
+# a golden); the API calls are recorded below. out.requests.txt shows the DELETE + POST
+# issued against the catalog with an identical, unchanged spec -- a destructive recreate of a
+# resource that did not change.
+title "Deploy the recreate and record the requests (see out.requests.txt)"
+$CLI bundle deploy --auto-approve &> LOG.deploy
+trace print_requests.py --sort '//postgres' > out.requests.txt

acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml

@@ -0,0 +1,3 @@
+# Direct-engine demonstration of a plan-time reference-resolution limitation. Not Cloud: the
+# behavior lives in the planner and is shown against the testserver.
+Cloud = false

acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl

@@ -0,0 +1,34 @@
+bundle:
+  name: schema-id-ref-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+# When a resource is recreated, the direct deploy engine cannot resolve a reference to its
+# output fields at plan time (a recreate does not preserve the id), so the reference stays the
+# literal "${...}" placeholder. A dependent that uses it then diffs its saved value against the
+# placeholder and plans a phantom change -- even when the referenced value is actually unchanged.
+#
+# This test shows that on UC schemas (Lakebase/postgres resources exhibit it too) and its
+# boundary: UC resources reference each other by NAME (catalog_name, schema_name) -- user-
+# provided INPUT fields the planner resolves from config even when the target is recreated, so
+# idiomatic UC wiring never hits it. The exception is a reference to an OUTPUT-only field such
+# as .id (a schema's full_name "catalog.schema").
+#
+# referent recreates on storage_root (immutable, non-identity), so its full_name / .id
+# (main.referent) is unchanged. consumer references ${referent.id} in its mutable comment, so
+# the direct engine plans a spurious update of consumer.
+#
+# This test runs the direct engine only. For context, the terraform engine (now being removed)
+# left consumer alone: it re-plans at apply, finds ${referent.id} resolved to the unchanged
+# value, and issues no request for consumer.
+resources:
+  schemas:
+    referent:
+      catalog_name: main
+      name: referent
+      storage_root: s3://my-bucket/v1
+    consumer:
+      catalog_name: main
+      name: consumer
+      comment: refers to ${resources.schemas.referent.id}

acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.json

@@ -0,0 +1,6 @@
+{
+  "action": "update",
+  "old": "refers to main.referent",
+  "new": "refers to ${resources.schemas.referent.id}",
+  "remote": "refers to main.referent"
+}

acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.txt

@@ -0,0 +1,24 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.1/unity-catalog/schemas/main.referent",
+  "q": {
+    "force": "true"
+  }
+}
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/schemas/main.consumer",
+  "body": {
+    "comment": "refers to main.referent",
+    "custom_max_retention_hours": 0
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/schemas",
+  "body": {
+    "catalog_name": "main",
+    "name": "referent",
+    "storage_root": "s3://my-bucket/v2"
+  }
+}

acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]