From ce7931fa28236c960e8a5c6e42b63db882c2de46 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 1 Oct 2026 11:09:42 +0200 Subject: [PATCH 1/9] acc: cover spurious recreate/update when referencing a recreated resource The direct engine cannot resolve ${resources.X.name}/.id while X is being recreated (a recreate does not preserve the id), so a dependent referencing X diffs the saved value against the unresolved "${...}" placeholder and plans a phantom change -- recreate for an immutable field, update for a mutable one -- even though the referenced value is unchanged. - postgres_projects/recreate_ref_both_engines: a pg_version bump recreates the project (project_id, hence name, unchanged); both engines spuriously recreate a catalog whose branch references the project name. Apples-to-apples, same behavior. - postgres_databases/recreate_ref_spurious_update: a postgres_role change recreates the role (role_id, hence name, unchanged); the direct engine spuriously updates a database whose mutable role field references the role name. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 40 +++++++++++++++++++ .../out.d_role_change.direct.json | 6 +++ .../out.d_role_change.terraform.json | 1 + .../out.plan.direct.txt | 4 ++ .../out.plan.terraform.txt | 3 ++ .../out.test.toml | 5 +++ .../recreate_ref_spurious_update/output.txt | 40 +++++++++++++++++++ .../recreate_ref_spurious_update/script | 20 ++++++++++ .../recreate_ref_spurious_update/test.toml | 6 +++ .../databricks.yml.tmpl | 24 +++++++++++ .../recreate_ref_both_engines/out.test.toml | 5 +++ .../recreate_ref_both_engines/output.txt | 30 ++++++++++++++ .../recreate_ref_both_engines/script | 18 +++++++++ .../recreate_ref_both_engines/test.toml | 3 ++ 14 files changed, 205 insertions(+) create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.direct.json create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.terraform.json create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml create mode 100644 acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt create mode 100644 acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script create mode 100644 acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl new file mode 100644 index 00000000000..097389b59a7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl @@ -0,0 +1,40 @@ +bundle: + name: recreate-ref-update-$UNIQUE_NAME + +sync: + paths: [] + +# Companion to postgres_projects/recreate_ref_both_engines (which shows the spurious +# RECREATE). Here the referrer field is MUTABLE, so the same phantom diff plans a +# spurious UPDATE instead of a recreate. +# +# Changing postgres_role recreates the role (the backend rejects it in the PATCH +# update_mask - see postgres_roles/recreate-postgres-role), but role_id is unchanged, so +# owner.name (projects/.../roles/app-owner) is stable. The direct engine cannot resolve +# ${owner.name} while owner is being recreated, so d.role diffs the saved name against +# the literal "${...}" placeholder and plans an unnecessary update of the database. +resources: + postgres_projects: + p: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: P + pg_version: 16 + + postgres_branches: + b: + parent: ${resources.postgres_projects.p.name} + branch_id: main + no_expiry: true + + postgres_roles: + owner: + parent: ${resources.postgres_branches.b.name} + role_id: app-owner + postgres_role: app_role + + postgres_databases: + d: + database_id: my-db + parent: ${resources.postgres_branches.b.name} + postgres_database: mydb + role: ${resources.postgres_roles.owner.name} diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.direct.json b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.direct.json new file mode 100644 index 00000000000..dfde20d76ab --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.direct.json @@ -0,0 +1,6 @@ +{ + "action": "update", + "old": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", + "new": "${resources.postgres_roles.owner.name}", + "remote": "" +} diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.terraform.json b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.terraform.json new file mode 100644 index 00000000000..19765bd501b --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.terraform.json @@ -0,0 +1 @@ +null diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt new file mode 100644 index 00000000000..1a9c8f02df2 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt @@ -0,0 +1,4 @@ +update postgres_databases.d +recreate postgres_roles.owner + +Plan: 1 to add, 1 to change, 1 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt new file mode 100644 index 00000000000..5e4935880ed --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt @@ -0,0 +1,3 @@ +update postgres_roles.owner + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml new file mode 100644 index 00000000000..db99a8e07fa --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml @@ -0,0 +1,5 @@ +Cloud = false +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt new file mode 100644 index 00000000000..4c1b8cae6f1 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt @@ -0,0 +1,40 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-ref-update-[UNIQUE_NAME]/default/files... +Created postgres_branches.b +Created postgres_databases.d +Created postgres_projects.p +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +=== Recreate the role; d.role references the role's (unchanged) name +>>> update_file.py databricks.yml postgres_role: app_role postgres_role: new_role + +>>> [CLI] bundle plan + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.b + delete resources.postgres_databases.d + delete resources.postgres_projects.p + delete resources.postgres_roles.owner + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.p + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.b + +This action will result in the deletion of the following Lakebase databases. +All data stored in them will be permanently lost: + delete resources.postgres_databases.d + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-ref-update-[UNIQUE_NAME]/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script new file mode 100644 index 00000000000..f007feccf3b --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +title "Initial deployment" +trace $CLI bundle deploy + +title "Recreate the role; d.role references the role's (unchanged) name" +trace update_file.py databricks.yml "postgres_role: app_role" "postgres_role: new_role" + +# The plan diverges by engine, so capture it per engine. The direct engine recreates the +# role and reports a spurious "update postgres_databases.d"; against the testserver +# terraform updates the role in place, resolves the stable name, and leaves d unchanged. +trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +# Field-level proof: old is the real name, new is the still-unresolved reference. +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.d".changes.role' > out.d_role_change.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml new file mode 100644 index 00000000000..ada4f066c37 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml @@ -0,0 +1,6 @@ +# Local-only demonstration of a direct-engine planner bug (reference to a resource being +# recreated reads as a phantom change). Not Cloud: terraform updating the role in place (vs +# the backend delete+create) is a testserver behavior. +Cloud = false +RecordRequests = false +EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl new file mode 100644 index 00000000000..c662ac17fa0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: recreate-ref-both-$UNIQUE_NAME + +sync: + paths: [] + +# pg_version is spec:immutable, so BOTH engines recreate the project on a bump - an +# apples-to-apples recreate of the referent, with no engine disagreement on the field. +# project_id is unchanged, so the project's name (projects/{project_id}) is stable across +# the recreate. The catalog's branch is built from that stable name, so the catalog's +# config value does not actually change. +resources: + postgres_projects: + p: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: P + pg_version: 16 + + postgres_catalogs: + cat: + catalog_id: mycat + branch: ${resources.postgres_projects.p.name}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml new file mode 100644 index 00000000000..db99a8e07fa --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml @@ -0,0 +1,5 @@ +Cloud = false +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt new file mode 100644 index 00000000000..77efbb4c5cf --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt @@ -0,0 +1,30 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-ref-both-[UNIQUE_NAME]/default/files... +Created postgres_catalogs.cat +Created postgres_projects.p +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Bump pg_version: recreates the project; project_id (hence name) is unchanged +>>> update_file.py databricks.yml pg_version: 16 pg_version: 17 + +>>> [CLI] bundle plan +recreate postgres_catalogs.cat +recreate postgres_projects.p + +Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_catalogs.cat + delete resources.postgres_projects.p + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.p + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-ref-both-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script new file mode 100644 index 00000000000..721a0067f85 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +title "Initial deployment" +trace $CLI bundle deploy + +# Both engines recreate the project (pg_version immutable). Neither can resolve +# ${p.name} for a resource being recreated, so both plan a recreate of the catalog - +# even though cat.branch resolves to the same value. The behavior is identical: this is +# a shared plan-time limitation, not a direct-only bug. +title "Bump pg_version: recreates the project; project_id (hence name) is unchanged" +trace update_file.py databricks.yml "pg_version: 16" "pg_version: 17" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml new file mode 100644 index 00000000000..310d0528708 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml @@ -0,0 +1,3 @@ +Cloud = false +RecordRequests = false +EnvMatrix.DMS = [] From 70a8bd8655713c2be7ee24fee1f463bee8044bfb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 2 Oct 2026 10:53:24 +0200 Subject: [PATCH 2/9] acc: use descriptive resource names in the recreate-reference tests Rename p/b/owner/d/cat to my_project/my_branch/my_role/my_database/my_catalog. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 24 +++++++++---------- ...ect.json => out.database_role.direct.json} | 2 +- ....json => out.database_role.terraform.json} | 0 .../out.plan.direct.txt | 4 ++-- .../out.plan.terraform.txt | 2 +- .../recreate_ref_spurious_update/output.txt | 24 +++++++++---------- .../recreate_ref_spurious_update/script | 9 +++---- .../databricks.yml.tmpl | 8 +++---- .../recreate_ref_both_engines/output.txt | 14 +++++------ .../recreate_ref_both_engines/script | 6 ++--- 10 files changed, 47 insertions(+), 46 deletions(-) rename acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/{out.d_role_change.direct.json => out.database_role.direct.json} (69%) rename acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/{out.d_role_change.terraform.json => out.database_role.terraform.json} (100%) diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl index 097389b59a7..b6e34f8a93e 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl @@ -10,31 +10,31 @@ sync: # # Changing postgres_role recreates the role (the backend rejects it in the PATCH # update_mask - see postgres_roles/recreate-postgres-role), but role_id is unchanged, so -# owner.name (projects/.../roles/app-owner) is stable. The direct engine cannot resolve -# ${owner.name} while owner is being recreated, so d.role diffs the saved name against -# the literal "${...}" placeholder and plans an unnecessary update of the database. +# my_role.name (projects/.../roles/app-owner) is stable. The direct engine cannot resolve +# ${my_role.name} while the role is being recreated, so my_database.role diffs the saved +# name against the literal "${...}" placeholder and plans an unnecessary update. resources: postgres_projects: - p: + my_project: project_id: test-pg-proj-$UNIQUE_NAME - display_name: P + display_name: My project pg_version: 16 postgres_branches: - b: - parent: ${resources.postgres_projects.p.name} + my_branch: + parent: ${resources.postgres_projects.my_project.name} branch_id: main no_expiry: true postgres_roles: - owner: - parent: ${resources.postgres_branches.b.name} + my_role: + parent: ${resources.postgres_branches.my_branch.name} role_id: app-owner postgres_role: app_role postgres_databases: - d: + my_database: database_id: my-db - parent: ${resources.postgres_branches.b.name} + parent: ${resources.postgres_branches.my_branch.name} postgres_database: mydb - role: ${resources.postgres_roles.owner.name} + role: ${resources.postgres_roles.my_role.name} diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.direct.json b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.direct.json similarity index 69% rename from acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.direct.json rename to acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.direct.json index dfde20d76ab..9f396c4425f 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.direct.json +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.direct.json @@ -1,6 +1,6 @@ { "action": "update", "old": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", - "new": "${resources.postgres_roles.owner.name}", + "new": "${resources.postgres_roles.my_role.name}", "remote": "" } diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.terraform.json b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.terraform.json similarity index 100% rename from acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.d_role_change.terraform.json rename to acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.terraform.json diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt index 1a9c8f02df2..1560fe029bb 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt @@ -1,4 +1,4 @@ -update postgres_databases.d -recreate postgres_roles.owner +update postgres_databases.my_database +recreate postgres_roles.my_role Plan: 1 to add, 1 to change, 1 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt index 5e4935880ed..3da0471b2bc 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt @@ -1,3 +1,3 @@ -update postgres_roles.owner +update postgres_roles.my_role Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt index 4c1b8cae6f1..1a1922d310a 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt @@ -2,14 +2,14 @@ === Initial deployment >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-ref-update-[UNIQUE_NAME]/default/files... -Created postgres_branches.b -Created postgres_databases.d -Created postgres_projects.p -Created postgres_roles.owner +Created postgres_branches.my_branch +Created postgres_databases.my_database +Created postgres_projects.my_project +Created postgres_roles.my_role Files: 0 uploaded, 0 deleted Resources: 4 created, 0 changed, 0 deleted, 0 unchanged -=== Recreate the role; d.role references the role's (unchanged) name +=== Recreate the role; my_database.role references the role's (unchanged) name >>> update_file.py databricks.yml postgres_role: app_role postgres_role: new_role >>> [CLI] bundle plan @@ -18,22 +18,22 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.postgres_branches.b - delete resources.postgres_databases.d - delete resources.postgres_projects.p - delete resources.postgres_roles.owner + delete resources.postgres_branches.my_branch + delete resources.postgres_databases.my_database + delete resources.postgres_projects.my_project + delete resources.postgres_roles.my_role This action will result in the deletion of the following Lakebase projects along with all their branches, databases, and endpoints. All data stored in them will be permanently lost: - delete resources.postgres_projects.p + delete resources.postgres_projects.my_project This action will result in the deletion of the following Lakebase branches. All data stored in them will be permanently lost: - delete resources.postgres_branches.b + delete resources.postgres_branches.my_branch This action will result in the deletion of the following Lakebase databases. All data stored in them will be permanently lost: - delete resources.postgres_databases.d + delete resources.postgres_databases.my_database All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-ref-update-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script index f007feccf3b..1b8e1389fd0 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script @@ -9,12 +9,13 @@ trap cleanup EXIT title "Initial deployment" trace $CLI bundle deploy -title "Recreate the role; d.role references the role's (unchanged) name" +title "Recreate the role; my_database.role references the role's (unchanged) name" trace update_file.py databricks.yml "postgres_role: app_role" "postgres_role: new_role" # The plan diverges by engine, so capture it per engine. The direct engine recreates the -# role and reports a spurious "update postgres_databases.d"; against the testserver -# terraform updates the role in place, resolves the stable name, and leaves d unchanged. +# role and reports a spurious "update postgres_databases.my_database"; against the +# testserver terraform updates the role in place, resolves the stable name, and leaves the +# database unchanged. trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt # Field-level proof: old is the real name, new is the still-unresolved reference. -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.d".changes.role' > out.d_role_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database".changes.role' > out.database_role.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl index c662ac17fa0..0d87c8a956b 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl @@ -11,14 +11,14 @@ sync: # config value does not actually change. resources: postgres_projects: - p: + my_project: project_id: test-pg-proj-$UNIQUE_NAME - display_name: P + display_name: My project pg_version: 16 postgres_catalogs: - cat: + my_catalog: catalog_id: mycat - branch: ${resources.postgres_projects.p.name}/branches/production + branch: ${resources.postgres_projects.my_project.name}/branches/production postgres_database: appdb create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt index 77efbb4c5cf..acba4e011c3 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt @@ -2,8 +2,8 @@ === Initial deployment >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-ref-both-[UNIQUE_NAME]/default/files... -Created postgres_catalogs.cat -Created postgres_projects.p +Created postgres_catalogs.my_catalog +Created postgres_projects.my_project Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged @@ -11,19 +11,19 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml pg_version: 16 pg_version: 17 >>> [CLI] bundle plan -recreate postgres_catalogs.cat -recreate postgres_projects.p +recreate postgres_catalogs.my_catalog +recreate postgres_projects.my_project Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.postgres_catalogs.cat - delete resources.postgres_projects.p + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project This action will result in the deletion of the following Lakebase projects along with all their branches, databases, and endpoints. All data stored in them will be permanently lost: - delete resources.postgres_projects.p + delete resources.postgres_projects.my_project All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-ref-both-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script index 721a0067f85..81260ab985c 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script @@ -10,9 +10,9 @@ title "Initial deployment" trace $CLI bundle deploy # Both engines recreate the project (pg_version immutable). Neither can resolve -# ${p.name} for a resource being recreated, so both plan a recreate of the catalog - -# even though cat.branch resolves to the same value. The behavior is identical: this is -# a shared plan-time limitation, not a direct-only bug. +# ${my_project.name} for a resource being recreated, so both plan a recreate of the +# catalog - even though my_catalog.branch resolves to the same value. The behavior is +# identical: this is a shared plan-time limitation, not a direct-only bug. title "Bump pg_version: recreates the project; project_id (hence name) is unchanged" trace update_file.py databricks.yml "pg_version: 16" "pg_version: 17" trace $CLI bundle plan From e118e31ab838361bc7f340fdff02758c0d5565f0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 2 Oct 2026 11:15:44 +0200 Subject: [PATCH 3/9] acc: capture deploy result and per-engine requests in the recreate-reference tests Deploy the spurious action and record the API calls per engine, so the goldens show not just the plan but what actually happens: - recreate test: both engines DELETE + POST the catalog with an identical, unchanged spec. - update test: direct issues a spurious PATCH on the database (writing back the unchanged role) while terraform issues no request for it at all. Deploy stdout goes to LOG.deploy (parallel completion order is nondeterministic). Scripts now explain what is planned vs. what happens, and the plan/request differences per engine. Co-authored-by: Isaac --- .../out.requests.direct.txt | 29 +++++++++++++++ .../out.requests.terraform.txt | 15 ++++++++ .../recreate_ref_spurious_update/output.txt | 3 ++ .../recreate_ref_spurious_update/script | 24 +++++++++--- .../recreate_ref_spurious_update/test.toml | 2 +- .../out.requests.direct.txt | 35 ++++++++++++++++++ .../out.requests.terraform.txt | 37 +++++++++++++++++++ .../recreate_ref_both_engines/output.txt | 6 ++- .../recreate_ref_both_engines/script | 26 ++++++++++--- .../recreate_ref_both_engines/test.toml | 5 ++- 10 files changed, 169 insertions(+), 13 deletions(-) create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.direct.txt create mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.terraform.txt create mode 100644 acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.direct.txt create mode 100644 acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.terraform.txt diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.direct.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.direct.txt new file mode 100644 index 00000000000..77f461b6b15 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.direct.txt @@ -0,0 +1,29 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" +} +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases/my-db", + "q": { + "update_mask": "spec.role" + }, + "body": { + "spec": { + "postgres_database": "mydb", + "role": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles", + "q": { + "role_id": "app-owner" + }, + "body": { + "spec": { + "postgres_role": "new_role" + } + } +} diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.terraform.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.terraform.txt new file mode 100644 index 00000000000..41f8bbd11a0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.terraform.txt @@ -0,0 +1,15 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "role_id": "app-owner", + "spec": { + "postgres_role": "new_role" + } + } +} diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt index 1a1922d310a..abae404b692 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt @@ -16,6 +16,9 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json +=== Deploy and record per-engine requests (see out.plan.*, out.requests.*) +>>> print_requests.py --sort //postgres + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_branches.my_branch diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script index 1b8e1389fd0..afd033e0408 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script @@ -8,14 +8,28 @@ trap cleanup EXIT title "Initial deployment" trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" # drop create requests; capture only the edit below +# Changing postgres_role recreates the role in the direct engine (the backend rejects this +# field in the PATCH update_mask, so it is delete + create -- see recreate-postgres-role). +# role_id is unchanged, so my_role.name (projects/.../roles/app-owner) is stable. title "Recreate the role; my_database.role references the role's (unchanged) name" trace update_file.py databricks.yml "postgres_role: app_role" "postgres_role: new_role" -# The plan diverges by engine, so capture it per engine. The direct engine recreates the -# role and reports a spurious "update postgres_databases.my_database"; against the -# testserver terraform updates the role in place, resolves the stable name, and leaves the -# database unchanged. +# PLANNED (diverges by engine, so captured per engine): +# - direct recreates the role, so it cannot resolve ${my_role.name} (a recreate does not +# preserve the id) and diffs the saved name against the literal "${...}" placeholder -> +# spurious "update postgres_databases.my_database". +# - terraform updates the role in place against the lenient testserver, resolves the stable +# name, and leaves my_database unchanged. trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -# Field-level proof: old is the real name, new is the still-unresolved reference. +# Field-level proof (direct): old is the real name, new is the still-unresolved reference. trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database".changes.role' > out.database_role.$DATABRICKS_BUNDLE_ENGINE.json + +# WHAT ACTUALLY HAPPENED: deploy and record per-engine requests (deploy stdout -> LOG.deploy; +# parallel completion order is nondeterministic). out.requests.* shows direct issues a PATCH +# against my_database -- a spurious update writing back the unchanged role -- whereas terraform +# issues no request for my_database at all. +title "Deploy and record per-engine requests (see out.plan.*, out.requests.*)" +$CLI bundle deploy --auto-approve &> LOG.deploy +trace print_requests.py --sort '//postgres' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml index ada4f066c37..33d0548f593 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml @@ -2,5 +2,5 @@ # recreated reads as a phantom change). Not Cloud: terraform updating the role in place (vs # the backend delete+create) is a testserver behavior. Cloud = false -RecordRequests = false +RecordRequests = true EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.direct.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.direct.txt new file mode 100644 index 00000000000..6b3a6bf988e --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.direct.txt @@ -0,0 +1,35 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/catalogs/mycat" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "mycat" + }, + "body": { + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "My project", + "pg_version": 17 + } + } +} diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.terraform.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.terraform.txt new file mode 100644 index 00000000000..34160c5469d --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.terraform.txt @@ -0,0 +1,37 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/catalogs/mycat" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "mycat" + }, + "body": { + "catalog_id": "mycat", + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "project_id": "test-pg-proj-[UNIQUE_NAME]", + "spec": { + "display_name": "My project", + "pg_version": 17 + } + } +} diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt index acba4e011c3..69d409ae1fe 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt @@ -7,15 +7,19 @@ Created postgres_projects.my_project Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -=== Bump pg_version: recreates the project; project_id (hence name) is unchanged +=== Bump pg_version: recreates the project; project_id and name are unchanged >>> update_file.py databricks.yml pg_version: 16 pg_version: 17 +=== Plan >>> [CLI] bundle plan recreate postgres_catalogs.my_catalog recreate postgres_projects.my_project Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged +=== Deploy the recreate and record per-engine requests (see out.requests.*) +>>> print_requests.py --sort //postgres + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_catalogs.my_catalog diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script index 81260ab985c..bed41e4558d 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script @@ -8,11 +8,27 @@ trap cleanup EXIT title "Initial deployment" trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" # drop create requests; capture only the recreate below -# Both engines recreate the project (pg_version immutable). Neither can resolve -# ${my_project.name} for a resource being recreated, so both plan a recreate of the -# catalog - even though my_catalog.branch resolves to the same value. The behavior is -# identical: this is a shared plan-time limitation, not a direct-only bug. -title "Bump pg_version: recreates the project; project_id (hence name) is unchanged" +# pg_version is spec:immutable, so BOTH engines recreate the project on a bump. project_id +# (hence the project name "projects/{project_id}") is unchanged, so my_catalog.branch -- +# built from ${my_project.name} -- resolves to the same value: the catalog does not change. +title "Bump pg_version: recreates the project; project_id and name are unchanged" trace update_file.py databricks.yml "pg_version: 16" "pg_version: 17" + +# PLANNED (identical on both engines): neither engine can resolve ${my_project.name} for a +# project being recreated (a recreate does not preserve the id), so both diff the saved +# branch against the literal "${...}" placeholder and plan a spurious recreate of the +# catalog alongside the project. The catalog's branch value never actually changes. +title "Plan" trace $CLI bundle plan + +# WHAT ACTUALLY HAPPENED: both engines perform a real delete + create of the catalog. Deploy +# stdout goes to LOG.deploy (parallel completion order is nondeterministic, so it is not a +# golden); the API calls are recorded per engine. out.requests.* shows the DELETE + POST +# issued against the catalog with an identical, unchanged spec -- a destructive recreate of a +# resource that did not change. The two engines send project_id/catalog_id differently (query +# string vs body), which is why the request files differ. +title "Deploy the recreate and record per-engine requests (see out.requests.*)" +$CLI bundle deploy --auto-approve &> LOG.deploy +trace print_requests.py --sort '//postgres' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml index 310d0528708..c715c3aea73 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml @@ -1,3 +1,6 @@ +# Local-only demonstration of a plan-time reference-resolution limitation shared by both +# engines. Not Cloud: terraform updating resources against the lenient testserver, and the +# recorded request shapes, are testserver behaviors. Cloud = false -RecordRequests = false +RecordRequests = true EnvMatrix.DMS = [] From 06bdd5c585a4454347b636bec388a576e547ead3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 2 Oct 2026 11:26:13 +0200 Subject: [PATCH 4/9] acc: cover the recreate-reference bug on UC schemas via a .id reference Adds schemas/recreate_id_ref_spurious_update. UC resources reference each other by name (user-provided inputs that resolve from config), so idiomatic UC wiring is immune; the bug only surfaces via an output-only .id reference. referent recreates on storage_root (keeping its full_name/.id), and consumer references its id in a comment, so both engines PLAN a spurious update of consumer. The recorded requests show the engines diverge at deploy: the direct engine PATCHes consumer (a spurious write), while terraform downgrades the planned update to a no-op once the id resolves. In the recreate test both engines instead execute the replace -- terraform cannot cancel it (separate destroy/create graph nodes), which is why the direct engine's apply-time downgrade would be strictly more capable. Co-authored-by: Isaac --- .../recreate_ref_both_engines/script | 6 ++++ .../databricks.yml.tmpl | 31 ++++++++++++++++ .../out.consumer_comment.direct.json | 6 ++++ .../out.consumer_comment.terraform.json | 1 + .../out.requests.direct.txt | 24 +++++++++++++ .../out.requests.terraform.txt | 16 +++++++++ .../out.test.toml | 3 ++ .../output.txt | 36 +++++++++++++++++++ .../recreate_id_ref_spurious_update/script | 34 ++++++++++++++++++ .../recreate_id_ref_spurious_update/test.toml | 7 ++++ 10 files changed, 164 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.direct.json create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.terraform.json create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.direct.txt create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.terraform.txt create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script create mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script index bed41e4558d..24febd937de 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script @@ -29,6 +29,12 @@ trace $CLI bundle plan # issued against the catalog with an identical, unchanged spec -- a destructive recreate of a # resource that did not change. The two engines send project_id/catalog_id differently (query # string vs body), which is why the request files differ. +# +# Neither engine downgrades this: both execute the planned recreate. Unlike a spurious UPDATE +# (which terraform re-plans away at apply -- see schemas/recreate_id_ref_spurious_update), +# terraform cannot cancel a replace because it is split into separate destroy/create graph +# nodes. The direct engine's proposed apply-time downgrade would skip even this destructive +# recreate, since it still holds the recreate as one cancelable unit before the delete. title "Deploy the recreate and record per-engine requests (see out.requests.*)" $CLI bundle deploy --auto-approve &> LOG.deploy trace print_requests.py --sort '//postgres' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl new file mode 100644 index 00000000000..794dae817a5 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl @@ -0,0 +1,31 @@ +bundle: + name: schema-id-ref-$UNIQUE_NAME + +sync: + paths: [] + +# Shows the same bug on a NON-postgres family (UC schemas), its boundary, and -- via the +# recorded requests -- how terraform already avoids it at deploy while the direct engine does not. +# +# UC resources reference each other by NAME (catalog_name, schema_name) -- user-provided INPUT +# fields the planner resolves from config even when the target is being recreated. So idiomatic +# UC wiring never hits this bug. The exception is a reference to an OUTPUT-only field such as +# .id (a schema's full_name "catalog.schema"): when the referent is recreated, .id cannot be +# resolved at plan time and reads as a phantom change on the referrer. +# +# referent recreates on storage_root (immutable, non-identity), so its full_name / .id +# (main.referent) is unchanged. consumer references ${referent.id} in its mutable comment, so +# BOTH engines PLAN a spurious update of consumer. They diverge at DEPLOY (see out.requests.*): +# terraform re-plans consumer after referent is recreated, finds ${referent.id} resolved to the +# unchanged value, and skips it (no request for consumer); the direct engine executes the planned +# PATCH -- a spurious write. That apply-time downgrade is exactly what the direct engine lacks. +resources: + schemas: + referent: + catalog_name: main + name: referent + storage_root: s3://my-bucket/v1 + consumer: + catalog_name: main + name: consumer + comment: refers to ${resources.schemas.referent.id} diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.direct.json b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.direct.json new file mode 100644 index 00000000000..e530378ac87 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.direct.json @@ -0,0 +1,6 @@ +{ + "action": "update", + "old": "refers to main.referent", + "new": "refers to ${resources.schemas.referent.id}", + "remote": "refers to main.referent" +} diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.terraform.json b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.terraform.json new file mode 100644 index 00000000000..19765bd501b --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.terraform.json @@ -0,0 +1 @@ +null diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.direct.txt b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.direct.txt new file mode 100644 index 00000000000..45b57602a4b --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.direct.txt @@ -0,0 +1,24 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/schemas/main.referent", + "q": { + "force": "true" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.consumer", + "body": { + "comment": "refers to main.referent", + "custom_max_retention_hours": 0 + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "referent", + "storage_root": "s3://my-bucket/v2" + } +} diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.terraform.txt b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.terraform.txt new file mode 100644 index 00000000000..a854fd6a4f0 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.terraform.txt @@ -0,0 +1,16 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/schemas/main.referent", + "q": { + "force": "true" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "referent", + "storage_root": "s3://my-bucket/v2" + } +} diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml new file mode 100644 index 00000000000..ba82ba326c6 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt new file mode 100644 index 00000000000..428e10ad455 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt @@ -0,0 +1,36 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-id-ref-[UNIQUE_NAME]/default/files... +Created schemas.consumer +Created schemas.referent +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Recreate referent via storage_root (full_name/.id unchanged) +>>> update_file.py databricks.yml s3://my-bucket/v1 s3://my-bucket/v2 + +=== Plan +>>> [CLI] bundle plan +update schemas.consumer +recreate schemas.referent + +Plan: 1 to add, 1 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle plan -o json + +=== Deploy and record per-engine requests (see out.requests.*) +>>> print_requests.py --sort //schemas + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.consumer + delete resources.schemas.referent + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.consumer + delete resources.schemas.referent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/schema-id-ref-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script new file mode 100644 index 00000000000..778ea7a0db5 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script @@ -0,0 +1,34 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +title "Initial deployment" +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" # drop create requests; capture only the edit below + +# storage_root is immutable, so changing it recreates the schema. name/catalog_name are +# unchanged, so referent's full_name (main.referent) -- and thus ${referent.id} -- is stable. +title "Recreate referent via storage_root (full_name/.id unchanged)" +trace update_file.py databricks.yml "s3://my-bucket/v1" "s3://my-bucket/v2" + +# PLANNED (identical on both engines): referent recreates, so ${referent.id} cannot be +# resolved (a recreate does not preserve the id) and consumer.comment diffs the saved value +# against the literal "${...}" placeholder -> spurious update of consumer. +title "Plan" +trace $CLI bundle plan +# Field-level proof: old is the resolved full_name, new is the still-unresolved reference. +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.schemas.consumer".changes.comment' > out.consumer_comment.$DATABRICKS_BUNDLE_ENGINE.json + +# WHAT ACTUALLY HAPPENED (deploy stdout -> LOG.deploy; parallel completion order is +# nondeterministic): the engines diverge, which the per-engine request logs capture. The direct +# engine issues a PATCH against the consumer schema -- writing back its unchanged comment, a +# spurious update. Terraform issues NO request for consumer: it downgraded the planned update to +# a no-op once ${referent.id} resolved to the unchanged value. The direct engine's missing +# apply-time downgrade is the bug. +title "Deploy and record per-engine requests (see out.requests.*)" +$CLI bundle deploy --auto-approve &> LOG.deploy +trace print_requests.py --sort '//schemas' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml new file mode 100644 index 00000000000..6e66d6ba0d4 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml @@ -0,0 +1,7 @@ +# Local-only demonstration that the recreate-reference bug is not postgres-specific. UC +# resources only hit it via .id (output) references; their idiomatic name references are +# user-provided inputs that resolve from config. catalogs are direct-only, so this test uses a +# literal catalog_name (main) to keep both engines runnable. +Cloud = false +RecordRequests = true +EnvMatrix.DMS = [] From 0929ff0a625a8adfc966b3d57a7522d511d9c763 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 2 Oct 2026 11:31:45 +0200 Subject: [PATCH 5/9] acc: make each recreate-reference test's comment self-contained Lead each databricks.yml with a standalone description of the bug (a reference to a recreated resource cannot be resolved at plan time, so a dependent sees a phantom change) instead of cross-referencing the sibling tests, so each file reads on its own. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 21 ++++++++++++------- .../databricks.yml.tmpl | 17 ++++++++++----- .../databricks.yml.tmpl | 16 +++++++------- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl index b6e34f8a93e..bc3100517f8 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl @@ -4,15 +4,20 @@ bundle: sync: paths: [] -# Companion to postgres_projects/recreate_ref_both_engines (which shows the spurious -# RECREATE). Here the referrer field is MUTABLE, so the same phantom diff plans a -# spurious UPDATE instead of a recreate. +# When a resource is recreated, the direct deploy engine cannot resolve a reference to its +# output fields at plan time (a recreate does not preserve the id), so the reference stays the +# literal "${...}" placeholder. A dependent that uses it then diffs its saved value against the +# placeholder and plans a phantom change -- even when the referenced value is actually +# unchanged. This test covers the mutable-field case (a spurious UPDATE); the immutable case +# (a spurious recreate) is postgres_projects/recreate_ref_both_engines. # -# Changing postgres_role recreates the role (the backend rejects it in the PATCH -# update_mask - see postgres_roles/recreate-postgres-role), but role_id is unchanged, so -# my_role.name (projects/.../roles/app-owner) is stable. The direct engine cannot resolve -# ${my_role.name} while the role is being recreated, so my_database.role diffs the saved -# name against the literal "${...}" placeholder and plans an unnecessary update. +# Changing postgres_role recreates the role in the direct engine (the backend rejects this +# field in the PATCH update_mask, so it is delete + create -- see postgres_roles/ +# recreate-postgres-role). role_id is unchanged, so my_role.name (projects/.../roles/app-owner) +# is stable. The direct engine cannot resolve ${my_role.name} while the role is recreated, so +# my_database.role diffs the saved name against the "${...}" placeholder and plans an +# unnecessary update. Terraform updates the role in place against the lenient testserver, +# resolves the stable name, and leaves my_database unchanged. resources: postgres_projects: my_project: diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl index 0d87c8a956b..f4e54dda1c2 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl @@ -4,11 +4,18 @@ bundle: sync: paths: [] -# pg_version is spec:immutable, so BOTH engines recreate the project on a bump - an -# apples-to-apples recreate of the referent, with no engine disagreement on the field. -# project_id is unchanged, so the project's name (projects/{project_id}) is stable across -# the recreate. The catalog's branch is built from that stable name, so the catalog's -# config value does not actually change. +# When a resource is recreated, the direct deploy engine cannot resolve a reference to its +# output fields at plan time (a recreate does not preserve the id), so the reference stays the +# literal "${...}" placeholder. A dependent that uses it then diffs its saved value against the +# placeholder and plans a phantom change -- even when the referenced value is actually +# unchanged. The dependent's action depends on the field: an immutable/id field -> recreate, a +# mutable field -> update. This test covers the immutable case (a spurious RECREATE). +# +# pg_version is spec:immutable, so BOTH engines recreate the project on a bump -- an +# apples-to-apples recreate with no engine disagreement on the field. project_id is unchanged, +# so the project name (projects/{project_id}) is stable, and my_catalog.branch -- built from +# ${my_project.name} -- resolves to the same value: the catalog does not actually change, yet +# both engines plan and execute its recreate. resources: postgres_projects: my_project: diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl index 794dae817a5..70f2ce39cab 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl @@ -4,14 +4,16 @@ bundle: sync: paths: [] -# Shows the same bug on a NON-postgres family (UC schemas), its boundary, and -- via the -# recorded requests -- how terraform already avoids it at deploy while the direct engine does not. +# When a resource is recreated, the direct deploy engine cannot resolve a reference to its +# output fields at plan time (a recreate does not preserve the id), so the reference stays the +# literal "${...}" placeholder. A dependent that uses it then diffs its saved value against the +# placeholder and plans a phantom change -- even when the referenced value is actually unchanged. # -# UC resources reference each other by NAME (catalog_name, schema_name) -- user-provided INPUT -# fields the planner resolves from config even when the target is being recreated. So idiomatic -# UC wiring never hits this bug. The exception is a reference to an OUTPUT-only field such as -# .id (a schema's full_name "catalog.schema"): when the referent is recreated, .id cannot be -# resolved at plan time and reads as a phantom change on the referrer. +# This test shows that on UC schemas (Lakebase/postgres resources exhibit it too) and its +# boundary: UC resources reference each other by NAME (catalog_name, schema_name) -- user- +# provided INPUT fields the planner resolves from config even when the target is recreated, so +# idiomatic UC wiring never hits it. The exception is a reference to an OUTPUT-only field such +# as .id (a schema's full_name "catalog.schema"). # # referent recreates on storage_root (immutable, non-identity), so its full_name / .id # (main.referent) is unchanged. consumer references ${referent.id} in its mutable comment, so From 391c86b4066492683ef56e2b503249d09ffe8644 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 2 Oct 2026 13:27:52 +0200 Subject: [PATCH 6/9] acc: run the recreate-reference tests on the direct engine only Terraform is being removed, so pin these tests to the direct engine and drop the terraform goldens. The comments describe terraform's behavior for context rather than running it, and the field-level proof (the saved value vs the still-unresolved reference, with the remote unchanged) demonstrates the bug without a terraform comparison. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 7 +++- .../out.database_role.terraform.json | 1 - .../out.plan.terraform.txt | 3 -- .../out.requests.terraform.txt | 15 -------- .../out.test.toml | 2 +- .../recreate_ref_spurious_update/output.txt | 3 +- .../recreate_ref_spurious_update/script | 21 +++++------ .../recreate_ref_spurious_update/test.toml | 1 + .../databricks.yml.tmpl | 11 ++++-- .../out.requests.terraform.txt | 37 ------------------- .../recreate_ref_both_engines/out.test.toml | 2 +- .../recreate_ref_both_engines/output.txt | 2 +- .../recreate_ref_both_engines/script | 31 ++++++---------- .../recreate_ref_both_engines/test.toml | 1 + .../databricks.yml.tmpl | 9 +++-- .../out.consumer_comment.terraform.json | 1 - .../out.requests.terraform.txt | 16 -------- .../out.test.toml | 2 +- .../output.txt | 2 +- .../recreate_id_ref_spurious_update/script | 17 ++++----- .../recreate_id_ref_spurious_update/test.toml | 1 + 21 files changed, 55 insertions(+), 130 deletions(-) delete mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.terraform.json delete mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt delete mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.terraform.txt delete mode 100644 acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.terraform.txt delete mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.terraform.json delete mode 100644 acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.terraform.txt diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl index bc3100517f8..c373b285fd4 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/databricks.yml.tmpl @@ -16,8 +16,11 @@ sync: # recreate-postgres-role). role_id is unchanged, so my_role.name (projects/.../roles/app-owner) # is stable. The direct engine cannot resolve ${my_role.name} while the role is recreated, so # my_database.role diffs the saved name against the "${...}" placeholder and plans an -# unnecessary update. Terraform updates the role in place against the lenient testserver, -# resolves the stable name, and leaves my_database unchanged. +# unnecessary update. +# +# This test runs the direct engine only. For context, the terraform engine (now being removed) +# left my_database alone: against the lenient testserver it updates the role in place, so the +# name reference resolved at plan and never diffed. resources: postgres_projects: my_project: diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.terraform.json b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.terraform.json deleted file mode 100644 index 19765bd501b..00000000000 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.terraform.json +++ /dev/null @@ -1 +0,0 @@ -null diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt deleted file mode 100644 index 3da0471b2bc..00000000000 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.terraform.txt +++ /dev/null @@ -1,3 +0,0 @@ -update postgres_roles.my_role - -Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.terraform.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.terraform.txt deleted file mode 100644 index 41f8bbd11a0..00000000000 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.terraform.txt +++ /dev/null @@ -1,15 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", - "q": { - "update_mask": "spec" - }, - "body": { - "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/app-owner", - "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", - "role_id": "app-owner", - "spec": { - "postgres_role": "new_role" - } - } -} diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml index db99a8e07fa..adc4cd4ee4a 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml @@ -1,5 +1,5 @@ Cloud = false CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt index abae404b692..f3e3168b804 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt @@ -12,11 +12,12 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === Recreate the role; my_database.role references the role's (unchanged) name >>> update_file.py databricks.yml postgres_role: app_role postgres_role: new_role +=== Plan >>> [CLI] bundle plan >>> [CLI] bundle plan -o json -=== Deploy and record per-engine requests (see out.plan.*, out.requests.*) +=== Deploy and record the requests (see out.plan.direct.txt, out.requests.direct.txt) >>> print_requests.py --sort //postgres >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script index afd033e0408..e500129d3b4 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script @@ -16,20 +16,17 @@ rm -f "$OUT_REQUESTS" # drop create requests; capture only the edit below title "Recreate the role; my_database.role references the role's (unchanged) name" trace update_file.py databricks.yml "postgres_role: app_role" "postgres_role: new_role" -# PLANNED (diverges by engine, so captured per engine): -# - direct recreates the role, so it cannot resolve ${my_role.name} (a recreate does not -# preserve the id) and diffs the saved name against the literal "${...}" placeholder -> -# spurious "update postgres_databases.my_database". -# - terraform updates the role in place against the lenient testserver, resolves the stable -# name, and leaves my_database unchanged. +# PLANNED: the direct engine recreates the role, so it cannot resolve ${my_role.name} (a +# recreate does not preserve the id) and diffs the saved name against the literal "${...}" +# placeholder -> spurious "update postgres_databases.my_database". +title "Plan" trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -# Field-level proof (direct): old is the real name, new is the still-unresolved reference. +# Field-level proof: old is the real name, new is the still-unresolved reference. trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database".changes.role' > out.database_role.$DATABRICKS_BUNDLE_ENGINE.json -# WHAT ACTUALLY HAPPENED: deploy and record per-engine requests (deploy stdout -> LOG.deploy; -# parallel completion order is nondeterministic). out.requests.* shows direct issues a PATCH -# against my_database -- a spurious update writing back the unchanged role -- whereas terraform -# issues no request for my_database at all. -title "Deploy and record per-engine requests (see out.plan.*, out.requests.*)" +# WHAT ACTUALLY HAPPENED: deploy and record the requests (deploy stdout -> LOG.deploy; parallel +# completion order is nondeterministic). out.requests.direct.txt shows the direct engine issues +# a PATCH against my_database -- a spurious update writing back the unchanged role. +title "Deploy and record the requests (see out.plan.direct.txt, out.requests.direct.txt)" $CLI bundle deploy --auto-approve &> LOG.deploy trace print_requests.py --sort '//postgres' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml index 33d0548f593..e8c39cc87ea 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml @@ -4,3 +4,4 @@ Cloud = false RecordRequests = true EnvMatrix.DMS = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl index f4e54dda1c2..febe35153a0 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/databricks.yml.tmpl @@ -11,11 +11,14 @@ sync: # unchanged. The dependent's action depends on the field: an immutable/id field -> recreate, a # mutable field -> update. This test covers the immutable case (a spurious RECREATE). # -# pg_version is spec:immutable, so BOTH engines recreate the project on a bump -- an -# apples-to-apples recreate with no engine disagreement on the field. project_id is unchanged, -# so the project name (projects/{project_id}) is stable, and my_catalog.branch -- built from +# pg_version is spec:immutable, so a bump recreates the project. project_id is unchanged, so the +# project name (projects/{project_id}) is stable, and my_catalog.branch -- built from # ${my_project.name} -- resolves to the same value: the catalog does not actually change, yet -# both engines plan and execute its recreate. +# the direct engine plans and executes its recreate. +# +# This test runs the direct engine only. For context, the terraform engine (now being removed) +# behaved the same here -- it also recreated the catalog, because it cannot cancel a replace +# once the plan commits it to separate destroy/create graph nodes. resources: postgres_projects: my_project: diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.terraform.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.terraform.txt deleted file mode 100644 index 34160c5469d..00000000000 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.terraform.txt +++ /dev/null @@ -1,37 +0,0 @@ -{ - "method": "DELETE", - "path": "/api/2.0/postgres/catalogs/mycat" -} -{ - "method": "DELETE", - "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" -} -{ - "method": "POST", - "path": "/api/2.0/postgres/catalogs", - "q": { - "catalog_id": "mycat" - }, - "body": { - "catalog_id": "mycat", - "spec": { - "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", - "create_database_if_missing": true, - "postgres_database": "appdb" - } - } -} -{ - "method": "POST", - "path": "/api/2.0/postgres/projects", - "q": { - "project_id": "test-pg-proj-[UNIQUE_NAME]" - }, - "body": { - "project_id": "test-pg-proj-[UNIQUE_NAME]", - "spec": { - "display_name": "My project", - "pg_version": 17 - } - } -} diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml index db99a8e07fa..adc4cd4ee4a 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml @@ -1,5 +1,5 @@ Cloud = false CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt index 69d409ae1fe..0055e0fca92 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt @@ -17,7 +17,7 @@ recreate postgres_projects.my_project Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged -=== Deploy the recreate and record per-engine requests (see out.requests.*) +=== Deploy the recreate and record the requests (see out.requests.direct.txt) >>> print_requests.py --sort //postgres >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script index 24febd937de..3c2b21386c2 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script @@ -10,31 +10,24 @@ title "Initial deployment" trace $CLI bundle deploy rm -f "$OUT_REQUESTS" # drop create requests; capture only the recreate below -# pg_version is spec:immutable, so BOTH engines recreate the project on a bump. project_id -# (hence the project name "projects/{project_id}") is unchanged, so my_catalog.branch -- -# built from ${my_project.name} -- resolves to the same value: the catalog does not change. +# pg_version is spec:immutable, so a bump recreates the project. project_id (hence the project +# name "projects/{project_id}") is unchanged, so my_catalog.branch -- built from +# ${my_project.name} -- resolves to the same value: the catalog does not change. title "Bump pg_version: recreates the project; project_id and name are unchanged" trace update_file.py databricks.yml "pg_version: 16" "pg_version: 17" -# PLANNED (identical on both engines): neither engine can resolve ${my_project.name} for a -# project being recreated (a recreate does not preserve the id), so both diff the saved -# branch against the literal "${...}" placeholder and plan a spurious recreate of the -# catalog alongside the project. The catalog's branch value never actually changes. +# PLANNED: the direct engine cannot resolve ${my_project.name} for a project being recreated +# (a recreate does not preserve the id), so it diffs the saved branch against the literal +# "${...}" placeholder and plans a spurious recreate of the catalog alongside the project. The +# catalog's branch value never actually changes. title "Plan" trace $CLI bundle plan -# WHAT ACTUALLY HAPPENED: both engines perform a real delete + create of the catalog. Deploy -# stdout goes to LOG.deploy (parallel completion order is nondeterministic, so it is not a -# golden); the API calls are recorded per engine. out.requests.* shows the DELETE + POST +# WHAT ACTUALLY HAPPENED: the direct engine performs a real delete + create of the catalog. +# Deploy stdout goes to LOG.deploy (parallel completion order is nondeterministic, so it is not +# a golden); the API calls are recorded below. out.requests.direct.txt shows the DELETE + POST # issued against the catalog with an identical, unchanged spec -- a destructive recreate of a -# resource that did not change. The two engines send project_id/catalog_id differently (query -# string vs body), which is why the request files differ. -# -# Neither engine downgrades this: both execute the planned recreate. Unlike a spurious UPDATE -# (which terraform re-plans away at apply -- see schemas/recreate_id_ref_spurious_update), -# terraform cannot cancel a replace because it is split into separate destroy/create graph -# nodes. The direct engine's proposed apply-time downgrade would skip even this destructive -# recreate, since it still holds the recreate as one cancelable unit before the delete. -title "Deploy the recreate and record per-engine requests (see out.requests.*)" +# resource that did not change. +title "Deploy the recreate and record the requests (see out.requests.direct.txt)" $CLI bundle deploy --auto-approve &> LOG.deploy trace print_requests.py --sort '//postgres' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml index c715c3aea73..9781ffd40a7 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml @@ -4,3 +4,4 @@ Cloud = false RecordRequests = true EnvMatrix.DMS = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl index 70f2ce39cab..15ad70be8fe 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/databricks.yml.tmpl @@ -17,10 +17,11 @@ sync: # # referent recreates on storage_root (immutable, non-identity), so its full_name / .id # (main.referent) is unchanged. consumer references ${referent.id} in its mutable comment, so -# BOTH engines PLAN a spurious update of consumer. They diverge at DEPLOY (see out.requests.*): -# terraform re-plans consumer after referent is recreated, finds ${referent.id} resolved to the -# unchanged value, and skips it (no request for consumer); the direct engine executes the planned -# PATCH -- a spurious write. That apply-time downgrade is exactly what the direct engine lacks. +# the direct engine plans a spurious update of consumer. +# +# This test runs the direct engine only. For context, the terraform engine (now being removed) +# left consumer alone: it re-plans at apply, finds ${referent.id} resolved to the unchanged +# value, and issues no request for consumer. resources: schemas: referent: diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.terraform.json b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.terraform.json deleted file mode 100644 index 19765bd501b..00000000000 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.terraform.json +++ /dev/null @@ -1 +0,0 @@ -null diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.terraform.txt b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.terraform.txt deleted file mode 100644 index a854fd6a4f0..00000000000 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.terraform.txt +++ /dev/null @@ -1,16 +0,0 @@ -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/schemas/main.referent", - "q": { - "force": "true" - } -} -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/schemas", - "body": { - "catalog_name": "main", - "name": "referent", - "storage_root": "s3://my-bucket/v2" - } -} diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml index ba82ba326c6..d0b77dbc454 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = [] diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt index 428e10ad455..b293c39708a 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt @@ -19,7 +19,7 @@ Plan: 1 to add, 1 to change, 1 to delete, 0 unchanged >>> [CLI] bundle plan -o json -=== Deploy and record per-engine requests (see out.requests.*) +=== Deploy and record the requests (see out.requests.direct.txt) >>> print_requests.py --sort //schemas >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script index 778ea7a0db5..6781c6e0a94 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script @@ -15,20 +15,17 @@ rm -f "$OUT_REQUESTS" # drop create requests; capture only the edit below title "Recreate referent via storage_root (full_name/.id unchanged)" trace update_file.py databricks.yml "s3://my-bucket/v1" "s3://my-bucket/v2" -# PLANNED (identical on both engines): referent recreates, so ${referent.id} cannot be -# resolved (a recreate does not preserve the id) and consumer.comment diffs the saved value -# against the literal "${...}" placeholder -> spurious update of consumer. +# PLANNED: referent recreates, so the direct engine cannot resolve ${referent.id} (a recreate +# does not preserve the id) and consumer.comment diffs the saved value against the literal +# "${...}" placeholder -> spurious update of consumer. title "Plan" trace $CLI bundle plan # Field-level proof: old is the resolved full_name, new is the still-unresolved reference. trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.schemas.consumer".changes.comment' > out.consumer_comment.$DATABRICKS_BUNDLE_ENGINE.json -# WHAT ACTUALLY HAPPENED (deploy stdout -> LOG.deploy; parallel completion order is -# nondeterministic): the engines diverge, which the per-engine request logs capture. The direct -# engine issues a PATCH against the consumer schema -- writing back its unchanged comment, a -# spurious update. Terraform issues NO request for consumer: it downgraded the planned update to -# a no-op once ${referent.id} resolved to the unchanged value. The direct engine's missing -# apply-time downgrade is the bug. -title "Deploy and record per-engine requests (see out.requests.*)" +# WHAT ACTUALLY HAPPENED: deploy and record the requests (deploy stdout -> LOG.deploy; parallel +# completion order is nondeterministic). out.requests.direct.txt shows the direct engine issues +# a PATCH against the consumer schema -- a spurious update writing back its unchanged comment. +title "Deploy and record the requests (see out.requests.direct.txt)" $CLI bundle deploy --auto-approve &> LOG.deploy trace print_requests.py --sort '//schemas' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml index 6e66d6ba0d4..50035f718c4 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml @@ -5,3 +5,4 @@ Cloud = false RecordRequests = true EnvMatrix.DMS = [] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 3de1a80db64ac1c222a270891839186338f1409b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 5 Oct 2026 11:58:02 +0200 Subject: [PATCH 7/9] acc: drop obsolete DATABRICKS_BUNDLE_ENGINE references The engine matrix has been removed from the acceptance suite, so the tests no longer pin EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] or carry an engine suffix on their golden files; they run with the (only) direct engine and plain golden names. Co-authored-by: Isaac --- ...atabase_role.direct.json => out.database_role.json} | 0 .../{out.plan.direct.txt => out.plan.txt} | 0 .../{out.requests.direct.txt => out.requests.txt} | 0 .../recreate_ref_spurious_update/out.test.toml | 3 +-- .../recreate_ref_spurious_update/output.txt | 2 +- .../recreate_ref_spurious_update/script | 10 +++++----- .../recreate_ref_spurious_update/test.toml | 7 ++----- .../{out.requests.direct.txt => out.requests.txt} | 0 .../recreate_ref_both_engines/out.test.toml | 3 +-- .../recreate_ref_both_engines/output.txt | 2 +- .../postgres_projects/recreate_ref_both_engines/script | 6 +++--- .../recreate_ref_both_engines/test.toml | 7 ++----- ...r_comment.direct.json => out.consumer_comment.json} | 0 .../{out.requests.direct.txt => out.requests.txt} | 0 .../recreate_id_ref_spurious_update/out.test.toml | 3 +-- .../schemas/recreate_id_ref_spurious_update/output.txt | 2 +- .../schemas/recreate_id_ref_spurious_update/script | 8 ++++---- .../schemas/recreate_id_ref_spurious_update/test.toml | 9 +++------ 18 files changed, 25 insertions(+), 37 deletions(-) rename acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/{out.database_role.direct.json => out.database_role.json} (100%) rename acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/{out.plan.direct.txt => out.plan.txt} (100%) rename acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/{out.requests.direct.txt => out.requests.txt} (100%) rename acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/{out.requests.direct.txt => out.requests.txt} (100%) rename acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/{out.consumer_comment.direct.json => out.consumer_comment.json} (100%) rename acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/{out.requests.direct.txt => out.requests.txt} (100%) diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.direct.json b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.json similarity index 100% rename from acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.direct.json rename to acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.database_role.json diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.txt similarity index 100% rename from acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.direct.txt rename to acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.txt diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.direct.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.txt similarity index 100% rename from acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.direct.txt rename to acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.requests.txt diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml index adc4cd4ee4a..0faf51e59a4 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.test.toml @@ -1,5 +1,4 @@ Cloud = false CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt index f3e3168b804..6a330cb7129 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt @@ -17,7 +17,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json -=== Deploy and record the requests (see out.plan.direct.txt, out.requests.direct.txt) +=== Deploy and record the requests (see out.plan.txt, out.requests.txt) >>> print_requests.py --sort //postgres >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script index e500129d3b4..e7bd202f782 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script @@ -20,13 +20,13 @@ trace update_file.py databricks.yml "postgres_role: app_role" "postgres_role: ne # recreate does not preserve the id) and diffs the saved name against the literal "${...}" # placeholder -> spurious "update postgres_databases.my_database". title "Plan" -trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan > out.plan.txt # Field-level proof: old is the real name, new is the still-unresolved reference. -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database".changes.role' > out.database_role.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database".changes.role' > out.database_role.json # WHAT ACTUALLY HAPPENED: deploy and record the requests (deploy stdout -> LOG.deploy; parallel -# completion order is nondeterministic). out.requests.direct.txt shows the direct engine issues +# completion order is nondeterministic). out.requests.txt shows the direct engine issues # a PATCH against my_database -- a spurious update writing back the unchanged role. -title "Deploy and record the requests (see out.plan.direct.txt, out.requests.direct.txt)" +title "Deploy and record the requests (see out.plan.txt, out.requests.txt)" $CLI bundle deploy --auto-approve &> LOG.deploy -trace print_requests.py --sort '//postgres' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py --sort '//postgres' > out.requests.txt diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml index e8c39cc87ea..5d1190e91a6 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml @@ -1,7 +1,4 @@ -# Local-only demonstration of a direct-engine planner bug (reference to a resource being -# recreated reads as a phantom change). Not Cloud: terraform updating the role in place (vs -# the backend delete+create) is a testserver behavior. +# Direct-engine demonstration of a plan-time reference-resolution limitation (a reference to a +# resource being recreated reads as a phantom change). Not Cloud: shown against the testserver. Cloud = false RecordRequests = true -EnvMatrix.DMS = [] -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.direct.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.txt similarity index 100% rename from acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.direct.txt rename to acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.requests.txt diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml index adc4cd4ee4a..0faf51e59a4 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/out.test.toml @@ -1,5 +1,4 @@ Cloud = false CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt index 0055e0fca92..664bec01dde 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/output.txt @@ -17,7 +17,7 @@ recreate postgres_projects.my_project Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged -=== Deploy the recreate and record the requests (see out.requests.direct.txt) +=== Deploy the recreate and record the requests (see out.requests.txt) >>> print_requests.py --sort //postgres >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script index 3c2b21386c2..96b1b3421c2 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/script @@ -25,9 +25,9 @@ trace $CLI bundle plan # WHAT ACTUALLY HAPPENED: the direct engine performs a real delete + create of the catalog. # Deploy stdout goes to LOG.deploy (parallel completion order is nondeterministic, so it is not -# a golden); the API calls are recorded below. out.requests.direct.txt shows the DELETE + POST +# a golden); the API calls are recorded below. out.requests.txt shows the DELETE + POST # issued against the catalog with an identical, unchanged spec -- a destructive recreate of a # resource that did not change. -title "Deploy the recreate and record the requests (see out.requests.direct.txt)" +title "Deploy the recreate and record the requests (see out.requests.txt)" $CLI bundle deploy --auto-approve &> LOG.deploy -trace print_requests.py --sort '//postgres' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py --sort '//postgres' > out.requests.txt diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml index 9781ffd40a7..5160755092d 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml @@ -1,7 +1,4 @@ -# Local-only demonstration of a plan-time reference-resolution limitation shared by both -# engines. Not Cloud: terraform updating resources against the lenient testserver, and the -# recorded request shapes, are testserver behaviors. +# Direct-engine demonstration of a plan-time reference-resolution limitation. Not Cloud: the +# behavior lives in the planner and is shown against the testserver. Cloud = false RecordRequests = true -EnvMatrix.DMS = [] -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.direct.json b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.json similarity index 100% rename from acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.direct.json rename to acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.consumer_comment.json diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.direct.txt b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.txt similarity index 100% rename from acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.direct.txt rename to acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.requests.txt diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml index d0b77dbc454..a927a5fbc06 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/out.test.toml @@ -1,3 +1,2 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt index b293c39708a..c6d1e785ef9 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/output.txt @@ -19,7 +19,7 @@ Plan: 1 to add, 1 to change, 1 to delete, 0 unchanged >>> [CLI] bundle plan -o json -=== Deploy and record the requests (see out.requests.direct.txt) +=== Deploy and record the requests (see out.requests.txt) >>> print_requests.py --sort //schemas >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script index 6781c6e0a94..f489860d60a 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/script @@ -21,11 +21,11 @@ trace update_file.py databricks.yml "s3://my-bucket/v1" "s3://my-bucket/v2" title "Plan" trace $CLI bundle plan # Field-level proof: old is the resolved full_name, new is the still-unresolved reference. -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.schemas.consumer".changes.comment' > out.consumer_comment.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.schemas.consumer".changes.comment' > out.consumer_comment.json # WHAT ACTUALLY HAPPENED: deploy and record the requests (deploy stdout -> LOG.deploy; parallel -# completion order is nondeterministic). out.requests.direct.txt shows the direct engine issues +# completion order is nondeterministic). out.requests.txt shows the direct engine issues # a PATCH against the consumer schema -- a spurious update writing back its unchanged comment. -title "Deploy and record the requests (see out.requests.direct.txt)" +title "Deploy and record the requests (see out.requests.txt)" $CLI bundle deploy --auto-approve &> LOG.deploy -trace print_requests.py --sort '//schemas' > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py --sort '//schemas' > out.requests.txt diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml index 50035f718c4..abc1b883df5 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml @@ -1,8 +1,5 @@ -# Local-only demonstration that the recreate-reference bug is not postgres-specific. UC -# resources only hit it via .id (output) references; their idiomatic name references are -# user-provided inputs that resolve from config. catalogs are direct-only, so this test uses a -# literal catalog_name (main) to keep both engines runnable. +# Direct-engine demonstration that the recreate-reference bug is not postgres-specific. UC +# resources only hit it via .id (output) references; their name references are user-provided +# inputs that resolve from config. Not Cloud: shown against the testserver. Cloud = false RecordRequests = true -EnvMatrix.DMS = [] -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 0329f9ff6de145cd79eee133833c850be3b86efe Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 5 Oct 2026 15:46:20 +0200 Subject: [PATCH 8/9] acc: record the plan inline in the update test With a single engine the plan no longer diverges, so there is no reason to route it to a separate file -- record it inline in output.txt, matching the recreate and UC tests. Co-authored-by: Isaac --- .../recreate_ref_spurious_update/out.plan.txt | 4 ---- .../recreate_ref_spurious_update/output.txt | 6 +++++- .../postgres_databases/recreate_ref_spurious_update/script | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.txt diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.txt deleted file mode 100644 index 1560fe029bb..00000000000 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/out.plan.txt +++ /dev/null @@ -1,4 +0,0 @@ -update postgres_databases.my_database -recreate postgres_roles.my_role - -Plan: 1 to add, 1 to change, 1 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt index 6a330cb7129..59cbcceab2e 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/output.txt @@ -14,10 +14,14 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === Plan >>> [CLI] bundle plan +update postgres_databases.my_database +recreate postgres_roles.my_role + +Plan: 1 to add, 1 to change, 1 to delete, 2 unchanged >>> [CLI] bundle plan -o json -=== Deploy and record the requests (see out.plan.txt, out.requests.txt) +=== Deploy and record the requests (see out.requests.txt) >>> print_requests.py --sort //postgres >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script index e7bd202f782..de79565f150 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/script @@ -20,13 +20,13 @@ trace update_file.py databricks.yml "postgres_role: app_role" "postgres_role: ne # recreate does not preserve the id) and diffs the saved name against the literal "${...}" # placeholder -> spurious "update postgres_databases.my_database". title "Plan" -trace $CLI bundle plan > out.plan.txt +trace $CLI bundle plan # Field-level proof: old is the real name, new is the still-unresolved reference. trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database".changes.role' > out.database_role.json # WHAT ACTUALLY HAPPENED: deploy and record the requests (deploy stdout -> LOG.deploy; parallel # completion order is nondeterministic). out.requests.txt shows the direct engine issues # a PATCH against my_database -- a spurious update writing back the unchanged role. -title "Deploy and record the requests (see out.plan.txt, out.requests.txt)" +title "Deploy and record the requests (see out.requests.txt)" $CLI bundle deploy --auto-approve &> LOG.deploy trace print_requests.py --sort '//postgres' > out.requests.txt From 9be9e493408458d14e023fd85f0657c75ee7bb97 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 5 Oct 2026 21:16:09 +0200 Subject: [PATCH 9/9] =?UTF-8?q?acc:=20address=20review=20=E2=80=94=20drop?= =?UTF-8?q?=20inherited=20RecordRequests,=20widen=20UC=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RecordRequests is already set in acceptance/bundle/resources/test.toml, so the leaf configs need not repeat it. Also reword the schemas test: the bug is hit by any output-only reference (e.g. .id or full_name), not only .id. Co-authored-by: Isaac --- .../recreate_ref_spurious_update/test.toml | 1 - .../postgres_projects/recreate_ref_both_engines/test.toml | 1 - .../schemas/recreate_id_ref_spurious_update/test.toml | 5 ++--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml index 5d1190e91a6..13969337856 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate_ref_spurious_update/test.toml @@ -1,4 +1,3 @@ # Direct-engine demonstration of a plan-time reference-resolution limitation (a reference to a # resource being recreated reads as a phantom change). Not Cloud: shown against the testserver. Cloud = false -RecordRequests = true diff --git a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml index 5160755092d..2c49ed36ba5 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate_ref_both_engines/test.toml @@ -1,4 +1,3 @@ # Direct-engine demonstration of a plan-time reference-resolution limitation. Not Cloud: the # behavior lives in the planner and is shown against the testserver. Cloud = false -RecordRequests = true diff --git a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml index abc1b883df5..3c3f9d6c718 100644 --- a/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml +++ b/acceptance/bundle/resources/schemas/recreate_id_ref_spurious_update/test.toml @@ -1,5 +1,4 @@ # Direct-engine demonstration that the recreate-reference bug is not postgres-specific. UC -# resources only hit it via .id (output) references; their name references are user-provided -# inputs that resolve from config. Not Cloud: shown against the testserver. +# resources only hit it via output-only references (e.g. .id / full_name); their name references +# are user-provided inputs that resolve from config. Not Cloud: shown against the testserver. Cloud = false -RecordRequests = true