Fix "Nothing to update" when a UC comment is set outside the bundle

.nextchanges/bundles/schema-comment-nothing-to-update.md

@@ -0,0 +1 @@
+Fixed `bundle deploy` failing with `Nothing to update` on the direct engine when a schema's, catalog's or volume's comment was set outside the bundle.

acceptance/bundle/resources/catalogs/drift/comment_out_of_band/databricks.yml.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  catalogs:
+    catalog1:
+      name: test-catalog-$UNIQUE_NAME

acceptance/bundle/resources/catalogs/drift/comment_out_of_band/output.txt

@@ -0,0 +1,57 @@
+
+=== Deploy a catalog that does not declare a comment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Created catalogs.catalog1
+Files: 0 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Set the comment out of band, the way Catalog Explorer does
+=== The remote comment is drift, so the plan updates the catalog
+>>> [CLI] bundle plan --output json
+{
+  "action": "update",
+  "remote": "set outside the bundle"
+}
+
+=== Redeploy clears the comment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Updated catalogs.catalog1
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+=== The comment is gone
+>>> [CLI] api get /api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]
+""
+
+=== Redeploy is a no-op
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.catalogs.catalog1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted
+
+=== The UpdateCatalog payloads the deploys sent
+>>> print_requests.py --method PATCH //unity-catalog/catalogs
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]",
+  "body": {
+    "comment": "set outside the bundle"
+  }
+}
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]",
+  "body": {
+    "comment": ""
+  }
+}

acceptance/bundle/resources/catalogs/drift/comment_out_of_band/script

@@ -0,0 +1,32 @@
+echo "*" > .gitignore
+envsubst < databricks.yml.tmpl > databricks.yml
+
+CATALOG="test-catalog-${UNIQUE_NAME}"
+
+# print_requests.py consumes out.requests.txt, so it has to run after the last
+# request the test makes, which is the destroy below.
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+
+    title "The UpdateCatalog payloads the deploys sent"
+    trace print_requests.py --method PATCH //unity-catalog/catalogs
+}
+trap cleanup EXIT
+
+title "Deploy a catalog that does not declare a comment"
+trace $CLI bundle deploy
+
+title "Set the comment out of band, the way Catalog Explorer does"
+MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/catalogs/$CATALOG" --json '{"comment":"set outside the bundle"}' > /dev/null
+
+title "The remote comment is drift, so the plan updates the catalog"
+trace $CLI bundle plan --output json | jq '.plan[].changes.comment'
+
+title "Redeploy clears the comment"
+trace $CLI bundle deploy
+
+title "The comment is gone"
+MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.1/unity-catalog/catalogs/$CATALOG" | jq '.comment'
+
+title "Redeploy is a no-op"
+trace $CLI bundle deploy

acceptance/bundle/resources/catalogs/drift/comment_out_of_band/test.toml

@@ -0,0 +1,11 @@
+Cloud = true
+RequiresUnityCatalog = true
+
+# UpdateCatalog payload construction is direct-engine-specific.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [
+    ".databricks",
+    ".gitignore",
+    "databricks.yml",
+]

acceptance/bundle/resources/schemas/drift/comment_out_of_band/databricks.yml.tmpl

@@ -0,0 +1,8 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  schemas:
+    schema1:
+      catalog_name: main
+      name: test-schema-$UNIQUE_NAME

acceptance/bundle/resources/schemas/drift/comment_out_of_band/output.txt

@@ -0,0 +1,60 @@
+
+=== Deploy a schema that does not declare a comment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Created schemas.schema1
+Files: 0 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Set the comment out of band, the way Catalog Explorer does
+=== The remote comment is drift, so the plan updates the schema
+>>> [CLI] bundle plan --output json
+{
+  "action": "update",
+  "remote": "set outside the bundle"
+}
+
+=== Redeploy clears the comment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Updated schemas.schema1
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+=== The comment is gone
+>>> [CLI] api get /api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]
+""
+
+=== Redeploy is a no-op
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.schemas.schema1
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.schema1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted
+
+=== The UpdateSchema payloads the deploys sent
+>>> print_requests.py --method PATCH //unity-catalog/schemas
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]",
+  "body": {
+    "comment": "set outside the bundle"
+  }
+}
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]",
+  "body": {
+    "comment": ""
+  }
+}

acceptance/bundle/resources/schemas/drift/comment_out_of_band/script

@@ -0,0 +1,32 @@
+echo "*" > .gitignore
+envsubst < databricks.yml.tmpl > databricks.yml
+
+SCHEMA="main.test-schema-${UNIQUE_NAME}"
+
+# print_requests.py consumes out.requests.txt, so it has to run after the last
+# request the test makes, which is the destroy below.
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+
+    title "The UpdateSchema payloads the deploys sent"
+    trace print_requests.py --method PATCH //unity-catalog/schemas
+}
+trap cleanup EXIT
+
+title "Deploy a schema that does not declare a comment"
+trace $CLI bundle deploy
+
+title "Set the comment out of band, the way Catalog Explorer does"
+MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/schemas/$SCHEMA" --json '{"comment":"set outside the bundle"}' > /dev/null
+
+title "The remote comment is drift, so the plan updates the schema"
+trace $CLI bundle plan --output json | jq '.plan[].changes.comment'
+
+title "Redeploy clears the comment"
+trace $CLI bundle deploy
+
+title "The comment is gone"
+MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.1/unity-catalog/schemas/$SCHEMA" | jq '.comment'
+
+title "Redeploy is a no-op"
+trace $CLI bundle deploy

acceptance/bundle/resources/schemas/drift/comment_out_of_band/test.toml

@@ -0,0 +1,11 @@
+Cloud = true
+RequiresUnityCatalog = true
+
+# UpdateSchema payload construction is direct-engine-specific.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [
+    ".databricks",
+    ".gitignore",
+    "databricks.yml",
+]

acceptance/bundle/resources/volumes/drift/comment_out_of_band/databricks.yml.tmpl

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  schemas:
+    schema1:
+      catalog_name: main
+      name: test-schema-$UNIQUE_NAME
+  volumes:
+    volume1:
+      catalog_name: main
+      schema_name: ${resources.schemas.schema1.name}
+      name: test-volume-$UNIQUE_NAME

acceptance/bundle/resources/volumes/drift/comment_out_of_band/output.txt

@@ -0,0 +1,68 @@
+
+=== Deploy a volume that does not declare a comment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Created schemas.schema1
+Created volumes.volume1
+Files: 0 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Set the comment out of band, the way Catalog Explorer does
+=== The remote comment is drift, so the plan updates the volume
+>>> [CLI] bundle plan --output json
+{
+  "action": "update",
+  "remote": "set outside the bundle"
+}
+
+=== Redeploy clears the comment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Updated volumes.volume1
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
+
+=== The comment is gone
+>>> [CLI] api get /api/2.1/unity-catalog/volumes/main.test-schema-[UNIQUE_NAME].test-volume-[UNIQUE_NAME]
+""
+
+=== Redeploy is a no-op
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.schemas.schema1
+  delete resources.volumes.volume1
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.schema1
+
+This action will result in the deletion of the following volumes.
+For managed volumes, the files stored in the volume are also deleted from your
+cloud tenant within 30 days. For external volumes, the metadata about the volume
+is removed from the catalog, but the underlying files are not deleted:
+  delete resources.volumes.volume1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Destroy: 2 deleted
+
+=== The UpdateVolume payloads the deploys sent
+>>> print_requests.py --method PATCH //unity-catalog/volumes
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/volumes/main.test-schema-[UNIQUE_NAME].test-volume-[UNIQUE_NAME]",
+  "body": {
+    "comment": "set outside the bundle"
+  }
+}
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/volumes/main.test-schema-[UNIQUE_NAME].test-volume-[UNIQUE_NAME]",
+  "body": {
+    "comment": ""
+  }
+}

acceptance/bundle/resources/volumes/drift/comment_out_of_band/script

@@ -0,0 +1,32 @@
+echo "*" > .gitignore
+envsubst < databricks.yml.tmpl > databricks.yml
+
+VOLUME="main.test-schema-${UNIQUE_NAME}.test-volume-${UNIQUE_NAME}"
+
+# print_requests.py consumes out.requests.txt, so it has to run after the last
+# request the test makes, which is the destroy below.
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+
+    title "The UpdateVolume payloads the deploys sent"
+    trace print_requests.py --method PATCH //unity-catalog/volumes
+}
+trap cleanup EXIT
+
+title "Deploy a volume that does not declare a comment"
+trace $CLI bundle deploy
+
+title "Set the comment out of band, the way Catalog Explorer does"
+MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/volumes/$VOLUME" --json '{"comment":"set outside the bundle"}' > /dev/null
+
+title "The remote comment is drift, so the plan updates the volume"
+trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.comment'
+
+title "Redeploy clears the comment"
+trace $CLI bundle deploy
+
+title "The comment is gone"
+MSYS_NO_PATHCONV=1 trace $CLI api get "/api/2.1/unity-catalog/volumes/$VOLUME" | jq '.comment'
+
+title "Redeploy is a no-op"
+trace $CLI bundle deploy

acceptance/bundle/resources/volumes/drift/comment_out_of_band/test.toml

@@ -0,0 +1,11 @@
+Cloud = true
+RequiresUnityCatalog = true
+
+# UpdateVolume payload construction is direct-engine-specific.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [
+    ".databricks",
+    ".gitignore",
+    "databricks.yml",
+]

bundle/direct/dresources/catalog.go

@@ -62,7 +62,7 @@ func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catal
 		Options:                      config.Options,
 		Owner:                        "", // Not supported by DABs
 		Properties:                   config.Properties,
-		ForceSendFields:              utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields, "EnablePredictiveOptimization", "IsolationMode", "Owner"),
+		ForceSendFields:              forceSendComment(utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields, "EnablePredictiveOptimization", "IsolationMode", "Owner")),
 	}
 
 	response, err := r.client.Catalogs.Update(ctx, updateRequest)
@@ -86,7 +86,7 @@ func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config
 		Options:                      config.Options,
 		Owner:                        "", // Not supported by DABs
 		Properties:                   config.Properties,
-		ForceSendFields:              utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields, "EnablePredictiveOptimization", "IsolationMode", "Owner"),
+		ForceSendFields:              forceSendComment(utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields, "EnablePredictiveOptimization", "IsolationMode", "Owner")),
 	}
 
 	if config.Name != id {

bundle/direct/dresources/schema.go

@@ -56,7 +56,7 @@ func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalo
 		NewName:                      "", // We recreate schemas on name change intentionally.
 		Owner:                        "", // Not supported by DABs
 		Properties:                   config.Properties,
-		ForceSendFields:              utils.FilterFields[catalog.UpdateSchema](config.ForceSendFields, "EnablePredictiveOptimization", "NewName", "Owner"),
+		ForceSendFields:              forceSendComment(utils.FilterFields[catalog.UpdateSchema](config.ForceSendFields, "EnablePredictiveOptimization", "NewName", "Owner")),
 	}
 
 	response, err := r.client.Schemas.Update(ctx, updateRequest)