Set default value (or empty list/dict) no longer unset the field

docs/changelog.rst

@@ -17,6 +17,24 @@ Changes in 1.0.0
     - make sure to read https://www.mongodb.com/docs/manual/core/transactions-in-applications/#callback-api-vs-core-api
     - run_in_transaction context manager relies on Pymongo coreAPI, it will retry automatically in case of ``UnknownTransactionCommitResult`` but not ``TransientTransactionError`` exceptions
     - Using .count() in a transaction will always use Collection.count_document (as estimated_document_count is not supported in transactions)
+- BREAKING CHANGE (#2928): Assigning a field value and deleting a field now
+  have distinct persistence semantics:
+
+  - Assigning empty default values, such as ``""``, ``[]``, or ``{}``, now
+    stores those values. Previous versions implicitly unset the field.  #267
+  - Assigning falsy values to dynamic fields now stores those values instead
+    of implicitly unsetting the field.
+
+  Existing documents with missing default-valued fields are not migrated
+  automatically. They must be migrated if queries for the default value need
+  to match them.
+
+- BREAKING CHANGE (internal API, supporting #2928):
+  ``_get_changed_fields()`` was removed and replaced by
+  ``_get_updated_fields()``, which returns a
+  ``(changed_fields, unset_fields)`` tuple containing two disjoint lists of
+  database paths. ``_clear_changed_fields()`` was renamed to
+  ``_clear_updated_fields()``.
 - Add a warning that ``mongoengine.org`` is no longer controlled by the MongoEngine
   project and appears to be an expired domain takeover.
 - Bug Fix - Fix querying GenericReferenceField with __in operator #2886

mongoengine/base/datastructures.py

@@ -38,6 +38,17 @@ def wrapper(self, key, *args, **kwargs):
     return wrapper
 
 
+def mark_key_as_unset_wrapper(parent_method):
+    """Decorator that ensures _mark_as_unset gets called after deleting a key."""
+
+    def wrapper(self, key, *args, **kwargs):
+        result = parent_method(self, key, *args, **kwargs)
+        self._mark_as_unset(key)
+        return result
+
+    return wrapper
+
+
 class BaseDict(dict):
     """A special dict so we can watch any changes."""
 
@@ -86,8 +97,7 @@ def __setstate__(self, state):
         return self
 
     __setitem__ = mark_key_as_changed_wrapper(dict.__setitem__)
-    __delattr__ = mark_key_as_changed_wrapper(dict.__delattr__)
-    __delitem__ = mark_key_as_changed_wrapper(dict.__delitem__)
+    __delitem__ = mark_key_as_unset_wrapper(dict.__delitem__)
     pop = mark_as_changed_wrapper(dict.pop)
     clear = mark_as_changed_wrapper(dict.clear)
     update = mark_as_changed_wrapper(dict.update)
@@ -101,6 +111,10 @@ def _mark_as_changed(self, key=None):
             else:
                 self._instance._mark_as_changed(self._name)
 
+    def _mark_as_unset(self, key):
+        if hasattr(self._instance, "_mark_as_unset"):
+            self._instance._mark_as_unset(f"{self._name}.{key}")
+
 
 class BaseList(list):
     """A special list so we can watch any changes."""