build_entities_from_data: stop a null object from corrupting the entities

CHANGELOG.md

@@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
 ### Fixed
 
 - GraphParameterType.autocomplete: results are now sorted by label and only include graphs matching all given query terms, instead of an unsorted, non-deterministic order matching any single term
+- `build_entities_from_data`: a relation whose value is null now leaves the path empty instead of filling it with `""`. That empty string was a reference to a sub entity that does not exist, and a JSON dataset written from such entities failed the whole write with `Current context not Array but Object`
+- `build_entities_from_data`: a key that is an object in one item and null in another, or a list of objects in one item and an empty list in another, is now described as a relation whichever item comes first. The type used to be taken from the last item seen, so in one of the two orders the objects were flattened into their `str()` - `{'city': 'San Francisco'}` - rather than becoming sub entities
 
 
 ## [4.20.0] 2026-08-14 - shipped with DI v26.2.0

cmem_plugin_base/dataintegration/utils/entity_builder.py

@@ -12,6 +12,13 @@ def merge_path_values(paths_map1: dict, paths_map2: dict) -> dict:
     each representing paths and corresponding values. It merges these dictionaries
     by combining values for common paths and returns the merged dictionary.
 
+    Args:
+        paths_map1 (dict): The first dictionary containing paths and values.
+        paths_map2 (dict): The second dictionary containing paths and values.
+
+    Where both dictionaries know a type for the same key, the more specific one wins
+    rather than the one seen last - see `_more_specific_type`.
+
     Args:
         paths_map1 (dict): The first dictionary containing paths and values.
         paths_map2 (dict): The second dictionary containing paths and values.
@@ -24,11 +31,36 @@ def merge_path_values(paths_map1: dict, paths_map2: dict) -> dict:
         current_path_map = {}
         if paths_map1.get(key) is not None:
             current_path_map = paths_map1[key]
-        current_path_map = current_path_map | value
+        current_path_map = current_path_map | {
+            _key: _more_specific_type(current_path_map.get(_key), _type)
+            for _key, _type in value.items()
+        }
         paths_map1[key] = current_path_map
     return paths_map1
 
 
+def _more_specific_type(known: str | None, found: str) -> str:
+    """Pick the type that says more about a key, out of the two seen for it.
+
+    A key's type is read off the values in the data, and two items can disagree about
+    it: one object carries a `lockedByUser` and the next has it as null, or one list is
+    empty and the next holds objects. Taking whichever came last then decides the schema
+    by the order of the data - and losing to `NoneType` or to an empty `list` is the
+    direction that loses information, because the object seen in the other item is no
+    longer described as a relation and gets flattened into its `str()` instead.
+
+    Genuinely conflicting types, say `str` against `int`, are left to the last one seen,
+    as before.
+    """
+    if known is None or known == "NoneType":
+        return found
+    if found == "NoneType":
+        return known
+    if {known, found} == {"list", "list_dict"}:
+        return "list_dict"
+    return found
+
+
 def generate_paths_from_data(data: dict | list, path: str | None = "root") -> dict:
     """Generate a dictionary representing paths and data types from a nested JSON structure.
 
@@ -128,7 +160,11 @@ def _get_entity(
     schema = path_to_schema_map[path_from_root]
     for _ in schema.paths:
         if data.get(_.path) is None:
-            values.append([""])
+            # A relation carries URIs pointing at sub entities, so a null has to leave the
+            # path empty. An "" there is a reference that resolves to nothing, which a
+            # dataset sink follows and fails on - a JSON dataset aborts the whole write
+            # with "Current context not Array but Object".
+            values.append([] if _.is_relation else [""])
         elif not _.is_relation:
             values.append(
                 [f"{data.get(_.path)}"]

tests/test_utils_build_entities_from_data.py

@@ -225,3 +225,88 @@ def test_empty_object() -> None:
     test_data = """[]"""
     data = json.loads(test_data)
     assert build_entities_from_data(data) is None
+
+
+def test_null_object_leaves_the_relation_empty() -> None:
+    """Test that an object which is null in one item does not fake a sub entity.
+
+    A relation path carries URIs pointing at sub entities. Writing "" there hands out
+    a reference that resolves to nothing, and a JSON dataset sink following it aborts
+    the whole write with "Current context not Array but Object".
+    """
+    test_data = """
+[{
+  "name": "seebi",
+  "address": null
+},
+{
+  "name": "sai",
+  "address": {"city": "San Francisco"}
+}]"""
+    entities = build_entities_from_json(test_data)
+    assert entities.schema == EntitySchema(
+        type_uri="",
+        paths=[
+            EntityPath("name", False, is_single_value=True),
+            EntityPath("address", True, is_single_value=True),
+        ],
+    )
+    without_address, with_address = list(entities.entities)
+    assert without_address.values == [["seebi"], []]
+    assert with_address.values[1][0].startswith("urn:x-ulid:")
+
+
+def test_an_object_is_not_flattened_by_a_later_null() -> None:
+    """Test that the item order does not decide whether a key is a relation.
+
+    The same two items as above, the other way round. Reading the type off the last
+    item seen made `address` a plain path, and the object in the first item was then
+    written as its `str()` - `{'city': 'San Francisco'}` - instead of a sub entity.
+    """
+    test_data = """
+[{
+  "name": "sai",
+  "address": {"city": "San Francisco"}
+},
+{
+  "name": "seebi",
+  "address": null
+}]"""
+    entities = build_entities_from_json(test_data)
+    assert entities.schema == EntitySchema(
+        type_uri="",
+        paths=[
+            EntityPath("name", False, is_single_value=True),
+            EntityPath("address", True, is_single_value=True),
+        ],
+    )
+    with_address, without_address = list(entities.entities)
+    assert with_address.values[1][0].startswith("urn:x-ulid:")
+    assert without_address.values == [["seebi"], []]
+    assert entities.sub_entities is not None
+    assert len(entities.sub_entities) == 1
+    assert [_.values for _ in entities.sub_entities[0].entities] == [[["San Francisco"]]]
+
+
+def test_an_empty_list_does_not_flatten_a_list_of_objects() -> None:
+    """Test that a key empty in one item stays a relation because another item fills it"""
+    test_data = """
+[{
+  "name": "sai",
+  "pets": [{"name": "cat"}]
+},
+{
+  "name": "seebi",
+  "pets": []
+}]"""
+    entities = build_entities_from_json(test_data)
+    assert entities.schema == EntitySchema(
+        type_uri="",
+        paths=[
+            EntityPath("name", False, is_single_value=True),
+            EntityPath("pets", True, is_single_value=False),
+        ],
+    )
+    with_pets, without_pets = list(entities.entities)
+    assert with_pets.values[1][0].startswith("urn:x-ulid:")
+    assert without_pets.values == [["seebi"], []]