diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b13ea..7a9b76f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p ### Fixed - GraphParameterType.autocomplete: results are now sorted by label and only include graphs matching all given query terms, instead of an unsorted, non-deterministic order matching any single term +- `build_entities_from_data`: a relation whose value is null now leaves the path empty instead of filling it with `""`. That empty string was a reference to a sub entity that does not exist, and a JSON dataset written from such entities failed the whole write with `Current context not Array but Object` +- `build_entities_from_data`: a key that is an object in one item and null in another, or a list of objects in one item and an empty list in another, is now described as a relation whichever item comes first. The type used to be taken from the last item seen, so in one of the two orders the objects were flattened into their `str()` - `{'city': 'San Francisco'}` - rather than becoming sub entities ## [4.20.0] 2026-08-14 - shipped with DI v26.2.0 diff --git a/cmem_plugin_base/dataintegration/utils/entity_builder.py b/cmem_plugin_base/dataintegration/utils/entity_builder.py index cd896ec..4b50f45 100644 --- a/cmem_plugin_base/dataintegration/utils/entity_builder.py +++ b/cmem_plugin_base/dataintegration/utils/entity_builder.py @@ -12,6 +12,13 @@ def merge_path_values(paths_map1: dict, paths_map2: dict) -> dict: each representing paths and corresponding values. It merges these dictionaries by combining values for common paths and returns the merged dictionary. + Args: + paths_map1 (dict): The first dictionary containing paths and values. + paths_map2 (dict): The second dictionary containing paths and values. + + Where both dictionaries know a type for the same key, the more specific one wins + rather than the one seen last - see `_more_specific_type`. + Args: paths_map1 (dict): The first dictionary containing paths and values. paths_map2 (dict): The second dictionary containing paths and values. @@ -24,11 +31,36 @@ def merge_path_values(paths_map1: dict, paths_map2: dict) -> dict: current_path_map = {} if paths_map1.get(key) is not None: current_path_map = paths_map1[key] - current_path_map = current_path_map | value + current_path_map = current_path_map | { + _key: _more_specific_type(current_path_map.get(_key), _type) + for _key, _type in value.items() + } paths_map1[key] = current_path_map return paths_map1 +def _more_specific_type(known: str | None, found: str) -> str: + """Pick the type that says more about a key, out of the two seen for it. + + A key's type is read off the values in the data, and two items can disagree about + it: one object carries a `lockedByUser` and the next has it as null, or one list is + empty and the next holds objects. Taking whichever came last then decides the schema + by the order of the data - and losing to `NoneType` or to an empty `list` is the + direction that loses information, because the object seen in the other item is no + longer described as a relation and gets flattened into its `str()` instead. + + Genuinely conflicting types, say `str` against `int`, are left to the last one seen, + as before. + """ + if known is None or known == "NoneType": + return found + if found == "NoneType": + return known + if {known, found} == {"list", "list_dict"}: + return "list_dict" + return found + + def generate_paths_from_data(data: dict | list, path: str | None = "root") -> dict: """Generate a dictionary representing paths and data types from a nested JSON structure. @@ -128,7 +160,11 @@ def _get_entity( schema = path_to_schema_map[path_from_root] for _ in schema.paths: if data.get(_.path) is None: - values.append([""]) + # A relation carries URIs pointing at sub entities, so a null has to leave the + # path empty. An "" there is a reference that resolves to nothing, which a + # dataset sink follows and fails on - a JSON dataset aborts the whole write + # with "Current context not Array but Object". + values.append([] if _.is_relation else [""]) elif not _.is_relation: values.append( [f"{data.get(_.path)}"] diff --git a/tests/test_utils_build_entities_from_data.py b/tests/test_utils_build_entities_from_data.py index c7c7218..3b6a0a0 100644 --- a/tests/test_utils_build_entities_from_data.py +++ b/tests/test_utils_build_entities_from_data.py @@ -225,3 +225,88 @@ def test_empty_object() -> None: test_data = """[]""" data = json.loads(test_data) assert build_entities_from_data(data) is None + + +def test_null_object_leaves_the_relation_empty() -> None: + """Test that an object which is null in one item does not fake a sub entity. + + A relation path carries URIs pointing at sub entities. Writing "" there hands out + a reference that resolves to nothing, and a JSON dataset sink following it aborts + the whole write with "Current context not Array but Object". + """ + test_data = """ +[{ + "name": "seebi", + "address": null +}, +{ + "name": "sai", + "address": {"city": "San Francisco"} +}]""" + entities = build_entities_from_json(test_data) + assert entities.schema == EntitySchema( + type_uri="", + paths=[ + EntityPath("name", False, is_single_value=True), + EntityPath("address", True, is_single_value=True), + ], + ) + without_address, with_address = list(entities.entities) + assert without_address.values == [["seebi"], []] + assert with_address.values[1][0].startswith("urn:x-ulid:") + + +def test_an_object_is_not_flattened_by_a_later_null() -> None: + """Test that the item order does not decide whether a key is a relation. + + The same two items as above, the other way round. Reading the type off the last + item seen made `address` a plain path, and the object in the first item was then + written as its `str()` - `{'city': 'San Francisco'}` - instead of a sub entity. + """ + test_data = """ +[{ + "name": "sai", + "address": {"city": "San Francisco"} +}, +{ + "name": "seebi", + "address": null +}]""" + entities = build_entities_from_json(test_data) + assert entities.schema == EntitySchema( + type_uri="", + paths=[ + EntityPath("name", False, is_single_value=True), + EntityPath("address", True, is_single_value=True), + ], + ) + with_address, without_address = list(entities.entities) + assert with_address.values[1][0].startswith("urn:x-ulid:") + assert without_address.values == [["seebi"], []] + assert entities.sub_entities is not None + assert len(entities.sub_entities) == 1 + assert [_.values for _ in entities.sub_entities[0].entities] == [[["San Francisco"]]] + + +def test_an_empty_list_does_not_flatten_a_list_of_objects() -> None: + """Test that a key empty in one item stays a relation because another item fills it""" + test_data = """ +[{ + "name": "sai", + "pets": [{"name": "cat"}] +}, +{ + "name": "seebi", + "pets": [] +}]""" + entities = build_entities_from_json(test_data) + assert entities.schema == EntitySchema( + type_uri="", + paths=[ + EntityPath("name", False, is_single_value=True), + EntityPath("pets", True, is_single_value=False), + ], + ) + with_pets, without_pets = list(entities.entities) + assert with_pets.values[1][0].startswith("urn:x-ulid:") + assert without_pets.values == [["seebi"], []]