From bd6014e0903b12386ac636e01f05fcef11042466 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 15:33:58 +0900 Subject: [PATCH 1/3] Tests: insert_many stores the same values on the typed-column route and the JSON route (refs #388) Parity (differential) tests for a typed-column route of the JVM insert_many that does not exist yet, so they fail on the parent: each stores the same rows with the typed route switched off and as shipped, and compares every record property by property through SQL (names, Java class of each value, value), for ints around 2^31 and 2^53 and 2^63, floats (NaN, infinity, subnormal, -0.0), strings, dates, date-times, nulls, None against a missing key, key order, mixed columns, date and Decimal inside lists, bytes, numpy scalars and aware date-times. Also the crossing counts, the order of mixed chunks, the commit boundary, the parallel mode, vertices, edge types, a duplicate key and a value no route can store. Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 1 + tests/test_insert_many_typed_columns.py | 619 ++++++++++++++++++++++++ 2 files changed, 620 insertions(+) create mode 100644 tests/test_insert_many_typed_columns.py diff --git a/tests/conftest.py b/tests/conftest.py index 22b929f..f8b209f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -168,6 +168,7 @@ "test_reader_identity_crossings.py": "crossing counts of get_rid, get_vertex, get_edge and get_element on the JVM build (JPype proxies of the Java row and the bridge class)", "test_typed_batch_guard.py": "the object_hook and the side array of the JVM build's TypedRows batches (JPype arrays, the bridge class, a spy on json.loads)", "test_jvm_ingest_integrity_faults.py": "rows-dropped faults put into the JVM build's column conversion and bulk vertex path (core._column_to_java, GraphBatch._create_vertices_json_bulk)", + "test_insert_many_typed_columns.py": "the typed-column route of the JVM wheel's insert_many (the bridge's DocumentBatcher.insertManyColumns, core._typed_chunk_columns, the Java classes of the stored values)", } if BACKEND == "native" and not os.environ.get("NATIVE_PARITY_ALL"): collect_ignore.extend(NATIVE_NOT_APPLICABLE) diff --git a/tests/test_insert_many_typed_columns.py b/tests/test_insert_many_typed_columns.py new file mode 100644 index 0000000..5b2250b --- /dev/null +++ b/tests/test_insert_many_typed_columns.py @@ -0,0 +1,619 @@ +"""insert_many (the JVM wheel): a chunk whose rows have the same keys and whose columns each hold one kind of value crosses as typed columns. + +Before, every chunk of 10,000 rows became one JSON text (``json_bulk_dumps_tagged``), which the engine parsed again into one JSON tree and +copied key by key (``DocumentBatcher.insertManyJson``). Now a homogeneous chunk takes ``DocumentBatcher.insertManyColumns``: one numpy +buffer per int, float or bool column, one UTF-8 buffer per string column, one ``TemporalColumn`` per date or date-time column, and no text. +Every other chunk keeps the JSON text exactly as it was. + +The stored data has to be the same on both routes, so each parity test stores the same rows twice, once with the typed route switched off +(``core._typed_chunk_columns`` returns None: the old route) and once as shipped, and compares every record property by property +through SQL: the property names in the order the record lists them, the Java class of every value (an Integer is not a Long), and the +value. The route that ran is read from the calls made on the bridge class. The Java types are read through the Java database +(``get_java_database``), so this file has no native counterpart (the native wheel carries homogeneous chunks through its own columnar +entry, ``arcadedb._native.bulk.insert_rows_columnar``). +""" + +import datetime as dt +import decimal +import math + +import jpype +import numpy as np +import pytest + +from arcadedb import core +from arcadedb.exceptions import ArcadeDBError + +BRIDGE = "com.arcadedb.python.DocumentBatcher" +UTC9 = dt.timezone(dt.timedelta(hours=9)) + + +class _Spy: + """Forwards every call of a Java class and records its name.""" + + def __init__(self, target, log): + self._target, self._log = target, log + + def __getattr__(self, name): + attribute = getattr(self._target, name) + if not callable(attribute): + return attribute + + def call(*args): + self._log.append(name) + return attribute(*args) + + return call + + +@pytest.fixture +def bridge(monkeypatch): + """The names of the DocumentBatcher calls of the test, in order.""" + log = [] + real = core._java_class + + def spy(name): + cls = real(name) + return _Spy(cls, log) if name == BRIDGE else cls + + monkeypatch.setattr(core, "_java_class", spy) + return log + + +def _old_route(monkeypatch): + """The route before this change: no chunk is typed, every chunk is one JSON text.""" + monkeypatch.setattr(core, "_typed_chunk_columns", lambda chunk: None) + + +def _describe(value): + if value is None: + return ("null",) + if isinstance(value, str): + return ("str", value) + if isinstance(value, jpype.JArray): + return (str(value.getClass().getName()), repr(list(value))) + if hasattr(value, "getClass"): + return (str(value.getClass().getName()), str(value)) + return ("py:" + type(value).__name__, repr(value)) + + +def _stored(db, type_name): + """Every record by id: [(property, (Java class, text of the value)), ...] in the order the record lists its properties.""" + result = db.get_java_database().query( + "sql", f"SELECT FROM {type_name} ORDER BY id" # nosec B608 - test-owned name + ) + records = [] + while result.hasNext(): + record = result.next() + records.append( + [ + (str(name), _describe(record.getProperty(name))) + for name in record.getPropertyNames() + ] + ) + return records + + +DECLARED = { + "i": "LONG", + "f": "DOUBLE", + "b": "BOOLEAN", + "s": "STRING", + "d": "DATE", + "t": "DATETIME", +} + + +def _make(db, name, declared=False, kind="DOCUMENT", buckets=None): + suffix = f" BUCKETS {buckets}" if buckets else "" + db.command("sql", f"CREATE {kind} TYPE {name}{suffix}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + if declared: + for prop, ty in DECLARED.items(): + db.command("sql", f"CREATE PROPERTY {name}.{prop} {ty}") + + +INT_EDGES = [ + 0, + 1, + -1, + 2**31 - 1, + 2**31, + -(2**31), + -(2**31) - 1, + 2**53 + 1, + -(2**53) - 1, + 2**63 - 1, + -(2**63), +] +FLOAT_EDGES = [ + 0.0, + -0.0, + 0.1, + 1.0, + 1e300, + -1e-300, + 5e-324, + 2.2250738585072014e-308, + 1.7976931348623157e308, + float("nan"), + float("inf"), + float("-inf"), +] +STRINGS = [ + "", + "a", + 'quo"te', + "back\\slash", + "new\nline", + "tab\t", + "emoji \U0001f600", + "é ü 中", + "\x01tag", + "x" * 5000, +] +DATES = [ + dt.date(1970, 1, 1), + dt.date(1969, 12, 31), + dt.date(2026, 11, 14), + dt.date(1, 1, 1), + dt.date(9999, 12, 31), + dt.date(2000, 2, 29), +] +DATETIMES = [ + dt.datetime(1970, 1, 1), + dt.datetime(1969, 12, 31, 23, 59, 59, 999999), + dt.datetime(2026, 11, 14, 12, 34, 56, 123456), + dt.datetime(2000, 2, 29, 0, 0, 0, 1), + dt.datetime(1, 1, 1), + dt.datetime(9999, 12, 31, 23, 59, 59, 999999), +] + + +def _one_column(values, name="v"): + return [{"id": i, name: v} for i, v in enumerate(values)] + + +def _cycle(values, n): + return [values[i % len(values)] for i in range(n)] + + +def _all_kinds(nulls): + n = 12 + rows = [] + for i in range(n): + null = nulls and i % 3 == 1 + rows.append( + { + "id": i, + "i": None if null else INT_EDGES[i % len(INT_EDGES)], + "f": None if null else FLOAT_EDGES[i % len(FLOAT_EDGES)], + "b": None if null else i % 2 == 0, + "s": None if null else STRINGS[i % len(STRINGS)], + "d": None if null else DATES[i % len(DATES)], + "t": None if null else DATETIMES[i % len(DATETIMES)], + } + ) + return rows + + +# name -> (rows, typed): the rows the chunk holds and the route they take +CASES = { + "ints": (_one_column(INT_EDGES), True), + "floats_nan_inf_subnormal": (_one_column(FLOAT_EDGES), True), + "bools": (_one_column([True, False, True, True]), True), + "strings": (_one_column(STRINGS), True), + "dates": (_one_column(DATES), True), + "datetimes": (_one_column(DATETIMES), True), + "all_kinds": (_all_kinds(False), True), + "all_kinds_with_nulls": (_all_kinds(True), True), + "null_in_every_row": (_one_column([None, None, None]), True), + "null_beside_ints": (_one_column([1, None, 2**40, None]), True), + "null_beside_floats": (_one_column([1.5, None, float("nan")]), True), + "null_beside_strings": (_one_column(["a", None, "b"]), True), + "single_row": (_one_column([7]), True), + # shapes the typed route cannot carry: the chunk keeps the JSON text and the per-row path for the rows the text cannot carry + "other_key_order": ( + [ + {"id": 0, "a": 1, "b": 2}, + {"id": 1, "b": 3, "a": 4}, + {"id": 2, "a": 5, "b": 6}, + ], + False, + ), + "key_missing_in_a_row": ([{"id": 0, "a": 1}, {"id": 1}, {"id": 2, "a": 3}], False), + "null_in_one_row_missing_in_another": ( + [{"id": 0, "a": None}, {"id": 1}, {"id": 2, "a": 2}], + False, + ), + "an_extra_key_in_a_row": ([{"id": 0, "a": 1}, {"id": 1, "a": 2, "b": 3}], False), + "int_beside_float": (_one_column([1, 2.5, 3]), False), + "bool_beside_int": (_one_column([True, 1, False]), False), + "str_beside_int": (_one_column(["a", 1]), False), + "date_beside_datetime": (_one_column([DATES[0], DATETIMES[0]]), False), + "date_inside_a_list": (_one_column([[DATES[2]], [DATES[0]]]), False), + "datetime_inside_a_list": (_one_column([[DATETIMES[2]], [1]]), False), + "decimal_inside_a_list": (_one_column([[decimal.Decimal("1.5")], [2]]), False), + "decimal": ( + _one_column([decimal.Decimal("12345.6789"), decimal.Decimal("-0.50")]), + False, + ), + "bytes": (_one_column([b"\x00\xff abc\x7f", b""]), False), + "numpy_scalars": ( + _one_column([np.int64(2**40 + 5), np.int64(-3), np.int64(7)]), + False, + ), + "numpy_float_scalars": (_one_column([np.float64(0.1), np.float64(2.5)]), False), + "aware_datetime": ( + _one_column([dt.datetime(2026, 5, 6, 7, 8, 9, 654321, tzinfo=UTC9)]), + False, + ), + "nested_lists_and_maps": ( + _one_column([[1, 2, [3, 4]], {"k": [1, {"z": 2}], "n": None}]), + False, + ), + "tuple": (_one_column([(1, 2, 3), (4, 5, 6)]), False), + "list_of_ints_and_a_float": (_one_column([[1, 2.5], [3, 4]]), False), +} + + +# every case on schemaless properties, and the rows of every kind on declared ones too (the engine converts to the declared type) +PAIRS = [(case, False) for case in CASES] + [ + ("all_kinds", True), + ("all_kinds_with_nulls", True), +] + + +@pytest.mark.parametrize( + "case, declared", + PAIRS, + ids=[f"{c}-{'declared' if d else 'free'}" for c, d in PAIRS], +) +def test_the_typed_route_stores_what_the_json_route_stores( + temp_db, monkeypatch, bridge, case, declared +): + rows, typed = CASES[case] + _make(temp_db, "Old", declared) + _make(temp_db, "New", declared) + with monkeypatch.context() as old: + _old_route(old) + bridge.clear() + assert temp_db.insert_many("Old", rows) == len(rows) + old_calls = list(bridge) + assert "insertManyColumns" not in old_calls + bridge.clear() + assert temp_db.insert_many("New", rows) == len(rows) + if typed: + assert bridge == ["insertManyColumns"], f"{case} left the typed route" + else: + # the chunk keeps the route it always had, call for call (the JSON text, and the per-row path for rows the text cannot carry) + assert ( + bridge == old_calls + ), f"{case} took the typed route, which cannot carry it" + new, old = _stored(temp_db, "New"), _stored(temp_db, "Old") + assert len(new) == len(rows) + assert new == old + + +def test_a_whole_number_is_an_integer_when_it_fits_an_int_and_a_long_beyond( + temp_db, bridge +): + """The classes themselves, not only old against new: what the JSON text stored (JSONObject.parseNumber).""" + _make(temp_db, "T") + temp_db.insert_many( + "T", _one_column([5, 2**31 - 1, 2**31, -(2**31), -(2**31) - 1, 2**62]) + ) + assert bridge == ["insertManyColumns"] + classes = [dict(r)["v"][0] for r in _stored(temp_db, "T")] + assert classes == [ + "java.lang.Integer", + "java.lang.Integer", + "java.lang.Long", + "java.lang.Integer", + "java.lang.Long", + "java.lang.Long", + ] + + +def test_insert_columns_still_stores_a_long_for_every_whole_number(temp_db): + """insert_columns shares fillRow with the new entry point; only insertManyColumns narrows (a regression test, green before this change).""" + _make(temp_db, "C") + temp_db.insert_columns("C", {"id": np.arange(3, dtype=np.int64), "v": [1, 2, 3]}) + for record in _stored(temp_db, "C"): + assert dict(record)["id"][0] == "java.lang.Long" + + +def test_a_chunk_that_takes_the_old_route_builds_no_bridge_array( + temp_db, monkeypatch, bridge +): + """A fallback chunk costs the census and nothing else: no column is built before the shape is known to qualify.""" + built = [] + real = core._typed_java_column + + def spy(*args): + built.append(args[0]) + return real(*args) + + monkeypatch.setattr(core, "_typed_java_column", spy) + _make(temp_db, "T") + # the first column is fine, the second holds a Decimal: nothing is built for either + temp_db.insert_many("T", [{"id": i, "v": decimal.Decimal(i)} for i in range(5)]) + assert built == [] + assert bridge == ["insertManyJson"] + + +def test_a_homogeneous_load_crosses_once_per_chunk_with_no_text_and_no_per_value_conversion( + temp_db, monkeypatch, bridge +): + monkeypatch.setattr(core, "_INSERT_MANY_CHUNK", 1000) + + def no_text(*args, **kwargs): + raise AssertionError("a homogeneous chunk went through the JSON text") + + monkeypatch.setattr(core, "json_bulk_dumps_tagged", no_text) + converted = [] + real = core.convert_python_to_java + + def counting(value): + converted.append(type(value).__name__) + return real(value) + + monkeypatch.setattr(core, "convert_python_to_java", counting) + _make(temp_db, "T") + rows = [ + { + "id": i, + "n": i * 3, + "x": i * 0.5, + "ok": i % 2 == 0, + "name": f"row {i}", + "day": DATES[i % len(DATES)], + "at": DATETIMES[i % len(DATETIMES)], + } + for i in range(2500) + ] + assert temp_db.insert_many("T", rows) == 2500 + assert bridge == ["insertManyColumns"] * 3 # chunks of 1000, 1000, 500 + assert converted == [], "a value was converted one at a time" + assert temp_db.query("sql", "SELECT count(*) AS c FROM T").first().get("c") == 2500 + record = dict( + _stored(temp_db, "T")[1235] + ) # 1235 % 6 == 5: the last date and date-time of the lists above + assert record == { + "id": ("java.lang.Integer", "1235"), + "n": ("java.lang.Integer", "3705"), + "x": ("java.lang.Double", "617.5"), + "ok": ("java.lang.Boolean", "false"), + "name": ("str", "row 1235"), + "day": ("java.time.LocalDate", "2000-02-29"), + "at": ("java.time.LocalDateTime", DATETIMES[5].isoformat()), + } + + +def test_chunks_of_both_routes_keep_the_order_of_the_rows(temp_db, monkeypatch, bridge): + monkeypatch.setattr(core, "_INSERT_MANY_CHUNK", 4) + _make(temp_db, "T", buckets=1) + rows = _one_column(list(range(12))) + rows[5] = {"id": 5, "v": decimal.Decimal(5)} # the middle chunk keeps the text + assert temp_db.insert_many("T", rows) == 12 + assert bridge == ["insertManyColumns", "insertManyJson", "insertManyColumns"] + ids = [r.get("id") for r in temp_db.query("sql", "SELECT id FROM T").to_list()] + assert ids == list(range(12)) + + +def test_the_commit_boundary_still_splits_a_chunk(temp_db, bridge): + _make(temp_db, "T") + assert temp_db.insert_many("T", _one_column(list(range(25))), commit_every=10) == 25 + assert ( + bridge == ["insertManyColumns"] * 3 + ) # 10, 10, 5: a chunk never straddles a commit + assert len(_stored(temp_db, "T")) == 25 + + +def test_an_empty_batch_and_a_batch_of_one(temp_db, bridge): + _make(temp_db, "T") + assert temp_db.insert_many("T", []) == 0 + assert bridge == [] + assert temp_db.insert_many("T", [{"id": 1, "v": 2}]) == 1 + assert bridge == ["insertManyColumns"] + assert len(_stored(temp_db, "T")) == 1 + + +def test_a_generator_is_read_chunk_by_chunk(temp_db, monkeypatch, bridge): + monkeypatch.setattr(core, "_INSERT_MANY_CHUNK", 100) + _make(temp_db, "T") + n = temp_db.insert_many("T", ({"id": i, "v": i} for i in range(250))) + assert n == 250 + assert bridge == ["insertManyColumns"] * 3 + + +@pytest.mark.parametrize("kind", ["DOCUMENT", "VERTEX"]) +def test_documents_and_vertices_are_the_same_on_both_routes( + temp_db, monkeypatch, bridge, kind +): + _make(temp_db, "Old", kind=kind) + _make(temp_db, "New", kind=kind) + rows = _one_column(INT_EDGES) + with monkeypatch.context() as old: + _old_route(old) + temp_db.insert_many("Old", rows) + temp_db.insert_many("New", rows) + assert _stored(temp_db, "New") == _stored(temp_db, "Old") + for name in ("Old", "New"): + sql = f"SELECT FROM {name}" # nosec B608 - test-owned name + result = temp_db.get_java_database().query("sql", sql) + n = 0 + while result.hasNext(): + assert result.next().isVertex() == (kind == "VERTEX") + n += 1 + assert n == len(rows) + + +def test_an_edge_type_is_refused_as_before(temp_db, monkeypatch): + temp_db.command("sql", "CREATE VERTEX TYPE V") + temp_db.command("sql", "CREATE EDGE TYPE E") + rows = _one_column([1, 2]) + messages = [] + for route in ("old", "new"): + with monkeypatch.context() as m: + if route == "old": + _old_route(m) + with pytest.raises(ArcadeDBError) as error: + temp_db.insert_many("E", rows) + messages.append(str(error.value)) + assert messages[0] == messages[1] + assert "edge type" in messages[0] + + +def test_a_duplicate_key_fails_the_call_and_rolls_it_back_on_both_routes( + temp_db, monkeypatch, bridge +): + outcomes = [] + for route in ("old", "new"): + name = f"T_{route}" + temp_db.command("sql", f"CREATE DOCUMENT TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + temp_db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + rows = _one_column([1, 2, 3, 4]) + [ + {"id": 2, "v": 9} + ] # the id 2 is there twice + with monkeypatch.context() as m: + if route == "old": + _old_route(m) + bridge.clear() + with pytest.raises( + Exception + ) as error: # noqa: B017 - the class is compared below + temp_db.insert_many(name, rows) + outcomes.append( + ( + type(error.value).__name__, + temp_db.query("sql", f"SELECT count(*) AS c FROM {name}") # nosec B608 + .first() + .get("c"), + list(bridge), + ) + ) + assert outcomes[0][:2] == outcomes[1][:2] + assert outcomes[1][1] == 0, "the call rolled back the transaction it opened" + assert outcomes[0][2] == ["insertManyJson"] + assert outcomes[1][2] == ["insertManyColumns"] + + +@pytest.mark.parametrize( + "values", + [[1, 2**64], [-(2**63) - 1, 1], ["ok", "a\ud800b"]], + ids=["beyond_int64", "below_int64", "lone_surrogate"], +) +def test_a_value_no_route_can_store_fails_as_before( + temp_db, monkeypatch, bridge, values +): + outcomes = [] + for route in ("old", "new"): + name = f"T_{route}" + _make(temp_db, name) + with monkeypatch.context() as m: + if route == "old": + _old_route(m) + bridge.clear() + try: + temp_db.insert_many(name, _one_column(values)) + raised = None + except Exception as error: # noqa: BLE001 - the same class is the claim + raised = type(error).__name__ + outcomes.append( + (raised, len(_stored(temp_db, name)), "insertManyColumns" in bridge) + ) + assert outcomes[0][:2] == outcomes[1][:2] + assert ( + outcomes[1][2] is False + ), "a value the text refuses must not take the typed route" + + +@pytest.mark.parametrize("case", ["ints", "all_kinds_with_nulls", "decimal"]) +def test_parallel_mode_stores_what_the_json_route_stores( + temp_db, monkeypatch, bridge, case +): + rows, typed = CASES[case] + _make(temp_db, "Old") + _make(temp_db, "New") + with monkeypatch.context() as old: + _old_route(old) + bridge.clear() + assert temp_db.insert_many("Old", rows, parallel=True) == len(rows) + assert "insertManyJsonParallel" in bridge + bridge.clear() + assert temp_db.insert_many("New", rows, parallel=True) == len(rows) + assert bridge[0] == ( + "insertManyColumnsParallel" if typed else "insertManyJsonParallel" + ) + assert _stored(temp_db, "New") == _stored(temp_db, "Old") + + +def test_a_caller_transaction_is_left_to_the_caller(temp_db, bridge): + _make(temp_db, "T") + with pytest.raises(RuntimeError): + with temp_db.transaction(): + temp_db.insert_many("T", _one_column(list(range(30))), commit_every=10) + raise RuntimeError("roll back") + assert bridge == ["insertManyColumns"] + assert temp_db.query("sql", "SELECT count(*) AS c FROM T").first().get("c") == 0 + + +@pytest.mark.parametrize( + "values, expected", + [ + ([1, 2, 3], ("int", False)), + ([1, None], ("int", True)), + ([2**63], None), + ([-(2**63) - 1, None], None), + ([1, 2.0], None), + ([True, False], ("bool", False)), + ([True, 1], None), + ([None, None], ("null", True)), + (["a", None], ("str", True)), + ([1.5, math.nan], ("float", False)), + ([dt.date(2020, 1, 2)], ("date", False)), + ([dt.datetime(2020, 1, 2)], ("datetime", False)), + ([dt.datetime(2020, 1, 2), None], ("datetime", True)), + ([dt.datetime(2020, 1, 2, tzinfo=dt.timezone.utc)], None), + ([dt.date(2020, 1, 2), dt.datetime(2020, 1, 2)], None), + ([decimal.Decimal(1)], None), + ([b"x"], None), + ([[1, 2]], None), + ([{"a": 1}], None), + ([np.int64(1)], None), + ([np.float64(1)], None), + ], +) +def test_which_columns_qualify(values, expected): + assert core._typed_kind(values) == expected + + +@pytest.mark.parametrize( + "chunk", + [ + [], + [{}, {}], + [{1: 1}], + [{"a": 1}, [("a", 2)]], + [{"a": 1, "b": 2}, {"b": 2, "a": 1}], + [{"a": 1}, {"a": 1, "b": 2}], + [{"a": 1}, {"b": 1}], + [{"a": 1}, {"a": 2.5}], + [{"a": [1]}, {"a": [2]}], + ], + ids=[ + "empty", + "no_keys", + "non_str_key", + "not_a_dict", + "key_order", + "missing_key", + "other_key", + "mixed_kind", + "list", + ], +) +def test_which_chunks_qualify(chunk): + assert core._typed_chunk_columns(chunk) is None From ab3f0bb8d8751b78db5ee2e53d3188e9b947c518 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 15:34:40 +0900 Subject: [PATCH 2/3] JVM insert_many: a homogeneous chunk crosses as typed columns, not as one JSON text (refs #388) A chunk whose rows are dicts with the same str keys in the same order, and whose columns each hold None and values of one exact type (int within 64 bits, float, bool, str, date, naive datetime), is read as columns: one numpy buffer per int, float or bool column, one StringColumn per string column, one TemporalColumn per date or datetime column, an Object[] only for a numeric column with a null. DocumentBatcher.insertManyColumns (and the parallel twin) builds the documents from them, and stores a whole number as the JSON text route did, an Integer when it fits an int and a Long beyond, so the stored type of a schemaless property does not depend on the route (insert_columns is unchanged and keeps Long). Every other chunk keeps the JSON text and the per-row fallback exactly as before: the whole chunk is judged first, from the types alone, before any bridge array is built. Bridge jar and Python only, no native image. Co-Authored-By: Claude Sonnet 5.5 --- .../modules/ROOT/pages/how-to/bulk-loads.adoc | 2 + src/arcadedb/core.py | 178 +++++++++++++++++- .../com/arcadedb/python/DocumentBatcher.java | 57 +++++- 3 files changed, 225 insertions(+), 12 deletions(-) diff --git a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc index 099184f..a0307c2 100644 --- a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc +++ b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc @@ -65,6 +65,8 @@ Because the two packages have different defaults, the same program can keep diff On both packages, `insert_many` keeps rows with `datetime`, `date`, `Decimal`, `bytes`, numpy scalar, or pandas `NA` and `NaT` values on the bulk path. On the `arcadedb` (native) package, a row with a value the bulk text cannot carry (NaN or infinity, an integer beyond 64 bits, a date-time or `Decimal` inside a list or dict, or a `datetime.time`) goes in with its own statement, in the order given. +On the `arcadedb-jvm` package, a chunk of 10,000 rows crosses as typed columns, with no JSON text, when every row is a `dict` with the same keys in the same order and each column holds `None` and values of one type: `int` within 64 bits, `float` (NaN and infinity included), `bool`, `str`, `date`, or a `datetime` without a time zone. The stored values are the same as the JSON text stores: a whole number is an `Integer` when it fits an int and a `Long` beyond, and `None` is a null property. Any other chunk keeps the JSON text, as before: a row that lacks a key or lists the keys in another order, a column that mixes types (an `int` beside a `float`), a `Decimal`, `bytes`, numpy or pandas value, a `datetime` with a time zone, a list, tuple or dict, and a float vector for a property declared `ARRAY_OF_FLOATS`. A chunk is judged as a whole, so one such row sends its 10,000 rows down the JSON route. The `arcadedb` (native) package already sends the rows of one shape through its columnar entry. + `insert_many` and `create_vertices` check their rows while they write the JSON text, in one pass over the rows, not in a loop before it. The same rows fail the check as before, and they take the same fallback path as before. The check raises no error to your code. [[columnar-loads]] diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index 29ef657..a653524 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -8,7 +8,7 @@ import itertools import operator from collections.abc import Mapping -from datetime import date +from datetime import date, datetime from decimal import Decimal from os import PathLike from typing import TYPE_CHECKING, Any, Callable, List, Optional @@ -36,6 +36,8 @@ from .results import ResultSet from .transactions import TransactionContext from .type_conversion import ( + _INT64_MAX, + _INT64_MIN, _is_numpy_bool, bulk_row_ok, convert_python_to_java, @@ -336,6 +338,129 @@ def _column_to_java(name, values): return jpype.JArray(jpype.JObject)(converted), len(converted) +# insert_many: the kinds of column a chunk can cross as typed columns (see _typed_chunk_columns). The exact types only: a subclass +# (pandas.Timestamp, numpy.float64) and every other type (Decimal, bytes, list, dict, tuple) keep the JSON text. +_TYPED_KINDS = { + int: "int", + float: "float", + bool: "bool", + str: "str", + date: "date", + datetime: "datetime", +} +_DICT_ONLY = frozenset((dict,)) +_STR_ONLY = frozenset((str,)) +_NO_ZONE = frozenset((None,)) +_EPOCH_ORDINAL = date(1970, 1, 1).toordinal() +_EPOCH_DATE = date(1970, 1, 1) +_EPOCH_DATETIME = datetime(1970, 1, 1) + + +def _typed_kind(values): + """``(kind, nullable)`` of one column of an insert_many chunk when it can cross as a typed column, else None. + + The column holds ``None`` and values of ONE exact type of ``_TYPED_KINDS`` (``"null"`` when there is nothing but ``None``). + A mixed column (an int beside a float, a bool beside an int) would store each value as its own type on the JSON route, which one + typed array cannot do; an int beyond 64 bits, a date-time with a zone and a type the table does not list keep the JSON route. + """ + kinds = set(map(type, values)) + nullable = _NONE_TYPE in kinds + kinds.discard(_NONE_TYPE) + if not kinds: + return "null", True + if len(kinds) != 1: + return None + kind = _TYPED_KINDS.get(next(iter(kinds))) + if kind == "int": + present = [v for v in values if v is not None] if nullable else values + if min(present) < _INT64_MIN or max(present) > _INT64_MAX: + return None + elif kind == "datetime": + present = [v for v in values if v is not None] if nullable else values + if set(map(operator.attrgetter("tzinfo"), present)) != _NO_ZONE: + return None + return (kind, nullable) if kind else None + + +def _typed_java_column(name, values, kind, nullable): + """One column of an insert_many chunk as the Java column ``DocumentBatcher.insertManyColumns`` takes, or None. + + A whole column of int, float or bool without a null is one numpy array and one buffer copy (``long[]``, ``double[]``, + ``boolean[]``); a column of str and None is one ``StringColumn``; date and naive date-time columns are one ``TemporalColumn`` + (a ``long[]`` of epoch days or microseconds, and one ``boolean[]`` when there is a null); a numeric column with a null converts + per element into an ``Object[]``, as ``insert_columns`` does, and a column of nothing but None is an ``Object[]`` of nulls. + """ + n = len(values) + if kind == "null": + return jpype.JArray(jpype.JObject)([None] * n) + if kind == "str": + return _string_column(values) + if kind == "date" or kind == "datetime": + mask = None + if nullable: + mask = jpype.JArray(jpype.JBoolean)( + _np.fromiter( + map(operator.is_, values, itertools.repeat(None)), + dtype=_np.bool_, + count=n, + ) + ) + filler = _EPOCH_DATE if kind == "date" else _EPOCH_DATETIME + values = [filler if v is None else v for v in values] + temporal = _java_class("com.arcadedb.python.TemporalColumn") + if kind == "date": + ticks = ( + _np.fromiter(map(date.toordinal, values), dtype=_np.int64, count=n) + - _EPOCH_ORDINAL + ) + return temporal.dates(jpype.JArray(jpype.JLong)(ticks), 1, mask) + ticks = _np.array(values, dtype="datetime64[us]").astype(_np.int64) + return temporal.dateTimes(jpype.JArray(jpype.JLong)(ticks), 1_000_000, mask) + if nullable: + return _column_to_java(name, values)[0] + dtype = {"int": _np.int64, "float": _np.float64, "bool": _np.bool_}[kind] + try: + array = _np.fromiter(values, dtype=dtype, count=n) + except (OverflowError, ValueError, TypeError): + return None + return _column_to_java(name, array)[0] + + +def _typed_chunk_columns(chunk): + """``(names, columns)`` for one chunk of insert_many rows that can cross as typed columns, else None (the chunk keeps the JSON text). + + The chunk qualifies when every row is a ``dict`` with the same str keys in the same order (a row that lacks a key, has another + one, or lists them in another order is a different shape, and a record keeps its properties in the order they were set), and every + column holds one kind of value (``_typed_kind``). Whether a column qualifies is decided from the types alone before any bridge + array is built, so a chunk that cannot cross this way costs the census and nothing else. The result is stored as the JSON text + stores it: ``DocumentBatcher.insertManyColumns`` stores a whole number as an Integer when it fits an int, as the text parser + does, and ``None`` is a null property, never a missing one. + """ + if _np is None or not chunk or set(map(type, chunk)) != _DICT_ONLY: + return None + names = tuple(chunk[0]) + if not names or set(map(type, names)) != _STR_ONLY: + return None + if len(set(map(tuple, chunk))) != 1: + return None + lists = [] + shapes = [] + for name in names: + values = list(map(operator.itemgetter(name), chunk)) + shape = _typed_kind(values) + if shape is None: + return None + lists.append(values) + shapes.append(shape) + columns = [] + for name, values, (kind, nullable) in zip(names, lists, shapes): + column = _typed_java_column(name, values, kind, nullable) + if column is None: + return None + columns.append(column) + return names, columns + + def _wrap_java_record(java_record, database=None): """Wrap a Java record in the matching Python class (Vertex/Edge/Document).""" if java_record is None: @@ -765,11 +890,17 @@ def insert_many( ) -> int: """Bulk-insert documents with one FFI crossing per chunk of rows. - The iterable is read 10,000 rows at a time; each chunk is serialized - to one JSON string and looped Java-side (``DocumentBatcher``), - avoiding the per-row JNI cost that caps ``new_document``-loop ingest. + The iterable is read 10,000 rows at a time; each chunk crosses once and + is looped Java-side (``DocumentBatcher``), avoiding the per-row JNI cost + that caps ``new_document``-loop ingest. A chunk whose rows are dicts with + the same keys in the same order, and whose columns each hold ``None`` and + values of one type (``int`` within 64 bits, ``float``, ``bool``, ``str``, + ``date``, or a ``datetime`` without a zone), crosses as typed columns, with + no text (``DocumentBatcher.insertManyColumns``); a whole number is stored + as an Integer when it fits an int and as a Long beyond, which is what the + JSON text stored. Any other chunk is serialized to one JSON string. Memory stays bounded by the chunk whatever the row count, so a - generator of any length can be loaded (#294). A value that sits directly + generator of any length can be loaded (#294). On the JSON route, a value that sits directly in a row may be JSON-representable (str/int/float/bool/None, nested lists/dicts), a ``datetime`` (also ``pandas.Timestamp`` and ``numpy.datetime64``), ``date``, ``Decimal``, ``bytes``/``bytearray``, @@ -944,6 +1075,12 @@ def _insert_many_chunk(self, type_name, chunk): still go in bulk, and every row is inserted in the order given. Dates, date-times, Decimals, bytes, numpy scalars and pandas NA and NaT sitting directly in a row cross inside the text as tagged values (see json_bulk_dumps_tagged). """ + # A chunk of rows with the same keys whose columns each hold one kind of value crosses as typed columns, with no text + # (see _typed_chunk_columns); every other chunk keeps the JSON text below, as it was. + typed = _typed_chunk_columns(chunk) + if typed is not None: + self._insert_many_typed(type_name, typed, len(chunk)) + return # Float vectors for ARRAY_OF_FLOATS properties cross as one binary matrix # beside the JSON, not as text (see split_float_vectors). split = split_float_vectors(self._java_db, type_name, chunk) @@ -954,6 +1091,21 @@ def _insert_many_chunk(self, type_name, chunk): return self._insert_many_bulk(type_name, payload, split) + def _insert_many_typed(self, type_name, typed, n): + """One chunk of insert_many as typed columns inside the open transaction (commitEvery 0: the transaction stays with insert_many).""" + names, columns = typed + try: + _java_class("com.arcadedb.python.DocumentBatcher").insertManyColumns( + self._java_db, + type_name, + jpype.JArray(jpype.JString)(names), + jpype.JArray(jpype.JObject)(columns), + n, + 0, + ) + except Exception as e: + raise ArcadeDBError(f"Failed to bulk-insert into '{type_name}': {e}") from e + def _insert_many_runs(self, type_name, chunk): """A chunk with a row the JSON text cannot carry: runs of carriable rows go in bulk, the other rows one by one, in order.""" skip = declared_float_vectors(self._java_db, type_name) @@ -1013,6 +1165,22 @@ def _insert_many_parallel(self, type_name, first, rows): chunk = first try: while chunk: + typed = _typed_chunk_columns(chunk) + if typed is not None: + try: + failures = batcher.insertManyColumnsParallel( + self._java_db, + type_name, + jpype.JArray(jpype.JString)(typed[0]), + jpype.JArray(jpype.JObject)(typed[1]), + len(chunk), + ) + except Exception as e: + raise self._parallel_load_failed(type_name, e) from e + chunk_failures.append(failures) + n += len(chunk) + chunk = list(itertools.islice(rows, _INSERT_MANY_CHUNK)) + continue bad = [] split = split_float_vectors(self._java_db, type_name, chunk) try: diff --git a/src/java/com/arcadedb/python/DocumentBatcher.java b/src/java/com/arcadedb/python/DocumentBatcher.java index 8d892a9..5ed7424 100644 --- a/src/java/com/arcadedb/python/DocumentBatcher.java +++ b/src/java/com/arcadedb/python/DocumentBatcher.java @@ -11,7 +11,8 @@ * the async executor's parallel bucket writers (insertManyJsonParallel; the * Python insert_many wrapper waits for completion itself, then reads the * failures the writers reported). insertColumns takes whole columns instead of - * JSON (Database.insert_columns). Float vectors for ARRAY_OF_FLOATS properties cross as binary matrices (FloatVectors), beside + * JSON (Database.insert_columns); insertManyColumns is the same for a chunk of insert_many rows whose columns each hold one kind of + * value, and stores every number as the JSON text route did (see narrowed). Float vectors for ARRAY_OF_FLOATS properties cross as binary matrices (FloatVectors), beside * the JSON or as a column. The boxDoubles/boxLongs * helpers below serve AsyncExecutor.append_samples' numpy fast path. * @@ -95,11 +96,26 @@ public static long insertManyJson(final Database db, final String typeName, fina */ public static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns, final int n, final int commitEvery) { + return insertColumns(db, typeName, names, columns, n, commitEvery, false); + } + + /** + * The columns of one insert_many chunk (every row has the same keys in the same order and each column holds one kind of value): as + * {@link #insertColumns}, except that a whole number is stored as the JSON text route stored it, an Integer when it fits an int and a + * Long beyond, so the type of a schemaless property does not depend on which route carried the chunk (see {@link #narrowed}). + */ + public static long insertManyColumns(final Database db, final String typeName, final String[] names, final Object[] columns, + final int n, final int commitEvery) { + return insertColumns(db, typeName, names, columns, n, commitEvery, true); + } + + private static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns, + final int n, final int commitEvery, final boolean narrowIntegers) { checkColumns(names, columns, n); final boolean vertex = n > 0 && isVertexType(db, typeName); writeChunks(db, n, commitEvery, i -> { final MutableDocument doc = newRecord(db, typeName, vertex); - fillRow(doc, names, columns, i); + fillRow(doc, names, columns, i, narrowIntegers); doc.save(); }); return n; @@ -180,13 +196,24 @@ private static void backoffBeforeRetry(final int attempt) { */ public static AsyncFailures insertColumnsParallel(final Database db, final String typeName, final String[] names, final Object[] columns, final int n) { + return insertColumnsParallel(db, typeName, names, columns, n, false); + } + + /** The parallel twin of {@link #insertManyColumns}. */ + public static AsyncFailures insertManyColumnsParallel(final Database db, final String typeName, final String[] names, + final Object[] columns, final int n) { + return insertColumnsParallel(db, typeName, names, columns, n, true); + } + + private static AsyncFailures insertColumnsParallel(final Database db, final String typeName, final String[] names, + final Object[] columns, final int n, final boolean narrowIntegers) { checkColumns(names, columns, n); final AsyncFailures failures = new AsyncFailures(); final ErrorCallback onError = failures::record; final boolean vertex = n > 0 && isVertexType(db, typeName); for (int i = 0; i < n; i++) { final MutableDocument doc = newRecord(db, typeName, vertex); - fillRow(doc, names, columns, i); + fillRow(doc, names, columns, i, narrowIntegers); db.async().createRecord(doc, null, onError); } return failures; @@ -239,12 +266,25 @@ else if (col instanceof TemporalColumn t) } } - private static void fillRow(final MutableDocument doc, final String[] names, final Object[] columns, final int i) { + /** + * The value a JSON number token becomes in the engine (JSONObject.parseNumber): an Integer when a whole number fits an int, a Long + * beyond. The columns of a numpy long[] hold every whole number as a long, which would store a Long where the text route stored an + * Integer, so the insert_many columns narrow it back. Written with returns, not a conditional: {@code c ? (int) v : v} would promote + * both arms to long. + */ + private static Object narrowed(final long v) { + if (v >= Integer.MIN_VALUE && v <= Integer.MAX_VALUE) + return (int) v; + return v; + } + + private static void fillRow(final MutableDocument doc, final String[] names, final Object[] columns, final int i, + final boolean narrowIntegers) { for (int c = 0; c < names.length; c++) { final Object col = columns[c]; - final Object v; + Object v; if (col instanceof long[] a) - v = a[i]; + v = narrowIntegers ? narrowed(a[i]) : (Object) a[i]; else if (col instanceof double[] a) v = a[i]; else if (col instanceof boolean[] a) @@ -255,8 +295,11 @@ else if (col instanceof StringColumn s) v = s.get(i); else if (col instanceof TemporalColumn t) v = t.get(i); - else + else { v = ((Object[]) col)[i]; + if (narrowIntegers && v instanceof Long l) + v = narrowed(l); + } doc.set(names[c], v); } } From 38f015bb3cb92099c37a8856133abe41515fce94 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 20:21:00 +0900 Subject: [PATCH 3/3] JVM insert_many typed columns: a row the JSON text cannot carry keeps its whole numbers Long, as the per-row path stores them (refs #388) The first CI run of the typed route (tagged 26.11.1, workflow_dispatch 43588034127, all 9 finished legs) failed 5 tests of tests/test_insert_many_typed_columns.py, the same 5 on every leg. Cause and evidence: - The parity failures (all_kinds-free, all_kinds_with_nulls-free, parallel all_kinds_with_nulls) differ in one value: row 11 (i = 0, f = -inf). The old route sends a row with a NaN, an infinity or a subnormal double down the per-row path (bulk_row_ok is False for rows 6, 9, 10 and 11 of that data, checked by script), and the per-row path stores a Python int as a Long. The text route stores the same number as an Integer. insertManyColumns narrowed every whole number to an Integer, so that one row differed. This is not engine-version specific: the per-row path is Python-side. - Fix in the code, not the test: fillRow keeps the whole numbers of such a row a Long (carriedByText: a double[] or Object[] column that holds a non-finite or subnormal double in the row). The stored data of the typed route is now the old route's for every row. The A/B candidate wheel changes (one pass over the columns per row in fillRow). - Two tests asserted the wrong thing. insert_columns still stores a Long: the type declared id INTEGER, which converts a Long to an Integer on any entry point, so it now declares nothing. The homogeneous-load test hard-coded how JPype prints a Java Boolean ('false') and a String ('str'); it now asserts the classes and the lower-cased texts. - New parity cases pin the rule: whole numbers that fit an int beside NaN, infinity, both signs of the smallest subnormal, the largest subnormal, the smallest normal value, -0.0 and a plain float, with and without nulls, on both the sync and the parallel route. - Docs: bulk-loads.adoc and the insert_many docstring state the exception. Co-Authored-By: Claude Sonnet 5.5 --- .../modules/ROOT/pages/how-to/bulk-loads.adoc | 2 +- src/arcadedb/core.py | 7 +- .../com/arcadedb/python/DocumentBatcher.java | 33 ++++++- tests/test_insert_many_typed_columns.py | 96 ++++++++++++++++--- 4 files changed, 116 insertions(+), 22 deletions(-) diff --git a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc index a0307c2..2d8c576 100644 --- a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc +++ b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc @@ -65,7 +65,7 @@ Because the two packages have different defaults, the same program can keep diff On both packages, `insert_many` keeps rows with `datetime`, `date`, `Decimal`, `bytes`, numpy scalar, or pandas `NA` and `NaT` values on the bulk path. On the `arcadedb` (native) package, a row with a value the bulk text cannot carry (NaN or infinity, an integer beyond 64 bits, a date-time or `Decimal` inside a list or dict, or a `datetime.time`) goes in with its own statement, in the order given. -On the `arcadedb-jvm` package, a chunk of 10,000 rows crosses as typed columns, with no JSON text, when every row is a `dict` with the same keys in the same order and each column holds `None` and values of one type: `int` within 64 bits, `float` (NaN and infinity included), `bool`, `str`, `date`, or a `datetime` without a time zone. The stored values are the same as the JSON text stores: a whole number is an `Integer` when it fits an int and a `Long` beyond, and `None` is a null property. Any other chunk keeps the JSON text, as before: a row that lacks a key or lists the keys in another order, a column that mixes types (an `int` beside a `float`), a `Decimal`, `bytes`, numpy or pandas value, a `datetime` with a time zone, a list, tuple or dict, and a float vector for a property declared `ARRAY_OF_FLOATS`. A chunk is judged as a whole, so one such row sends its 10,000 rows down the JSON route. The `arcadedb` (native) package already sends the rows of one shape through its columnar entry. +On the `arcadedb-jvm` package, a chunk of 10,000 rows crosses as typed columns, with no JSON text, when every row is a `dict` with the same keys in the same order and each column holds `None` and values of one type: `int` within 64 bits, `float` (NaN and infinity included), `bool`, `str`, `date`, or a `datetime` without a time zone. The stored values are the same as the JSON text stores: a whole number is an `Integer` when it fits an int and a `Long` beyond, and `None` is a null property. A row with a NaN, an infinity, or a subnormal `float` stores its whole numbers as `Long`, which is what the per-row path stores for such a row. Any other chunk keeps the JSON text, as before: a row that lacks a key or lists the keys in another order, a column that mixes types (an `int` beside a `float`), a `Decimal`, `bytes`, numpy or pandas value, a `datetime` with a time zone, a list, tuple or dict, and a float vector for a property declared `ARRAY_OF_FLOATS`. A chunk is judged as a whole, so one such row sends its 10,000 rows down the JSON route. The `arcadedb` (native) package already sends the rows of one shape through its columnar entry. `insert_many` and `create_vertices` check their rows while they write the JSON text, in one pass over the rows, not in a loop before it. The same rows fail the check as before, and they take the same fallback path as before. The check raises no error to your code. diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index a653524..79d8c85 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -434,7 +434,9 @@ def _typed_chunk_columns(chunk): column holds one kind of value (``_typed_kind``). Whether a column qualifies is decided from the types alone before any bridge array is built, so a chunk that cannot cross this way costs the census and nothing else. The result is stored as the JSON text stores it: ``DocumentBatcher.insertManyColumns`` stores a whole number as an Integer when it fits an int, as the text parser - does, and ``None`` is a null property, never a missing one. + does, and ``None`` is a null property, never a missing one. A row with a NaN, an infinity or a subnormal float is the exception: the + text cannot carry it, so the JSON route inserts that row on the per-row path, which stores its whole numbers as Long, and the + bridge stores them as Long for that row too. """ if _np is None or not chunk or set(map(type, chunk)) != _DICT_ONLY: return None @@ -898,7 +900,8 @@ def insert_many( ``date``, or a ``datetime`` without a zone), crosses as typed columns, with no text (``DocumentBatcher.insertManyColumns``); a whole number is stored as an Integer when it fits an int and as a Long beyond, which is what the - JSON text stored. Any other chunk is serialized to one JSON string. + JSON text stored (a row with a NaN, an infinity or a subnormal ``float`` + stores its whole numbers as Long, as the per-row path always did). Any other chunk is serialized to one JSON string. Memory stays bounded by the chunk whatever the row count, so a generator of any length can be loaded (#294). On the JSON route, a value that sits directly in a row may be JSON-representable (str/int/float/bool/None, nested diff --git a/src/java/com/arcadedb/python/DocumentBatcher.java b/src/java/com/arcadedb/python/DocumentBatcher.java index 5ed7424..d70fd9b 100644 --- a/src/java/com/arcadedb/python/DocumentBatcher.java +++ b/src/java/com/arcadedb/python/DocumentBatcher.java @@ -12,7 +12,7 @@ * Python insert_many wrapper waits for completion itself, then reads the * failures the writers reported). insertColumns takes whole columns instead of * JSON (Database.insert_columns); insertManyColumns is the same for a chunk of insert_many rows whose columns each hold one kind of - * value, and stores every number as the JSON text route did (see narrowed). Float vectors for ARRAY_OF_FLOATS properties cross as binary matrices (FloatVectors), beside + * value, and stores every number as the JSON text route did (see narrowed and carriedByText). Float vectors for ARRAY_OF_FLOATS properties cross as binary matrices (FloatVectors), beside * the JSON or as a column. The boxDoubles/boxLongs * helpers below serve AsyncExecutor.append_samples' numpy fast path. * @@ -102,7 +102,9 @@ public static long insertColumns(final Database db, final String typeName, final /** * The columns of one insert_many chunk (every row has the same keys in the same order and each column holds one kind of value): as * {@link #insertColumns}, except that a whole number is stored as the JSON text route stored it, an Integer when it fits an int and a - * Long beyond, so the type of a schemaless property does not depend on which route carried the chunk (see {@link #narrowed}). + * Long beyond, so the type of a schemaless property does not depend on which route carried the chunk (see {@link #narrowed}). A row + * the text could not have carried (a NaN, an infinity or a subnormal double) keeps every whole number a Long, as the per-row path + * stores it (see {@link #carriedByText}). */ public static long insertManyColumns(final Database db, final String typeName, final String[] names, final Object[] columns, final int n, final int commitEvery) { @@ -278,13 +280,36 @@ private static Object narrowed(final long v) { return v; } + /** + * Whether the JSON text route could have carried row i. The text cannot carry a NaN, an infinity or a subnormal double (the engine's + * parser reads 5e-324 as a BigDecimal), so Python sends a row that has one down the per-row path, which stores every whole number of + * that row as a Long: the same value the text would have made an Integer. insert_many columns keep that for such a row, so what a + * record stores does not depend on which route carried its chunk. Only a double[] and an Object[] (a column with a null) can hold one. + */ + private static boolean carriedByText(final Object[] columns, final int i) { + for (final Object col : columns) { + if (col instanceof double[] a) { + if (!textDouble(a[i])) + return false; + } else if (col instanceof Object[] o && o[i] instanceof Double d && !textDouble(d)) + return false; + } + return true; + } + + /** A double the JSON text keeps as a double: finite, and zero or at least the smallest normal value (Double.MIN_NORMAL). */ + private static boolean textDouble(final double v) { + return Double.isFinite(v) && (v == 0.0 || Math.abs(v) >= Double.MIN_NORMAL); + } + private static void fillRow(final MutableDocument doc, final String[] names, final Object[] columns, final int i, final boolean narrowIntegers) { + final boolean narrow = narrowIntegers && carriedByText(columns, i); for (int c = 0; c < names.length; c++) { final Object col = columns[c]; Object v; if (col instanceof long[] a) - v = narrowIntegers ? narrowed(a[i]) : (Object) a[i]; + v = narrow ? narrowed(a[i]) : (Object) a[i]; else if (col instanceof double[] a) v = a[i]; else if (col instanceof boolean[] a) @@ -297,7 +322,7 @@ else if (col instanceof TemporalColumn t) v = t.get(i); else { v = ((Object[]) col)[i]; - if (narrowIntegers && v instanceof Long l) + if (narrow && v instanceof Long l) v = narrowed(l); } doc.set(names[c], v); diff --git a/tests/test_insert_many_typed_columns.py b/tests/test_insert_many_typed_columns.py index 5b2250b..9f923f3 100644 --- a/tests/test_insert_many_typed_columns.py +++ b/tests/test_insert_many_typed_columns.py @@ -16,6 +16,7 @@ import datetime as dt import decimal import math +import sys import jpype import numpy as np @@ -68,8 +69,8 @@ def _old_route(monkeypatch): def _describe(value): if value is None: return ("null",) - if isinstance(value, str): - return ("str", value) + if isinstance(value, str): # a String JPype already converted + return ("java.lang.String", value) if isinstance(value, jpype.JArray): return (str(value.getClass().getName()), repr(list(value))) if hasattr(value, "getClass"): @@ -178,6 +179,40 @@ def _cycle(values, n): return [values[i % len(values)] for i in range(n)] +# floats the JSON text cannot carry (NaN, infinity, a subnormal: the row goes down the per-row path, which stores its whole numbers as +# Long) beside floats it carries (the smallest normal value, -0.0, a plain value) +NON_TEXT_FLOATS = [ + float("nan"), + float("inf"), + float("-inf"), + 5e-324, + -5e-324, + math.nextafter(sys.float_info.min, 0.0), # the largest subnormal + sys.float_info.min, + -0.0, + 0.5, +] +SMALL_INTS = [3, -3, 0, 2**31 - 1, -(2**31), 5, 6, 7, 8] + + +def _ints_beside_floats(nulls): + """Whole numbers that fit an int in the same rows as the floats above: the rows the text cannot carry keep them Long.""" + rows = [] + for i in range(2 * len(NON_TEXT_FLOATS)): + rows.append( + { + "id": i, + "i": None if nulls and i % 4 == 1 else SMALL_INTS[i % len(SMALL_INTS)], + "f": ( + None + if nulls and i % 4 == 2 + else NON_TEXT_FLOATS[i % len(NON_TEXT_FLOATS)] + ), + } + ) + return rows + + def _all_kinds(nulls): n = 12 rows = [] @@ -211,6 +246,11 @@ def _all_kinds(nulls): "null_beside_ints": (_one_column([1, None, 2**40, None]), True), "null_beside_floats": (_one_column([1.5, None, float("nan")]), True), "null_beside_strings": (_one_column(["a", None, "b"]), True), + "ints_beside_nan_inf_subnormal": (_ints_beside_floats(False), True), + "nullable_ints_beside_nullable_nan_inf_subnormal": ( + _ints_beside_floats(True), + True, + ), "single_row": (_one_column([7]), True), # shapes the typed route cannot carry: the chunk keeps the JSON text and the per-row path for the rows the text cannot carry "other_key_order": ( @@ -316,11 +356,18 @@ def test_a_whole_number_is_an_integer_when_it_fits_an_int_and_a_long_beyond( def test_insert_columns_still_stores_a_long_for_every_whole_number(temp_db): - """insert_columns shares fillRow with the new entry point; only insertManyColumns narrows (a regression test, green before this change).""" - _make(temp_db, "C") + """insert_columns shares fillRow with the new entry point; only insertManyColumns narrows (a regression test, green before this change). + + The type declares no property: a declared INTEGER would convert the Long to an Integer whichever entry point stored it. + """ + temp_db.command("sql", "CREATE DOCUMENT TYPE C") temp_db.insert_columns("C", {"id": np.arange(3, dtype=np.int64), "v": [1, 2, 3]}) - for record in _stored(temp_db, "C"): - assert dict(record)["id"][0] == "java.lang.Long" + records = _stored(temp_db, "C") + assert len(records) == 3 + for record in records: + assert {name for name, _ in record} == {"id", "v"} + for name, (kind, _) in record: + assert kind == "java.lang.Long", name def test_a_chunk_that_takes_the_old_route_builds_no_bridge_array( @@ -379,14 +426,24 @@ def counting(value): record = dict( _stored(temp_db, "T")[1235] ) # 1235 % 6 == 5: the last date and date-time of the lists above - assert record == { - "id": ("java.lang.Integer", "1235"), - "n": ("java.lang.Integer", "3705"), - "x": ("java.lang.Double", "617.5"), - "ok": ("java.lang.Boolean", "false"), - "name": ("str", "row 1235"), - "day": ("java.time.LocalDate", "2000-02-29"), - "at": ("java.time.LocalDateTime", DATETIMES[5].isoformat()), + assert {name: kind for name, (kind, _) in record.items()} == { + "id": "java.lang.Integer", + "n": "java.lang.Integer", + "x": "java.lang.Double", + "ok": "java.lang.Boolean", + "name": "java.lang.String", + "day": "java.time.LocalDate", + "at": "java.time.LocalDateTime", + } + # the text of a value is how JPype prints it (a Java Boolean prints as False), so the case is not part of the claim + assert {name: text.lower() for name, (_, text) in record.items()} == { + "id": "1235", + "n": "3705", + "x": "617.5", + "ok": "false", + "name": "row 1235", + "day": "2000-02-29", + "at": DATETIMES[5].isoformat().lower(), } @@ -530,7 +587,16 @@ def test_a_value_no_route_can_store_fails_as_before( ), "a value the text refuses must not take the typed route" -@pytest.mark.parametrize("case", ["ints", "all_kinds_with_nulls", "decimal"]) +@pytest.mark.parametrize( + "case", + [ + "ints", + "all_kinds_with_nulls", + "ints_beside_nan_inf_subnormal", + "nullable_ints_beside_nullable_nan_inf_subnormal", + "decimal", + ], +) def test_parallel_mode_stores_what_the_json_route_stores( temp_db, monkeypatch, bridge, case ):