From c1a78388510abc29fca69f4620628289e65309f1 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 02:43:51 +0900 Subject: [PATCH 01/18] Test: a TypedRows batch with no tagged value is decoded without an object_hook and without reading its side array (refs #315) to_list() and iter_dicts() of the JVM wheel build a json object_hook for every batch and read the side array of every batch, whether or not the batch holds a tagged value. These tests count the hook factory, the object_hook argument of json.loads and the batch elements that are read, for a result of plain values (no hook, no read of the side array) and for every kind of value the bridge tags (the hook stays). They also compare to_list() and iter_dicts() with the decoding that always builds the hook, over single shapes, a mixed result and batch boundaries. They fail on the current decoder (every batch gets a hook and a read of the side array) and pass once the decoder looks for the tag escape in the batch text first. Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 1 + tests/test_typed_batch_guard.py | 401 ++++++++++++++++++++++++++++++++ 2 files changed, 402 insertions(+) create mode 100644 tests/test_typed_batch_guard.py diff --git a/tests/conftest.py b/tests/conftest.py index 32ddf8f..e7d904c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -153,6 +153,7 @@ "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", + "test_typed_batch_guard.py": "the object_hook and the side array of the JVM build's TypedRows batches (JPype arrays, the bridge class, a spy on json.loads)", "test_jvm_ingest_integrity_faults.py": "rows-dropped faults put into the JVM build's column conversion and bulk vertex path (core._column_to_java, GraphBatch._create_vertices_json_bulk)", } if BACKEND == "native" and not os.environ.get("NATIVE_PARITY_ALL"): diff --git a/tests/test_typed_batch_guard.py b/tests/test_typed_batch_guard.py new file mode 100644 index 0000000..688a587 --- /dev/null +++ b/tests/test_typed_batch_guard.py @@ -0,0 +1,401 @@ +"""A TypedRows batch with no tagged value is decoded without a ``json`` object_hook and without reading its side array (#315). + +``ResultSet.to_list()`` and ``iter_dicts()`` read rows through the bridge's ``TypedRows``: a batch is +``{String json, Object[] side, Boolean drained}``. Java writes a value JSON cannot carry exactly as +``{"\\u0001": [kind, payload]}`` and Python restores it with an ``object_hook``, which reads the side +array. A batch of plain values (strings, integers, finite floats, booleans, nulls, lists, maps) holds no +tag, so the hook and the read of the side array are two JPype operations and a Python call per object +for nothing. The decoder now looks for the tag escape in the batch text first, as ``_loads_typed_row`` +already did for one row. + +Two things are pinned here, and neither may cost the other: + +* the guard fires: a batch without the tag escape is parsed by plain ``json.loads`` and ``batch[1]`` is + not read (the hook factory and the ``object_hook`` argument are counted); +* the guard never fires on a tagged batch: every shape the bridge tags keeps the hook, and the rows are + the ones the always-hooked decoder gives, which is what ``to_list()`` did before. + +Only the JVM build has this reader (``NATIVE_NOT_APPLICABLE`` in ``conftest.py``). +""" + +import contextlib +import json +import types + +import jpype +import pytest + +from arcadedb import results + +# the wide table with every property type (fixture ``db``), the row-by-row reference and the in-memory result sets +from tests.test_typed_rows import _in_memory, db, exact, reference # noqa: F401 + +ESCAPE = "\\u0001" +BATCH = 1000 # results._TYPED_BATCH_ROWS: the rows of one to_list() crossing + + +class _Batch: + """A TypedRows batch that logs which of its three elements are read (0 json, 1 side, 2 drained).""" + + def __init__(self, target, reads): + self._target = target + self._reads = reads + + def __getitem__(self, index): + self._reads.append(index) + return self._target[index] + + +class _Bridge: + """The bridge class TypedRows, with every batch it returns wrapped in a _Batch.""" + + def __init__(self, real, reads): + self._real = real + self._reads = reads + + def nextRows(self, java_result_set, size): + return _Batch(self._real.nextRows(java_result_set, size), self._reads) + + +class _Json: + """The ``json`` module as results.py sees it, logging the keyword arguments of every ``loads``.""" + + def __init__(self, calls): + self._calls = calls + + def loads(self, text, **kwargs): + self._calls.append(kwargs) + return json.loads(text, **kwargs) + + def __getattr__(self, name): + return getattr(json, name) + + +@contextlib.contextmanager +def watching(): + """What the decoder of results.py does while the block runs. + + ``loads``: the keyword arguments of every json.loads call, ``hooks``: the side arrays the hook factory + was built with (one entry per batch decoded with a hook), ``reads``: the batch elements read. + Nothing is recorded outside the block, so the reference rows are computed first. + """ + seen = types.SimpleNamespace(loads=[], hooks=[], reads=[]) + real_hook = results._typed_hook + + def counting_hook(side): + seen.hooks.append(side) + return real_hook(side) + + bridge = _Bridge(results._bridge_class("TypedRows"), seen.reads) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(results, "_typed_hook", counting_hook) + patch.setattr(results, "json", _Json(seen.loads)) + patch.setitem(results._BRIDGE_CLASSES, "TypedRows", bridge) + yield seen + + +def _hooked(seen): + """How many json.loads calls got an object_hook.""" + return sum(1 for kwargs in seen.loads if kwargs.get("object_hook") is not None) + + +def _texts(java_result_set, size): + """The JSON text of every batch TypedRows writes for the result set.""" + typed_rows = jpype.JClass("com.arcadedb.python.TypedRows") + texts = [] + while True: + batch = typed_rows.nextRows(java_result_set, size) + texts.append(str(batch[0])) + if batch[2]: + return texts + + +def _decoded_with_the_hook_always(java_result_set, size): + """The rows decoded as to_list() did before #315: an object_hook on every batch, tagged or not.""" + typed_rows = jpype.JClass("com.arcadedb.python.TypedRows") + rows = [] + while True: + batch = typed_rows.nextRows(java_result_set, size) + rows.extend( + json.loads(str(batch[0]), object_hook=results._typed_hook(batch[1])) + ) + if batch[2]: + return rows + + +def _all(rows): + return [exact(row) for row in rows] + + +def _java(name): + return jpype.JClass(name) + + +def _java_list(*items): + java_list = _java("java.util.ArrayList")() + for item in items: + java_list.add(item) + return java_list + + +def _java_map(**entries): + java_map = _java("java.util.LinkedHashMap")() + for key, value in entries.items(): + java_map.put(key, value) + return java_map + + +def _java_set(*items): + java_set = _java("java.util.LinkedHashSet")() + for item in items: + java_set.add(item) + return java_set + + +def _int_key_map(): + java_map = _java("java.util.HashMap")() + java_map.put(jpype.JInt(1), "one") + return java_map + + +def _tag_key_map(): + java_map = _java("java.util.HashMap")() + java_map.put("\u0001", "tag") + return java_map + + +# Every kind of value the bridge tags: the decoder must keep the hook for a batch that holds one. The +# lambdas build the Java objects when the test runs, as the JVM is not started when this module loads. +TAGGED = { + "decimal": lambda: _java("java.math.BigDecimal")("1.25"), + "decimal_exponent": lambda: _java("java.math.BigDecimal")("1E+3"), + "big_integer": lambda: _java("java.math.BigInteger")( + "123456789012345678901234567890" + ), + "date": lambda: _java("java.time.LocalDate").of(2020, 1, 2), + "date_year_one": lambda: _java("java.time.LocalDate").of(1, 1, 1), + "datetime": lambda: _java("java.time.LocalDateTime").of( + 2021, 2, 3, 4, 5, 6, 789123000 + ), + "instant": lambda: _java("java.time.Instant").parse( + "2021-02-03T04:05:06.789123456Z" + ), + "offset_datetime": lambda: _java("java.time.OffsetDateTime").parse( + "2021-02-03T04:05:06.5+09:00" + ), + "util_date": lambda: _java("java.util.Date")(1612325106789), + "set": lambda: _java_set("a", "b"), + "nan": lambda: float("nan"), + "inf": lambda: float("inf"), + "ninf": lambda: float("-inf"), + "float_array": lambda: jpype.JArray(jpype.JFloat)([0.1, 0.2, 0.3]), + "float_array_nan": lambda: jpype.JArray(jpype.JFloat)([0.5, float("nan")]), + "double_array": lambda: jpype.JArray(jpype.JDouble)([0.25, 0.5]), + "map_with_an_int_key": _int_key_map, + "map_with_the_tag_as_key": _tag_key_map, + "decimal_in_a_list": lambda: _java_list(1, _java("java.math.BigDecimal")("2.5")), + "date_in_a_map": lambda: _java_map( + k=_java_list(_java("java.time.LocalDate").of(1999, 12, 31)) + ), + "nan_in_a_map": lambda: _java_map(a=1, b=_java_list(float("nan"))), +} + +# Values that hold no tag; two of them put the escape into the text anyway (a control character and the +# text of the escape in a string), so the decoder keeps the hook and must still return the plain value. +PLAIN = { + "ints": lambda: 7, + "long": lambda: jpype.JLong(9223372036854775807), + "float": lambda: 0.1, + "negative_zero": lambda: -0.0, + "bool": lambda: True, + "string": lambda: 'q"uote\\ \n\t é 日本 \U0001f600', + "list": lambda: _java_list(1, "a", 2.5, True), + "nested_map": lambda: _java_map(a=1, b=_java_map(c=_java_list(1, 2, 3))), + "int_array": lambda: jpype.JArray(jpype.JInt)([1, 2, 3]), + "byte_array": lambda: jpype.JArray(jpype.JByte)([0, 1, -1]), + "control_character_in_a_string": lambda: "a\u0001b", + "the_escape_text_in_a_string": lambda: "x\\u0001y", +} + + +def _rows(value, count=3): + return [{"v": value} for _ in range(count)] + + +# ---------------------------------------------------------------- a batch with no tagged value + + +@pytest.fixture +def plain(temp_db): + """2,100 rows (three batches) of plain values only.""" + temp_db.command("sql", "CREATE DOCUMENT TYPE Plain") + with temp_db.transaction(): + for i in range(2100): + temp_db.command( + "sql", + "INSERT INTO Plain SET k = :k, s = :s, b = :b, x = :x, z = null, li = :li, ma = :ma", + { + "k": i, + "s": f'row {i} "q" \\ é', + "b": i % 2 == 0, + "x": i / 8, + "li": [i, "a", 0.5], + "ma": {"a": i, "b": {"c": [1, 2, 3]}}, + }, + ) + return temp_db + + +PLAIN_QUERY = "SELECT k, s, b, x, z, li, ma FROM Plain ORDER BY k" + + +def test_to_list_of_untagged_batches_uses_no_hook_and_reads_no_side_array(plain): + result_set = plain.query("sql", PLAIN_QUERY) # held: freeing it closes the Java one + texts = _texts(result_set._java_result_set, BATCH) + assert len(texts) == 3 + assert not any(ESCAPE in text for text in texts), "the premise: nothing is tagged" + expected = _all(reference(plain, PLAIN_QUERY)) + + with watching() as seen: + rows = plain.query("sql", PLAIN_QUERY).to_list() + + assert _all(rows) == expected + assert len(seen.loads) == 3 # one parse per batch + assert _hooked(seen) == 0 + assert seen.hooks == [] + assert set(seen.reads) == {0, 2}, "only the text and the drained flag are read" + + +def test_iter_dicts_of_untagged_rows_uses_no_hook_and_reads_no_side_array(plain): + expected = _all(reference(plain, PLAIN_QUERY)) + + with watching() as seen: + rows = list(plain.query("sql", PLAIN_QUERY).iter_dicts()) + + assert _all(rows) == expected + assert ( + len(seen.loads) == 2100 + 1 + ) # a row per crossing, and the empty batch at the end + assert _hooked(seen) == 0 + assert seen.hooks == [] + assert set(seen.reads) == {0}, "only the text is read" + + +def test_an_empty_result_is_decoded_without_a_hook(plain): + query = "SELECT k FROM Plain WHERE k < 0" + with watching() as seen: + assert plain.query("sql", query).to_list() == [] + assert list(plain.query("sql", query).iter_dicts()) == [] + assert _hooked(seen) == 0 and seen.hooks == [] + + +# ---------------------------------------------------------------- a batch with a tagged value + + +@pytest.mark.parametrize("name", sorted(TAGGED)) +def test_every_tagged_shape_keeps_the_hook(db, name): + rows = _rows(TAGGED[name]()) + assert any( + ESCAPE in text for text in _texts(_in_memory(rows), BATCH) + ), "the bridge writes the tag escape for this value" + expected = _all(r.to_dict() for r in results.ResultSet(_in_memory(rows), db)) + + with watching() as seen: + listed = results.ResultSet(_in_memory(rows), db).to_list() + assert _all(listed) == expected + assert len(seen.hooks) == 1 and _hooked(seen) == 1 # the one batch + assert 1 in seen.reads + + with watching() as seen: + iterated = list(results.ResultSet(_in_memory(rows), db).iter_dicts()) + assert _all(iterated) == expected + assert len(seen.hooks) == 3 and _hooked(seen) == 3 # every row is tagged + assert seen.loads[-1] == {}, "the empty batch at the end holds nothing to restore" + + +def test_the_decision_is_made_for_each_batch(db): + """One tagged row among 2,100 gives one hooked batch of three for to_list(), one hooked row for iter_dicts().""" + decimal = _java("java.math.BigDecimal")("1.5") + rows = [{"n": decimal if i == 1500 else i} for i in range(2100)] + expected = _all(r.to_dict() for r in results.ResultSet(_in_memory(rows), db)) + + with watching() as seen: + listed = results.ResultSet(_in_memory(rows), db).to_list() + assert _all(listed) == expected + assert type(listed[1500]["n"]).__name__ == "Decimal" + assert len(seen.loads) == 3 + assert [kwargs.get("object_hook") is not None for kwargs in seen.loads] == [ + False, + True, + False, + ] + assert len(seen.hooks) == 1 + assert seen.reads.count(1) == 1 + + with watching() as seen: + iterated = list(results.ResultSet(_in_memory(rows), db).iter_dicts()) + assert _all(iterated) == expected + assert len(seen.loads) == 2100 + 1 + assert len(seen.hooks) == 1 and _hooked(seen) == 1 + assert seen.reads.count(1) == 1 + + +# ---------------------------------------------------------------- the rows are the ones the hook gave + + +def _parity(make_result_set): + """to_list() and iter_dicts() against the always-hooked decoding of the same batches. + + ``make_result_set`` returns a new ResultSet on every call. Each one is held while it is read: a + ResultSet that is freed closes its Java result set. + """ + first = make_result_set() + old = _all(_decoded_with_the_hook_always(first._java_result_set, BATCH)) + second = make_result_set() + one_row_at_a_time = _all(_decoded_with_the_hook_always(second._java_result_set, 1)) + assert one_row_at_a_time == old + assert _all(make_result_set().to_list()) == old + assert _all(make_result_set().iter_dicts()) == old + return len(old) + + +@pytest.mark.parametrize("name", sorted(TAGGED) + sorted(PLAIN)) +def test_each_shape_decodes_as_with_the_hook_always(db, name): + value = (TAGGED.get(name) or PLAIN[name])() + assert _parity(lambda: results.ResultSet(_in_memory(_rows(value)), db)) == 3 + + +def test_a_mixed_batch_decodes_as_with_the_hook_always(db): + """Every shape in one result set: tagged and plain values side by side, and a row of its own for each.""" + values = {name: make() for name, make in {**PLAIN, **TAGGED}.items()} + rows = [values] + [{"v": value} for value in values.values()] + assert _parity(lambda: results.ResultSet(_in_memory(rows), db)) == len(rows) + + +def test_the_wide_table_decodes_as_with_the_hook_always(db): + """Every property type, NaN and the infinities, dates at both ends of the range, a Decimal, a vector, a link.""" + assert _parity(lambda: db.query("sql", "SELECT FROM Wide")) == 7 + + +@pytest.mark.parametrize( + "query", + [ + "SELECT i, s FROM Wide WHERE i IS NOT NULL", + "SELECT s FROM Wide WHERE s IS NOT NULL", + "SELECT x, f FROM Wide WHERE x IS NOT NULL", + "SELECT [1, 2] AS a, {'x': {'y': 1}} AS m", + "SELECT s AS `\u0001`, i FROM Wide LIMIT 3", + "SELECT count(*) AS c FROM Wide", + ], +) +def test_query_shapes_decode_as_with_the_hook_always(db, query): + assert _parity(lambda: db.query("sql", query)) >= 1 + + +def test_batches_across_the_boundary_decode_as_with_the_hook_always(db): + """2,500 rows with a tagged value in the first, the middle and the last batch.""" + decimal = _java("java.math.BigDecimal")("1.5") + date = _java("java.time.LocalDate").of(2020, 1, 2) + tagged = {3: decimal, 1700: float("nan"), 2499: date} + rows = [{"n": tagged.get(i, i)} for i in range(2500)] + assert _parity(lambda: results.ResultSet(_in_memory(rows), db)) == 2500 From 650c3eeefb393808339370a1d3efa3c15c4aca26 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 02:45:18 +0900 Subject: [PATCH 02/18] JVM to_list and iter_dicts: parse a TypedRows batch with no tag in its text without an object_hook and without reading its side array (refs #315) Every batch was decoded with json.loads(text, object_hook=_typed_hook(side)) and its side array was read first, also when the batch held only plain values. The hook is a Python call per JSON object and the read of the side array is a JPype operation per batch, which for iter_dicts (one row per crossing) is one per row. TypedRows writes every tag as {"\u0001": [kind, payload]} with the escape spelled out, writes a control character of a string as the same escape, and never writes the raw character, and every side array entry is referred to by a tag in the text. A text without the escape therefore holds no tag and no side index, the hook would return each object unchanged, and plain json.loads gives the same value. The check is a substring scan of a str that already exists (_loads_typed_row did it for one row). A string that holds U+0001 or the text of the escape only keeps the hook, which then changes nothing. No Java change, and the native wheel is untouched. The speed effect is not measured: it waits for a timed A/B of a candidate wheel against 26.11.1.post2. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/results.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index a17f8bc..20c0ee3 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -121,6 +121,24 @@ def _loads_typed_row(java_row): return json.loads(text, object_hook=_typed_hook(java_row[1])) +def _loads_typed_batch(java_batch): + """The rows of one ``TypedRows.nextRows`` batch, ``{String json, Object[] side, Boolean drained}``. + + The hook, and the read of the side array that builds it, are needed only when the text holds a + tag. TypedRows writes every tag as ``{"\\u0001": [kind, payload]}`` with the escape spelled out + (``tagged``, and the ``r`` row of ``appendRow``), writes a control character of a string as the + same escape (``quote``), and never writes the raw character, and every entry of the side array + is referred to by such a tag. So a text without the escape holds no tag and no side index: the + hook would return every object unchanged, and ``json.loads(text)`` is the same value without it. + A string that holds U+0001, or the text of the escape, also puts the escape into the text; that + only keeps the hook, which then changes nothing. + """ + text = str(java_batch[0]) + if "\\u0001" not in text: + return json.loads(text) + return json.loads(text, object_hook=_typed_hook(java_batch[1])) + + def _loads_names(java_text): """The property names ``ReaderCalls.rowNames`` / ``documentNames`` wrote, as a tuple.""" return tuple(json.loads(str(java_text))) @@ -340,7 +358,9 @@ def _to_list_typed(self, typed_rows) -> List[Dict[str, Any]]: else (RIDs, embedded documents, non-finite floats, ...) by index into a side array of the engine's own objects, so every value comes back as the type ``convert_java_to_python`` gives it. The C ``json`` module - parses the text and one hook call per tagged value restores it. + parses the text and one hook call per tagged value restores it; a batch + with no tag in its text is parsed without the hook and its side array + is not read (``_loads_typed_batch``). """ out: List[Dict[str, Any]] = [] if not self._readable(): @@ -352,9 +372,7 @@ def _to_list_typed(self, typed_rows) -> List[Dict[str, Any]]: ) except JException as exc: raise _read_error(exc) from exc - side = java_batch[1] - rows = json.loads(str(java_batch[0]), object_hook=_typed_hook(side)) - out.extend(rows) + out.extend(_loads_typed_batch(java_batch)) if java_batch[2]: # the result set is drained, and the bridge closed it: no # second call just to see an empty batch. (A batch also ends @@ -392,9 +410,7 @@ def iter_dicts(self, convert_types: bool = True) -> Iterator[Dict[str, Any]]: java_batch = typed_rows.nextRows(self._java_result_set, 1) except JException as exc: raise _read_error(exc) from exc - rows = json.loads( - str(java_batch[0]), object_hook=_typed_hook(java_batch[1]) - ) + rows = _loads_typed_batch(java_batch) if not rows: # nextRows closed the drained result set itself self._drained() From 04fd969c0a8e732c55e6ccccf23514d17f0915f7 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 02:47:57 +0900 Subject: [PATCH 03/18] Tests first: timestamp and date columns of insert_arrow and insert_dataframe on the JVM wheel (refs #315) Round trips of every Arrow temporal type through a real load on both wheels (every timestamp unit with and without a zone, to the nanosecond; date32 and date64 before 1970; nulls; batches; vertex and parallel loads; pandas and Polars frames), and JVM-only tests that count what one load does: no java.time call and no convert_python_to_java per cell, and each timestamp or date column reaching the bridge as one TemporalColumn per batch. The counting tests fail on the code before, which builds one LocalDateTime or LocalDate per cell in Python and converts every element again. Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 1 + tests/test_arrow_temporal_column_jvm.py | 271 ++++++++++++++++++++++++ tests/test_dataframe_ingest.py | 229 ++++++++++++++++++++ 3 files changed, 501 insertions(+) create mode 100644 tests/test_arrow_temporal_column_jvm.py diff --git a/tests/conftest.py b/tests/conftest.py index e7d904c..3aa8a12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -150,6 +150,7 @@ "test_graph_batch_vector_crossings_jvm.py": "the JPype crossings of the JVM wheel's create_vertices vector path (the bridge classes, the property-matrix fallback)", "test_insert_columns_string_blob.py": "the bridge's StringColumn and the JVM wheel's core._column_to_java (JPype arrays, a spy on the per-element conversion)", "test_arrow_vector_column_jvm.py": "the JPype crossings of the JVM wheel's Arrow vector column (core.convert_python_to_java, the bridge's FloatVectors.Matrix) and its Python heap", + "test_arrow_temporal_column_jvm.py": "the JPype calls of the JVM wheel's Arrow timestamp and date columns (the bridge's TemporalColumn, core._column_to_java, java.time classes)", "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", diff --git a/tests/test_arrow_temporal_column_jvm.py b/tests/test_arrow_temporal_column_jvm.py new file mode 100644 index 0000000..ef7c6dc --- /dev/null +++ b/tests/test_arrow_temporal_column_jvm.py @@ -0,0 +1,271 @@ +"""insert_arrow and insert_dataframe with timestamp and date columns on the JVM wheel: a column crosses as one array of epoch ticks per batch, +not as one JPype call per cell, and what is stored is unchanged. + +``_column_values`` turned every timestamp or date column into a Python list of ``LocalDateTime`` or ``LocalDate``, one +``ofEpochSecond`` or ``ofEpochDay`` call per cell, and ``_column_to_java`` then converted every element again with +``convert_python_to_java``: a million-row timestamp column was two million JPype calls. Now the column is the integer epoch values in ONE +``long[]`` (plus one ``boolean[]`` when a row is null) wrapped in the bridge's ``TemporalColumn``, built with one call per column and batch, and +``DocumentBatcher`` builds each ``LocalDateTime`` or ``LocalDate`` as it fills the document of that row. + +Two tests fail on the code before (they count the per-cell ``java.time`` calls, the per-element conversions and the columns that reach the bridge +as a ``TemporalColumn``); the others pin what the old route stored, so the new one must store the same: the bridge column against the per-cell +reading it replaced (every unit, negative ticks, nulls) and the refusal of a bad unit and of a timestamp past the years of ``java.time`` +before a row is written. The round trips of every Arrow temporal type through a real load, with the stored wall clock to the nanosecond, +are in ``test_dataframe_ingest.py::TestTemporalColumns``, which runs on both wheels. The JVM build only: it counts JPype calls and builds +bridge objects, so conftest.py does not collect this file on the native wheel. +""" + +from collections import Counter + +import jpype +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +from arcadedb import core, type_conversion +from arcadedb.jvm import start_jvm + +TEMPORAL_COLUMN = "com.arcadedb.python.TemporalColumn" +UNITS = {"s": 1, "ms": 1_000, "us": 1_000_000, "ns": 1_000_000_000} +TEMPORAL_COLUMNS = 7 # the five timestamp columns and the two date columns of _table() + + +@pytest.fixture(scope="module", autouse=True) +def _jvm(): + """The tests that build a bridge object without a database need the JVM too (a no-op when it already runs).""" + start_jvm() + + +def _column_class(): + return jpype.JClass(TEMPORAL_COLUMN) + + +def _table(n, start=0): + """n rows: a timestamp column of every unit, one with a zone, a date32 and a date64, a null in every seventh row.""" + idx = np.arange(start, start + n, dtype=np.int64) + null = idx % 7 == 3 + stamp = idx * 1_000 - 5_000_000 + columns = {"idx": pa.array(idx)} + for unit in UNITS: + columns[f"t_{unit}"] = pa.array(stamp, mask=null).cast(pa.timestamp(unit)) + columns["zoned"] = pa.array(stamp, mask=null).cast( + pa.timestamp("us", tz="Asia/Karachi") + ) + columns["d32"] = pa.array(idx - 300, mask=null).cast(pa.int32()).cast(pa.date32()) + columns["d64"] = pa.array((idx - 300) * 86_400_000, mask=null).cast(pa.date64()) + return pa.table(columns) + + +def _make(db, name="Stamp"): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.idx LONG") + + +def _count(db, name): + sql = f"SELECT count(*) AS c FROM {name}" # nosec B608 - test-owned name + return db.query("sql", sql).first().get("c") + + +@pytest.fixture +def crossings(monkeypatch): + """What one load does on the JPype side: the ``java.time`` calls Python makes, the per-value conversions, and the type and length of every column handed to the bridge.""" + # the type conversion caches the classes it fetched first: fetch them now, so the counting proxy below never lands in that cache + type_conversion._get_java_time_types() + type_conversion._get_java_python_types() + log = {"java_time": [], "convert": [], "columns": []} + real_convert = core.convert_python_to_java + real_column = core._column_to_java + real_jclass = jpype.JClass + + class _Counting: + def __init__(self, name): + self._name = name + self._target = real_jclass(name) + + def __getattr__(self, attr): + member = getattr(self._target, attr) + if not callable(member): + return member + + def call(*args): + log["java_time"].append(f"{self._name}.{attr}") + return member(*args) + + return call + + def convert(value): + log["convert"].append(type(value).__name__) + return real_convert(value) + + def column_to_java(name, values): + array, n = real_column(name, values) + log["columns"].append((type(array).__name__.rsplit(".", 1)[-1], n)) + return array, n + + def jclass(name, *args, **kwargs): + if isinstance(name, str) and name.startswith("java.time."): + return _Counting(name) + return real_jclass(name, *args, **kwargs) + + monkeypatch.setattr(core, "convert_python_to_java", convert) + monkeypatch.setattr(core, "_column_to_java", column_to_java) + monkeypatch.setattr(jpype, "JClass", jclass) + return log + + +def _temporal_columns(log): + """The lengths of the columns that reached the bridge as a TemporalColumn.""" + return Counter(n for kind, n in log["columns"] if kind == "TemporalColumn") + + +# ---- the tests that fail on the code before the fix: they count what the load does, never how long it takes + + +def test_a_temporal_column_makes_no_java_time_call_and_no_conversion_per_cell( + temp_db, crossings +): + """Before: one ``ofEpochSecond`` or ``ofEpochDay`` and one ``convert_python_to_java`` per cell. After: neither, for any of the seven columns.""" + n = 600 + _make(temp_db) + assert temp_db.insert_arrow("Stamp", _table(n), batch_size=n) == n + assert crossings["java_time"] == [] + assert crossings["convert"] == [] + assert _temporal_columns(crossings) == {n: TEMPORAL_COLUMNS} + + +def test_each_temporal_column_crosses_as_one_temporal_column_per_batch( + temp_db, crossings +): + """1,300 rows in batches of 500 (500, 500, 300): each of the seven columns is ONE bridge column of the batch's rows, three times.""" + _make(temp_db) + assert temp_db.insert_arrow("Stamp", _table(1_300), batch_size=500) == 1_300 + assert _temporal_columns(crossings) == { + 500: 2 * TEMPORAL_COLUMNS, + 300: TEMPORAL_COLUMNS, + } + + +def test_a_dataframe_and_a_parallel_load_take_the_same_route(temp_db, crossings): + """``insert_dataframe`` reads the frame as an Arrow table and ``parallel=True`` fills the same columns: no per-cell call on either.""" + _make(temp_db) + frame = _table(400).to_pandas() + assert isinstance(frame, pd.DataFrame) + assert temp_db.insert_dataframe("Stamp", frame, batch_size=200) == 400 + _make(temp_db, "Par") + assert ( + temp_db.insert_arrow("Par", _table(400), batch_size=200, parallel=True) == 400 + ) + assert crossings["java_time"] == [] and crossings["convert"] == [] + assert _temporal_columns(crossings) == {200: 4 * TEMPORAL_COLUMNS} + + +# ---- what the bridge column holds + + +def _reference(ticks, per_second): + """The per-cell reading the old route made: ``LocalDateTime.ofEpochSecond`` of the floor quotient, the remainder in nanoseconds.""" + local_datetime = jpype.JClass("java.time.LocalDateTime") + utc = jpype.JClass("java.time.ZoneOffset").UTC + seconds, rest = np.divmod(np.asarray(ticks, dtype=np.int64), per_second) + nanos = rest * (1_000_000_000 // per_second) + return [ + local_datetime.ofEpochSecond(int(s), int(n), utc) + for s, n in zip(seconds, nanos) + ] + + +@pytest.mark.parametrize("unit", list(UNITS)) +def test_a_date_time_column_reads_as_the_per_cell_reading_did(unit): + per_second = UNITS[unit] + limit = min(100_000_000_000 * per_second, 2**62) # about the year 5000 + ticks = np.random.default_rng(7).integers(-limit, limit, size=2_000) + ticks[:4] = [ + -1, + 0, + 1, + -per_second, + ] # one tick before the epoch is a floor, not a truncation + nulls = np.zeros(len(ticks), dtype=np.bool_) + nulls[5::11] = True + column = _column_class().dateTimes( + jpype.JArray(jpype.JLong)(ticks), + per_second, + jpype.JArray(jpype.JBoolean)(nulls), + ) + assert int(column.rows()) == len(ticks) + expected = _reference(ticks, per_second) + for i, want in enumerate(expected): + got = column.get(i) + if nulls[i]: + assert got is None + else: + assert bool(got.equals(want)), (i, int(ticks[i]), str(got), str(want)) + + +def test_a_nanosecond_tick_keeps_its_nanosecond_and_a_null_flag_is_optional(): + ns = 1_577_934_245_123_456_789 # 2020-01-02 03:04:05.123456789 + column = _column_class().dateTimes( + jpype.JArray(jpype.JLong)([ns, -1]), 1_000_000_000, None + ) + assert str(column.get(0)) == "2020-01-02T03:04:05.123456789" + assert str(column.get(1)) == "1969-12-31T23:59:59.999999999" + + +@pytest.mark.parametrize( + "per_day, days", + [(1, [-1, 0, 1, 18_322, -135_140]), (86_400_000, [-86_400_000, 0, 86_400_000])], + ids=["date32", "date64"], +) +def test_a_date_column_reads_the_day(per_day, days): + column = _column_class().dates(jpype.JArray(jpype.JLong)(days), per_day, None) + local_date = jpype.JClass("java.time.LocalDate") + for i, value in enumerate(days): + assert bool(column.get(i).equals(local_date.ofEpochDay(value // per_day))) + + +def test_a_date64_value_is_milliseconds_divided_with_a_floor(): + """The numpy ``//`` the old route used: a millisecond before midnight is still the day before.""" + column = _column_class().dates( + jpype.JArray(jpype.JLong)([86_400_000 - 1, -1]), 86_400_000, None + ) + assert str(column.get(0)) == "1970-01-01" and str(column.get(1)) == "1969-12-31" + + +def test_the_bridge_refuses_what_it_cannot_read(): + longs = jpype.JArray(jpype.JLong)([1, 2]) + for per_second in ( + 0, + -1, + 7, + 3_000_000_000, + ): # none divides a second into whole nanoseconds + with pytest.raises(jpype.JException): + _column_class().dateTimes(longs, per_second, None) + with pytest.raises(jpype.JException): + _column_class().dates(longs, 0, None) + with pytest.raises(jpype.JException): # one null flag for two rows + _column_class().dateTimes(longs, 1_000, jpype.JArray(jpype.JBoolean)([False])) + + +def test_column_to_java_hands_a_temporal_column_over_as_it_is(): + column = _column_class().dates(jpype.JArray(jpype.JLong)([0, 1, 2]), 1, None) + array, n = core._column_to_java("day", column) + assert n == 3 and array is column + + +def test_a_timestamp_past_the_years_of_java_time_is_refused_before_a_row_is_written( + temp_db, +): + """``timestamp[s]`` holds 2**62 seconds, which no ``LocalDateTime`` does: the call raises what it raised before, with the batch unwritten.""" + _make(temp_db) + table = pa.table( + { + "idx": pa.array([0, 1], pa.int64()), + "t": pa.array([0, 2**62], pa.int64()).cast(pa.timestamp("s")), + } + ) + with pytest.raises(jpype.JException): + temp_db.insert_arrow("Stamp", table) + assert _count(temp_db, "Stamp") == 0 + assert not temp_db.is_transaction_active() diff --git a/tests/test_dataframe_ingest.py b/tests/test_dataframe_ingest.py index 449e6d7..6b6c8db 100644 --- a/tests/test_dataframe_ingest.py +++ b/tests/test_dataframe_ingest.py @@ -212,6 +212,235 @@ def test_dates_before_the_epoch(self, temp_db): assert _one_column(temp_db, pa.timestamp("ms"), values) == values +# The engine pattern that shows a stored wall clock to the nanosecond: a read through Result.get returns a Python datetime, +# which holds microseconds and cuts the rest. +_WALL_CLOCK = "yyyy-MM-dd HH:mm:ss.SSSSSSSSS" +_TICKS_PER_SECOND = {"s": 1, "ms": 1_000, "us": 1_000_000, "ns": 1_000_000_000} +# Epoch ticks of each unit: one tick before the epoch (a floor, never a truncation toward zero), the epoch, one tick after +# it, 2020-01-02 03:04:05.123456789 cut to the unit, 1900-01-01 00:00:00 plus one tick, a null, and the year 2255 (the last +# year a nanosecond column holds, so every unit stays inside the range of a DATETIME_NANOS property). +_STAMPS = { + "s": [-1, 0, 1, 1_577_934_245, -2_208_988_799, None, 9_000_000_000], + "ms": [-1, 0, 1, 1_577_934_245_123, -2_208_988_799_999, None, 9_000_000_000_000], + "us": [ + -1, + 0, + 1, + 1_577_934_245_123_456, + -2_208_988_799_999_999, + None, + 9_000_000_000_000_000, + ], + "ns": [ + -1, + 0, + 1, + 1_577_934_245_123_456_789, + -2_208_988_799_999_999_999, + None, + 9_000_000_000_000_000_000, + ], +} +_DAYS = [ + dt.date(1600, 3, 1), + dt.date(1969, 12, 31), + dt.date(1970, 1, 1), + dt.date(2020, 2, 29), + None, + dt.date(2999, 12, 31), +] + + +def _wall_clock(ticks, unit): + """The UTC wall clock of ``ticks`` of ``unit`` since the epoch as ``_WALL_CLOCK`` prints it (a None stays None).""" + if ticks is None: + return None + seconds, rest = divmod(ticks, _TICKS_PER_SECOND[unit]) + moment = dt.datetime(1970, 1, 1) + dt.timedelta(seconds=seconds) + nanos = rest * (1_000_000_000 // _TICKS_PER_SECOND[unit]) + return f"{moment.isoformat(sep=' ', timespec='seconds')}.{nanos:09d}" + + +def _timestamps(ticks, unit, tz=None): + """An Arrow timestamp column of integer ticks (None is a null), so that no Python datetime cuts a nanosecond.""" + return pa.array(ticks, pa.int64()).cast(pa.timestamp(unit, tz=tz)) + + +def _wall_clocks(db, type_name, *columns): + """Each column as the text of its stored wall clock, in the order of ``idx``: {column: [text or None, ...]}.""" + shown = ", ".join(f"{c}.format('{_WALL_CLOCK}') AS {c}" for c in columns) + sql = f"SELECT idx, {shown} FROM {type_name} ORDER BY idx" # nosec B608 - test-controlled + rows = _rows(db, sql) + return {c: [r.get(c) for r in rows] for c in columns} + + +class TestTemporalColumns: + """A timestamp or date column of a load keeps what the Arrow integers say: the unit, the zone rule, the nulls, the dates before 1970. + + The JVM wheel sends such a column to the bridge as one array of epoch ticks and builds the date-times in Java; the native + wheel lays it out as columnar buffers. The expected values come from the integers and the UTC rule, never from a + database read, so both wheels must store the same wall clock. + """ + + @pytest.mark.parametrize( + "tz", [None, "UTC", "Asia/Karachi"], ids=["naive", "utc", "zoned"] + ) + @pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"]) + def test_a_timestamp_keeps_its_wall_clock_to_the_nanosecond( + self, temp_db, unit, tz + ): + """A zoned timestamp is already UTC epoch time and a naive one is the clock it shows: both read the integer.""" + ticks = _STAMPS[unit] + temp_db.command("sql", "CREATE DOCUMENT TYPE Stamp") + temp_db.command("sql", "CREATE PROPERTY Stamp.val DATETIME_NANOS") + table = pa.table( + { + "idx": pa.array(range(len(ticks)), pa.int64()), + "val": _timestamps(ticks, unit, tz), + } + ) + assert temp_db.insert_arrow("Stamp", table) == len(ticks) + assert _wall_clocks(temp_db, "Stamp", "val") == { + "val": [_wall_clock(t, unit) for t in ticks] + } + + @pytest.mark.parametrize("declared", [None, "DATE"], ids=["undeclared", "DATE"]) + @pytest.mark.parametrize( + "arrow_type", [pa.date32(), pa.date64()], ids=["date32", "date64"] + ) + def test_a_date_before_and_after_the_epoch_with_a_null( + self, temp_db, arrow_type, declared + ): + assert _one_column(temp_db, arrow_type, _DAYS, declared) == _DAYS + + @pytest.mark.parametrize( + "kind, parallel", + [("DOCUMENT", False), ("DOCUMENT", True), ("VERTEX", False)], + ids=["document", "document_parallel", "vertex"], + ) + def test_every_temporal_column_of_a_load_in_batches(self, temp_db, kind, parallel): + """23 rows in batches of 5 (the last of 3): six timestamp columns and two date columns, with their nulls in different rows.""" + n = 23 + idx = np.arange(n, dtype=np.int64) + units = { + "t_s": "s", + "t_ms": "ms", + "t_us": "us", + "t_ns": "ns", + "t_zoned": "us", + "t_null": "us", + } + ticks = { + "t_s": (idx * 86_400 - 1_000_000_000).tolist(), + "t_ms": (idx * 1_000_003 - 1_000_000_000_001).tolist(), + "t_us": (idx * 1_000_000_007 - 1_000_000_000_000_001).tolist(), + "t_ns": (idx * 1_000_000_000_011 - 1_000_000_000_000_000_001).tolist(), + "t_zoned": (idx * 777_777_777 - 5).tolist(), + "t_null": [None] * n, + } + for name, every in ( + ("t_s", 4), + ("t_ms", 5), + ("t_us", 6), + ("t_ns", 7), + ("t_zoned", 3), + ): + ticks[name] = [ + None if i % every == 2 else t for i, t in enumerate(ticks[name]) + ] + days = [ + ( + None + if i % 6 == 1 + else dt.date(2000, 1, 1) + dt.timedelta(days=i * 997 - 30_000) + ) + for i in range(n) + ] + temp_db.command("sql", f"CREATE {kind} TYPE Wide") + for name in units: + temp_db.command("sql", f"CREATE PROPERTY Wide.{name} DATETIME_NANOS") + columns = {"idx": pa.array(idx)} + for name, unit in units.items(): + tz = "Asia/Karachi" if name == "t_zoned" else None + columns[name] = _timestamps(ticks[name], unit, tz) + columns["d32"] = pa.array(days, pa.date32()) + columns["d64"] = pa.array(days, pa.date64()) + assert ( + temp_db.insert_arrow( + "Wide", pa.table(columns), batch_size=5, parallel=parallel + ) + == n + ) + assert _count(temp_db, "Wide") == n + assert _wall_clocks(temp_db, "Wide", *units) == { + name: [_wall_clock(t, unit) for t in ticks[name]] + for name, unit in units.items() + } + stored = _rows(temp_db, "SELECT idx, d32, d64 FROM Wide ORDER BY idx") + assert [r.get("d32") for r in stored] == days + assert [r.get("d64") for r in stored] == days + + def test_pandas_and_polars_columns_keep_their_unit_zone_and_nulls(self, temp_db): + """pandas datetime64[ns] with a NaT, the same instants with a zone, and Polars Datetime of two units and a Date.""" + ticks = _STAMPS["ns"] + nat = np.iinfo(np.int64).min + values = np.array([nat if t is None else t for t in ticks], dtype="int64") + naive = pd.Series(values.view("datetime64[ns]")) + pandas_frame = pd.DataFrame( + { + "idx": np.arange(len(ticks), dtype=np.int64), + "t_naive": naive, + "t_zoned": naive.dt.tz_localize("UTC").dt.tz_convert("Etc/GMT-5"), + } + ) + whole_ms = [ + dt.datetime(1969, 12, 31, 23, 59, 59, 999000), + dt.datetime(2020, 1, 2, 3, 4, 5, 123000), + None, + dt.datetime(1900, 1, 1, 0, 0, 0, 1000), + ] + micros = [ + dt.datetime(1969, 12, 31, 23, 59, 59, 999999), + dt.datetime(2020, 1, 2, 3, 4, 5, 123456), + None, + dt.datetime(1900, 1, 1, 0, 0, 0, 1), + ] + polars_frame = pl.DataFrame( + { + "idx": pl.Series(range(4), dtype=pl.Int64), + "t_ms": pl.Series(whole_ms, dtype=pl.Datetime("ms")), + "t_ns": pl.Series(micros, dtype=pl.Datetime("ns")), + "day": pl.Series([m and m.date() for m in micros], dtype=pl.Date), + } + ) + for name, columns in ( + ("FromPandas", ("t_naive", "t_zoned")), + ("FromPolars", ("t_ms", "t_ns")), + ): + temp_db.command("sql", f"CREATE DOCUMENT TYPE {name}") + for column in columns: + temp_db.command( + "sql", f"CREATE PROPERTY {name}.{column} DATETIME_NANOS" + ) + assert temp_db.insert_dataframe("FromPandas", pandas_frame) == len(ticks) + assert temp_db.insert_dataframe("FromPolars", polars_frame) == 4 + shown = [_wall_clock(t, "ns") for t in ticks] + assert _wall_clocks(temp_db, "FromPandas", "t_naive", "t_zoned") == { + "t_naive": shown, + "t_zoned": shown, # the zone is only how pandas shows the instant: it is stored as the UTC wall clock + } + got = _wall_clocks(temp_db, "FromPolars", "t_ms", "t_ns") + assert got["t_ms"] == [ + m and m.isoformat(sep=" ", timespec="milliseconds") + "000000" + for m in whole_ms + ] + assert got["t_ns"] == [ + m and m.isoformat(sep=" ", timespec="microseconds") + "000" for m in micros + ] + days = _rows(temp_db, "SELECT idx, day FROM FromPolars ORDER BY idx") + assert [r.get("day") for r in days] == [m and m.date() for m in micros] + + class TestNulls: def test_every_kind_stores_nulls_as_missing(self, temp_db): temp_db.command("sql", "CREATE DOCUMENT TYPE Nul") From ef382809d54648c053b449b59b4a1942460ca184 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 02:50:26 +0900 Subject: [PATCH 04/18] JVM insert_arrow, insert_dataframe, insert_columns: a timestamp or date column crosses as one long[] of epoch ticks, not one JPype call per cell (refs #315) _temporal_to_java made one LocalDateTime.ofEpochSecond or LocalDate.ofEpochDay call per cell, and insert_columns converted every element of that list again with convert_python_to_java: a million-row timestamp column was two million JPype calls. The column is now the integer epoch values in one long[] (plus one boolean[] when a row is null) wrapped in the new bridge class TemporalColumn (dateTimes(long[] ticks, long ticksPerSecond, boolean[] nulls) and dates(long[] ticks, long ticksPerDay, boolean[] nulls)), and DocumentBatcher builds each LocalDateTime or LocalDate as it fills the document of that row, as StringColumn does for text. The reading rules are the ones of the code before: UTC wall clock, a zoned timestamp is already UTC epoch time, a naive one is the clock it shows, the unit's precision is kept (a nanosecond tick is not cut to microseconds), a tick before the epoch floors, date64 is milliseconds divided by 86,400,000 with a floor. The factory checks the smallest and the largest tick, so a value past the years of java.time still raises before a row is written. The per-cell path is retired: the wheel carries its own bridge jar, so no fallback for an older one. Not measured: the speed claim waits for a timed A/B against 26.11.1.post2. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/core.py | 5 +- src/arcadedb/dataframe.py | 60 ++++----- .../com/arcadedb/python/DocumentBatcher.java | 16 ++- .../com/arcadedb/python/TemporalColumn.java | 118 ++++++++++++++++++ 4 files changed, 163 insertions(+), 36 deletions(-) create mode 100644 src/java/com/arcadedb/python/TemporalColumn.java diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index a050b72..68fc806 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -241,7 +241,8 @@ def _column_to_java(name, values): into a ``long[]``, ``double[]``, or ``boolean[]``; a string or object array and any other sequence of mixed values convert per element into an ``Object[]`` (a ``str`` reuses one Java String per distinct value, ``None`` is a null); a column of nothing but ``str`` and ``None`` crosses instead as one - ``StringColumn`` (``_string_column``). A pandas + ``StringColumn`` (``_string_column``). A bridge ``TemporalColumn``, which ``insert_arrow`` and ``insert_dataframe`` build from a + timestamp or date column (``dataframe._temporal_column``), crosses as it is. A pandas ``Series`` is read through ``to_numpy``, with a nullable dtype's ```` as ``None``. A 2-D ``float32`` array is a column of vectors: it crosses as one ``float[]`` and each document gets one row of it. Kinds that do not cross natively raise ``TypeError`` rather @@ -313,6 +314,8 @@ def _column_to_java(name, values): # a string, bytes, or object array: its elements are Python objects (str, None, ...), converted one by one below values = list(values.astype(object, copy=False)) elif not isinstance(values, (list, tuple)): + if isinstance(values, _java_class("com.arcadedb.python.TemporalColumn")): + return values, int(values.rows()) values = list(values) column = _string_column(values) if column is not None: diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 4c7beb2..6fa3de5 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -245,43 +245,46 @@ def _check_declared( ) -def _temporal_to_java(chunk, pa): - """A timestamp or date chunk as a list of ``LocalDateTime``/``LocalDate``, keeping the unit's precision. +def _temporal_column(chunk, pa): + """A timestamp or date chunk as one ``TemporalColumn`` of the bridge, keeping the unit's precision. + + The integer epoch values cross as ONE ``long[]`` (a buffer copy), the null rows as one ``boolean[]`` when there is a null, and the + bridge builds each ``LocalDateTime`` or ``LocalDate`` as it fills the document of that row: one call per column and batch, where a + ``LocalDateTime.ofEpochSecond`` or ``LocalDate.ofEpochDay`` call per cell (and one conversion per element in ``insert_columns``) came + before. ``insert_columns`` takes the column as it is. DATETIME is a UTC wall clock: a timestamp with a zone is already UTC epoch time, and a timestamp without one is taken as the wall clock it shows (the same rule ``Document.set`` applies to a ``datetime``), so both - are read from the integer epoch value. Going through ``datetime`` would cut a nanosecond value to microseconds. + are read from the integer epoch value, in the unit of the column. Going through ``datetime`` would cut a nanosecond value to + microseconds. A date32 is days, a date64 is milliseconds (the bridge divides by 86,400,000, rounding toward negative infinity). """ import numpy as np types = pa.types - mask = chunk.is_null().to_numpy(zero_copy_only=False) if types.is_date(chunk.type): - local_date = jpype.JClass("java.time.LocalDate") - raw = chunk.cast(pa.int64() if types.is_date64(chunk.type) else pa.int32()) - days = raw.fill_null(0).to_numpy(zero_copy_only=False).astype(np.int64) - if types.is_date64(chunk.type): - days = days // 86_400_000 - return [ - None if m else local_date.ofEpochDay(int(d)) for d, m in zip(days, mask) - ] - per_second = {"s": 1, "ms": 1_000, "us": 1_000_000, "ns": 1_000_000_000}[ - chunk.type.unit - ] - values = chunk.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) - seconds, rest = np.divmod(values.astype(np.int64), per_second) - nanos = rest * (1_000_000_000 // per_second) - local_datetime = jpype.JClass("java.time.LocalDateTime") - utc = jpype.JClass("java.time.ZoneOffset").UTC - return [ - None if m else local_datetime.ofEpochSecond(int(s), int(n), utc) - for s, n, m in zip(seconds, nanos, mask) - ] + date64 = types.is_date64(chunk.type) + raw = chunk.cast(pa.int64() if date64 else pa.int32()) + ticks, per_unit, factory = raw, 86_400_000 if date64 else 1, "dates" + else: + per_second = {"s": 1, "ms": 1_000, "us": 1_000_000, "ns": 1_000_000_000} + ticks = chunk.cast(pa.int64()) + per_unit, factory = per_second[chunk.type.unit], "dateTimes" + values = ticks.fill_null(0).to_numpy(zero_copy_only=False) + nulls = None + if chunk.null_count: + nulls = jpype.JArray(jpype.JBoolean)( + chunk.is_null().to_numpy(zero_copy_only=False) + ) + return getattr(jpype.JClass("com.arcadedb.python.TemporalColumn"), factory)( + jpype.JArray(jpype.JLong)(np.ascontiguousarray(values, dtype=np.int64)), + per_unit, + nulls, + ) def _temporal_to_tagged(chunk, pa): """A timestamp or date chunk as tagged values (``T`` date-time, ``D`` date) for the native backend, with the same - reading rules as :func:`_temporal_to_java`: ISO text of the integer epoch value, so a nanosecond value keeps its + reading rules as :func:`_temporal_column`: ISO text of the integer epoch value, so a nanosecond value keeps its nanoseconds (a Python ``datetime`` would cut them to microseconds). """ import numpy as np @@ -381,7 +384,8 @@ def _column_values( float_vectors: bool = False, ): """One Arrow array as what ``Database.insert_columns`` takes: a numpy array when the column crosses as one - buffer (integers, floats and booleans without nulls), else a list of values (``None`` is a null). + buffer (integers, floats and booleans without nulls), else a list of values (``None`` is a null). On the JVM backend a timestamp or + date column is one bridge ``TemporalColumn`` instead (``_temporal_column``: one ``long[]`` of epoch ticks, no value per cell). ``float_vectors`` (the JVM backend, a property declared ``ARRAY_OF_FLOATS``): a float32 vector column of one length and no null is one 2-D float32 matrix, which ``insert_columns`` sends as one ``float[]``; ``to_pylist`` made one Python float per element and @@ -434,9 +438,7 @@ def _column_values( if matrix is not None: return matrix if kind in ("timestamp", "date"): - return ( - _temporal_to_tagged(chunk, pa) if native else _temporal_to_java(chunk, pa) - ) + return _temporal_to_tagged(chunk, pa) if native else _temporal_column(chunk, pa) if native and kind in ("decimal", "binary"): return _scalars_to_tagged(kind, chunk) if kind == "null": diff --git a/src/java/com/arcadedb/python/DocumentBatcher.java b/src/java/com/arcadedb/python/DocumentBatcher.java index 0edeb45..8d892a9 100644 --- a/src/java/com/arcadedb/python/DocumentBatcher.java +++ b/src/java/com/arcadedb/python/DocumentBatcher.java @@ -81,17 +81,17 @@ public static long insertManyJson(final Database db, final String typeName, fina /** * Columnar insert: the documents are built Java-side from whole columns, so each column crosses the FFI once (a long[] or - * double[] copied from a numpy buffer, a boolean[], a StringColumn (one UTF-8 byte[] plus the int[] offsets), or an Object[] of - * boxed numbers, Strings of a mixed column, and nulls) instead of one + * double[] copied from a numpy buffer, a boolean[], a StringColumn (one UTF-8 byte[] plus the int[] offsets), a TemporalColumn (one + * long[] of epoch ticks), or an Object[] of boxed numbers, Strings of a mixed column, and nulls) instead of one * JSON text per batch (bindings issue #150: 2.24x over insertManyJson on the first * 2,000,000 TPC-H SF1 line items, nine typed properties, commit every 10,000, same sums and count). * * Same failure contract as insertManyJson (#7882): batch commits and the failure rollback apply only to a transaction * opened here; a caller's own transaction is the caller's to commit or roll back. * - * @param columns one array per name, each of length n: long[], double[], boolean[], Object[], a FloatVectors.Matrix or a - * StringColumn; a null element (or null row of a StringColumn) sets the property to null, as a JSON null does - * on the insertManyJson path + * @param columns one array per name, each of length n: long[], double[], boolean[], Object[], a FloatVectors.Matrix, a + * StringColumn or a TemporalColumn; a null element (or null row of a StringColumn or TemporalColumn) sets the + * property to null, as a JSON null does on the insertManyJson path */ public static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns, final int n, final int commitEvery) { @@ -229,9 +229,11 @@ else if (col instanceof FloatVectors.Matrix m) len = m.rows(); else if (col instanceof StringColumn s) len = s.rows(); + else if (col instanceof TemporalColumn t) + len = t.rows(); else throw new IllegalArgumentException("column '" + names[c] + "' is a " + (col == null ? "null" : col.getClass().getName()) - + ", not a long[], double[], boolean[], Object[], FloatVectors.Matrix or StringColumn"); + + ", not a long[], double[], boolean[], Object[], FloatVectors.Matrix, StringColumn or TemporalColumn"); if (len != n) throw new IllegalArgumentException("column '" + names[c] + "' has " + len + " values for " + n + " rows"); } @@ -251,6 +253,8 @@ else if (col instanceof FloatVectors.Matrix m) v = m.row(i); else if (col instanceof StringColumn s) v = s.get(i); + else if (col instanceof TemporalColumn t) + v = t.get(i); else v = ((Object[]) col)[i]; doc.set(names[c], v); diff --git a/src/java/com/arcadedb/python/TemporalColumn.java b/src/java/com/arcadedb/python/TemporalColumn.java new file mode 100644 index 0000000..533fb3c --- /dev/null +++ b/src/java/com/arcadedb/python/TemporalColumn.java @@ -0,0 +1,118 @@ +/* + * Python-bindings bridge: a timestamp or date column of Database.insert_columns that crosses as one long[] of epoch ticks. + * + * Building a LocalDateTime or LocalDate in Python costs one JPype call per cell (and the list of them one conversion per element + * again), so a million-row timestamp column was two million calls. Here the whole column is ONE long[] (the integer epoch value of every + * row, as Arrow holds it), the tick size, and optionally ONE boolean[] marking the null rows. Row i is read when the document of that + * row is built, so the values are short-lived. + * + * A date-time is {@code ticks / ticksPerSecond} whole seconds (rounded toward negative infinity, so a tick before the epoch is the + * last tick of 1969) and the rest as nanoseconds, read as the UTC wall clock: the engine stores a DATETIME as a UTC wall clock, a + * zone-aware Arrow timestamp is already UTC epoch time and a zone-less one is taken as the wall clock it shows. The unit's precision is + * kept: a nanosecond tick is not cut to microseconds. A date is {@code ticks / ticksPerDay} days since the epoch, rounded the same way + * (a date32 is days, a date64 is milliseconds). + * + * Compiled into arcadedb-python-bridge.jar during the wheel build and consumed by DocumentBatcher.insertColumns() and + * insertColumnsParallel(). + */ +package com.arcadedb.python; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +public final class TemporalColumn { + private static final long NANOS_PER_SECOND = 1_000_000_000L; + + private final long[] ticks; + private final boolean[] nulls; + private final boolean date; + /** Ticks in one second (a date-time) or in one day (a date). */ + private final long ticksPerUnit; + /** Nanoseconds in one tick, for a date-time. */ + private final long nanosPerTick; + + private TemporalColumn(final long[] ticks, final boolean[] nulls, final boolean date, final long ticksPerUnit, + final long nanosPerTick) { + if (nulls != null && nulls.length != ticks.length) + throw new IllegalArgumentException("a temporal column has " + nulls.length + " null flags for " + ticks.length + " rows"); + this.ticks = ticks; + this.nulls = nulls; + this.date = date; + this.ticksPerUnit = ticksPerUnit; + this.nanosPerTick = nanosPerTick; + checkRange(); + } + + /** + * A column of date-times. + * + * @param ticks the epoch value of every row, in units of {@code 1 / ticksPerSecond} second since 1970-01-01T00:00:00 + * @param ticksPerSecond 1 (seconds), 1,000, 1,000,000 or 1,000,000,000: any divisor of a second into whole nanoseconds + * @param nulls null for a column without a null, else one flag per row (a flagged row reads as null and its tick is ignored) + * @throws IllegalArgumentException for a tick size that is not a whole number of nanoseconds, or a wrong number of null flags. + * A row past the years a LocalDateTime holds raises the DateTimeException of LocalDateTime itself, + * here, before anything is written + */ + public static TemporalColumn dateTimes(final long[] ticks, final long ticksPerSecond, final boolean[] nulls) { + if (ticksPerSecond < 1 || NANOS_PER_SECOND % ticksPerSecond != 0) + throw new IllegalArgumentException( + "ticks per second must divide 1,000,000,000 into whole nanoseconds, got " + ticksPerSecond); + return new TemporalColumn(ticks, nulls, false, ticksPerSecond, NANOS_PER_SECOND / ticksPerSecond); + } + + /** + * A column of dates. + * + * @param ticks the epoch value of every row, in units of {@code 1 / ticksPerDay} day since 1970-01-01 + * @param ticksPerDay 1 for a date32 (days), 86,400,000 for a date64 (milliseconds) + * @param nulls null for a column without a null, else one flag per row (a flagged row reads as null and its tick is ignored) + * @throws IllegalArgumentException for a tick size below 1, or a wrong number of null flags. A row past the days a LocalDate holds + * raises the DateTimeException of LocalDate itself, here, before anything is written + */ + public static TemporalColumn dates(final long[] ticks, final long ticksPerDay, final boolean[] nulls) { + if (ticksPerDay < 1) + throw new IllegalArgumentException("ticks per day must be at least 1, got " + ticksPerDay); + return new TemporalColumn(ticks, nulls, true, ticksPerDay, 0L); + } + + public int rows() { + return ticks.length; + } + + /** The LocalDateTime or LocalDate of row {@code i}, or null for a null row. */ + public Object get(final int i) { + if (nulls != null && nulls[i]) + return null; + return valueOf(ticks[i]); + } + + private Object valueOf(final long tick) { + if (date) + return LocalDate.ofEpochDay(Math.floorDiv(tick, ticksPerUnit)); + return LocalDateTime.ofEpochSecond(Math.floorDiv(tick, ticksPerUnit), (int) (Math.floorMod(tick, ticksPerUnit) * nanosPerTick), + ZoneOffset.UTC); + } + + /** + * The value of a row grows with its tick, so the smallest and the largest tick of the non-null rows are the two that can fall outside + * the years of java.time: a value past them raises here, as one call per cell did in Python, and nothing of the batch is written. + */ + private void checkRange() { + long min = Long.MAX_VALUE; + long max = Long.MIN_VALUE; + for (int i = 0; i < ticks.length; i++) { + if (nulls != null && nulls[i]) + continue; + final long tick = ticks[i]; + if (tick < min) + min = tick; + if (tick > max) + max = tick; + } + if (min <= max) { + valueOf(min); + valueOf(max); + } + } +} From a17daaed3feac3526c4bd5e0e8a7ed47c61fe555 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 02:58:42 +0900 Subject: [PATCH 05/18] Tests: a native transaction crossing is one foreign call, with no reply text, no free and no JSON decode (refs #315) arcadedb_n_tx returns the transaction state as one integer (bit 0 active, bits 1 to 31 depth, bits 32 to 62 commits, -1 for an error whose text lib.last_error holds), as arcadedb_n_cmd returns a cursor id. These tests pin that and that an error still reaches Python as the same typed exception with the same message: - with db.transaction() is three foreign calls (mark, begin, commit), all arcadedb_n_tx: no strlen, no copy into a str, no arcadedb_free (was twelve); - begin, commit, rollback, is_transaction_active and the mark each make exactly one foreign call; - no JSON decode in a transaction, not even for the mark that reads active and depth; - the layout of the integer, -1 for an unknown handle with the error in last_error, commit without begin, rollback with nothing to roll back, a failed statement in a block, nested with blocks, an injected failure on each entry point, a database closed with a transaction open. They fail on the old code: lib.tx does not exist there, and the foreign-call and decode counts are higher. Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 2 +- tests/test_native_one_row_crossings.py | 8 +- tests/test_native_tx_crossings.py | 271 +++++++++++++++++++++---- 3 files changed, 241 insertions(+), 40 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3aa8a12..d59db09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -93,7 +93,7 @@ collect_ignore.append("test_native_vector_provider.py") # the copy of a record's values as read that save() compares with the stored row (arcadedb._native.records), counted through its seams collect_ignore.append("test_native_save_baseline.py") - # crossing counts of the transaction entry points (lib.call is the native library's ctypes door) + # crossing and foreign-call counts of the transaction entry points (lib.tx and lib.call are the native library's ctypes doors) collect_ignore.append("test_native_tx_crossings.py") # counts the statement texts the native backend hands to the engine (the JPype backend has no such layer) collect_ignore.append("test_native_statement_texts.py") diff --git a/tests/test_native_one_row_crossings.py b/tests/test_native_one_row_crossings.py index 73f32a9..e65e3e9 100644 --- a/tests/test_native_one_row_crossings.py +++ b/tests/test_native_one_row_crossings.py @@ -7,7 +7,7 @@ reads the stored record first: the engine's reply to an UPDATE comes after the write, and the check must refuse before writing anything inside the caller's transaction, so it cannot be read from the update's own result. -The counts are crossings (the ctypes doors ``cmd``, ``call``, ``call_bin``, ``exec_query`` and the batched cursor close), not seconds: +The counts are crossings (the ctypes doors ``cmd``, ``call``, ``call_bin``, ``exec_query``, ``tx`` and the batched cursor close), not seconds: deterministic. The oracle for the values and the errors is the lazy form itself (``db.command(...).first()``), kept for every other caller. """ @@ -27,7 +27,7 @@ class _Crossings: def __init__(self, monkeypatch): self.names = [] - for attr in ("cmd", "call", "call_bin", "exec_query"): + for attr in ("cmd", "call", "call_bin", "exec_query", "tx"): monkeypatch.setattr(lib, attr, self._spy(attr, getattr(lib, attr))) real_close = lib._lib.arcadedb_n_rs_close_many @@ -39,7 +39,9 @@ def close_many(*args): def _spy(self, attr, real): def spy(*args, **kwargs): - self.names.append(args[0] if attr in ("call", "call_bin") else attr) + # the transaction door is named by its entry point, as it was when it went through lib.call + name = "arcadedb_n_tx" if attr == "tx" else attr + self.names.append(args[0] if attr in ("call", "call_bin") else name) return real(*args, **kwargs) return spy diff --git a/tests/test_native_tx_crossings.py b/tests/test_native_tx_crossings.py index 7f9f618..a218d61 100644 --- a/tests/test_native_tx_crossings.py +++ b/tests/test_native_tx_crossings.py @@ -1,11 +1,15 @@ -"""The native wheel spends the fewest library crossings a transaction needs. +"""The native wheel spends the fewest library crossings a transaction needs, and each one is a single foreign call. -Every ``arcadedb_n_tx`` call is one trip into the native library: a ctypes call, a malloc'd JSON reply, ``string_at``, a decode and a -``json.loads``. The reply of begin, commit and rollback already carries the whole transaction state (``active``, ``depth`` and -``commits``), so a caller that needs the state right after one of these calls must not ask for it again with a mark. +Every ``arcadedb_n_tx`` call is one trip into the native library. It used to return a malloc'd JSON text (about 60 bytes) that the +caller then read with ``strlen``, a copy into a ``str`` and ``arcadedb_free``: four foreign calls per crossing, twelve for +``with db.transaction()``. It now returns the transaction state as one integer (bit 0 ``active``, bits 1 to 31 ``depth``, bits 32 to +62 ``commits``, -1 for an error whose text ``lib.last_error`` holds), the way ``arcadedb_n_cmd`` returns a cursor id: one foreign +call, no reply text, no free, no JSON decode. The reply of begin, commit and rollback carries the whole state, so a caller that +needs the state right after one of these calls must not ask for it again with a mark. -The counts are crossings, not seconds: they are deterministic. The behaviour tests next to them pin what the marks are for (a caller's -transaction is never rolled back by a nested block, a retry rolls the failed attempt back), so a lower count can not buy a different result. +The counts are crossings and foreign calls, not seconds: they are deterministic. The behaviour tests next to them pin what the +marks are for (a caller's transaction is never rolled back by a nested block, a retry rolls the failed attempt back) and that an +error reaches Python as the same typed exception with the same message, so a lower count can not buy a different result. Needs the native library, so conftest.py does not collect this file on the JPype backend (a skip there would fail scripts/check_test_skips.py); it runs in the native parity run. @@ -20,20 +24,32 @@ from arcadedb._native import lib _OPS = {0: "begin", 1: "commit", 2: "rollback", 3: "mark"} +_DEPTH_MASK = 0x7FFFFFFF + + +def _fields(state): + """``(active, depth, commits)`` of the integer ``arcadedb_n_tx`` returns (the layout is pinned here, not read from the wheel).""" + return bool(state & 1), (state >> 1) & _DEPTH_MASK, state >> 32 class _Crossings: - """Counts every ``lib.call`` made while it is installed; ``tx`` is the sequence of ``arcadedb_n_tx`` operations.""" + """Counts every ``lib.tx`` and ``lib.call`` made while it is installed; ``tx`` is the sequence of transaction operations.""" def __init__(self, monkeypatch): self.calls = [] - real = lib.call + real_tx = lib.tx + real_call = lib.call + + def counting_tx(handle, op): + self.calls.append(_OPS[op]) + return real_tx(handle, op) - def counting(name, *args): - self.calls.append(_OPS[args[1]] if name == "arcadedb_n_tx" else name) - return real(name, *args) + def counting_call(name, *args): + self.calls.append(name) + return real_call(name, *args) - monkeypatch.setattr(lib, "call", counting) + monkeypatch.setattr(lib, "tx", counting_tx) + monkeypatch.setattr(lib, "call", counting_call) @property def tx(self): @@ -43,6 +59,51 @@ def reset(self): self.calls.clear() +class _ForeignCalls: + """Every foreign (ctypes) call the transaction path can make, by name, while it is installed. + + The entry points of the library are counted through a proxy of the loaded library object, and the two helpers that turn a reply + text into a ``str`` (``strlen`` and ``PyUnicode_FromStringAndSize``) by name. ``arcadedb_free`` is an entry point, so a reply + text that is read and released shows up as ``arcadedb_n_tx``, ``_strlen``, ``_str_from_utf8``, ``arcadedb_free``. + """ + + def __init__(self, monkeypatch): + self.names = [] + names = self.names + real_lib = lib._lib + + class _Proxy: + def __getattr__(self, attr): + function = getattr(real_lib, attr) + + def counted(*args): + names.append(attr) + return function(*args) + + return counted + + monkeypatch.setattr(lib, "_lib", _Proxy()) + for helper in ("_strlen", "_str_from_utf8"): + monkeypatch.setattr( + lib, helper, self._counted(helper, getattr(lib, helper)) + ) + + def _counted(self, name, function): + def counted(*args): + self.names.append(name) + return function(*args) + + return counted + + @property + def entries(self): + # a batched cursor close that an earlier statement left waiting runs before a commit: it belongs to that statement + return [n for n in self.names if n != "arcadedb_n_rs_close_many"] + + def reset(self): + del self.names[:] + + @pytest.fixture def db(temp_db_path): database = arcadedb.create_database(temp_db_path) @@ -51,6 +112,7 @@ def db(temp_db_path): database.transaction() ): # the thread's transaction context exists from here on, as it does in a running application pass + lib.flush_closes() # a cursor closed earlier is not part of a call measured below yield database database.drop() @@ -60,6 +122,11 @@ def crossings(db, monkeypatch): return _Crossings(monkeypatch) +@pytest.fixture +def foreign(db, monkeypatch): + return _ForeignCalls(monkeypatch) + + def _count(db): return db.query("sql", "SELECT count(*) AS c FROM R").first().get("c") @@ -119,6 +186,51 @@ def fn(): ] +# -- foreign calls: a crossing is one call into the library, with no reply text to read and free + + +def test_with_transaction_is_three_foreign_calls(db, foreign): + # was twelve: each of the three crossings (mark, begin, commit) was the entry, strlen, the copy into a str and arcadedb_free + with db.transaction(): + pass + assert foreign.entries == ["arcadedb_n_tx"] * 3 + + +def test_each_transaction_call_is_one_foreign_call_and_frees_nothing(db, foreign): + for call in ( + db.is_transaction_active, + db.begin, + db.is_transaction_active, + db.rollback, + db.begin, + db.commit, + db._transaction_stack_mark, + ): + foreign.reset() + call() + assert foreign.entries == ["arcadedb_n_tx"], call.__name__ + + +def test_a_transaction_that_raises_is_three_foreign_calls(db, foreign): + with pytest.raises(ValueError): + with db.transaction(): + raise ValueError("boom") + assert foreign.entries == ["arcadedb_n_tx"] * 3 + + +def test_a_transaction_never_reads_a_reply_text(db, monkeypatch): + taken = [] + real = lib._take + monkeypatch.setattr(lib, "_take", lambda *a: taken.append(1) or real(*a)) + with db.transaction(): + pass + db.begin() + assert db.is_transaction_active() is True + db.rollback() + assert db.run_in_transaction(lambda: 1) == 1 + assert taken == [] + + # -- what the marks are for: the same results at the lower count @@ -143,6 +255,18 @@ def test_a_nested_block_that_succeeds_commits_with_the_caller(db): assert _count(db) == 2 +def test_nested_with_blocks_push_and_pop_one_level_each(db): + base = db._transaction_stack_mark()[1] + with db.transaction(): + assert db._transaction_stack_mark() == (True, base + 1) + with db.transaction(): + assert db._transaction_stack_mark() == (True, base + 2) + db.command("sql", "INSERT INTO R SET k = 1") + assert db._transaction_stack_mark() == (True, base + 1) + assert db._transaction_stack_mark() == (False, base) + assert _count(db) == 1 + + def test_a_refused_attempt_is_rolled_back_before_the_retry(db): attempts = [] @@ -193,11 +317,11 @@ def test_begin_commit_rollback_return_nothing(db): assert db.commit() is None -# -- decodes: a status the caller does not read is not a JSON decode +# -- decodes: a status the caller does not read is not a JSON decode, and a state the caller reads is not one either class _Decodes: - """Counts every ``json.loads`` while it is installed (the transaction replies are decoded with it).""" + """Counts every ``json.loads`` while it is installed (an error reply is decoded with it, nothing else is).""" def __init__(self, monkeypatch): self.n = 0 @@ -232,40 +356,99 @@ def test_is_transaction_active_decodes_nothing_and_reports_the_state(db, decodes assert decodes.n == 0 -def test_with_transaction_decodes_only_the_mark(db, decodes): - # the mark reads two fields (active, depth) and is decoded; begin and commit read none +def test_with_transaction_decodes_nothing(db, decodes): + # the mark reads two fields (active, depth) out of the integer, begin and commit read none: no JSON anywhere with db.transaction(): pass - assert decodes.n == 1 + assert decodes.n == 0 -def test_a_state_reply_still_carries_depth_and_commits(db): +def test_a_state_still_carries_active_depth_and_commits(db): db.begin() db.begin() assert db._transaction_stack_mark() == (True, 2) db.rollback() db.rollback() assert db._transaction_stack_mark()[0] is False - assert db._begin_state()["depth"] == 1 - assert db._rollback_state()["active"] is False + active, depth, _ = _fields(db._begin_state()) + assert (active, depth) == (True, 1) + active, _, _ = _fields(db._rollback_state()) + assert active is False + + +def test_the_state_comes_back_as_one_integer(db): + handle = db._h + idle = lib.tx(handle, 3) + assert type(idle) is int and idle >= 0 + active, base, _ = _fields(idle) + assert active is False + one = lib.tx(handle, 0) + assert type(one) is int and one >= 0 + assert _fields(one)[:2] == (True, base + 1) + two = lib.tx(handle, 0) + assert _fields(two)[:2] == (True, base + 2) + assert _fields(two)[2] == _fields(lib.tx(handle, 3))[2] # a mark reads the count + assert _fields(lib.tx(handle, 2))[:2] == (True, base + 1) + assert _fields(lib.tx(handle, 2))[:2] == (False, base) + + +# -- errors: the same typed exception with the same message + + +def test_a_failed_call_returns_minus_one_and_the_error_is_in_last_error(db): + assert lib.tx(2**62, 3) == -1 # a handle no database has + error = json.loads(lib.last_error()) + assert error["ok"] is False + assert error["cls"] == "java.lang.IllegalArgumentException" + assert "unknown database handle" in error["error"] + assert error["chain"][0][0] == error["cls"] + + +def test_a_commit_without_begin_raises_the_engines_error_unchanged(db): + with pytest.raises(ArcadeDBError) as caught: + db.commit() + native = caught.value.native + assert native["ok"] is False + assert native["cls"] and "Transaction not begun" in native["error"] + assert ( + str(caught.value) + == f"Failed to commit transaction: {native['cls']}: {native['error']}" + ) + assert caught.value.java_chain == native["chain"] + assert caught.value.java_chain[0] == [native["cls"], native["error"]] + assert db.is_transaction_active() is False -def test_an_error_reply_is_decoded_into_the_same_error(db, monkeypatch): +def test_a_rollback_with_nothing_to_roll_back_is_not_an_error(db): + db.rollback() + assert db.is_transaction_active() is False + + +def test_a_failed_statement_in_a_block_rolls_the_block_back_and_leaves_no_level(db): + db.command("sql", "CREATE PROPERTY R.k INTEGER") + db.command("sql", "CREATE INDEX ON R (k) UNIQUE") + base = db._transaction_stack_mark() + with pytest.raises(ArcadeDBError) as caught: + with db.transaction(): + db.command("sql", "INSERT INTO R (k) VALUES (1), (1)") + assert "DuplicatedKeyException" in str(caught.value) + assert db._transaction_stack_mark() == base + assert _count(db) == 0 + + +def test_a_failure_reply_is_read_from_last_error_into_the_same_error(db, monkeypatch): reply = ( '{"ok":false,"error":"boom","cls":"x.Y","chain":[["x.Y","boom"]],' '"retry":"need_retry","retry_ms":3}' ) - real = lib.call - monkeypatch.setattr( - lib, - "call", - lambda name, *a: reply if name == "arcadedb_n_tx" else real(name, *a), - ) + monkeypatch.setattr(lib, "tx", lambda handle, op: -1) + monkeypatch.setattr(lib, "last_error", lambda: reply) for op, prefix in ( (db.begin, "Failed to begin transaction"), (db.commit, "Failed to commit transaction"), (db.rollback, "Failed to rollback transaction"), (db.is_transaction_active, "Failed to check transaction status"), + (db._transaction_stack_mark, "Failed to read the transaction state"), ): with pytest.raises(ArcadeDBError) as caught: op() @@ -274,12 +457,28 @@ def test_an_error_reply_is_decoded_into_the_same_error(db, monkeypatch): assert caught.value.java_chain == [["x.Y", "boom"]] -def test_a_differently_spaced_ok_reply_is_still_read(db, monkeypatch): - monkeypatch.setattr( - lib, - "call", - lambda name, *a: '{"ok": true, "active": true, "depth": 1, "commits": 0}', - ) - assert db.is_transaction_active() is True - db.begin() - db.commit() +# -- a closed database + + +def test_a_transaction_left_open_is_gone_after_a_close_and_reopen(temp_db_path): + database = arcadedb.create_database(temp_db_path) + database.command("sql", "CREATE DOCUMENT TYPE R") + database.begin() + database.command("sql", "INSERT INTO R SET k = 1") + assert database.is_transaction_active() is True + database.close() + for call in ( + database.begin, + database.commit, + database.rollback, + database.is_transaction_active, + database._transaction_stack_mark, + ): + with pytest.raises(ArcadeDBError, match="Database is closed"): + call() + reopened = arcadedb.open_database(temp_db_path) + try: + assert reopened.is_transaction_active() is False + assert _count(reopened) == 0 + finally: + reopened.drop() From 452e93c4178311fbf0eccb3301b5e1e099ee0e7c Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 03:00:20 +0900 Subject: [PATCH 06/18] Native transactions: arcadedb_n_tx returns the state as one integer, not a JSON text (refs #315) A begin, a commit, a rollback and a mark each crossed into the library and got back a malloc'd JSON text of about 60 bytes that Python read with strlen, a copy into a str and arcadedb_free, and mostly never looked at. with db.transaction() is three crossings (the mark, begin, commit), so twelve foreign calls and three strings built, copied and freed. arcadedb_n_tx now returns a long: bit 0 active, bits 1 to 31 depth, bits 32 to 62 the low 31 bits of the commit count, -1 for an error. It is the way arcadedb_n_cmd returns a cursor id: the error text is kept per thread and read with arcadedb_n_last_error, built by the same errorJson as before, so the typed exception and its message do not change. Each crossing is one foreign call, and with db.transaction() is three. The Python side reads the fields of the integer inline, so the mark, the begin and rollback states the unwind and run_in_transaction read, and is_transaction_active no longer decode anything. Not measured: the speed claim waits for a timed A/B of a candidate wheel against 26.11.1.post2. Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/ArcadeNative.java | 24 ++++-- src/arcadedb/_native/database.py | 80 ++++++++----------- src/arcadedb/_native/lib.py | 17 +++- 3 files changed, 66 insertions(+), 55 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index 5ccd0c3..2ec6c1f 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -1,9 +1,10 @@ /* * SPDX-License-Identifier: Apache-2.0 * - * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Every function returns a + * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Most functions return a * malloc'd UTF-8 JSON text that the caller releases with arcadedb_free: {"ok":true,...} or * {"ok":false,"error":msg,"cls":java class,"chain":[[class,msg],..],"retry":"need_retry"|"duplicated_key"|null,"retry_ms":n}. + * arcadedb_n_cmd and arcadedb_n_tx return a long (-1 for an error, whose text in that form arcadedb_n_last_error returns). * Databases, result sets and servers are long handles. Any thread attached to the isolate may call any function; a * transaction belongs to the (Java) thread that began it, as in the embedded API. */ @@ -184,9 +185,15 @@ public static CCharPointer info(final IsolateThread thread, final long h) { } } - /** op 0 begin, 1 commit, 2 rollback, 3 mark. Every reply carries {"active":b,"depth":n,"commits":n}. */ + /** + * op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one non-negative long: bit 0 is + * isTransactionActive, bits 1 to 31 the nesting depth, bits 32 to 62 the low 31 bits of the commit count of the thread's + * transaction context (callers compare two counts for equality). -1 on error: the error text is then read with + * arcadedb_n_last_error on the same thread, as for arcadedb_n_cmd. No reply text, so no malloc, no arcadedb_free and no JSON + * decode in the caller. + */ @CEntryPoint(name = "arcadedb_n_tx") - public static CCharPointer tx(final IsolateThread thread, final long h, final int op) { + public static long tx(final IsolateThread thread, final long h, final int op) { try { final Database db = db(h); switch (op) { @@ -196,14 +203,15 @@ public static CCharPointer tx(final IsolateThread thread, final long h, final in default -> { } } - return out(txState(db)); + return txState(db); } catch (final Throwable t) { - return err(t); + LAST_ERROR.set(errorJson(t)); + return -1; } } - private static String txState(final Database db) { - int depth = 0; + private static long txState(final Database db) { + long depth = 0; long commits = 0; if (db instanceof LocalDatabase local) depth = local.getNestedTransactions(); @@ -212,7 +220,7 @@ private static String txState(final Database db) { if (ctx != null) commits = ctx.getCommitCount(); } - return "{\"ok\":true,\"active\":" + db.isTransactionActive() + ",\"depth\":" + depth + ",\"commits\":" + commits + "}"; + return (db.isTransactionActive() ? 1L : 0L) | ((depth & 0x7FFFFFFFL) << 1) | ((commits & 0x7FFFFFFFL) << 32); } // ---- statements diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index c02ac66..fc9ae74 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -74,12 +74,10 @@ def _call(prefix: str, name: str, *args) -> dict: return reply -# Every successful ``arcadedb_n_tx`` reply is ``{"ok":true,"active":,"depth":,"commits":}`` (``ArcadeNative.txState``). A caller -# that reads none of the state (begin, commit, rollback), or only ``active``, tells success and that bool from the text it already holds -# instead of decoding a JSON object. An error reply is ``{"ok":false,...}``: it fails the prefix test and is decoded, as before. -_TX_OK = '{"ok":true,' -_TX_ACTIVE = '{"ok":true,"active":true,' -_TX_IDLE = '{"ok":true,"active":false,' +# ``arcadedb_n_tx`` answers with the transaction state after the call as one int (``ArcadeNative.tx``, no reply text to copy, free or +# decode): bit 0 is ``active``, bits 1 to 31 the nesting ``depth``, bits 32 to 62 the ``commits`` of the transaction context (its low 31 +# bits: callers compare two of them for equality). Callers read the fields inline: ``state & 1``, ``(state >> 1) & 0x7FFFFFFF``, +# ``state >> 32``. -1 is an error: its text, the ``{"ok":false,...}`` reply of every other entry point, is ``lib.last_error()``. def _first_row(database, reply: dict): @@ -793,73 +791,63 @@ def _command_first_row(self, language: str, command: str, *args) -> NativeResult # -- transactions - def _tx_text(self, op: int, prefix: str) -> str: - """One ``arcadedb_n_tx`` crossing: the reply text of a success (not decoded), an ArcadeDBError for a failure.""" + def _tx_state(self, op: int, prefix: str) -> int: + """One ``arcadedb_n_tx`` crossing, one foreign call: the transaction state after the operation as an int (its layout is + described at the top of this module), an ArcadeDBError for a failure. + """ if self._closed: raise ArcadeDBError("Database is closed") if (op == 1 or op == 2) and lib.CLOSE_BATCH: lib.flush_closes() - text = lib.call("arcadedb_n_tx", self._h, op) - if text.startswith(_TX_OK): - return text - reply = json.loads(text) - if not reply.get("ok"): - raise _error(prefix, reply) - return text - - def _tx(self, op: int, prefix: str) -> dict: - """The decoded state reply: for a caller that reads ``depth`` or ``commits``.""" - return json.loads(self._tx_text(op, prefix)) + state = lib.tx(self._h, op) + if state < 0: + raise _error(prefix, json.loads(lib.last_error())) + return state def begin(self): - self._tx_text(0, "Failed to begin transaction") + self._tx_state(0, "Failed to begin transaction") - def _begin_state(self) -> dict: - """begin(), and the state right after it: the reply of ``arcadedb_n_tx`` carries ``active``, ``depth`` and ``commits``.""" - return self._tx(0, "Failed to begin transaction") + def _begin_state(self) -> int: + """begin(), and the state right after it: the int of ``arcadedb_n_tx`` carries ``active``, ``depth`` and ``commits``.""" + return self._tx_state(0, "Failed to begin transaction") def commit(self): - self._tx_text(1, "Failed to commit transaction") + self._tx_state(1, "Failed to commit transaction") def rollback(self): - self._tx_text(2, "Failed to rollback transaction") + self._tx_state(2, "Failed to rollback transaction") - def _rollback_state(self) -> dict: - """rollback(), and the state right after it (the reply carries it, as begin's does): the unwind reads it instead of asking.""" - return self._tx(2, "Failed to rollback transaction") + def _rollback_state(self) -> int: + """rollback(), and the state right after it (the int carries it, as begin's does): the unwind reads it instead of asking.""" + return self._tx_state(2, "Failed to rollback transaction") def transaction(self) -> TransactionContext: return TransactionContext(self) def is_transaction_active(self) -> bool: - text = self._tx_text(3, "Failed to check transaction status") - if text.startswith(_TX_ACTIVE): - return True - if text.startswith(_TX_IDLE): - return False - return json.loads(text)["active"] + return bool(self._tx_state(3, "Failed to check transaction status") & 1) def _transaction_stack_mark(self): - state = self._tx(3, "Failed to read the transaction state") - return state["active"], state["depth"] + state = self._tx_state(3, "Failed to read the transaction state") + return bool(state & 1), (state >> 1) & 0x7FFFFFFF def _transaction_commit_mark(self): """The commit count of the current transaction (the witness that a BATCH boundary published part of its work).""" - return self._tx(3, "Failed to read the transaction state")["commits"] + return self._tx_state(3, "Failed to read the transaction state") >> 32 def _unwind_opened_since(self, mark): """Roll back every level opened since ``mark``, and no other; returns ``(state, first_error)``. - ``state`` is the transaction state after the last crossing (every rollback reply carries it, so no read sits between two + ``state`` is the transaction state int after the last crossing (every rollback carries it, so no read sits between two steps), or None when that crossing failed. A level the caller held at the mark is theirs, as in the JPype ``Database``. """ was_active, depth_before = mark first_error = None - state = self._tx(3, "Failed to read the transaction state") - for _ in range(state["depth"] - depth_before): + state = self._tx_state(3, "Failed to read the transaction state") + for _ in range(((state >> 1) & 0x7FFFFFFF) - depth_before): if state is None: - state = self._tx(3, "Failed to check transaction status") - if not state["active"]: + state = self._tx_state(3, "Failed to check transaction status") + if not state & 1: break try: state = self._rollback_state() @@ -868,8 +856,8 @@ def _unwind_opened_since(self, mark): first_error = first_error or e if not was_active: if state is None: - state = self._tx(3, "Failed to check transaction status") - if state["active"]: + state = self._tx_state(3, "Failed to check transaction status") + if state & 1: try: state = self._rollback_state() except Exception as e: @@ -893,7 +881,7 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005): mark = self._transaction_stack_mark() commits_at_start = None try: - commits_at_start = self._begin_state()["commits"] + commits_at_start = self._begin_state() >> 32 result = fn() self.commit() return result @@ -919,7 +907,7 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005): # retry, as before); the raise sits outside that try, or its own except would swallow it. try: commits_now = ( - state["commits"] + state >> 32 if state is not None else self._transaction_commit_mark() ) diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 394bd32..ebff0d4 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -259,6 +259,9 @@ class _Params(ctypes.Structure): "arcadedb_n_set_upcall": [P, P], "arcadedb_free": [P, P], } + # the entry points that answer with a number, not with the address of a reply text: a cursor id (arcadedb_n_cmd) and the + # transaction state (arcadedb_n_tx); -1 is an error for both, its text is read with arcadedb_n_last_error + returns_long = ("arcadedb_n_cmd", "arcadedb_n_tx") for name, argtypes in sigs.items(): fn = getattr(lib, name, None) if fn is None: # a library built before this entry point existed @@ -272,7 +275,7 @@ class _Params(ctypes.Structure): "arcadedb_n_rs_close_many", "arcadedb_n_set_upcall", ) - else ctypes.c_longlong if name == "arcadedb_n_cmd" else ctypes.c_void_p + else ctypes.c_longlong if name in returns_long else ctypes.c_void_p ) global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN, GB_BIN, TS_BIN if not hasattr( @@ -488,7 +491,19 @@ def cmd(handle, flags, language, command, params, buf, count, first) -> int: ) +def tx(handle, op) -> int: + """``arcadedb_n_tx``: op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one int (bit 0 + ``active``, bits 1 to 31 ``depth``, bits 32 to 62 ``commits``, as ``ArcadeNative.tx`` writes it), or -1 for an error (read it + with :func:`last_error`). One foreign call: no reply text to copy, free or decode. + """ + lib = _lib or _load() + t = _local.__dict__.get("thread") + th = t.handle if t is not None else _thread_handle() + return lib.arcadedb_n_tx(th, handle, op) + + def last_error() -> str: + """The error of the last failed :func:`cmd` or :func:`tx` on this thread, as the JSON text of an error reply.""" return call("arcadedb_n_last_error") From 3db17be8b4b0f1b0281b5fe910a1ef5e02b287ee Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 02:59:46 +0900 Subject: [PATCH 07/18] Test: save() of a saved record with auto-transaction on spends two crossings, not three (refs #317) The transaction check of a saved record's save() cannot refuse while auto-transaction is on, yet it still crossed into the library (arcadedb_n_tx) before the SELECT and the UPDATE. These tests pin two crossings there, bare and inside a transaction, with and without the read-back of the row, and pin what must not move: the check is back the moment auto-transaction is off (first, before any read), a stale record is refused under auto-transaction too, and the stored values and the next save's baseline are those of an update inside a transaction for NULL, float, text, date-time, date, Decimal, nested values, a float32 array and a value set unchanged. The two-crossing tests fail on the code before the change (three crossings); the rest pass on both. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_one_row_crossings.py | 109 +++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/tests/test_native_one_row_crossings.py b/tests/test_native_one_row_crossings.py index e65e3e9..d24cde4 100644 --- a/tests/test_native_one_row_crossings.py +++ b/tests/test_native_one_row_crossings.py @@ -7,6 +7,11 @@ reads the stored record first: the engine's reply to an UPDATE comes after the write, and the check must refuse before writing anything inside the caller's transaction, so it cannot be read from the update's own result. +With auto-transaction on, the transaction check cannot refuse (the engine opens a transaction for a bare write), so ``save()`` does not +ask for the state: two crossings, the SELECT and the UPDATE. With it off (the engine default) the check still comes first, outside a +transaction and inside one. The engine runs the before-update callbacks and the property validation before it looks for a transaction, +and an UPDATE of a record that is gone raises nothing, so the engine's own refusal cannot stand in for the check. + The counts are crossings (the ctypes doors ``cmd``, ``call``, ``call_bin``, ``exec_query``, ``tx`` and the batched cursor close), not seconds: deterministic. The oracle for the values and the errors is the lazy form itself (``db.command(...).first()``), kept for every other caller. """ @@ -192,6 +197,110 @@ def test_an_update_that_reads_its_row_back_is_three_crossings(db, crossings): assert db.lookup_by_rid(rid).get("brand_new") == "value" +@pytest.mark.parametrize("in_transaction", [False, True], ids=["bare", "in_tx"]) +def test_updating_a_saved_record_with_auto_transaction_is_two_crossings( + db, crossings, in_transaction +): + rid = _saved(db, a=1, name="x") + db.set_auto_transaction(True) + record = db.lookup_by_rid(rid).modify() + record.set("a", 2) + with contextlib.ExitStack() as stack: + if in_transaction: + stack.enter_context(db.transaction()) + crossings.reset() + record.save() + # the SELECT of the stale-record check and the UPDATE: the transaction check cannot refuse, so it is not asked (was three) + assert crossings.names == ["exec_query", "exec_query"] + assert db.lookup_by_rid(rid).get("a") == 2 + + +@pytest.mark.parametrize("in_transaction", [False, True], ids=["bare", "in_tx"]) +def test_an_update_that_reads_its_row_back_with_auto_transaction_is_two_crossings( + db, crossings, in_transaction +): + rid = _saved(db, a=1) + db.set_auto_transaction(True) + record = db.lookup_by_rid(rid).modify() + record.set("brand_new", "value") # a new property is read back (RETURN AFTER) + with contextlib.ExitStack() as stack: + if in_transaction: + stack.enter_context(db.transaction()) + crossings.reset() + record.save() + assert crossings.names == ["exec_query", "exec_query"] + assert db.lookup_by_rid(rid).get("brand_new") == "value" + + +def test_with_auto_transaction_off_again_the_transaction_check_comes_back( + db, crossings +): + rid = _saved(db, a=1) + db.set_auto_transaction(True) + db.set_auto_transaction(False) + record = db.lookup_by_rid(rid).modify() + record.set("a", 2) + crossings.reset() + with pytest.raises(ArcadeDBError, match="Transaction not begun"): + record.save() + assert crossings.names == ["arcadedb_n_tx"] # refused before it reads anything + with db.transaction(): + crossings.reset() + record.save() + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert db.lookup_by_rid(rid).get("a") == 2 + + +def test_a_stale_record_is_refused_under_auto_transaction_too(db): + rid = _saved(db, a=1, b=1) + mine = db.lookup_by_rid(rid).modify() + with db.transaction(): + bump_sql = f"UPDATE {rid} SET b = 9" # nosec B608 - a record id this test read + db.command("sql", bump_sql).close() + db.set_auto_transaction(True) + mine.set("a", 2) + with pytest.raises(ArcadeDBError, match="ConcurrentModificationException"): + mine.save() + stored = db.lookup_by_rid(rid) + assert (stored.get("a"), stored.get("b")) == (1, 9) # nothing of mine was written + + +UPDATE_CASES = { + "null": None, + "float": 2.5, + "text": "new", + "date and time": dt.datetime(2026, 11, 9, 12, 30, 15, 123000), + "date": dt.date(2026, 11, 9), + "decimal": Decimal("1.50"), + "list and map": {"l": [1, 2, 3], "m": {"k": [1.5, "v"]}}, + "float array": np.arange(16, dtype=np.float32), + "as read": "old", # the value the record already holds: set() of an unchanged value +} + + +@pytest.mark.parametrize("case", list(UPDATE_CASES)) +def test_an_update_under_auto_transaction_stores_what_one_in_a_transaction_stores( + db, case +): + value = UPDATE_CASES[case] + inside, bare = _saved(db, k="old", n=0), _saved(db, k="old", n=0) + record = db.lookup_by_rid(inside).modify() + record.set("k", value) + with db.transaction(): + record.save() + db.set_auto_transaction(True) + held = db.lookup_by_rid(bare).modify() + held.set("k", value) + held.save() + assert _plain(db.lookup_by_rid(bare).to_dict()) == _plain( + db.lookup_by_rid(inside).to_dict() + ) + # the copy the save left is the stored row: the next save of the same object is no conflict + held.set("n", 1) + held.save() + assert db.lookup_by_rid(bare).get("n") == 1 + + def test_a_stale_record_is_still_refused_before_anything_is_written(db): rid = _saved(db, a=1, b=1) mine = db.lookup_by_rid(rid).modify() From e8e0c450f99c9fe88a892686710a2aef4a0c8d38 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 03:00:30 +0900 Subject: [PATCH 08/18] Native: save() of a saved record does not ask for the transaction state when auto-transaction is on (refs #317) _save_update asked is_transaction_active() (an arcadedb_n_tx crossing) first and only then looked at the auto-transaction flag, so the answer was thrown away whenever the flag was on. The operands are swapped: the flag is a Python attribute, the state is a crossing. With auto-transaction on, an update of a saved record is two crossings (the SELECT of the stale-record check and the UPDATE), not three. With it off nothing changes: the check still comes first, outside a transaction and inside one. The check is not replaced by the engine's refusal for the default mode: the engine runs the before-update callbacks and the property validation before it looks for a transaction (read in LocalDatabase.updateRecord of the 26.12.1-SNAPSHOT source), and an UPDATE of a record that is gone raises nothing, so a save outside a transaction would run callbacks, or pass silently, instead of raising "Transaction not begun" before it reads. A Python-side record of the transaction state is not exact either: a SQL COMMIT or ROLLBACK statement ends the transaction without passing the wheel's transaction calls. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/records.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/arcadedb/_native/records.py b/src/arcadedb/_native/records.py index c42d973..9bb732d 100644 --- a/src/arcadedb/_native/records.py +++ b/src/arcadedb/_native/records.py @@ -359,8 +359,11 @@ def _save_update(self, db) -> None: Like every write (and like the JVM build), it needs an active transaction while auto-transaction is off, the engine default: outside one it raises ArcadeDBError ("Transaction not begun") before it reads or writes anything. This method opens no - transaction of its own.""" - if not db.is_transaction_active() and not db._auto_transaction: + transaction of its own. With auto-transaction on the check cannot refuse, so the state is not asked for (one crossing less). + The engine's own refusal cannot stand in for the check with it off: the engine runs the before-update callbacks and the + validation before it looks for a transaction, and an UPDATE of a record that is gone raises nothing. + """ + if not db._auto_transaction and not db.is_transaction_active(): raise ArcadeDBError( "Failed to save record: com.arcadedb.exception.TransactionException: Transaction not begun " "(wrap the write in `with db.transaction():` or call begin() first)" From 55b427a51d2e0b7a6c8c4a95c97c06ded38fb10d Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 03:54:08 +0900 Subject: [PATCH 09/18] Tests: the nesting depth the native mark reports is the engine's, and no failed assert leaves a transaction open (refs #315) CI of the first push (run 43478436168) failed two tests on a wrong assumption of mine: I expected the outer begin to raise the depth by one. LocalDatabase.getNestedTransactions() is the size of the thread's stack of transaction contexts (DatabaseContext.DatabaseContextTL.transactions). The stack always holds the thread's base context once the thread has used the database, so the depth is 1 while idle. The outer begin reuses that base context (depth 1, active). Only a begin while a transaction is active pushes a context (2), the commit or rollback of that level pops it (1, still active), and the outermost end leaves the base context on the stack (1, idle). popIfNotLastTransaction never removes the last one. The native tx entry point returns exactly that number, as the old JSON reply did, so the change is right and the two expectations were not. The fixture now rolls back whatever a test left open before it drops the database, so a failed assert is one failure and not also a "Cannot drop the database in transaction" teardown error. The close-and-reopen test does the same in a finally. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_tx_crossings.py | 78 ++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/tests/test_native_tx_crossings.py b/tests/test_native_tx_crossings.py index a218d61..78a1635 100644 --- a/tests/test_native_tx_crossings.py +++ b/tests/test_native_tx_crossings.py @@ -15,6 +15,7 @@ scripts/check_test_skips.py); it runs in the native parity run. """ +import contextlib import json import pytest @@ -113,8 +114,18 @@ def db(temp_db_path): ): # the thread's transaction context exists from here on, as it does in a running application pass lib.flush_closes() # a cursor closed earlier is not part of a call measured below - yield database - database.drop() + try: + yield database + finally: + # a test that fails with a transaction open (a begin without its end, a level a direct lib.tx call pushed) must not turn into + # a teardown error as well: the engine refuses to drop a database that has a transaction open on the thread + try: + for _ in range(16): + if not database.is_transaction_active(): + break + database.rollback() + finally: + database.drop() @pytest.fixture @@ -255,14 +266,19 @@ def test_a_nested_block_that_succeeds_commits_with_the_caller(db): assert _count(db) == 2 -def test_nested_with_blocks_push_and_pop_one_level_each(db): +def test_nested_with_blocks_push_one_level_for_the_inner_block_only(db): + # The depth is the size of the thread's stack of transaction contexts (LocalDatabase.getNestedTransactions). The stack always holds + # the thread's base context once the thread has used the database (the fixture did), so it is 1 while idle. The outer begin + # reuses that base context (depth stays 1, active); only a begin while a transaction is active pushes a context (2), and the commit + # of that inner level pops it (1, still active). The outer commit ends the base context and leaves it on the stack (1, idle). base = db._transaction_stack_mark()[1] + assert db._transaction_stack_mark() == (False, 1) with db.transaction(): - assert db._transaction_stack_mark() == (True, base + 1) + assert db._transaction_stack_mark() == (True, base) with db.transaction(): - assert db._transaction_stack_mark() == (True, base + 2) + assert db._transaction_stack_mark() == (True, base + 1) db.command("sql", "INSERT INTO R SET k = 1") - assert db._transaction_stack_mark() == (True, base + 1) + assert db._transaction_stack_mark() == (True, base) assert db._transaction_stack_mark() == (False, base) assert _count(db) == 1 @@ -366,6 +382,7 @@ def test_with_transaction_decodes_nothing(db, decodes): def test_a_state_still_carries_active_depth_and_commits(db): db.begin() db.begin() + # the first begin reuses the thread's base context (depth 1), the second pushes one (2) assert db._transaction_stack_mark() == (True, 2) db.rollback() db.rollback() @@ -382,14 +399,15 @@ def test_the_state_comes_back_as_one_integer(db): assert type(idle) is int and idle >= 0 active, base, _ = _fields(idle) assert active is False + # the first begin reuses the thread's base context (the depth stays), a begin inside it pushes one more (see the nested with test) one = lib.tx(handle, 0) assert type(one) is int and one >= 0 - assert _fields(one)[:2] == (True, base + 1) + assert _fields(one)[:2] == (True, base) two = lib.tx(handle, 0) - assert _fields(two)[:2] == (True, base + 2) + assert _fields(two)[:2] == (True, base + 1) assert _fields(two)[2] == _fields(lib.tx(handle, 3))[2] # a mark reads the count - assert _fields(lib.tx(handle, 2))[:2] == (True, base + 1) - assert _fields(lib.tx(handle, 2))[:2] == (False, base) + assert _fields(lib.tx(handle, 2))[:2] == (True, base) # the inner level is gone + assert _fields(lib.tx(handle, 2))[:2] == (False, base) # the outer one ended # -- errors: the same typed exception with the same message @@ -462,23 +480,31 @@ def test_a_failure_reply_is_read_from_last_error_into_the_same_error(db, monkeyp def test_a_transaction_left_open_is_gone_after_a_close_and_reopen(temp_db_path): database = arcadedb.create_database(temp_db_path) - database.command("sql", "CREATE DOCUMENT TYPE R") - database.begin() - database.command("sql", "INSERT INTO R SET k = 1") - assert database.is_transaction_active() is True - database.close() - for call in ( - database.begin, - database.commit, - database.rollback, - database.is_transaction_active, - database._transaction_stack_mark, - ): - with pytest.raises(ArcadeDBError, match="Database is closed"): - call() - reopened = arcadedb.open_database(temp_db_path) + reopened = None try: + database.command("sql", "CREATE DOCUMENT TYPE R") + database.begin() + database.command("sql", "INSERT INTO R SET k = 1") + assert database.is_transaction_active() is True + database.close() + for call in ( + database.begin, + database.commit, + database.rollback, + database.is_transaction_active, + database._transaction_stack_mark, + ): + with pytest.raises(ArcadeDBError, match="Database is closed"): + call() + reopened = arcadedb.open_database(temp_db_path) assert reopened.is_transaction_active() is False assert _count(reopened) == 0 finally: - reopened.drop() + if reopened is not None: + reopened.drop() + else: + # a failure before the reopen: end what is open and drop, ignoring the refusals of a database already closed + with contextlib.suppress(ArcadeDBError): + database.rollback() + with contextlib.suppress(ArcadeDBError): + database.drop() From 1fe1160a0559b5b31c5cfaa12b0730fd155b2529 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 15:33:49 +0900 Subject: [PATCH 10/18] Tests: native GraphBatch.create_vertices sends columns and reads RIDs from an int64 buffer; the stored vertices match the JSON route (refs #388, #315) Stub-library tests (both backends) for the routing, the header, the chunking, the RID text and every shape that keeps the JSON route; a native-only parity test that loads the same rows with the route on and off and compares the stored records, their property order and the RIDs. Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 2 + tests/test_native_graph_batch_frame.py | 223 ++++++++++++++ tests/test_native_graph_batch_frame_pack.py | 305 ++++++++++++++++++++ 3 files changed, 530 insertions(+) create mode 100644 tests/test_native_graph_batch_frame.py create mode 100644 tests/test_native_graph_batch_frame_pack.py diff --git a/tests/conftest.py b/tests/conftest.py index d59db09..7a92406 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,6 +67,8 @@ collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library collect_ignore.append("test_native_ingest_defaults.py") + # create_vertices on the native library: the columnar route (arcadedb_n_gb_colp) stores what the JSON route stores + collect_ignore.append("test_native_graph_batch_frame.py") # the float32 side buffer of the native GraphBatch entry (arcadedb_n_gb_bin), observed on the native library collect_ignore.append("test_native_graph_batch_binary_vectors.py") # the native library's start-up output (#477) can only be checked with the native library loaded diff --git a/tests/test_native_graph_batch_frame.py b/tests/test_native_graph_batch_frame.py new file mode 100644 index 0000000..dc27713 --- /dev/null +++ b/tests/test_native_graph_batch_frame.py @@ -0,0 +1,223 @@ +"""Native GraphBatch.create_vertices: the vertices that the columnar route stores are the ones the JSON route stores (ArcadeData/arcadedb-python #388, rank 7). + +The same rows are loaded into two vertex types of one schema, once with ``lib.GB_COLP`` on (``arcadedb_n_gb_colp``: columns by address, the +RIDs back in an int64 buffer) and once with it off (the JSON text of ``arcadedb_n_gb``), and every stored record is compared: the property +order, the type and the value of each property (``None`` and ``NaN`` included), and the RIDs the call returned (a ``list`` of ``str`` in row +order). A shape the columns cannot carry must come out the same because it takes the JSON route; the entry that each shape crossed is checked +too, so a shape cannot move from one route to the other unseen. Needs the native library, so conftest.py collects this file on the native +backend only; the routing, the header and the RID decode are in test_native_graph_batch_frame_pack.py (both backends, no library). +""" + +import re +from datetime import date, datetime, timedelta, timezone +from decimal import Decimal + +import numpy as np +import pandas as pd +import pytest + +from arcadedb._native import lib + +COLP = "arcadedb_n_gb_colp" + + +@pytest.fixture +def db(temp_db): + for name in ( + "A", + "B", + ): # the same schema twice: A is loaded with the columnar route, B with the JSON route + temp_db.command("sql", f"CREATE VERTEX TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.i LONG") + temp_db.command("sql", f"CREATE PROPERTY {name}.n INTEGER") + temp_db.command("sql", f"CREATE PROPERTY {name}.f FLOAT") + temp_db.command("sql", f"CREATE PROPERTY {name}.v ARRAY_OF_FLOATS") + return temp_db + + +def _vectors(n, dim=6): + x = np.random.default_rng(5).standard_normal((n, dim)).astype(np.float32) + x[0, 0] = np.float32(-0.0) + return x + + +def _rows(n, make): + return [{"i": i, **make(i)} for i in range(n)] + + +SHAPES = { + "scalars": (lambda i: {"s": f"né{i}", "d": i * 0.5, "b": i % 2 == 0}, True), + "big integers": ( + lambda i: {"big": [2**53 + 1, -(2**63), 2**63 - 1, 7][i % 4]}, + True, + ), + "nan inf and negative zero": ( + lambda i: {"x": [float("nan"), float("inf"), float("-inf"), -0.0, 1.5][i % 5]}, + True, + ), + "empty and odd strings": ( + lambda i: {"s": ["", "a\u0000b", "\U0001f600", "x" * 300][i % 4]}, + True, + ), + "nulls in a column": ( + lambda i: {"s": None if i % 3 == 0 else f"s{i}", "d": None if i % 2 else 1.0}, + True, + ), + "only nulls": (lambda i: {"s": None}, True), + "dates and datetimes": ( + lambda i: { + "day": date(2020, 1, 1) + timedelta(days=i), + "at": datetime(2020, 1, 2, 3, 4, 5, 123456 + i), + "utc": datetime(2020, 1, 2, 3, 4, 5, tzinfo=timezone(timedelta(hours=9))), + }, + True, + ), + "decimals and bytes": ( + lambda i: {"dec": Decimal(f"{i}.50"), "raw": bytes([i, 0, 255])}, + True, + ), + "declared integer and float": (lambda i: {"n": i, "f": i * 0.1}, True), + "float32 vectors": (lambda i: {"emb": _vectors(40)[i]}, True), + "declared vectors from lists": (lambda i: {"v": [i * 0.5, i + 0.25, 3.0]}, True), + "declared vectors from float32 arrays": (lambda i: {"v": _vectors(40)[i]}, True), + "float64 vectors": (lambda i: {"emb": _vectors(40)[i].astype(np.float64)}, False), + "nested lists and maps": ( + lambda i: {"l": [i, "x", [1, 2]], "m": {"k": i, "z": None}}, + False, + ), + "numpy scalars": ( + lambda i: {"a": np.int64(i), "b": np.float64(i) / 3, "c": np.bool_(i % 2)}, + False, + ), + "mixed int and float": (lambda i: {"a": i if i % 2 else i + 0.5}, False), + "a set and a tuple": (lambda i: {"s": {i}, "t": (i, 1)}, False), + "a pandas timestamp with nanoseconds": ( + lambda i: { + "t": pd.Timestamp("2020-01-01 00:00:00.000000123") + pd.Timedelta(i, "ns") + }, + False, + ), +} + + +def _dump(db, type_name): + """Every stored record as (property, type, repr) in its stored order, sorted by the row index.""" + out = {} + sql = f"SELECT FROM {type_name}" # nosec B608 - test-owned name + for row in db.query("sql", sql).to_list(): + props = [ + (k, type(v).__name__, repr(v)) + for k, v in row.items() + if not str(k).startswith("@") + ] + out[row["i"]] = props + return [out[i] for i in sorted(out)] + + +def _load(db, monkeypatch, type_name, rows, columnar): + calls = [] + real = lib.call + + def spy(name, *args): + calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", spy) + monkeypatch.setattr(lib, "GB_COLP", columnar) + try: + with db.graph_batch() as gb: + rids = gb.create_vertices(type_name, rows) + finally: + monkeypatch.undo() + return rids, calls + + +@pytest.mark.parametrize("shape", sorted(SHAPES)) +@pytest.mark.parametrize("n", [1, 40]) +def test_the_columnar_route_stores_what_the_json_route_stores( + db, monkeypatch, shape, n +): + make, carried = SHAPES[shape] + rows = _rows(n, make) + rids_on, calls_on = _load(db, monkeypatch, "A", rows, True) + rids_off, calls_off = _load(db, monkeypatch, "B", rows, False) + assert (COLP in calls_on) is carried, (shape, calls_on) + assert COLP not in calls_off + assert _dump(db, "A") == _dump(db, "B") + for rids in (rids_on, rids_off): + assert type(rids) is list and len(rids) == n + assert all(type(r) is str and re.fullmatch(r"#\d+:\d+", r) for r in rids) + + +def test_the_rids_are_the_vertices_in_row_order_across_chunks(db, monkeypatch): + rows = [ + {"i": i, "s": f"p{i}"} for i in range(2500) + ] # the default chunk is 1000 rows: three crossings + rids, calls = _load(db, monkeypatch, "A", rows, True) + assert calls.count(COLP) == 3 + by_rid = { + str(r["rid"]): r["i"] + for r in db.query("sql", "SELECT i, @rid AS rid FROM A").to_list() + } + assert [by_rid[r] for r in rids] == list(range(2500)) + assert len(set(rids)) == 2500 + + +@pytest.mark.parametrize( + "rows", + [ + [ + {"i": 0, "a": 1, "b": 2}, + {"i": 1, "b": 4, "a": 3}, + ], # the same keys in another order + [{"i": 0, "a": 1, "b": 2}, {"i": 1, "a": 3}], # a missing key + [{"i": 0, "a": None}, {"i": 1}], # None against a missing key + ], +) +def test_rows_with_other_keys_keep_the_json_route_and_the_stored_order( + db, monkeypatch, rows +): + _, calls_on = _load(db, monkeypatch, "A", rows, True) + _load(db, monkeypatch, "B", rows, False) + assert COLP not in calls_on + assert _dump(db, "A") == _dump(db, "B") + + +def test_a_none_row_among_dict_rows_is_an_empty_vertex_on_both_routes(db, monkeypatch): + rows = [{"i": 0, "a": 1}, None, {"i": 2, "a": 3}] + rids_on, calls_on = _load(db, monkeypatch, "A", rows, True) + rids_off, _ = _load(db, monkeypatch, "B", rows, False) + assert COLP not in calls_on and len(rids_on) == len(rids_off) == 3 + for type_name in ("A", "B"): + sql = f"SELECT count(*) AS c FROM {type_name}" # nosec B608 - test-owned name + assert db.query("sql", sql).first().get("c") == 3 + + +def test_an_integer_past_64_bits_is_refused_by_both_routes_alike(db, monkeypatch): + rows = [{"i": 0, "a": 2**70}] + errors = [] + for type_name, columnar in (("A", True), ("B", False)): + with pytest.raises(Exception) as raised: + _load(db, monkeypatch, type_name, rows, columnar) + errors.append(str(raised.value)) + assert errors[0] == errors[1] + assert db.query("sql", "SELECT count(*) AS c FROM A").first().get("c") == 0 + + +def test_an_empty_batch_and_a_batch_of_one(db, monkeypatch): + rids, calls = _load(db, monkeypatch, "A", [], True) + assert rids == [] and COLP not in calls + rids, calls = _load(db, monkeypatch, "A", [{"i": 0, "s": "one"}], True) + assert COLP in calls and len(rids) == 1 and rids[0].startswith("#") + + +def test_a_failing_chunk_stores_nothing_of_itself_and_keeps_the_chunks_before_it( + db, monkeypatch +): + db.command("sql", "CREATE INDEX ON A (i) UNIQUE") + rows = [{"i": i, "s": "x"} for i in range(1000)] + [ + {"i": i, "s": "y"} for i in range(5) + ] # the 2nd chunk repeats keys + with pytest.raises(Exception): + _load(db, monkeypatch, "A", rows, True) + assert db.query("sql", "SELECT count(*) AS c FROM A").first().get("c") == 1000 diff --git a/tests/test_native_graph_batch_frame_pack.py b/tests/test_native_graph_batch_frame_pack.py new file mode 100644 index 0000000..7069647 --- /dev/null +++ b/tests/test_native_graph_batch_frame_pack.py @@ -0,0 +1,305 @@ +"""Native GraphBatch.create_vertices sends a chunk of rows as columns and gets the RIDs back in an int64 buffer (ArcadeData/arcadedb-python #388, rank 7). + +The route is ``bulk.create_vertices_columns``: ``columnar.rows_to_columns`` and ``columnar.pack`` lay the rows out as the columns of +``arcadedb_n_insert_colp``, one crossing of ``arcadedb_n_gb_colp`` per chunk, and the library writes ``(bucket, position)`` as two int64 per +row into a buffer that Python hands over; no JSON text goes out and none comes back. A chunk the columns cannot carry (rows with other keys or +the same keys in another order, a nested list or map, a column of mixed types, a set, a NaT, a numpy scalar, an integer past 64 bits, a lone +surrogate, an empty chunk) keeps the JSON route it had, so nothing is lost or changed. + +No native library is needed here: ``lib.call`` is replaced by a stand-in that records the crossing, reads the addresses it is given and +writes RIDs into the buffer, as the library does. What the library stores is compared between the two routes in +``test_native_graph_batch_frame.py`` (native backend only). +""" + +import ctypes +import json +from datetime import date, datetime +from decimal import Decimal +from types import SimpleNamespace + +import numpy as np +import pytest + +from arcadedb._native import bulk, codec, columnar, extras, lib + +COLP = "arcadedb_n_gb_colp" +OLD = ("arcadedb_n_gb", "arcadedb_n_gb_bin") +BASE = 2**40 # a position that needs more than 32 bits + + +class _Db: + def is_transaction_active(self): + return False + + def query(self, language, text): # the schema read of columnar.Declared + self.schema_reads = getattr(self, "schema_reads", 0) + 1 + rows = [ + {"name": "P", "properties": [{"name": "n32", "type": "INTEGER"}]}, + ] + return SimpleNamespace(to_list=lambda: rows) + + +class _Library: + """Stands in for ``lib.call``: records every GraphBatch crossing and answers as the library does.""" + + def __init__(self, monkeypatch): + self.crossings, self.columns, self.next = [], [], 0 + monkeypatch.setattr(lib, "call", self.call) + monkeypatch.setattr( + lib, "_lib", SimpleNamespace(arcadedb_n_insert_colp=1, arcadedb_n_gb_colp=1) + ) + monkeypatch.setattr(lib, "GB_COLP", True) + monkeypatch.setattr(lib, "GB_BIN", True) + monkeypatch.setattr(bulk, "_ENABLED", True) + + def call(self, name, *args): + if name == COLP: + _gb, type_name, header, spec, nspec, out, slots = args + head = json.loads(header) + n = head["n"] + assert slots == 2 * n and nspec == 6 * len(head["cols"]) + self.crossings.append((name, type_name.decode(), head)) + self.columns.append( + [ + ctypes.string_at(spec[6 * c], spec[6 * c + 1]) + for c in range(len(head["cols"])) + ] + ) + buf = (ctypes.c_int64 * slots).from_address(out) + for i in range( + n + ): # row i is #12:(BASE + next + i), as the engine hands them out + buf[2 * i], buf[2 * i + 1] = 12, BASE + self.next + i + self.next += n + return json.dumps({"ok": True, "n": n}) + assert name in OLD, name + rows = json.loads(args[2])["rows"] + rows = len(rows) if isinstance(rows, list) else rows + rids = [f"#1:{self.next + i}" for i in range(rows)] + self.crossings.append((name, args[1].decode(), None)) + self.next += rows + return json.dumps({"ok": True, "rids": rids}) + + def entries(self): + return [c[0] for c in self.crossings if c[0] in (COLP, *OLD)] + + +@pytest.fixture +def library(monkeypatch): + return _Library(monkeypatch) + + +def _batch(chunk=100): + return extras.NativeGraphBatch(_Db(), 7, commit_every=chunk) + + +def _expect_columns(library, rows, chunk=100): + gb = _batch(chunk) + rids = gb.create_vertices("P", rows) + assert set(library.entries()) == {COLP}, library.entries() + return rids + + +def test_plain_rows_are_one_columnar_crossing_and_the_rids_come_back_as_text(library): + rows = [ + {"id": i, "name": f"né{i}", "score": i * 0.5, "flag": i % 2 == 0} + for i in range(30) + ] + rids = _expect_columns(library, rows) + assert library.entries() == [COLP] + head = library.crossings[0][2] + assert head["n"] == 30 + assert head["cols"] == [ + ["id", "q", 0], + ["name", "s", 0], + ["score", "d", 0], + ["flag", "b", 0], + ] + assert library.columns[0][0] == np.arange(30, dtype=np.int64).tobytes() + assert type(rids) is list and all(type(r) is str for r in rids) + assert rids == [f"#12:{BASE + i}" for i in range(30)] # row order, bucket:position + + +def test_the_chunks_of_one_call_are_one_crossing_each_and_the_rids_stay_in_order( + library, +): + rows = [{"id": i} for i in range(250)] + rids = _expect_columns(library, rows, chunk=100) + assert [c[2]["n"] for c in library.crossings] == [100, 100, 50] + assert rids == [f"#12:{BASE + i}" for i in range(250)] + library.crossings.clear() + assert _batch(0)._vertex_chunk == extras.NativeGraphBatch._BULK_CHUNK + _batch(0).create_vertices("P", rows) # commit_every 0: all 250 rows in one chunk + assert [c[2]["n"] for c in library.crossings] == [250] + + +def test_a_batch_of_one_row_and_no_per_row_python_in_the_codec(library, monkeypatch): + seen = {"prep": 0, "default": 0, "dumps": 0} + for key, module, name in (("prep", codec, "_prep"), ("default", codec, "_default")): + real = getattr(module, name) + + def counting(*args, _real=real, _key=key, **kwargs): + seen[_key] += 1 + return _real(*args, **kwargs) + + monkeypatch.setattr(module, name, counting) + assert _expect_columns(library, [{"id": 1}]) == [f"#12:{BASE}"] + rows = [ + {"id": i, "name": f"p{i}", "born": datetime(2020, 1, 2, 3, 4, 5, i)} + for i in range(100) + ] + _expect_columns(library, rows) + assert seen == { + "prep": 0, + "default": 0, + "dumps": 0, + } # the rows were never walked or written as JSON + + +@pytest.mark.parametrize( + "rows, kinds", + [ + ([{"a": 2**53 + 1}, {"a": -(2**63)}, {"a": 2**63 - 1}], "q"), + ([{"a": float("nan")}, {"a": float("inf")}, {"a": -0.0}], "d"), + ([{"a": None}, {"a": 1}, {"a": None}], "q"), + ([{"a": None}, {"a": None}], "s"), + ([{"a": date(2020, 1, 2)}, {"a": date(1969, 12, 31)}], "D"), + ([{"a": datetime(2020, 1, 2, 3, 4, 5, 6)}, {"a": None}], "T"), + ([{"a": Decimal("1.50")}, {"a": Decimal("-0.001")}], "n"), + ([{"a": b"\x00\xff"}, {"a": b""}], "x"), + ([{"a": True}, {"a": False}], "b"), + ([{"a": "x", "b": 1}, {"a": "y", "b": 2}], "s"), + ], +) +def test_every_value_a_column_holds_crosses_as_a_column(library, rows, kinds): + _expect_columns(library, rows) + assert library.crossings[0][2]["cols"][0][1] == kinds + + +def test_float32_vectors_cross_as_one_matrix_column(library): + x = np.random.default_rng(1).standard_normal((12, 8)).astype(np.float32) + rows = [{"id": i, "embedding": x[i]} for i in range(12)] + _expect_columns(library, rows) + cols = library.crossings[0][2]["cols"] + assert cols == [["id", "q", 0], ["embedding", "f", 8]] + assert library.columns[0][1] == x.tobytes() + + +def test_a_declared_integer_is_read_once_for_all_chunks_of_a_call(library): + db = _Db() + gb = extras.NativeGraphBatch(db, 7, commit_every=10) + gb.create_vertices("P", [{"n32": np.int32(i)} for i in range(25)]) + assert library.entries() == [COLP] * 3 + assert db.schema_reads == 1 + assert [c[2]["cols"][0] for c in library.crossings] == [["n32", "i", 0]] * 3 + + +FALLBACKS = { + "other keys": [{"a": 1}, {"a": 2, "b": 3}], + "same keys, another order": [{"a": 1, "b": 2}, {"b": 3, "a": 4}], + "missing key": [{"a": 1, "b": 2}, {"a": 3}], + "a none row among dict rows": [{"a": 1}, None], + "only none rows": [None, None], + "nested list": [{"a": [1, 2]}, {"a": [3]}], + "nested map": [{"a": {"x": 1}}, {"a": {"x": 2}}], + "int and float in one column": [{"a": 1}, {"a": 2.5}], + "bool and int in one column": [{"a": True}, {"a": 2}], + "numpy scalars": [{"a": np.int64(1)}, {"a": np.int64(2)}], + "a set": [{"a": {1, 2}}, {"a": {3}}], + "integer past 64 bits": [{"a": 1}, {"a": 2**70}], + "non-finite decimal": [{"a": Decimal("NaN")}, {"a": Decimal(1)}], + "lone surrogate": [{"a": "ok"}, {"a": "\ud800"}], + "float64 vectors": [{"a": np.zeros(3)}, {"a": np.ones(3)}], + "ragged vectors": [{"a": np.zeros(3, np.float32)}, {"a": np.zeros(4, np.float32)}], + "two-dimensional value": [{"a": np.zeros((2, 2), np.float32)}] * 2, + "list of floats on an undeclared property": [{"a": [0.5, 1.5]}, {"a": [2.5, 3.5]}], +} + + +@pytest.mark.parametrize("shape", sorted(FALLBACKS)) +def test_a_shape_the_columns_cannot_carry_keeps_the_json_route(library, shape): + rows = FALLBACKS[shape] + rids = _batch().create_vertices("P", list(rows)) + assert set(library.entries()) <= set(OLD) and library.entries(), library.entries() + assert len(rids) == len(rows) and all(type(r) is str for r in rids) + + +def test_no_rows_and_a_count_keep_the_json_route(library): + assert _batch().create_vertices("P", []) == [] + assert len(_batch().create_vertices("P", 3)) == 3 + assert set(library.entries()) <= set(OLD) + + +def test_a_chunk_that_cannot_be_carried_does_not_stop_the_others(library): + rows = ( + [{"id": i} for i in range(10)] + + [{"id": 10, "x": 1}] + + [{"id": i} for i in range(11, 20)] + ) + gb = _batch(chunk=10) + rids = gb.create_vertices("P", rows) + assert library.entries() == [COLP, OLD[1] if False else library.entries()[1], COLP] + assert library.entries()[1] in OLD + assert len(rids) == 20 + + +@pytest.mark.parametrize("switch", ["env", "bulk", "library"]) +def test_the_switches_turn_the_route_off(library, monkeypatch, switch): + if switch == "env": + monkeypatch.setattr(lib, "GB_COLP", False) + elif switch == "bulk": + monkeypatch.setattr(bulk, "_ENABLED", False) + else: + monkeypatch.setattr(lib, "_lib", SimpleNamespace(arcadedb_n_insert_colp=1)) + monkeypatch.setattr( + lib, "GB_COLP", False + ) # what lib._load sets for a library without the entry + _batch().create_vertices("P", [{"id": 1}, {"id": 2}]) + assert set(library.entries()) <= set(OLD) and library.entries() + + +def test_a_count_that_disagrees_is_an_error_not_a_short_list(library, monkeypatch): + real = library.call + + def short(name, *args): + if name == COLP: + return json.dumps({"ok": True, "n": 1}) + return real(name, *args) + + monkeypatch.setattr(lib, "call", short) + with pytest.raises(Exception, match="created 1 vertices for 2 rows"): + _batch().create_vertices("P", [{"id": 1}, {"id": 2}]) + + +def test_the_rid_text_is_the_engines_text_for_any_bucket_and_position(): + out = np.array([0, 0, 7, 2**62, 2**31 - 1, 2**31], dtype=np.int64) + assert bulk.rids_of(out) == ["#0:0", f"#7:{2**62}", f"#{2**31 - 1}:{2**31}"] + assert bulk.rids_of(np.empty(0, dtype=np.int64)) == [] + + +def test_same_order_is_a_rule_of_the_row_check_only(): + rows = [{"a": 1, "b": 2}, {"b": 3, "a": 4}] + assert columnar.rows_to_columns(rows) == { + "a": [1, 4], + "b": [2, 3], + } # insert_many's rule: the same keys + assert columnar.rows_to_columns(rows, same_order=True) is None + assert columnar.rows_to_columns(rows[:1] * 3, same_order=True) == { + "a": [1] * 3, + "b": [2] * 3, + } + + +def test_a_pandas_timestamp_with_nanoseconds_is_not_a_column(): + """The JSON route keeps the nanoseconds of a Timestamp (``isoformat``); microsecond ticks cannot, so the chunk keeps that route.""" + import pandas as pd + + declared = columnar.Declared(known={}) + assert ( + columnar.pack({"t": [pd.Timestamp("2020-01-01 00:00:00.000000123")]}, declared) + is None + ) + assert ( + columnar.pack({"t": [pd.Timestamp("2020-01-01 00:00:00.000123")]}, declared) + is not None + ) From c8d2b8726550471cdf00e8cc3dd66e21b5d6734c Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 15:33:49 +0900 Subject: [PATCH 11/18] Native: GraphBatch.create_vertices sends a chunk as columns and gets the RIDs in an int64 buffer (refs #388, #315) arcadedb_n_gb_colp takes the address table of arcadedb_n_insert_colp, builds the rows with the objects and the property order the JSON rows give (BulkOps.Table.cell, BulkOps.vertexRows) and calls the same GraphBatch.createVertices(type, Object[][]); (bucket, position) of each new vertex are written into a buffer of the caller. Rows with other keys or another key order, a nested value, a mixed column, a set, a numpy scalar, an integer past 64 bits, a lone surrogate or a pandas Timestamp with nanoseconds keep the JSON route. ARCADEDB_NATIVE_GB_COLP=0 turns it off for A/B runs. Co-Authored-By: Claude Sonnet 5.5 --- .../java/org/humemai/nativelib/BulkOps.java | 256 +++++++++++------- .../org/humemai/nativelib/NativeExtras.java | 35 +++ src/arcadedb/_native/bulk.py | 45 +++ src/arcadedb/_native/columnar.py | 14 +- src/arcadedb/_native/extras.py | 28 +- src/arcadedb/_native/lib.py | 10 +- 6 files changed, 284 insertions(+), 104 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index c703d87..2371067 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -26,6 +26,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.IntConsumer; @@ -103,73 +104,9 @@ static String insertColumnsReply(final Database db, final String type, final Str @SuppressWarnings("unchecked") private static long insertColumns(final Database db, final String type, final Map h, final long[] spec, final ErrorCallback onError) { - final long nRows = ((Number) h.get("n")).longValue(); - if (nRows < 0 || nRows > Integer.MAX_VALUE) - throw new IllegalArgumentException("row count out of range: " + nRows); - final int n = (int) nRows; final int commitEvery = ((Number) h.get("commit")).intValue(); - final List cols = (List) h.get("cols"); - final int k = cols.size(); - if (spec.length != SLOTS * k) - throw new IllegalArgumentException("address table has " + spec.length + " entries for " + k + " columns, expected " + SLOTS * k); - final String[] names = new String[k]; - final char[] kinds = new char[k]; - final int[] dims = new int[k]; - final long[] data = new long[k]; // addresses: a Word value cannot live in an array or be compared with null - final long[] nulls = new long[k]; // 0 = no nulls - final ByteBuffer[] blobs = new ByteBuffer[k]; - int longest = 0; - for (int c = 0; c < k; c++) { - final List col = (List) cols.get(c); - names[c] = (String) col.get(0); - kinds[c] = ((String) col.get(1)).charAt(0); - dims[c] = ((Number) col.get(2)).intValue(); - final long dataAddr = spec[SLOTS * c], dataBytes = spec[SLOTS * c + 1]; - final long auxAddr = spec[SLOTS * c + 2], auxBytes = spec[SLOTS * c + 3]; - final long nullAddr = spec[SLOTS * c + 4], nullBytes = spec[SLOTS * c + 5]; - final long need = switch (kinds[c]) { - case 'q', 'd' -> 8L * n; - case 'b' -> n; - case 'i', 'e', 'D' -> 4L * n; - case 'T' -> { - if (dims[c] != 1 && dims[c] != 1_000 && dims[c] != 1_000_000 && dims[c] != 1_000_000_000) - throw new IllegalArgumentException("column '" + names[c] + "': a date-time column has 1, 1000, 10^6 or 10^9 ticks per second, not " + dims[c]); - yield 8L * n; - } - case 'f' -> { - if (dims[c] <= 0) - throw new IllegalArgumentException("column '" + names[c] + "': a vector column needs a dimension above 0"); - yield 4L * n * dims[c]; - } - case 's', 'n', 'x' -> 4L * (n + 1L); - default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]); - }; - if (dataBytes < need || (need > 0 && dataAddr == 0)) - throw new IllegalArgumentException("column '" + names[c] + "' has " + dataBytes + " bytes, " + need + " needed"); - data[c] = dataAddr; - if (nullAddr != 0) { - if (nullBytes < (n + 7L) / 8) - throw new IllegalArgumentException("column '" + names[c] + "' has a null bitmap of " + nullBytes + " bytes, " + (n + 7L) / 8 + " needed"); - nulls[c] = nullAddr; - } - if (kinds[c] == 's' || kinds[c] == 'n' || kinds[c] == 'x') { - if (auxBytes < 0 || auxBytes > Integer.MAX_VALUE || (auxBytes > 0 && auxAddr == 0)) - throw new IllegalArgumentException("column '" + names[c] + "': the string bytes are missing or beyond 2 GiB"); - blobs[c] = auxBytes == 0 ? ByteBuffer.allocate(0) : CTypeConversion.asByteBuffer(WordFactory.pointer(auxAddr), (int) auxBytes); - final Pointer offsets = WordFactory.pointer(data[c]); - int prev = offsets.readInt(WordFactory.signed(0)); - if (prev < 0) - throw new IllegalArgumentException("column '" + names[c] + "': negative string offset"); - for (int i = 1; i <= n; i++) { - final int off = offsets.readInt(WordFactory.signed(4L * i)); - if (off < prev || off > auxBytes) - throw new IllegalArgumentException("column '" + names[c] + "': string offsets are not increasing within the " + auxBytes + " bytes (row " + (i - 1) + ")"); - longest = Math.max(longest, off - prev); - prev = off; - } - } - } - final byte[] scratch = new byte[longest]; + final Table table = Table.of(h, spec); + final int n = table.n; final Object schemaType = db.getSchema().getType(type); final boolean vertex = schemaType instanceof VertexType; if (onError != null) { @@ -181,7 +118,7 @@ private static long insertColumns(final Database db, final String type, final Ma try { for (; handed < n; handed++) { final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); - setRow(doc, handed, names, kinds, dims, data, nulls, blobs, scratch); + setRow(doc, handed, table); async.createRecord(doc, null, onError); } } catch (final Throwable t) { @@ -192,7 +129,7 @@ private static long insertColumns(final Database db, final String type, final Ma } writeChunks(db, n, commitEvery, i -> { final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); - setRow(doc, i, names, kinds, dims, data, nulls, blobs, scratch); + setRow(doc, i, table); doc.save(); }); return n; @@ -266,58 +203,181 @@ private static void backoffBeforeRetry(final int attempt) { } } - /** Sets the properties of row i on doc from the columns: the one place that reads a value out of a buffer. */ - private static void setRow(final MutableDocument doc, final int i, final String[] names, final char[] kinds, final int[] dims, - final long[] data, final long[] nulls, final ByteBuffer[] blobs, final byte[] scratch) { - final int k = names.length; - for (int c = 0; c < k; c++) { - final Pointer base = WordFactory.pointer(data[c]); - if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) { - doc.set(names[c], null); - continue; + /** Sets the properties of row i on doc from the columns. */ + private static void setRow(final MutableDocument doc, final int i, final Table t) { + for (int c = 0; c < t.k; c++) + doc.set(t.names[c], t.cell(c, i)); + } + + /** + * The rows of a column table as GraphBatch.createVertices takes them: per row {name, value, name, value, ...}. The names are in the + * iteration order of a java.util.HashMap that took them in column order: the order in which the JSON rows of the same call set their + * properties (Json.rows reads every row, which has the keys of the first row in the same order, into a fresh HashMap), so a vertex is + * stored with the property order it has always had. The values are the objects the JSON rows give (Table.cell). + */ + static Object[][] vertexRows(final Table t) { + final HashMap byName = new HashMap<>(); + for (int c = 0; c < t.k; c++) + byName.put(t.names[c], c); + if (byName.size() != t.k) + throw new IllegalArgumentException("two columns have the same name"); + final int[] order = new int[t.k]; + int at = 0; + for (final Integer c : byName.values()) + order[at++] = c; + final Object[][] rows = new Object[t.n][]; + for (int i = 0; i < t.n; i++) { + final Object[] row = new Object[2 * t.k]; + for (int p = 0; p < t.k; p++) { + row[2 * p] = t.names[order[p]]; + row[2 * p + 1] = t.cell(order[p], i); + } + rows[i] = row; + } + return rows; + } + + /** + * The columns of one call: the header ("n" rows, "cols" [name, kind, dim]) and the address table (SLOTS longs per column), validated + * before anything is built, and the one place that reads a value out of a buffer ({@link #cell}). The memory belongs to the caller for + * the duration of the call. + */ + static final class Table { + final int n, k; + final String[] names; + final char[] kinds; + final int[] dims; + final long[] data; // addresses: a Word value cannot live in an array or be compared with null + final long[] nulls; // 0 = no nulls + final ByteBuffer[] blobs; + final byte[] scratch; + + @SuppressWarnings("unchecked") + static Table of(final Map h, final long[] spec) { + final long nRows = ((Number) h.get("n")).longValue(); + if (nRows < 0 || nRows > Integer.MAX_VALUE) + throw new IllegalArgumentException("row count out of range: " + nRows); + final List cols = (List) h.get("cols"); + if (spec.length != SLOTS * cols.size()) + throw new IllegalArgumentException("address table has " + spec.length + " entries for " + cols.size() + " columns, expected " + SLOTS * cols.size()); + return new Table((int) nRows, cols, spec); + } + + @SuppressWarnings("unchecked") + private Table(final int n, final List cols, final long[] spec) { + this.n = n; + k = cols.size(); + names = new String[k]; + kinds = new char[k]; + dims = new int[k]; + data = new long[k]; + nulls = new long[k]; + blobs = new ByteBuffer[k]; + int longest = 0; + for (int c = 0; c < k; c++) { + final List col = (List) cols.get(c); + names[c] = (String) col.get(0); + kinds[c] = ((String) col.get(1)).charAt(0); + dims[c] = ((Number) col.get(2)).intValue(); + final long dataAddr = spec[SLOTS * c], dataBytes = spec[SLOTS * c + 1]; + final long auxAddr = spec[SLOTS * c + 2], auxBytes = spec[SLOTS * c + 3]; + final long nullAddr = spec[SLOTS * c + 4], nullBytes = spec[SLOTS * c + 5]; + final long need = switch (kinds[c]) { + case 'q', 'd' -> 8L * n; + case 'b' -> n; + case 'i', 'e', 'D' -> 4L * n; + case 'T' -> { + if (dims[c] != 1 && dims[c] != 1_000 && dims[c] != 1_000_000 && dims[c] != 1_000_000_000) + throw new IllegalArgumentException("column '" + names[c] + "': a date-time column has 1, 1000, 10^6 or 10^9 ticks per second, not " + dims[c]); + yield 8L * n; + } + case 'f' -> { + if (dims[c] <= 0) + throw new IllegalArgumentException("column '" + names[c] + "': a vector column needs a dimension above 0"); + yield 4L * n * dims[c]; + } + case 's', 'n', 'x' -> 4L * (n + 1L); + default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]); + }; + if (dataBytes < need || (need > 0 && dataAddr == 0)) + throw new IllegalArgumentException("column '" + names[c] + "' has " + dataBytes + " bytes, " + need + " needed"); + data[c] = dataAddr; + if (nullAddr != 0) { + if (nullBytes < (n + 7L) / 8) + throw new IllegalArgumentException("column '" + names[c] + "' has a null bitmap of " + nullBytes + " bytes, " + (n + 7L) / 8 + " needed"); + nulls[c] = nullAddr; + } + if (kinds[c] == 's' || kinds[c] == 'n' || kinds[c] == 'x') { + if (auxBytes < 0 || auxBytes > Integer.MAX_VALUE || (auxBytes > 0 && auxAddr == 0)) + throw new IllegalArgumentException("column '" + names[c] + "': the string bytes are missing or beyond 2 GiB"); + blobs[c] = auxBytes == 0 ? ByteBuffer.allocate(0) : CTypeConversion.asByteBuffer(WordFactory.pointer(auxAddr), (int) auxBytes); + final Pointer offsets = WordFactory.pointer(data[c]); + int prev = offsets.readInt(WordFactory.signed(0)); + if (prev < 0) + throw new IllegalArgumentException("column '" + names[c] + "': negative string offset"); + for (int i = 1; i <= n; i++) { + final int off = offsets.readInt(WordFactory.signed(4L * i)); + if (off < prev || off > auxBytes) + throw new IllegalArgumentException("column '" + names[c] + "': string offsets are not increasing within the " + auxBytes + " bytes (row " + (i - 1) + ")"); + longest = Math.max(longest, off - prev); + prev = off; + } + } } + scratch = new byte[longest]; + } + + /** The value of column c in row i (null for a null), as the JSON rows give it: Long, Double, Boolean, String, LocalDateTime, LocalDate, BigDecimal, byte[], float[]. */ + Object cell(final int c, final int i) { + if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) + return null; + final Pointer base = WordFactory.pointer(data[c]); switch (kinds[c]) { - case 'q' -> doc.set(names[c], base.readLong(WordFactory.signed(8L * i))); - case 'd' -> doc.set(names[c], base.readDouble(WordFactory.signed(8L * i))); - case 'b' -> doc.set(names[c], base.readByte(WordFactory.signed(i)) != 0); - case 'i' -> doc.set(names[c], base.readInt(WordFactory.signed(4L * i))); - case 'e' -> doc.set(names[c], base.readFloat(WordFactory.signed(4L * i))); - case 'T' -> { + case 'q': + return Long.valueOf(base.readLong(WordFactory.signed(8L * i))); + case 'd': + return Double.valueOf(base.readDouble(WordFactory.signed(8L * i))); + case 'b': + return Boolean.valueOf(base.readByte(WordFactory.signed(i)) != 0); + case 'i': + return Integer.valueOf(base.readInt(WordFactory.signed(4L * i))); + case 'e': + return Float.valueOf(base.readFloat(WordFactory.signed(4L * i))); + case 'T': { final long ticks = base.readLong(WordFactory.signed(8L * i)); final long perSecond = dims[c]; - doc.set(names[c], LocalDateTime.ofEpochSecond(Math.floorDiv(ticks, perSecond), - (int) (Math.floorMod(ticks, perSecond) * (1_000_000_000L / perSecond)), ZoneOffset.UTC)); + return LocalDateTime.ofEpochSecond(Math.floorDiv(ticks, perSecond), + (int) (Math.floorMod(ticks, perSecond) * (1_000_000_000L / perSecond)), ZoneOffset.UTC); } - case 'D' -> doc.set(names[c], LocalDate.ofEpochDay(base.readInt(WordFactory.signed(4L * i)))); - case 'n' -> { + case 'D': + return LocalDate.ofEpochDay(base.readInt(WordFactory.signed(4L * i))); + case 'n': { final int from = base.readInt(WordFactory.signed(4L * i)); final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; blobs[c].get(from, scratch, 0, len); - doc.set(names[c], new BigDecimal(new String(scratch, 0, len, StandardCharsets.US_ASCII))); + return new BigDecimal(new String(scratch, 0, len, StandardCharsets.US_ASCII)); } - case 'x' -> { + case 'x': { final int from = base.readInt(WordFactory.signed(4L * i)); final byte[] bytes = new byte[base.readInt(WordFactory.signed(4L * (i + 1L))) - from]; blobs[c].get(from, bytes, 0, bytes.length); - doc.set(names[c], bytes); + return bytes; } - case 's' -> { + case 's': { final int from = base.readInt(WordFactory.signed(4L * i)); final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; if (len == 0) - doc.set(names[c], ""); - else { - blobs[c].get(from, scratch, 0, len); - doc.set(names[c], new String(scratch, 0, len, StandardCharsets.UTF_8)); - } + return ""; + blobs[c].get(from, scratch, 0, len); + return new String(scratch, 0, len, StandardCharsets.UTF_8); } - default -> { + default: { final int dim = dims[c]; final float[] v = new float[dim]; final long at = 4L * i * dim; for (int j = 0; j < dim; j++) v[j] = base.readFloat(WordFactory.signed(at + 4L * j)); - doc.set(names[c], v); + return v; } } } diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index d230919..61993b5 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -501,6 +501,41 @@ public static CCharPointer gbBin(final IsolateThread thread, final long h, final return gbImpl(h, op, argsJson, buf.rawValue(), bufFloats); } + /** + * create_vertices from columns, by pointer, as arcadedb_n_insert_colp takes them (see BulkOps.Table): header JSON {"n":rows,"cols":[[name, + * kind, dim],..]} and an address table of nspec longs (BulkOps.SLOTS per column). The rows are built from the columns (BulkOps.vertexRows: + * the objects, and the property order, that the JSON rows of arcadedb_n_gb give) and go to the same GraphBatch.createVertices(type, + * Object[][]), so the transaction, the retries and the commit are the engine's and nothing differs from the JSON route but the transport. + * The RIDs do not come back as text: rids points at ridLongs int64 slots of the caller's (at least 2 per row), and row i gets bucket id + * at slot 2i and position at slot 2i + 1. Returns {"ok":true,"n":rows}. The caller keeps every buffer alive until the call returns. + */ + @CEntryPoint(name = "arcadedb_n_gb_colp") + public static CCharPointer gbColp(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header, + final CCharPointer spec, final int nspec, final CCharPointer rids, final long ridLongs) { + try { + final BatchEntry entry = BATCHES.get(h); + if (entry == null) + throw new IllegalArgumentException("unknown GraphBatch " + h); + if (nspec < 0 || nspec % BulkOps.SLOTS != 0) + throw new IllegalArgumentException("address table of " + nspec + " entries is not a multiple of " + BulkOps.SLOTS); + final long[] table = new long[nspec]; + for (int i = 0; i < nspec; i++) + table[i] = ((Pointer) spec).readLong(WordFactory.signed(8L * i)); + final BulkOps.Table columns = BulkOps.Table.of(args(header), table); + if (rids.isNull() || ridLongs < 2L * columns.n) + throw new IllegalArgumentException("the RID buffer holds " + ridLongs + " int64 slots, " + 2L * columns.n + " needed"); + final RID[] created = entry.batch().createVertices(ArcadeNative.str(type), BulkOps.vertexRows(columns)); + final Pointer out = (Pointer) rids; + for (int i = 0; i < created.length; i++) { + out.writeLong(WordFactory.signed(16L * i), created[i].getBucketId()); + out.writeLong(WordFactory.signed(16L * i + 8L), created[i].getPosition()); + } + return ArcadeNative.out("{\"ok\":true,\"n\":" + created.length + "}"); + } catch (final Throwable t) { + return ArcadeNative.err(t); + } + } + @SuppressWarnings("unchecked") private static CCharPointer gbImpl(final long h, final CCharPointer op, final CCharPointer argsJson, final long buf, final long bufFloats) { diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index c16d225..c3bc1dd 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -95,3 +95,48 @@ def insert_rows_columnar(db, type_name, chunk): rows, cols = packed _send(db, type_name, cols, rows, 0, f"Failed to bulk-insert into '{type_name}'") return True + + +def rids_of(out): + """The RID texts (``#bucket:position``, the ``RID.toString`` text) of an int64 array of ``(bucket, position)`` pairs, in row order.""" + pairs = out.reshape(-1, 2) + return list(map("#{}:{}".format, pairs[:, 0].tolist(), pairs[:, 1].tolist())) + + +def create_vertices_columns(gb, type_name, rows, declared, prefix): + """One ``GraphBatch.create_vertices`` chunk (a list of dicts) through ``arcadedb_n_gb_colp``: the rows cross as columns by address and + the RIDs come back in an int64 buffer, one crossing and one transaction, as the JSON route has it. Returns the list of RID texts, or None + when the chunk needs the JSON route (nothing was sent then): rows that do not all have the same keys in the same order, a value no column + holds (a nested list or map, a mixed-type column, a set, a NaT, a numpy scalar, an integer past 64 bits, a lone surrogate) or an empty chunk. + + ``declared``: a :class:`columnar.Declared` of the type, read once by the caller for all chunks of a call. + """ + if not (lib.GB_COLP and _usable()): + return None + columns = columnar.rows_to_columns(rows, same_order=True) + if columns is None: + return None + packed = columnar.pack(columns, declared) + if packed is None: + return None + count, cols = packed + spec = columnar.spec_of(cols) + out = columnar._np.empty(2 * count, dtype=columnar._np.int64) + from .database import _call + + reply = _call( + prefix, + "arcadedb_n_gb_colp", + gb._gb, + lib.b(type_name), + lib.b(columnar.header_of(count, 0, cols)), + spec, + len(spec), + out.ctypes.data, + out.size, + ) # `cols` and `out` hold the memory the addresses point into until the call returns + if reply["n"] != count: + raise ArcadeDBError( + f"{prefix}: the library created {reply['n']} vertices for {count} rows" + ) + return rids_of(out) diff --git a/src/arcadedb/_native/columnar.py b/src/arcadedb/_native/columnar.py index d001b3d..77bcb66 100644 --- a/src/arcadedb/_native/columnar.py +++ b/src/arcadedb/_native/columnar.py @@ -255,6 +255,10 @@ def _list_column(name, values, declared): if issubclass(only, datetime): if only.__name__ == "NaTType": return None # the JSON path stores it as a null + if only is not datetime and any( + getattr(v, "nanosecond", 0) for v in values if v is not None + ): + return None # a pandas Timestamp with nanoseconds: the JSON path keeps them, microsecond ticks cannot try: ticks = [0 if v is None else _epoch_micros(v) for v in values] except (OverflowError, ValueError): @@ -472,8 +476,12 @@ def header_of(rows, commit_every, packed, fid=None): return json.dumps(header) -def rows_to_columns(rows): - """A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key).""" +def rows_to_columns(rows, same_order=False): + """A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key). + + ``same_order``: every row must also list its keys in the order of the first row (a row with the same keys in another order gives + None), for a caller whose stored property order follows the key order of its rows. + """ if not rows: return None try: @@ -482,6 +490,8 @@ def rows_to_columns(rows): return None if set(map(len, rows)) != {len(keys)}: return None + if same_order and not all(map(keys.__eq__, map(list, rows))): + return None return {key: [row[key] for row in rows] for key in keys} except ( KeyError, diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index a0aa517..1130beb 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -20,7 +20,7 @@ from ..exceptions import ArcadeDBError from ..graph_batch import GraphBatch as _BaseGraphBatch from ..type_conversion import take_vector_columns -from . import interrupts, lib, not_yet +from . import bulk, columnar, interrupts, lib, not_yet from .codec import ( _PLAIN, _default, @@ -191,13 +191,32 @@ def _op(self, prefix: str, op: str, **args) -> dict: return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(text)) return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(_enc(args))) - def _create_chunk(self, prefix: str, type_name: str, rows: list) -> list: + def _create_chunk( + self, prefix: str, type_name: str, rows: list, declared=None + ) -> list: """One crossing and one transaction for `rows` (dicts): the RIDs of the new vertices. + Rows with the same keys in the same order, whose values are scalars, strings, dates, decimals, bytes or float32 vectors of one type + per property, cross as columns by address and the RIDs come back in an int64 buffer (``bulk.create_vertices_columns``, no JSON text in + either direction). Any other chunk goes as JSON text, as follows. + A property whose value in every row is a float32 numpy array of one length crosses as one float32 matrix in the side buffer (``take_vector_columns``, ``encode_vertex_args``), with no tag, hook call or Python code per row; every other value is written as :func:`encode_gb_args` writes it. """ + rids = bulk.create_vertices_columns( + self, + type_name, + rows, + ( + declared + if declared is not None + else columnar.Declared(self._db, type_name) + ), + prefix, + ) + if rids is not None: + return rids taken = take_vector_columns(rows, (), undeclared=True) if lib.GB_BIN else None if taken is None: return self._op(prefix, "create_vertices", type=type_name, rows=rows)[ @@ -308,12 +327,15 @@ def create_vertices(self, type_name: str, count_or_properties) -> list: # chunked, as the JPype wrapper does: one crossing, one JSON text and one reply per chunk, so a 1M-vertex load # never holds the text, the parsed rows and the reply of all of it at once (the Serial GC copies what lives) rids = [] + declared = columnar.Declared( + db, type_name + ) # the schema is read at most once for the chunks of this call has_none = type(None) in kinds for start in range(0, len(rows), chunk): part = rows[start : start + chunk] if has_none: part = [r or {} for r in part] - rids.extend(self._create_chunk(prefix, type_name, part)) + rids.extend(self._create_chunk(prefix, type_name, part, declared)) if not rows: rids = self._op(prefix, "create_vertices", type=type_name, rows=[])[ "rids" diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index ebff0d4..e730d86 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -36,6 +36,9 @@ # ARCADEDB_NATIVE_GB_BIN: the float32 vectors of a GraphBatch op (create_vertices rows, new_edges props) go through arcadedb_n_gb_bin in a float32 side buffer, not as number text GB_BIN = os.environ.get("ARCADEDB_NATIVE_GB_BIN", "1") != "0" +# ARCADEDB_NATIVE_GB_COLP: the rows of a GraphBatch create_vertices chunk cross as columns by address (arcadedb_n_gb_colp, the layout of +# arcadedb_n_insert_colp) and the RIDs come back in an int64 buffer, not as JSON text both ways; a chunk the columns cannot carry keeps the JSON route +GB_COLP = os.environ.get("ARCADEDB_NATIVE_GB_COLP", "1") != "0" # the directory the engine log, logging.properties and the crash files go to (None until the library is loaded) LOG_DIR_USED = None @@ -249,6 +252,7 @@ class _Params(ctypes.Structure): "arcadedb_n_gb_create": [P, L, S], "arcadedb_n_gb": [P, L, S, S], "arcadedb_n_gb_bin": [P, L, S, S, S, L], + "arcadedb_n_gb_colp": [P, L, S, S, P, I, P, L], "arcadedb_n_async": [P, L, S, S], "arcadedb_n_async_bin": [P, L, S, S, P, I], "arcadedb_n_vindex": [P, L, S, S], @@ -277,7 +281,7 @@ class _Params(ctypes.Structure): ) else ctypes.c_longlong if name in returns_long else ctypes.c_void_p ) - global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN, GB_BIN, TS_BIN + global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN, GB_BIN, GB_COLP, TS_BIN if not hasattr( lib, "arcadedb_n_exec_bin" ): # a library built before the binary path @@ -298,6 +302,10 @@ class _Params(ctypes.Structure): lib, "arcadedb_n_gb_bin" ): # a library built before the side buffer of GraphBatch ops GB_BIN = False + if not hasattr( + lib, "arcadedb_n_gb_colp" + ): # a library built before the columnar vertex load of GraphBatch + GB_COLP = False if not hasattr( lib, "arcadedb_n_async_bin" ): # a library built before the address table of append_samples From 0966a23077ad419580905119d63fc13395146850 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 16:10:57 +0900 Subject: [PATCH 12/18] Tests: fix the stand-in and three expectations of the native GraphBatch frame routing test (refs #388, #315) A patch of the first commit never applied (the test runner call in the same command was blocked), so the file kept: a schema row for a type with an INTEGER property while the test sent numpy int32 scalars (a list of numpy scalars is no column: the JSON route, correctly), a stand-in that read body["rows"] for the count form (KeyError), and a 21-row load in chunks of 10 that made two chunks where the test expected three. None of these was JVM-specific and none was a fault of the route. The declared-property test now sends a declared ARRAY_OF_FLOATS list column, the stand-in reads the count form, the chunk test has three chunks with the odd key in the middle one. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_graph_batch_frame_pack.py | 41 +++++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/tests/test_native_graph_batch_frame_pack.py b/tests/test_native_graph_batch_frame_pack.py index 7069647..6568466 100644 --- a/tests/test_native_graph_batch_frame_pack.py +++ b/tests/test_native_graph_batch_frame_pack.py @@ -34,7 +34,7 @@ def is_transaction_active(self): def query(self, language, text): # the schema read of columnar.Declared self.schema_reads = getattr(self, "schema_reads", 0) + 1 rows = [ - {"name": "P", "properties": [{"name": "n32", "type": "INTEGER"}]}, + {"name": "P", "properties": [{"name": "v", "type": "ARRAY_OF_FLOATS"}]}, ] return SimpleNamespace(to_list=lambda: rows) @@ -73,8 +73,10 @@ def call(self, name, *args): self.next += n return json.dumps({"ok": True, "n": n}) assert name in OLD, name - rows = json.loads(args[2])["rows"] - rows = len(rows) if isinstance(rows, list) else rows + body = json.loads(args[2]) + rows = ( + len(body["rows"]) if "rows" in body else body["count"] + ) # the count form has no rows rids = [f"#1:{self.next + i}" for i in range(rows)] self.crossings.append((name, args[1].decode(), None)) self.next += rows @@ -134,7 +136,7 @@ def test_the_chunks_of_one_call_are_one_crossing_each_and_the_rids_stay_in_order def test_a_batch_of_one_row_and_no_per_row_python_in_the_codec(library, monkeypatch): - seen = {"prep": 0, "default": 0, "dumps": 0} + seen = {"prep": 0, "default": 0} for key, module, name in (("prep", codec, "_prep"), ("default", codec, "_default")): real = getattr(module, name) @@ -152,7 +154,6 @@ def counting(*args, _real=real, _key=key, **kwargs): assert seen == { "prep": 0, "default": 0, - "dumps": 0, } # the rows were never walked or written as JSON @@ -185,13 +186,15 @@ def test_float32_vectors_cross_as_one_matrix_column(library): assert library.columns[0][1] == x.tobytes() -def test_a_declared_integer_is_read_once_for_all_chunks_of_a_call(library): +def test_a_declared_vector_property_is_read_once_for_all_chunks_of_a_call(library): db = _Db() gb = extras.NativeGraphBatch(db, 7, commit_every=10) - gb.create_vertices("P", [{"n32": np.int32(i)} for i in range(25)]) + gb.create_vertices("P", [{"v": [i * 0.5, i + 0.25, 3.0]} for i in range(25)]) assert library.entries() == [COLP] * 3 - assert db.schema_reads == 1 - assert [c[2]["cols"][0] for c in library.crossings] == [["n32", "i", 0]] * 3 + assert db.schema_reads == 1 # the schema of the type, once for the three chunks + assert [c[2]["cols"][0] for c in library.crossings] == [["v", "f", 3]] * 3 + expected = np.array([[i * 0.5, i + 0.25, 3.0] for i in range(10)], np.float32) + assert library.columns[0][0] == expected.tobytes() FALLBACKS = { @@ -233,14 +236,20 @@ def test_no_rows_and_a_count_keep_the_json_route(library): def test_a_chunk_that_cannot_be_carried_does_not_stop_the_others(library): rows = ( [{"id": i} for i in range(10)] - + [{"id": 10, "x": 1}] - + [{"id": i} for i in range(11, 20)] + + [ + {"id": 10 + i, **({"x": 1} if i == 3 else {})} for i in range(10) + ] # a key the others lack + + [{"id": i} for i in range(20, 30)] ) - gb = _batch(chunk=10) - rids = gb.create_vertices("P", rows) - assert library.entries() == [COLP, OLD[1] if False else library.entries()[1], COLP] - assert library.entries()[1] in OLD - assert len(rids) == 20 + rids = _batch(chunk=10).create_vertices("P", rows) + entries = library.entries() + assert ( + len(entries) == 3 + and entries[0] == COLP + and entries[1] in OLD + and entries[2] == COLP + ) + assert len(rids) == 30 @pytest.mark.parametrize("switch", ["env", "bulk", "library"]) From c0b3aaeca4627c6ccc57bc65c3b610f489daa564 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 17:22:15 +0900 Subject: [PATCH 13/18] Native: create_vertices stores float32 array vectors after the other properties, and only this route refuses Timestamps with nanoseconds (refs #388, #315) Two faults of the first commit that the native parity run showed: 1. Property order. The JSON route took float32 array columns out of the rows (take_vector_columns) and appended them after the other properties (NativeExtras.withVectors), so a vertex of {i, v} was stored [i, v], while the frame route stored the HashMap order [v, i]. The header now carries "last" (the float32 array columns), BulkOps.vertexRows puts them after the HashMap-ordered rest, in column order. Declared list-of-float columns stay in the HashMap order, as in the JSON rows (the parity shapes 'declared vectors from lists' and 'from float32 arrays' cover both). 2. Timestamps. The nanosecond guard sat in the shared packer and so changed insert_many, whose JSON route keeps microseconds (test_native_columnar_temporal.py::test_pandas_timestamps_keep_microseconds_as_the_json_path_does and test_native_mixed_chunk.py::test_a_pandas_timestamp_is_stored_as_the_statement_path_stores_it, red since that commit). pack() and _list_column() take exact_nanos (default off); only create_vertices_columns sets it, because GraphBatch's JSON route writes isoformat() and keeps the nanoseconds. Own tests fixed: 'mixed int and float' with one row is a column of one float and is carried; the two refusals differ only by the type name in the message; the pack test now pins both packer modes and the 'last' header; new end-to-end test of the paper shapes (Person rows in chunks, Product rows with float32 vectors: only arcadedb_n_gb_colp crossings, no create_vertices JSON crossing, no codec hook, vectors stored bit-equal). Co-Authored-By: Claude Sonnet 5.5 --- .../java/org/humemai/nativelib/BulkOps.java | 24 ++++-- .../org/humemai/nativelib/NativeExtras.java | 7 +- src/arcadedb/_native/bulk.py | 13 +++- src/arcadedb/_native/columnar.py | 22 ++++-- tests/test_native_graph_batch_frame.py | 75 ++++++++++++++++++- tests/test_native_graph_batch_frame_pack.py | 32 +++++--- 6 files changed, 144 insertions(+), 29 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index 2371067..3e880ae 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -27,8 +27,10 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.IntConsumer; final class BulkOps { @@ -213,18 +215,30 @@ private static void setRow(final MutableDocument doc, final int i, final Table t * The rows of a column table as GraphBatch.createVertices takes them: per row {name, value, name, value, ...}. The names are in the * iteration order of a java.util.HashMap that took them in column order: the order in which the JSON rows of the same call set their * properties (Json.rows reads every row, which has the keys of the first row in the same order, into a fresh HashMap), so a vertex is - * stored with the property order it has always had. The values are the objects the JSON rows give (Table.cell). + * stored with the property order it has always had. The values are the objects the JSON rows give (Table.cell). The columns that + * {@code last} names (float32 array vectors: the old route took them out of the rows and appended them, in column order) follow the others. */ - static Object[][] vertexRows(final Table t) { + static Object[][] vertexRows(final Table t, final Object last) { + final Set tail = new HashSet<>(); + if (last instanceof List names) + for (final Object name : names) + tail.add((String) name); final HashMap byName = new HashMap<>(); for (int c = 0; c < t.k; c++) - byName.put(t.names[c], c); - if (byName.size() != t.k) - throw new IllegalArgumentException("two columns have the same name"); + if (!tail.contains(t.names[c])) + byName.put(t.names[c], c); final int[] order = new int[t.k]; int at = 0; for (final Integer c : byName.values()) order[at++] = c; + int tailFound = 0; + for (int c = 0; c < t.k; c++) + if (tail.contains(t.names[c])) { + order[at++] = c; + tailFound++; + } + if (at != t.k || tailFound != tail.size()) + throw new IllegalArgumentException("two columns have the same name, or \"last\" names a column that is not there"); final Object[][] rows = new Object[t.n][]; for (int i = 0; i < t.n; i++) { final Object[] row = new Object[2 * t.k]; diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index 61993b5..68d1f80 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -503,7 +503,7 @@ public static CCharPointer gbBin(final IsolateThread thread, final long h, final /** * create_vertices from columns, by pointer, as arcadedb_n_insert_colp takes them (see BulkOps.Table): header JSON {"n":rows,"cols":[[name, - * kind, dim],..]} and an address table of nspec longs (BulkOps.SLOTS per column). The rows are built from the columns (BulkOps.vertexRows: + * kind, dim],..],"last":[names]} (the columns stored after the others, see BulkOps.vertexRows) and an address table of nspec longs (BulkOps.SLOTS per column). The rows are built from the columns (BulkOps.vertexRows: * the objects, and the property order, that the JSON rows of arcadedb_n_gb give) and go to the same GraphBatch.createVertices(type, * Object[][]), so the transaction, the retries and the commit are the engine's and nothing differs from the JSON route but the transport. * The RIDs do not come back as text: rids points at ridLongs int64 slots of the caller's (at least 2 per row), and row i gets bucket id @@ -521,10 +521,11 @@ public static CCharPointer gbColp(final IsolateThread thread, final long h, fina final long[] table = new long[nspec]; for (int i = 0; i < nspec; i++) table[i] = ((Pointer) spec).readLong(WordFactory.signed(8L * i)); - final BulkOps.Table columns = BulkOps.Table.of(args(header), table); + final Map head = args(header); + final BulkOps.Table columns = BulkOps.Table.of(head, table); if (rids.isNull() || ridLongs < 2L * columns.n) throw new IllegalArgumentException("the RID buffer holds " + ridLongs + " int64 slots, " + 2L * columns.n + " needed"); - final RID[] created = entry.batch().createVertices(ArcadeNative.str(type), BulkOps.vertexRows(columns)); + final RID[] created = entry.batch().createVertices(ArcadeNative.str(type), BulkOps.vertexRows(columns, head.get("last"))); final Pointer out = (Pointer) rids; for (int i = 0; i < created.length; i++) { out.writeLong(WordFactory.signed(16L * i), created[i].getBucketId()); diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index c3bc1dd..79cc32b 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -116,10 +116,19 @@ def create_vertices_columns(gb, type_name, rows, declared, prefix): columns = columnar.rows_to_columns(rows, same_order=True) if columns is None: return None - packed = columnar.pack(columns, declared) + packed = columnar.pack( + columns, declared, exact_nanos=True + ) # the JSON route keeps the nanoseconds of a Timestamp if packed is None: return None count, cols = packed + # a column of float32 arrays is stored after the other properties of a row: the old route took those vectors out of the JSON rows and + # appended them (NativeExtras.withVectors); the other columns keep the order of a HashMap of their names, as the JSON rows do + last = [ + c.name + for c in cols + if c.kind == "f" and isinstance(columns[c.name][0], columnar._np.ndarray) + ] spec = columnar.spec_of(cols) out = columnar._np.empty(2 * count, dtype=columnar._np.int64) from .database import _call @@ -129,7 +138,7 @@ def create_vertices_columns(gb, type_name, rows, declared, prefix): "arcadedb_n_gb_colp", gb._gb, lib.b(type_name), - lib.b(columnar.header_of(count, 0, cols)), + lib.b(columnar.header_of(count, 0, cols, last=last)), spec, len(spec), out.ctypes.data, diff --git a/src/arcadedb/_native/columnar.py b/src/arcadedb/_native/columnar.py index 77bcb66..dad4d67 100644 --- a/src/arcadedb/_native/columnar.py +++ b/src/arcadedb/_native/columnar.py @@ -215,7 +215,7 @@ def _epoch_micros(value): return seconds * 1_000_000 + value.microsecond -def _list_column(name, values, declared): +def _list_column(name, values, declared, exact_nanos=False): """A list or tuple of Python values as a column: one type (None allowed) or a list of equal-length float arrays; else None.""" n = len(values) types = set(map(type, values)) @@ -255,10 +255,12 @@ def _list_column(name, values, declared): if issubclass(only, datetime): if only.__name__ == "NaTType": return None # the JSON path stores it as a null - if only is not datetime and any( - getattr(v, "nanosecond", 0) for v in values if v is not None + if ( + exact_nanos + and only is not datetime + and any(getattr(v, "nanosecond", 0) for v in values if v is not None) ): - return None # a pandas Timestamp with nanoseconds: the JSON path keeps them, microsecond ticks cannot + return None # a pandas Timestamp with nanoseconds, for a caller whose JSON route keeps them (insert_many's keeps microseconds) try: ticks = [0 if v is None else _epoch_micros(v) for v in values] except (OverflowError, ValueError): @@ -411,7 +413,7 @@ def _arrow_column(name, arr, declared): return None -def pack(columns, declared): +def pack(columns, declared, exact_nanos=False): """``(rows, [Packed])`` for a dict of name -> column, or None when any column cannot be laid out or the columns differ in length. A column is a numpy array (1-D ints, floats and bools, or a 2-D float32 matrix), a list or tuple of Python values of one type (None @@ -429,14 +431,14 @@ def pack(columns, declared): elif isinstance(values, _np.ndarray): if values.dtype.kind in "OUS": col = ( - _list_column(name, values.tolist(), declared) + _list_column(name, values.tolist(), declared, exact_nanos) if values.ndim == 1 else None ) else: col = _numpy_column(name, values, declared) elif isinstance(values, (list, tuple)) and values: - col = _list_column(name, values, declared) + col = _list_column(name, values, declared, exact_nanos) else: col = None # a pandas Series, a generator, an empty list: the row path handles them if col is None: @@ -458,7 +460,7 @@ def spec_of(packed): return (ctypes.c_int64 * len(flat))(*flat) -def header_of(rows, commit_every, packed, fid=None): +def header_of(rows, commit_every, packed, fid=None, last=()): """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column; with a failure token ``fid`` it also says ``"parallel": true``, which hands the records to the async executor (the image then ignores the commit interval and counts the failed records on that token). @@ -470,6 +472,10 @@ def header_of(rows, commit_every, packed, fid=None): "commit": min(int(commit_every), 2**31 - 1), "cols": [[c.name, c.kind, c.dim] for c in packed], } + if ( + last + ): # create_vertices: the columns the old route appended to the end of each row (float32 array vectors) + header["last"] = list(last) if fid is not None: header["parallel"] = True header["fid"] = fid diff --git a/tests/test_native_graph_batch_frame.py b/tests/test_native_graph_batch_frame.py index dc27713..e8f5138 100644 --- a/tests/test_native_graph_batch_frame.py +++ b/tests/test_native_graph_batch_frame.py @@ -138,6 +138,8 @@ def test_the_columnar_route_stores_what_the_json_route_stores( db, monkeypatch, shape, n ): make, carried = SHAPES[shape] + if shape == "mixed int and float" and n == 1: + carried = True # one row holds one type: a column of one float rows = _rows(n, make) rids_on, calls_on = _load(db, monkeypatch, "A", rows, True) rids_off, calls_off = _load(db, monkeypatch, "B", rows, False) @@ -199,7 +201,9 @@ def test_an_integer_past_64_bits_is_refused_by_both_routes_alike(db, monkeypatch for type_name, columnar in (("A", True), ("B", False)): with pytest.raises(Exception) as raised: _load(db, monkeypatch, type_name, rows, columnar) - errors.append(str(raised.value)) + errors.append( + str(raised.value).replace(f"'{type_name}'", "'T'") + ) # the type name is the only difference assert errors[0] == errors[1] assert db.query("sql", "SELECT count(*) AS c FROM A").first().get("c") == 0 @@ -221,3 +225,72 @@ def test_a_failing_chunk_stores_nothing_of_itself_and_keeps_the_chunks_before_it with pytest.raises(Exception): _load(db, monkeypatch, "A", rows, True) assert db.query("sql", "SELECT count(*) AS c FROM A").first().get("c") == 1000 + + +def test_the_paper_shaped_loads_take_the_columnar_route_end_to_end( + temp_db, monkeypatch +): + """l2 (Person rows in chunks) and e2 (Product rows with a float32 vector): every chunk is one arcadedb_n_gb_colp crossing, no create_vertices + crossing of the JSON entries, no codec hook; the vertices and the vectors are stored, the RIDs come back as a list of str. + """ + from arcadedb._native import codec + + temp_db.command("sql", "CREATE VERTEX TYPE Person") + temp_db.command("sql", "CREATE VERTEX TYPE Product") + temp_db.command("sql", "CREATE PROPERTY Product.embedding ARRAY_OF_FLOATS") + calls, hooks = [], {"prep": 0, "default": 0} + real_call, real_prep, real_default = lib.call, codec._prep, codec._default + + def spy(name, *args): + calls.append( + (name, args[1] if name in ("arcadedb_n_gb", "arcadedb_n_gb_bin") else None) + ) + return real_call(name, *args) + + def counting(key, real): + def wrapper(*args, **kwargs): + hooks[key] += 1 + return real(*args, **kwargs) + + return wrapper + + people = [ + {"id": i, "name": f"p{i}", "age": 20 + i % 50, "city": f"c{i % 7}"} + for i in range(2500) + ] + x = _vectors(2100, 16) + products = [ + {"id": i, "title": f"t{i}", "price": i * 0.25, "embedding": x[i]} + for i in range(2100) + ] + with ( + temp_db.graph_batch() as gb + ): # the default commit_every: 1000 rows per crossing + monkeypatch.setattr(lib, "call", spy) + monkeypatch.setattr(codec, "_prep", counting("prep", real_prep)) + monkeypatch.setattr(codec, "_default", counting("default", real_default)) + rids_people = gb.create_vertices("Person", people) + rids_products = gb.create_vertices("Product", products) + monkeypatch.undo() + assert [ + c for c in calls if c[1] == b"create_vertices" + ] == [] # no JSON entry carried a chunk + assert [c[0] for c in calls].count( + COLP + ) == 3 + 3 # 1000, 1000, 500 and 1000, 1000, 100 + assert hooks == {"prep": 0, "default": 0} + assert len(rids_people) == 2500 and len(rids_products) == 2100 + assert all(type(r) is str for r in rids_people + rids_products) + stored = { + r["id"]: r["embedding"] + for r in temp_db.query("sql", "SELECT id, embedding FROM Product").to_list() + } + assert len(stored) == 2100 + assert all( + np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + for i in range(2100) + ) + assert ( + temp_db.query("sql", "SELECT count(*) AS c FROM Person").first().get("c") + == 2500 + ) diff --git a/tests/test_native_graph_batch_frame_pack.py b/tests/test_native_graph_batch_frame_pack.py index 6568466..28ee316 100644 --- a/tests/test_native_graph_batch_frame_pack.py +++ b/tests/test_native_graph_batch_frame_pack.py @@ -183,6 +183,9 @@ def test_float32_vectors_cross_as_one_matrix_column(library): _expect_columns(library, rows) cols = library.crossings[0][2]["cols"] assert cols == [["id", "q", 0], ["embedding", "f", 8]] + assert library.crossings[0][2]["last"] == [ + "embedding" + ] # stored after the other properties, as the old route did assert library.columns[0][1] == x.tobytes() @@ -193,6 +196,9 @@ def test_a_declared_vector_property_is_read_once_for_all_chunks_of_a_call(librar assert library.entries() == [COLP] * 3 assert db.schema_reads == 1 # the schema of the type, once for the three chunks assert [c[2]["cols"][0] for c in library.crossings] == [["v", "f", 3]] * 3 + assert ( + "last" not in library.crossings[0][2] + ) # a list column stays in the HashMap order, as in the JSON rows expected = np.array([[i * 0.5, i + 0.25, 3.0] for i in range(10)], np.float32) assert library.columns[0][0] == expected.tobytes() @@ -299,16 +305,22 @@ def test_same_order_is_a_rule_of_the_row_check_only(): } -def test_a_pandas_timestamp_with_nanoseconds_is_not_a_column(): - """The JSON route keeps the nanoseconds of a Timestamp (``isoformat``); microsecond ticks cannot, so the chunk keeps that route.""" +def test_a_pandas_timestamp_with_nanoseconds_keeps_the_json_route_of_a_vertex_load( + library, +): + """GraphBatch's JSON route writes a Timestamp with isoformat(), which keeps the nanoseconds, and microsecond ticks cannot; insert_many's + JSON route keeps microseconds (test_native_columnar_temporal.py), so only this route asks the packer to refuse them. + """ import pandas as pd declared = columnar.Declared(known={}) - assert ( - columnar.pack({"t": [pd.Timestamp("2020-01-01 00:00:00.000000123")]}, declared) - is None - ) - assert ( - columnar.pack({"t": [pd.Timestamp("2020-01-01 00:00:00.000123")]}, declared) - is not None - ) + stamp = pd.Timestamp("2020-01-01 00:00:00.000000123") + assert columnar.pack({"t": [stamp]}, declared) is not None # insert_many + assert columnar.pack({"t": [stamp]}, declared, exact_nanos=True) is None + micros = pd.Timestamp("2020-01-01 00:00:00.000123") + assert columnar.pack({"t": [micros]}, declared, exact_nanos=True) is not None + _batch().create_vertices("P", [{"t": stamp}, {"t": stamp}]) + assert set(library.entries()) <= set(OLD) and library.entries() + library.crossings.clear() + _batch().create_vertices("P", [{"t": micros}, {"t": micros}]) + assert library.entries() == [COLP] From 4f5e5fb860b5f23661c77814c8a3eddc980d94f7 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 14 Nov 2026 17:22:15 +0900 Subject: [PATCH 14/18] Tests: three native files that pinned the JSON route of create_vertices follow the new route (refs #388, #315) test_native_ingest_defaults.py (3 tests: default commit_every is 1000, explicit commit_every 250 and 0): asserted the rows per create_vertices crossing [1000,1000,500,1000,1000,500], [250]*10 and [2500] by wrapping NativeGraphBatch._op. Plain rows now cross as columns and do not pass through _op, so the spy also records len(rows) at bulk.create_vertices_columns; the expected numbers are unchanged (the chunking is the same). test_native_graph_batch_binary_vectors.py (7 tests) and test_native_vertex_vector_crossings.py (6 tests): asserted that create_vertices of float32 vector rows crossed arcadedb_n_gb_bin (or arcadedb_n_gb for the text arm) with the side buffer, the "vecs" entry and no per-row codec hook. A plain vector load now crosses arcadedb_n_gb_colp (tests/test_native_graph_batch_frame*.py assert that, and the stored bits). These files' subject is the side-buffer route, which a chunk the columns cannot carry (a nested value, ragged vectors) still takes, so their db fixture turns the columnar route off (lib.GB_COLP False); no assertion is loosened. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_graph_batch_binary_vectors.py | 5 ++++- tests/test_native_ingest_defaults.py | 12 +++++++++++- tests/test_native_vertex_vector_crossings.py | 5 ++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/test_native_graph_batch_binary_vectors.py b/tests/test_native_graph_batch_binary_vectors.py index c554310..99dd913 100644 --- a/tests/test_native_graph_batch_binary_vectors.py +++ b/tests/test_native_graph_batch_binary_vectors.py @@ -22,7 +22,10 @@ @pytest.fixture -def db(temp_db): +def db(temp_db, monkeypatch): + # this file pins the side-buffer route of arcadedb_n_gb_bin, which the chunks that the columnar entry cannot carry still take; a plain + # vector load takes arcadedb_n_gb_colp now (test_native_graph_batch_frame*.py), so the columnar route is off here + monkeypatch.setattr(lib, "GB_COLP", False) temp_db.command("sql", "CREATE VERTEX TYPE Vx") temp_db.command("sql", "CREATE PROPERTY Vx.id LONG") temp_db.command("sql", "CREATE PROPERTY Vx.embedding ARRAY_OF_FLOATS") diff --git a/tests/test_native_ingest_defaults.py b/tests/test_native_ingest_defaults.py index 9234bb7..e7519f6 100644 --- a/tests/test_native_ingest_defaults.py +++ b/tests/test_native_ingest_defaults.py @@ -9,7 +9,7 @@ import pytest import arcadedb -from arcadedb._native import extras +from arcadedb._native import bulk, extras from arcadedb.exceptions import ArcadeDBError _COUNT_SQL = { # fixed statements, one for each type the tests create @@ -85,6 +85,16 @@ def op(this, prefix, name, **args): monkeypatch.setattr(extras, "_enc", enc) monkeypatch.setattr(extras.NativeGraphBatch, "_op", op) + real_columns = bulk.create_vertices_columns + + def columns(gb, type_name, rows, declared, prefix): + # plain rows now cross as columns (arcadedb_n_gb_colp), not through _op: the rows per crossing are read here, same numbers + rids = real_columns(gb, type_name, rows, declared, prefix) + if rids is not None: + self.vertex_calls.append(len(rows)) + return rids + + monkeypatch.setattr(bulk, "create_vertices_columns", columns) def commit_every_given(self): return [d["commit_every"] for d in self.created if "commit_every" in d] diff --git a/tests/test_native_vertex_vector_crossings.py b/tests/test_native_vertex_vector_crossings.py index e684845..4db0e5a 100644 --- a/tests/test_native_vertex_vector_crossings.py +++ b/tests/test_native_vertex_vector_crossings.py @@ -79,7 +79,10 @@ def spy(monkeypatch): @pytest.fixture -def db(temp_db): +def db(temp_db, monkeypatch): + # this file pins the vector-matrix route of arcadedb_n_gb_bin ("vecs"), which the chunks that the columnar entry cannot carry still take; a + # plain vector load takes arcadedb_n_gb_colp now (test_native_graph_batch_frame*.py), so the columnar route is off here + monkeypatch.setattr(lib, "GB_COLP", False) for name in ("Declared", "Free"): temp_db.command("sql", f"CREATE VERTEX TYPE {name}") temp_db.command("sql", f"CREATE PROPERTY {name}.id LONG") From 013da60131ee1f4a04c0117ce30e3422e984631c Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 15 Nov 2026 17:47:21 +0900 Subject: [PATCH 15/18] Native create_vertices: a chunk of fewer than 64 rows keeps the JSON route of post3 (refs #388, #401) The laptop A/B of this pull request measured a single row 1.63 times slower than post3 (n_gb_v_1) and batches of 100 rows and more faster: the columnar entry has a fixed cost (a header, the column specs, the buffers) that a handful of rows does not repay. A chunk below lib.GB_COLP_MIN_ROWS (64, ARCADEDB_NATIVE_GB_COLP_MIN_ROWS, 0 turns the floor off) now takes the post3 JSON route; at or above it the columns, as before. Sizes 1 and 100 are measured and the crossover lies between them; below 64 the path is the post3 path, 100 and up is measured faster, 64 to 99 is not measured. The check is in bulk.create_vertices_columns, so a small chunk reads no schema and builds no columns. Tests: the stand-in library of the pack tests turns the floor off so the existing tests keep pinning the columnar route; new tests pin the floor at 1, 63, 64, 65 and 100 rows, that a small chunk builds nothing, that chunks are judged one by one, the switch and the environment variable. The native parity test loads the same rows with the default floor, with the JSON route and with the columns forced, at 1, 63, 64, 65 and 100 rows, and compares every stored record and the RIDs. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/bulk.py | 7 +- src/arcadedb/_native/lib.py | 5 ++ tests/test_native_graph_batch_frame.py | 29 +++++++- tests/test_native_graph_batch_frame_pack.py | 82 +++++++++++++++++++++ 4 files changed, 118 insertions(+), 5 deletions(-) diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index 79cc32b..b372344 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -107,12 +107,13 @@ def create_vertices_columns(gb, type_name, rows, declared, prefix): """One ``GraphBatch.create_vertices`` chunk (a list of dicts) through ``arcadedb_n_gb_colp``: the rows cross as columns by address and the RIDs come back in an int64 buffer, one crossing and one transaction, as the JSON route has it. Returns the list of RID texts, or None when the chunk needs the JSON route (nothing was sent then): rows that do not all have the same keys in the same order, a value no column - holds (a nested list or map, a mixed-type column, a set, a NaT, a numpy scalar, an integer past 64 bits, a lone surrogate) or an empty chunk. + holds (a nested list or map, a mixed-type column, a set, a NaT, a numpy scalar, an integer past 64 bits, a lone surrogate), an empty chunk or a + chunk of fewer rows than ``lib.GB_COLP_MIN_ROWS`` (64 by default: a handful of rows does not repay the fixed cost of the columnar entry). ``declared``: a :class:`columnar.Declared` of the type, read once by the caller for all chunks of a call. """ - if not (lib.GB_COLP and _usable()): - return None + if not (lib.GB_COLP and _usable()) or len(rows) < lib.GB_COLP_MIN_ROWS: + return None # a chunk below the floor keeps the JSON route of post3 (lib.GB_COLP_MIN_ROWS), as does a chunk the columns cannot carry columns = columnar.rows_to_columns(rows, same_order=True) if columns is None: return None diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index e730d86..0bf3987 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -39,6 +39,11 @@ # ARCADEDB_NATIVE_GB_COLP: the rows of a GraphBatch create_vertices chunk cross as columns by address (arcadedb_n_gb_colp, the layout of # arcadedb_n_insert_colp) and the RIDs come back in an int64 buffer, not as JSON text both ways; a chunk the columns cannot carry keeps the JSON route GB_COLP = os.environ.get("ARCADEDB_NATIVE_GB_COLP", "1") != "0" +# ARCADEDB_NATIVE_GB_COLP_MIN_ROWS: a create_vertices chunk of fewer rows than this takes the JSON route of 26.11.1.post3, whatever the columns could carry (0: no floor). +# The columnar entry has a fixed cost (a header, the column specs, the buffers) that a handful of rows does not repay: the laptop A/B of this change measured a single row +# 1.63 times slower than post3 and a batch of 100 rows faster. Sizes 1 and 100 are measured and the crossover lies between them; below 64 the path is the post3 path, so it +# cannot be slower; 100 rows and more are measured faster; 64 to 99 rows are not measured. +GB_COLP_MIN_ROWS = int(os.environ.get("ARCADEDB_NATIVE_GB_COLP_MIN_ROWS", "64")) # the directory the engine log, logging.properties and the crash files go to (None until the library is loaded) LOG_DIR_USED = None diff --git a/tests/test_native_graph_batch_frame.py b/tests/test_native_graph_batch_frame.py index e8f5138..f75cfc5 100644 --- a/tests/test_native_graph_batch_frame.py +++ b/tests/test_native_graph_batch_frame.py @@ -26,7 +26,8 @@ def db(temp_db): for name in ( "A", "B", - ): # the same schema twice: A is loaded with the columnar route, B with the JSON route + "C", + ): # the same schema three times: A is loaded with the columnar route, B with the JSON route, C with the columns forced temp_db.command("sql", f"CREATE VERTEX TYPE {name}") temp_db.command("sql", f"CREATE PROPERTY {name}.i LONG") temp_db.command("sql", f"CREATE PROPERTY {name}.n INTEGER") @@ -114,7 +115,7 @@ def _dump(db, type_name): return [out[i] for i in sorted(out)] -def _load(db, monkeypatch, type_name, rows, columnar): +def _load(db, monkeypatch, type_name, rows, columnar, floor=None): calls = [] real = lib.call @@ -124,6 +125,9 @@ def spy(name, *args): monkeypatch.setattr(lib, "call", spy) monkeypatch.setattr(lib, "GB_COLP", columnar) + monkeypatch.setattr( + lib, "GB_COLP_MIN_ROWS", 0 if floor is None else floor + ) # the route tests force the columns; floor=64 is the default try: with db.graph_batch() as gb: rids = gb.create_vertices(type_name, rows) @@ -294,3 +298,24 @@ def wrapper(*args, **kwargs): temp_db.query("sql", "SELECT count(*) AS c FROM Person").first().get("c") == 2500 ) + + +@pytest.mark.parametrize("n", [1, 63, 64, 65, 100]) +def test_the_floor_picks_the_route_by_size_and_both_routes_store_the_same( + db, monkeypatch, n +): + """#388: the default floor of 64 rows: below it the JSON route of post3, from it the columns; the stored data and the RIDs are those of the JSON route either way.""" + rows = _rows(n, SHAPES["scalars"][0]) + rids_floor, calls_floor = _load(db, monkeypatch, "A", rows, True, floor=64) + rids_json, calls_json = _load(db, monkeypatch, "B", rows, False) + assert (COLP in calls_floor) is (n >= 64), (n, calls_floor) + assert COLP not in calls_json + assert _dump(db, "A") == _dump(db, "B") + for rids in (rids_floor, rids_json): + assert type(rids) is list and len(rids) == n + assert all(type(r) is str and re.fullmatch(r"#\d+:\d+", r) for r in rids) + rids_cols, calls_cols = _load( + db, monkeypatch, "C", rows, True, floor=0 + ) # the columns forced, even for one row + assert COLP in calls_cols and len(rids_cols) == n + assert _dump(db, "C") == _dump(db, "B") diff --git a/tests/test_native_graph_batch_frame_pack.py b/tests/test_native_graph_batch_frame_pack.py index 28ee316..2f18707 100644 --- a/tests/test_native_graph_batch_frame_pack.py +++ b/tests/test_native_graph_batch_frame_pack.py @@ -49,6 +49,9 @@ def __init__(self, monkeypatch): lib, "_lib", SimpleNamespace(arcadedb_n_insert_colp=1, arcadedb_n_gb_colp=1) ) monkeypatch.setattr(lib, "GB_COLP", True) + monkeypatch.setattr( + lib, "GB_COLP_MIN_ROWS", 0 + ) # the tests below pin the columnar route on small batches; the floor has its own tests at the end monkeypatch.setattr(lib, "GB_BIN", True) monkeypatch.setattr(bulk, "_ENABLED", True) @@ -324,3 +327,82 @@ def test_a_pandas_timestamp_with_nanoseconds_keeps_the_json_route_of_a_vertex_lo library.crossings.clear() _batch().create_vertices("P", [{"t": micros}, {"t": micros}]) assert library.entries() == [COLP] + + +# the floor (ArcadeData/arcadedb-python #388, the laptop A/B of this route: one row 1.63 times slower than post3, 100 rows faster): a chunk below +# lib.GB_COLP_MIN_ROWS (64) takes the JSON route of 26.11.1.post3, exactly as before this change, and a chunk at or above it the columnar route + + +@pytest.mark.parametrize( + "n,route", [(1, OLD), (63, OLD), (64, (COLP,)), (65, (COLP,)), (100, (COLP,))] +) +def test_the_floor_keeps_small_chunks_on_the_json_route_and_the_rest_on_columns( + library, monkeypatch, n, route +): + monkeypatch.setattr( + lib, "GB_COLP_MIN_ROWS", 64 + ) # the default, whatever the environment says + rows = [{"id": i, "name": f"n{i}"} for i in range(n)] + rids = _batch(1000).create_vertices("P", rows) + assert library.entries()[0] in route and len(library.entries()) == 1, ( + n, + library.entries(), + ) + assert type(rids) is list and len(rids) == n and all(type(r) is str for r in rids) + + +def test_a_small_chunk_below_the_floor_reads_no_schema_and_builds_no_columns( + library, monkeypatch +): + """The post3 path costs nothing more than before for a small chunk: no schema read, no column layout, no packing.""" + monkeypatch.setattr(lib, "GB_COLP_MIN_ROWS", 64) + for name in ("rows_to_columns", "pack"): + monkeypatch.setattr( + columnar, + name, + lambda *a, _n=name, **k: pytest.fail( + f"{_n} ran for a chunk below the floor" + ), + ) + db = _Db() + extras.NativeGraphBatch(db, 7, commit_every=100).create_vertices( + "P", [{"id": i} for i in range(63)] + ) + assert getattr(db, "schema_reads", 0) == 0 + assert library.entries() == [OLD[0]] or library.entries() == [OLD[1]] + + +def test_the_chunks_of_a_load_are_judged_one_by_one(library, monkeypatch): + monkeypatch.setattr(lib, "GB_COLP_MIN_ROWS", 64) + _batch(100).create_vertices( + "P", [{"id": i} for i in range(230)] + ) # 100, 100 and a last chunk of 30 + assert [c[0] for c in library.crossings] == [COLP, COLP, OLD[0]] + + +def test_zero_turns_the_floor_off_and_the_switch_still_turns_the_route_off( + library, monkeypatch +): + monkeypatch.setattr(lib, "GB_COLP_MIN_ROWS", 0) + _batch().create_vertices("P", [{"id": 1}]) + assert library.entries() == [COLP] + library.crossings.clear() + monkeypatch.setattr(lib, "GB_COLP", False) + _batch().create_vertices("P", [{"id": i} for i in range(100)]) + assert COLP not in library.entries() + + +def test_the_floor_is_read_from_the_environment_and_is_64_by_default(): + import os + import subprocess + import sys + + code = "from arcadedb._native import lib; print(lib.GB_COLP_MIN_ROWS)" + env = { + k: v for k, v in os.environ.items() if k != "ARCADEDB_NATIVE_GB_COLP_MIN_ROWS" + } + run = lambda e: subprocess.run( + [sys.executable, "-c", code], env=e, capture_output=True, text=True, timeout=120 + ).stdout.strip() + assert run(env) == "64" + assert run({**env, "ARCADEDB_NATIVE_GB_COLP_MIN_ROWS": "5"}) == "5" From 1fd3ac5f04dd372e809660aa7990be7053c25102 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 15 Nov 2026 17:53:50 +0900 Subject: [PATCH 16/18] Native create_vertices: the comment of the 64 row floor names the micro that placed the crossover (refs #388, #401) An untimed micro of create_vertices calls in one open graph batch, this change against post3, native: slower below 40 rows, faster from 64 rows. No code change. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/lib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 0bf3987..875366e 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -41,8 +41,9 @@ GB_COLP = os.environ.get("ARCADEDB_NATIVE_GB_COLP", "1") != "0" # ARCADEDB_NATIVE_GB_COLP_MIN_ROWS: a create_vertices chunk of fewer rows than this takes the JSON route of 26.11.1.post3, whatever the columns could carry (0: no floor). # The columnar entry has a fixed cost (a header, the column specs, the buffers) that a handful of rows does not repay: the laptop A/B of this change measured a single row -# 1.63 times slower than post3 and a batch of 100 rows faster. Sizes 1 and 100 are measured and the crossover lies between them; below 64 the path is the post3 path, so it -# cannot be slower; 100 rows and more are measured faster; 64 to 99 rows are not measured. +# 1.63 times slower than post3 and a batch of 100 rows faster. An untimed micro of create_vertices calls in one open graph batch (this change against post3, native, cores 14 and 15) +# measured slower for 1, 8, 16 and 32 rows (1.7x, 1.4x, 1.2x, 1.03x) and faster for 64 and 100 rows (0.90, 0.86): the crossover is near 40 rows, so 64 is above it and a chunk below +# 64 rows takes the post3 path and cannot be slower. 64 to 99 rows were measured in the micro only (not in the pre-registered A/B, which has sizes 1 and 100). GB_COLP_MIN_ROWS = int(os.environ.get("ARCADEDB_NATIVE_GB_COLP_MIN_ROWS", "64")) # the directory the engine log, logging.properties and the crash files go to (None until the library is loaded) From ae0623bef6296742d808dbc0448aaba0e87c1889 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 15 Nov 2026 17:58:33 +0900 Subject: [PATCH 17/18] Tests: a def in the floor test, not an assigned lambda (ruff E731 of main; refs #388, #401) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_graph_batch_frame_pack.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_native_graph_batch_frame_pack.py b/tests/test_native_graph_batch_frame_pack.py index 2f18707..35ebe54 100644 --- a/tests/test_native_graph_batch_frame_pack.py +++ b/tests/test_native_graph_batch_frame_pack.py @@ -401,8 +401,15 @@ def test_the_floor_is_read_from_the_environment_and_is_64_by_default(): env = { k: v for k, v in os.environ.items() if k != "ARCADEDB_NATIVE_GB_COLP_MIN_ROWS" } - run = lambda e: subprocess.run( - [sys.executable, "-c", code], env=e, capture_output=True, text=True, timeout=120 - ).stdout.strip() + + def run(e): + return subprocess.run( + [sys.executable, "-c", code], + env=e, + capture_output=True, + text=True, + timeout=120, + ).stdout.strip() + assert run(env) == "64" assert run({**env, "ARCADEDB_NATIVE_GB_COLP_MIN_ROWS": "5"}) == "5" From 2b2eed10cc90f1cff866ec638d87e57ee2f70970 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 15 Nov 2026 18:46:38 +0900 Subject: [PATCH 18/18] Tests: nosec B404 and B603 on the subprocess of the floor-from-environment test, as the repo does (refs #388, #401) The Bandit job of CI (bandit==1.9.4 -c pyproject.toml -r src tests, low severity and confidence) flagged the import and the call of the test that starts a fresh interpreter to read lib.GB_COLP_MIN_ROWS. The call is an argv list with no shell: this interpreter and a fixed one-line program. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_graph_batch_frame_pack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_native_graph_batch_frame_pack.py b/tests/test_native_graph_batch_frame_pack.py index 35ebe54..8a7e935 100644 --- a/tests/test_native_graph_batch_frame_pack.py +++ b/tests/test_native_graph_batch_frame_pack.py @@ -394,7 +394,7 @@ def test_zero_turns_the_floor_off_and_the_switch_still_turns_the_route_off( def test_the_floor_is_read_from_the_environment_and_is_64_by_default(): import os - import subprocess + import subprocess # nosec B404 - runs this interpreter on a fixed one-line program import sys code = "from arcadedb._native import lib; print(lib.GB_COLP_MIN_ROWS)" @@ -403,7 +403,7 @@ def test_the_floor_is_read_from_the_environment_and_is_64_by_default(): } def run(e): - return subprocess.run( + return subprocess.run( # nosec B603 - argv list, no shell: this interpreter, a fixed program [sys.executable, "-c", code], env=e, capture_output=True,