diff --git a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc index 099184f..2d8c576 100644 --- a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc +++ b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc @@ -65,6 +65,8 @@ Because the two packages have different defaults, the same program can keep diff On both packages, `insert_many` keeps rows with `datetime`, `date`, `Decimal`, `bytes`, numpy scalar, or pandas `NA` and `NaT` values on the bulk path. On the `arcadedb` (native) package, a row with a value the bulk text cannot carry (NaN or infinity, an integer beyond 64 bits, a date-time or `Decimal` inside a list or dict, or a `datetime.time`) goes in with its own statement, in the order given. +On the `arcadedb-jvm` package, a chunk of 10,000 rows crosses as typed columns, with no JSON text, when every row is a `dict` with the same keys in the same order and each column holds `None` and values of one type: `int` within 64 bits, `float` (NaN and infinity included), `bool`, `str`, `date`, or a `datetime` without a time zone. The stored values are the same as the JSON text stores: a whole number is an `Integer` when it fits an int and a `Long` beyond, and `None` is a null property. A row with a NaN, an infinity, or a subnormal `float` stores its whole numbers as `Long`, which is what the per-row path stores for such a row. Any other chunk keeps the JSON text, as before: a row that lacks a key or lists the keys in another order, a column that mixes types (an `int` beside a `float`), a `Decimal`, `bytes`, numpy or pandas value, a `datetime` with a time zone, a list, tuple or dict, and a float vector for a property declared `ARRAY_OF_FLOATS`. A chunk is judged as a whole, so one such row sends its 10,000 rows down the JSON route. The `arcadedb` (native) package already sends the rows of one shape through its columnar entry. + `insert_many` and `create_vertices` check their rows while they write the JSON text, in one pass over the rows, not in a loop before it. The same rows fail the check as before, and they take the same fallback path as before. The check raises no error to your code. [[columnar-loads]] diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index 29ef657..79d8c85 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -8,7 +8,7 @@ import itertools import operator from collections.abc import Mapping -from datetime import date +from datetime import date, datetime from decimal import Decimal from os import PathLike from typing import TYPE_CHECKING, Any, Callable, List, Optional @@ -36,6 +36,8 @@ from .results import ResultSet from .transactions import TransactionContext from .type_conversion import ( + _INT64_MAX, + _INT64_MIN, _is_numpy_bool, bulk_row_ok, convert_python_to_java, @@ -336,6 +338,131 @@ def _column_to_java(name, values): return jpype.JArray(jpype.JObject)(converted), len(converted) +# insert_many: the kinds of column a chunk can cross as typed columns (see _typed_chunk_columns). The exact types only: a subclass +# (pandas.Timestamp, numpy.float64) and every other type (Decimal, bytes, list, dict, tuple) keep the JSON text. +_TYPED_KINDS = { + int: "int", + float: "float", + bool: "bool", + str: "str", + date: "date", + datetime: "datetime", +} +_DICT_ONLY = frozenset((dict,)) +_STR_ONLY = frozenset((str,)) +_NO_ZONE = frozenset((None,)) +_EPOCH_ORDINAL = date(1970, 1, 1).toordinal() +_EPOCH_DATE = date(1970, 1, 1) +_EPOCH_DATETIME = datetime(1970, 1, 1) + + +def _typed_kind(values): + """``(kind, nullable)`` of one column of an insert_many chunk when it can cross as a typed column, else None. + + The column holds ``None`` and values of ONE exact type of ``_TYPED_KINDS`` (``"null"`` when there is nothing but ``None``). + A mixed column (an int beside a float, a bool beside an int) would store each value as its own type on the JSON route, which one + typed array cannot do; an int beyond 64 bits, a date-time with a zone and a type the table does not list keep the JSON route. + """ + kinds = set(map(type, values)) + nullable = _NONE_TYPE in kinds + kinds.discard(_NONE_TYPE) + if not kinds: + return "null", True + if len(kinds) != 1: + return None + kind = _TYPED_KINDS.get(next(iter(kinds))) + if kind == "int": + present = [v for v in values if v is not None] if nullable else values + if min(present) < _INT64_MIN or max(present) > _INT64_MAX: + return None + elif kind == "datetime": + present = [v for v in values if v is not None] if nullable else values + if set(map(operator.attrgetter("tzinfo"), present)) != _NO_ZONE: + return None + return (kind, nullable) if kind else None + + +def _typed_java_column(name, values, kind, nullable): + """One column of an insert_many chunk as the Java column ``DocumentBatcher.insertManyColumns`` takes, or None. + + A whole column of int, float or bool without a null is one numpy array and one buffer copy (``long[]``, ``double[]``, + ``boolean[]``); a column of str and None is one ``StringColumn``; date and naive date-time columns are one ``TemporalColumn`` + (a ``long[]`` of epoch days or microseconds, and one ``boolean[]`` when there is a null); a numeric column with a null converts + per element into an ``Object[]``, as ``insert_columns`` does, and a column of nothing but None is an ``Object[]`` of nulls. + """ + n = len(values) + if kind == "null": + return jpype.JArray(jpype.JObject)([None] * n) + if kind == "str": + return _string_column(values) + if kind == "date" or kind == "datetime": + mask = None + if nullable: + mask = jpype.JArray(jpype.JBoolean)( + _np.fromiter( + map(operator.is_, values, itertools.repeat(None)), + dtype=_np.bool_, + count=n, + ) + ) + filler = _EPOCH_DATE if kind == "date" else _EPOCH_DATETIME + values = [filler if v is None else v for v in values] + temporal = _java_class("com.arcadedb.python.TemporalColumn") + if kind == "date": + ticks = ( + _np.fromiter(map(date.toordinal, values), dtype=_np.int64, count=n) + - _EPOCH_ORDINAL + ) + return temporal.dates(jpype.JArray(jpype.JLong)(ticks), 1, mask) + ticks = _np.array(values, dtype="datetime64[us]").astype(_np.int64) + return temporal.dateTimes(jpype.JArray(jpype.JLong)(ticks), 1_000_000, mask) + if nullable: + return _column_to_java(name, values)[0] + dtype = {"int": _np.int64, "float": _np.float64, "bool": _np.bool_}[kind] + try: + array = _np.fromiter(values, dtype=dtype, count=n) + except (OverflowError, ValueError, TypeError): + return None + return _column_to_java(name, array)[0] + + +def _typed_chunk_columns(chunk): + """``(names, columns)`` for one chunk of insert_many rows that can cross as typed columns, else None (the chunk keeps the JSON text). + + The chunk qualifies when every row is a ``dict`` with the same str keys in the same order (a row that lacks a key, has another + one, or lists them in another order is a different shape, and a record keeps its properties in the order they were set), and every + column holds one kind of value (``_typed_kind``). Whether a column qualifies is decided from the types alone before any bridge + array is built, so a chunk that cannot cross this way costs the census and nothing else. The result is stored as the JSON text + stores it: ``DocumentBatcher.insertManyColumns`` stores a whole number as an Integer when it fits an int, as the text parser + does, and ``None`` is a null property, never a missing one. A row with a NaN, an infinity or a subnormal float is the exception: the + text cannot carry it, so the JSON route inserts that row on the per-row path, which stores its whole numbers as Long, and the + bridge stores them as Long for that row too. + """ + if _np is None or not chunk or set(map(type, chunk)) != _DICT_ONLY: + return None + names = tuple(chunk[0]) + if not names or set(map(type, names)) != _STR_ONLY: + return None + if len(set(map(tuple, chunk))) != 1: + return None + lists = [] + shapes = [] + for name in names: + values = list(map(operator.itemgetter(name), chunk)) + shape = _typed_kind(values) + if shape is None: + return None + lists.append(values) + shapes.append(shape) + columns = [] + for name, values, (kind, nullable) in zip(names, lists, shapes): + column = _typed_java_column(name, values, kind, nullable) + if column is None: + return None + columns.append(column) + return names, columns + + def _wrap_java_record(java_record, database=None): """Wrap a Java record in the matching Python class (Vertex/Edge/Document).""" if java_record is None: @@ -765,11 +892,18 @@ def insert_many( ) -> int: """Bulk-insert documents with one FFI crossing per chunk of rows. - The iterable is read 10,000 rows at a time; each chunk is serialized - to one JSON string and looped Java-side (``DocumentBatcher``), - avoiding the per-row JNI cost that caps ``new_document``-loop ingest. + The iterable is read 10,000 rows at a time; each chunk crosses once and + is looped Java-side (``DocumentBatcher``), avoiding the per-row JNI cost + that caps ``new_document``-loop ingest. A chunk whose rows are dicts with + the same keys in the same order, and whose columns each hold ``None`` and + values of one type (``int`` within 64 bits, ``float``, ``bool``, ``str``, + ``date``, or a ``datetime`` without a zone), crosses as typed columns, with + no text (``DocumentBatcher.insertManyColumns``); a whole number is stored + as an Integer when it fits an int and as a Long beyond, which is what the + JSON text stored (a row with a NaN, an infinity or a subnormal ``float`` + stores its whole numbers as Long, as the per-row path always did). Any other chunk is serialized to one JSON string. Memory stays bounded by the chunk whatever the row count, so a - generator of any length can be loaded (#294). A value that sits directly + generator of any length can be loaded (#294). On the JSON route, a value that sits directly in a row may be JSON-representable (str/int/float/bool/None, nested lists/dicts), a ``datetime`` (also ``pandas.Timestamp`` and ``numpy.datetime64``), ``date``, ``Decimal``, ``bytes``/``bytearray``, @@ -944,6 +1078,12 @@ def _insert_many_chunk(self, type_name, chunk): still go in bulk, and every row is inserted in the order given. Dates, date-times, Decimals, bytes, numpy scalars and pandas NA and NaT sitting directly in a row cross inside the text as tagged values (see json_bulk_dumps_tagged). """ + # A chunk of rows with the same keys whose columns each hold one kind of value crosses as typed columns, with no text + # (see _typed_chunk_columns); every other chunk keeps the JSON text below, as it was. + typed = _typed_chunk_columns(chunk) + if typed is not None: + self._insert_many_typed(type_name, typed, len(chunk)) + return # Float vectors for ARRAY_OF_FLOATS properties cross as one binary matrix # beside the JSON, not as text (see split_float_vectors). split = split_float_vectors(self._java_db, type_name, chunk) @@ -954,6 +1094,21 @@ def _insert_many_chunk(self, type_name, chunk): return self._insert_many_bulk(type_name, payload, split) + def _insert_many_typed(self, type_name, typed, n): + """One chunk of insert_many as typed columns inside the open transaction (commitEvery 0: the transaction stays with insert_many).""" + names, columns = typed + try: + _java_class("com.arcadedb.python.DocumentBatcher").insertManyColumns( + self._java_db, + type_name, + jpype.JArray(jpype.JString)(names), + jpype.JArray(jpype.JObject)(columns), + n, + 0, + ) + except Exception as e: + raise ArcadeDBError(f"Failed to bulk-insert into '{type_name}': {e}") from e + def _insert_many_runs(self, type_name, chunk): """A chunk with a row the JSON text cannot carry: runs of carriable rows go in bulk, the other rows one by one, in order.""" skip = declared_float_vectors(self._java_db, type_name) @@ -1013,6 +1168,22 @@ def _insert_many_parallel(self, type_name, first, rows): chunk = first try: while chunk: + typed = _typed_chunk_columns(chunk) + if typed is not None: + try: + failures = batcher.insertManyColumnsParallel( + self._java_db, + type_name, + jpype.JArray(jpype.JString)(typed[0]), + jpype.JArray(jpype.JObject)(typed[1]), + len(chunk), + ) + except Exception as e: + raise self._parallel_load_failed(type_name, e) from e + chunk_failures.append(failures) + n += len(chunk) + chunk = list(itertools.islice(rows, _INSERT_MANY_CHUNK)) + continue bad = [] split = split_float_vectors(self._java_db, type_name, chunk) try: diff --git a/src/java/com/arcadedb/python/DocumentBatcher.java b/src/java/com/arcadedb/python/DocumentBatcher.java index 8d892a9..d70fd9b 100644 --- a/src/java/com/arcadedb/python/DocumentBatcher.java +++ b/src/java/com/arcadedb/python/DocumentBatcher.java @@ -11,7 +11,8 @@ * the async executor's parallel bucket writers (insertManyJsonParallel; the * Python insert_many wrapper waits for completion itself, then reads the * failures the writers reported). insertColumns takes whole columns instead of - * JSON (Database.insert_columns). Float vectors for ARRAY_OF_FLOATS properties cross as binary matrices (FloatVectors), beside + * JSON (Database.insert_columns); insertManyColumns is the same for a chunk of insert_many rows whose columns each hold one kind of + * value, and stores every number as the JSON text route did (see narrowed and carriedByText). Float vectors for ARRAY_OF_FLOATS properties cross as binary matrices (FloatVectors), beside * the JSON or as a column. The boxDoubles/boxLongs * helpers below serve AsyncExecutor.append_samples' numpy fast path. * @@ -95,11 +96,28 @@ public static long insertManyJson(final Database db, final String typeName, fina */ public static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns, final int n, final int commitEvery) { + return insertColumns(db, typeName, names, columns, n, commitEvery, false); + } + + /** + * The columns of one insert_many chunk (every row has the same keys in the same order and each column holds one kind of value): as + * {@link #insertColumns}, except that a whole number is stored as the JSON text route stored it, an Integer when it fits an int and a + * Long beyond, so the type of a schemaless property does not depend on which route carried the chunk (see {@link #narrowed}). A row + * the text could not have carried (a NaN, an infinity or a subnormal double) keeps every whole number a Long, as the per-row path + * stores it (see {@link #carriedByText}). + */ + public static long insertManyColumns(final Database db, final String typeName, final String[] names, final Object[] columns, + final int n, final int commitEvery) { + return insertColumns(db, typeName, names, columns, n, commitEvery, true); + } + + private static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns, + final int n, final int commitEvery, final boolean narrowIntegers) { checkColumns(names, columns, n); final boolean vertex = n > 0 && isVertexType(db, typeName); writeChunks(db, n, commitEvery, i -> { final MutableDocument doc = newRecord(db, typeName, vertex); - fillRow(doc, names, columns, i); + fillRow(doc, names, columns, i, narrowIntegers); doc.save(); }); return n; @@ -180,13 +198,24 @@ private static void backoffBeforeRetry(final int attempt) { */ public static AsyncFailures insertColumnsParallel(final Database db, final String typeName, final String[] names, final Object[] columns, final int n) { + return insertColumnsParallel(db, typeName, names, columns, n, false); + } + + /** The parallel twin of {@link #insertManyColumns}. */ + public static AsyncFailures insertManyColumnsParallel(final Database db, final String typeName, final String[] names, + final Object[] columns, final int n) { + return insertColumnsParallel(db, typeName, names, columns, n, true); + } + + private static AsyncFailures insertColumnsParallel(final Database db, final String typeName, final String[] names, + final Object[] columns, final int n, final boolean narrowIntegers) { checkColumns(names, columns, n); final AsyncFailures failures = new AsyncFailures(); final ErrorCallback onError = failures::record; final boolean vertex = n > 0 && isVertexType(db, typeName); for (int i = 0; i < n; i++) { final MutableDocument doc = newRecord(db, typeName, vertex); - fillRow(doc, names, columns, i); + fillRow(doc, names, columns, i, narrowIntegers); db.async().createRecord(doc, null, onError); } return failures; @@ -239,12 +268,48 @@ else if (col instanceof TemporalColumn t) } } - private static void fillRow(final MutableDocument doc, final String[] names, final Object[] columns, final int i) { + /** + * The value a JSON number token becomes in the engine (JSONObject.parseNumber): an Integer when a whole number fits an int, a Long + * beyond. The columns of a numpy long[] hold every whole number as a long, which would store a Long where the text route stored an + * Integer, so the insert_many columns narrow it back. Written with returns, not a conditional: {@code c ? (int) v : v} would promote + * both arms to long. + */ + private static Object narrowed(final long v) { + if (v >= Integer.MIN_VALUE && v <= Integer.MAX_VALUE) + return (int) v; + return v; + } + + /** + * Whether the JSON text route could have carried row i. The text cannot carry a NaN, an infinity or a subnormal double (the engine's + * parser reads 5e-324 as a BigDecimal), so Python sends a row that has one down the per-row path, which stores every whole number of + * that row as a Long: the same value the text would have made an Integer. insert_many columns keep that for such a row, so what a + * record stores does not depend on which route carried its chunk. Only a double[] and an Object[] (a column with a null) can hold one. + */ + private static boolean carriedByText(final Object[] columns, final int i) { + for (final Object col : columns) { + if (col instanceof double[] a) { + if (!textDouble(a[i])) + return false; + } else if (col instanceof Object[] o && o[i] instanceof Double d && !textDouble(d)) + return false; + } + return true; + } + + /** A double the JSON text keeps as a double: finite, and zero or at least the smallest normal value (Double.MIN_NORMAL). */ + private static boolean textDouble(final double v) { + return Double.isFinite(v) && (v == 0.0 || Math.abs(v) >= Double.MIN_NORMAL); + } + + private static void fillRow(final MutableDocument doc, final String[] names, final Object[] columns, final int i, + final boolean narrowIntegers) { + final boolean narrow = narrowIntegers && carriedByText(columns, i); for (int c = 0; c < names.length; c++) { final Object col = columns[c]; - final Object v; + Object v; if (col instanceof long[] a) - v = a[i]; + v = narrow ? narrowed(a[i]) : (Object) a[i]; else if (col instanceof double[] a) v = a[i]; else if (col instanceof boolean[] a) @@ -255,8 +320,11 @@ else if (col instanceof StringColumn s) v = s.get(i); else if (col instanceof TemporalColumn t) v = t.get(i); - else + else { v = ((Object[]) col)[i]; + if (narrow && v instanceof Long l) + v = narrowed(l); + } doc.set(names[c], v); } } diff --git a/tests/conftest.py b/tests/conftest.py index 22b929f..f8b209f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -168,6 +168,7 @@ "test_reader_identity_crossings.py": "crossing counts of get_rid, get_vertex, get_edge and get_element on the JVM build (JPype proxies of the Java row and the bridge class)", "test_typed_batch_guard.py": "the object_hook and the side array of the JVM build's TypedRows batches (JPype arrays, the bridge class, a spy on json.loads)", "test_jvm_ingest_integrity_faults.py": "rows-dropped faults put into the JVM build's column conversion and bulk vertex path (core._column_to_java, GraphBatch._create_vertices_json_bulk)", + "test_insert_many_typed_columns.py": "the typed-column route of the JVM wheel's insert_many (the bridge's DocumentBatcher.insertManyColumns, core._typed_chunk_columns, the Java classes of the stored values)", } if BACKEND == "native" and not os.environ.get("NATIVE_PARITY_ALL"): collect_ignore.extend(NATIVE_NOT_APPLICABLE) diff --git a/tests/test_insert_many_typed_columns.py b/tests/test_insert_many_typed_columns.py new file mode 100644 index 0000000..9f923f3 --- /dev/null +++ b/tests/test_insert_many_typed_columns.py @@ -0,0 +1,685 @@ +"""insert_many (the JVM wheel): a chunk whose rows have the same keys and whose columns each hold one kind of value crosses as typed columns. + +Before, every chunk of 10,000 rows became one JSON text (``json_bulk_dumps_tagged``), which the engine parsed again into one JSON tree and +copied key by key (``DocumentBatcher.insertManyJson``). Now a homogeneous chunk takes ``DocumentBatcher.insertManyColumns``: one numpy +buffer per int, float or bool column, one UTF-8 buffer per string column, one ``TemporalColumn`` per date or date-time column, and no text. +Every other chunk keeps the JSON text exactly as it was. + +The stored data has to be the same on both routes, so each parity test stores the same rows twice, once with the typed route switched off +(``core._typed_chunk_columns`` returns None: the old route) and once as shipped, and compares every record property by property +through SQL: the property names in the order the record lists them, the Java class of every value (an Integer is not a Long), and the +value. The route that ran is read from the calls made on the bridge class. The Java types are read through the Java database +(``get_java_database``), so this file has no native counterpart (the native wheel carries homogeneous chunks through its own columnar +entry, ``arcadedb._native.bulk.insert_rows_columnar``). +""" + +import datetime as dt +import decimal +import math +import sys + +import jpype +import numpy as np +import pytest + +from arcadedb import core +from arcadedb.exceptions import ArcadeDBError + +BRIDGE = "com.arcadedb.python.DocumentBatcher" +UTC9 = dt.timezone(dt.timedelta(hours=9)) + + +class _Spy: + """Forwards every call of a Java class and records its name.""" + + def __init__(self, target, log): + self._target, self._log = target, log + + def __getattr__(self, name): + attribute = getattr(self._target, name) + if not callable(attribute): + return attribute + + def call(*args): + self._log.append(name) + return attribute(*args) + + return call + + +@pytest.fixture +def bridge(monkeypatch): + """The names of the DocumentBatcher calls of the test, in order.""" + log = [] + real = core._java_class + + def spy(name): + cls = real(name) + return _Spy(cls, log) if name == BRIDGE else cls + + monkeypatch.setattr(core, "_java_class", spy) + return log + + +def _old_route(monkeypatch): + """The route before this change: no chunk is typed, every chunk is one JSON text.""" + monkeypatch.setattr(core, "_typed_chunk_columns", lambda chunk: None) + + +def _describe(value): + if value is None: + return ("null",) + if isinstance(value, str): # a String JPype already converted + return ("java.lang.String", value) + if isinstance(value, jpype.JArray): + return (str(value.getClass().getName()), repr(list(value))) + if hasattr(value, "getClass"): + return (str(value.getClass().getName()), str(value)) + return ("py:" + type(value).__name__, repr(value)) + + +def _stored(db, type_name): + """Every record by id: [(property, (Java class, text of the value)), ...] in the order the record lists its properties.""" + result = db.get_java_database().query( + "sql", f"SELECT FROM {type_name} ORDER BY id" # nosec B608 - test-owned name + ) + records = [] + while result.hasNext(): + record = result.next() + records.append( + [ + (str(name), _describe(record.getProperty(name))) + for name in record.getPropertyNames() + ] + ) + return records + + +DECLARED = { + "i": "LONG", + "f": "DOUBLE", + "b": "BOOLEAN", + "s": "STRING", + "d": "DATE", + "t": "DATETIME", +} + + +def _make(db, name, declared=False, kind="DOCUMENT", buckets=None): + suffix = f" BUCKETS {buckets}" if buckets else "" + db.command("sql", f"CREATE {kind} TYPE {name}{suffix}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + if declared: + for prop, ty in DECLARED.items(): + db.command("sql", f"CREATE PROPERTY {name}.{prop} {ty}") + + +INT_EDGES = [ + 0, + 1, + -1, + 2**31 - 1, + 2**31, + -(2**31), + -(2**31) - 1, + 2**53 + 1, + -(2**53) - 1, + 2**63 - 1, + -(2**63), +] +FLOAT_EDGES = [ + 0.0, + -0.0, + 0.1, + 1.0, + 1e300, + -1e-300, + 5e-324, + 2.2250738585072014e-308, + 1.7976931348623157e308, + float("nan"), + float("inf"), + float("-inf"), +] +STRINGS = [ + "", + "a", + 'quo"te', + "back\\slash", + "new\nline", + "tab\t", + "emoji \U0001f600", + "é ü 中", + "\x01tag", + "x" * 5000, +] +DATES = [ + dt.date(1970, 1, 1), + dt.date(1969, 12, 31), + dt.date(2026, 11, 14), + dt.date(1, 1, 1), + dt.date(9999, 12, 31), + dt.date(2000, 2, 29), +] +DATETIMES = [ + dt.datetime(1970, 1, 1), + dt.datetime(1969, 12, 31, 23, 59, 59, 999999), + dt.datetime(2026, 11, 14, 12, 34, 56, 123456), + dt.datetime(2000, 2, 29, 0, 0, 0, 1), + dt.datetime(1, 1, 1), + dt.datetime(9999, 12, 31, 23, 59, 59, 999999), +] + + +def _one_column(values, name="v"): + return [{"id": i, name: v} for i, v in enumerate(values)] + + +def _cycle(values, n): + return [values[i % len(values)] for i in range(n)] + + +# floats the JSON text cannot carry (NaN, infinity, a subnormal: the row goes down the per-row path, which stores its whole numbers as +# Long) beside floats it carries (the smallest normal value, -0.0, a plain value) +NON_TEXT_FLOATS = [ + float("nan"), + float("inf"), + float("-inf"), + 5e-324, + -5e-324, + math.nextafter(sys.float_info.min, 0.0), # the largest subnormal + sys.float_info.min, + -0.0, + 0.5, +] +SMALL_INTS = [3, -3, 0, 2**31 - 1, -(2**31), 5, 6, 7, 8] + + +def _ints_beside_floats(nulls): + """Whole numbers that fit an int in the same rows as the floats above: the rows the text cannot carry keep them Long.""" + rows = [] + for i in range(2 * len(NON_TEXT_FLOATS)): + rows.append( + { + "id": i, + "i": None if nulls and i % 4 == 1 else SMALL_INTS[i % len(SMALL_INTS)], + "f": ( + None + if nulls and i % 4 == 2 + else NON_TEXT_FLOATS[i % len(NON_TEXT_FLOATS)] + ), + } + ) + return rows + + +def _all_kinds(nulls): + n = 12 + rows = [] + for i in range(n): + null = nulls and i % 3 == 1 + rows.append( + { + "id": i, + "i": None if null else INT_EDGES[i % len(INT_EDGES)], + "f": None if null else FLOAT_EDGES[i % len(FLOAT_EDGES)], + "b": None if null else i % 2 == 0, + "s": None if null else STRINGS[i % len(STRINGS)], + "d": None if null else DATES[i % len(DATES)], + "t": None if null else DATETIMES[i % len(DATETIMES)], + } + ) + return rows + + +# name -> (rows, typed): the rows the chunk holds and the route they take +CASES = { + "ints": (_one_column(INT_EDGES), True), + "floats_nan_inf_subnormal": (_one_column(FLOAT_EDGES), True), + "bools": (_one_column([True, False, True, True]), True), + "strings": (_one_column(STRINGS), True), + "dates": (_one_column(DATES), True), + "datetimes": (_one_column(DATETIMES), True), + "all_kinds": (_all_kinds(False), True), + "all_kinds_with_nulls": (_all_kinds(True), True), + "null_in_every_row": (_one_column([None, None, None]), True), + "null_beside_ints": (_one_column([1, None, 2**40, None]), True), + "null_beside_floats": (_one_column([1.5, None, float("nan")]), True), + "null_beside_strings": (_one_column(["a", None, "b"]), True), + "ints_beside_nan_inf_subnormal": (_ints_beside_floats(False), True), + "nullable_ints_beside_nullable_nan_inf_subnormal": ( + _ints_beside_floats(True), + True, + ), + "single_row": (_one_column([7]), True), + # shapes the typed route cannot carry: the chunk keeps the JSON text and the per-row path for the rows the text cannot carry + "other_key_order": ( + [ + {"id": 0, "a": 1, "b": 2}, + {"id": 1, "b": 3, "a": 4}, + {"id": 2, "a": 5, "b": 6}, + ], + False, + ), + "key_missing_in_a_row": ([{"id": 0, "a": 1}, {"id": 1}, {"id": 2, "a": 3}], False), + "null_in_one_row_missing_in_another": ( + [{"id": 0, "a": None}, {"id": 1}, {"id": 2, "a": 2}], + False, + ), + "an_extra_key_in_a_row": ([{"id": 0, "a": 1}, {"id": 1, "a": 2, "b": 3}], False), + "int_beside_float": (_one_column([1, 2.5, 3]), False), + "bool_beside_int": (_one_column([True, 1, False]), False), + "str_beside_int": (_one_column(["a", 1]), False), + "date_beside_datetime": (_one_column([DATES[0], DATETIMES[0]]), False), + "date_inside_a_list": (_one_column([[DATES[2]], [DATES[0]]]), False), + "datetime_inside_a_list": (_one_column([[DATETIMES[2]], [1]]), False), + "decimal_inside_a_list": (_one_column([[decimal.Decimal("1.5")], [2]]), False), + "decimal": ( + _one_column([decimal.Decimal("12345.6789"), decimal.Decimal("-0.50")]), + False, + ), + "bytes": (_one_column([b"\x00\xff abc\x7f", b""]), False), + "numpy_scalars": ( + _one_column([np.int64(2**40 + 5), np.int64(-3), np.int64(7)]), + False, + ), + "numpy_float_scalars": (_one_column([np.float64(0.1), np.float64(2.5)]), False), + "aware_datetime": ( + _one_column([dt.datetime(2026, 5, 6, 7, 8, 9, 654321, tzinfo=UTC9)]), + False, + ), + "nested_lists_and_maps": ( + _one_column([[1, 2, [3, 4]], {"k": [1, {"z": 2}], "n": None}]), + False, + ), + "tuple": (_one_column([(1, 2, 3), (4, 5, 6)]), False), + "list_of_ints_and_a_float": (_one_column([[1, 2.5], [3, 4]]), False), +} + + +# every case on schemaless properties, and the rows of every kind on declared ones too (the engine converts to the declared type) +PAIRS = [(case, False) for case in CASES] + [ + ("all_kinds", True), + ("all_kinds_with_nulls", True), +] + + +@pytest.mark.parametrize( + "case, declared", + PAIRS, + ids=[f"{c}-{'declared' if d else 'free'}" for c, d in PAIRS], +) +def test_the_typed_route_stores_what_the_json_route_stores( + temp_db, monkeypatch, bridge, case, declared +): + rows, typed = CASES[case] + _make(temp_db, "Old", declared) + _make(temp_db, "New", declared) + with monkeypatch.context() as old: + _old_route(old) + bridge.clear() + assert temp_db.insert_many("Old", rows) == len(rows) + old_calls = list(bridge) + assert "insertManyColumns" not in old_calls + bridge.clear() + assert temp_db.insert_many("New", rows) == len(rows) + if typed: + assert bridge == ["insertManyColumns"], f"{case} left the typed route" + else: + # the chunk keeps the route it always had, call for call (the JSON text, and the per-row path for rows the text cannot carry) + assert ( + bridge == old_calls + ), f"{case} took the typed route, which cannot carry it" + new, old = _stored(temp_db, "New"), _stored(temp_db, "Old") + assert len(new) == len(rows) + assert new == old + + +def test_a_whole_number_is_an_integer_when_it_fits_an_int_and_a_long_beyond( + temp_db, bridge +): + """The classes themselves, not only old against new: what the JSON text stored (JSONObject.parseNumber).""" + _make(temp_db, "T") + temp_db.insert_many( + "T", _one_column([5, 2**31 - 1, 2**31, -(2**31), -(2**31) - 1, 2**62]) + ) + assert bridge == ["insertManyColumns"] + classes = [dict(r)["v"][0] for r in _stored(temp_db, "T")] + assert classes == [ + "java.lang.Integer", + "java.lang.Integer", + "java.lang.Long", + "java.lang.Integer", + "java.lang.Long", + "java.lang.Long", + ] + + +def test_insert_columns_still_stores_a_long_for_every_whole_number(temp_db): + """insert_columns shares fillRow with the new entry point; only insertManyColumns narrows (a regression test, green before this change). + + The type declares no property: a declared INTEGER would convert the Long to an Integer whichever entry point stored it. + """ + temp_db.command("sql", "CREATE DOCUMENT TYPE C") + temp_db.insert_columns("C", {"id": np.arange(3, dtype=np.int64), "v": [1, 2, 3]}) + records = _stored(temp_db, "C") + assert len(records) == 3 + for record in records: + assert {name for name, _ in record} == {"id", "v"} + for name, (kind, _) in record: + assert kind == "java.lang.Long", name + + +def test_a_chunk_that_takes_the_old_route_builds_no_bridge_array( + temp_db, monkeypatch, bridge +): + """A fallback chunk costs the census and nothing else: no column is built before the shape is known to qualify.""" + built = [] + real = core._typed_java_column + + def spy(*args): + built.append(args[0]) + return real(*args) + + monkeypatch.setattr(core, "_typed_java_column", spy) + _make(temp_db, "T") + # the first column is fine, the second holds a Decimal: nothing is built for either + temp_db.insert_many("T", [{"id": i, "v": decimal.Decimal(i)} for i in range(5)]) + assert built == [] + assert bridge == ["insertManyJson"] + + +def test_a_homogeneous_load_crosses_once_per_chunk_with_no_text_and_no_per_value_conversion( + temp_db, monkeypatch, bridge +): + monkeypatch.setattr(core, "_INSERT_MANY_CHUNK", 1000) + + def no_text(*args, **kwargs): + raise AssertionError("a homogeneous chunk went through the JSON text") + + monkeypatch.setattr(core, "json_bulk_dumps_tagged", no_text) + converted = [] + real = core.convert_python_to_java + + def counting(value): + converted.append(type(value).__name__) + return real(value) + + monkeypatch.setattr(core, "convert_python_to_java", counting) + _make(temp_db, "T") + rows = [ + { + "id": i, + "n": i * 3, + "x": i * 0.5, + "ok": i % 2 == 0, + "name": f"row {i}", + "day": DATES[i % len(DATES)], + "at": DATETIMES[i % len(DATETIMES)], + } + for i in range(2500) + ] + assert temp_db.insert_many("T", rows) == 2500 + assert bridge == ["insertManyColumns"] * 3 # chunks of 1000, 1000, 500 + assert converted == [], "a value was converted one at a time" + assert temp_db.query("sql", "SELECT count(*) AS c FROM T").first().get("c") == 2500 + record = dict( + _stored(temp_db, "T")[1235] + ) # 1235 % 6 == 5: the last date and date-time of the lists above + assert {name: kind for name, (kind, _) in record.items()} == { + "id": "java.lang.Integer", + "n": "java.lang.Integer", + "x": "java.lang.Double", + "ok": "java.lang.Boolean", + "name": "java.lang.String", + "day": "java.time.LocalDate", + "at": "java.time.LocalDateTime", + } + # the text of a value is how JPype prints it (a Java Boolean prints as False), so the case is not part of the claim + assert {name: text.lower() for name, (_, text) in record.items()} == { + "id": "1235", + "n": "3705", + "x": "617.5", + "ok": "false", + "name": "row 1235", + "day": "2000-02-29", + "at": DATETIMES[5].isoformat().lower(), + } + + +def test_chunks_of_both_routes_keep_the_order_of_the_rows(temp_db, monkeypatch, bridge): + monkeypatch.setattr(core, "_INSERT_MANY_CHUNK", 4) + _make(temp_db, "T", buckets=1) + rows = _one_column(list(range(12))) + rows[5] = {"id": 5, "v": decimal.Decimal(5)} # the middle chunk keeps the text + assert temp_db.insert_many("T", rows) == 12 + assert bridge == ["insertManyColumns", "insertManyJson", "insertManyColumns"] + ids = [r.get("id") for r in temp_db.query("sql", "SELECT id FROM T").to_list()] + assert ids == list(range(12)) + + +def test_the_commit_boundary_still_splits_a_chunk(temp_db, bridge): + _make(temp_db, "T") + assert temp_db.insert_many("T", _one_column(list(range(25))), commit_every=10) == 25 + assert ( + bridge == ["insertManyColumns"] * 3 + ) # 10, 10, 5: a chunk never straddles a commit + assert len(_stored(temp_db, "T")) == 25 + + +def test_an_empty_batch_and_a_batch_of_one(temp_db, bridge): + _make(temp_db, "T") + assert temp_db.insert_many("T", []) == 0 + assert bridge == [] + assert temp_db.insert_many("T", [{"id": 1, "v": 2}]) == 1 + assert bridge == ["insertManyColumns"] + assert len(_stored(temp_db, "T")) == 1 + + +def test_a_generator_is_read_chunk_by_chunk(temp_db, monkeypatch, bridge): + monkeypatch.setattr(core, "_INSERT_MANY_CHUNK", 100) + _make(temp_db, "T") + n = temp_db.insert_many("T", ({"id": i, "v": i} for i in range(250))) + assert n == 250 + assert bridge == ["insertManyColumns"] * 3 + + +@pytest.mark.parametrize("kind", ["DOCUMENT", "VERTEX"]) +def test_documents_and_vertices_are_the_same_on_both_routes( + temp_db, monkeypatch, bridge, kind +): + _make(temp_db, "Old", kind=kind) + _make(temp_db, "New", kind=kind) + rows = _one_column(INT_EDGES) + with monkeypatch.context() as old: + _old_route(old) + temp_db.insert_many("Old", rows) + temp_db.insert_many("New", rows) + assert _stored(temp_db, "New") == _stored(temp_db, "Old") + for name in ("Old", "New"): + sql = f"SELECT FROM {name}" # nosec B608 - test-owned name + result = temp_db.get_java_database().query("sql", sql) + n = 0 + while result.hasNext(): + assert result.next().isVertex() == (kind == "VERTEX") + n += 1 + assert n == len(rows) + + +def test_an_edge_type_is_refused_as_before(temp_db, monkeypatch): + temp_db.command("sql", "CREATE VERTEX TYPE V") + temp_db.command("sql", "CREATE EDGE TYPE E") + rows = _one_column([1, 2]) + messages = [] + for route in ("old", "new"): + with monkeypatch.context() as m: + if route == "old": + _old_route(m) + with pytest.raises(ArcadeDBError) as error: + temp_db.insert_many("E", rows) + messages.append(str(error.value)) + assert messages[0] == messages[1] + assert "edge type" in messages[0] + + +def test_a_duplicate_key_fails_the_call_and_rolls_it_back_on_both_routes( + temp_db, monkeypatch, bridge +): + outcomes = [] + for route in ("old", "new"): + name = f"T_{route}" + temp_db.command("sql", f"CREATE DOCUMENT TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + temp_db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + rows = _one_column([1, 2, 3, 4]) + [ + {"id": 2, "v": 9} + ] # the id 2 is there twice + with monkeypatch.context() as m: + if route == "old": + _old_route(m) + bridge.clear() + with pytest.raises( + Exception + ) as error: # noqa: B017 - the class is compared below + temp_db.insert_many(name, rows) + outcomes.append( + ( + type(error.value).__name__, + temp_db.query("sql", f"SELECT count(*) AS c FROM {name}") # nosec B608 + .first() + .get("c"), + list(bridge), + ) + ) + assert outcomes[0][:2] == outcomes[1][:2] + assert outcomes[1][1] == 0, "the call rolled back the transaction it opened" + assert outcomes[0][2] == ["insertManyJson"] + assert outcomes[1][2] == ["insertManyColumns"] + + +@pytest.mark.parametrize( + "values", + [[1, 2**64], [-(2**63) - 1, 1], ["ok", "a\ud800b"]], + ids=["beyond_int64", "below_int64", "lone_surrogate"], +) +def test_a_value_no_route_can_store_fails_as_before( + temp_db, monkeypatch, bridge, values +): + outcomes = [] + for route in ("old", "new"): + name = f"T_{route}" + _make(temp_db, name) + with monkeypatch.context() as m: + if route == "old": + _old_route(m) + bridge.clear() + try: + temp_db.insert_many(name, _one_column(values)) + raised = None + except Exception as error: # noqa: BLE001 - the same class is the claim + raised = type(error).__name__ + outcomes.append( + (raised, len(_stored(temp_db, name)), "insertManyColumns" in bridge) + ) + assert outcomes[0][:2] == outcomes[1][:2] + assert ( + outcomes[1][2] is False + ), "a value the text refuses must not take the typed route" + + +@pytest.mark.parametrize( + "case", + [ + "ints", + "all_kinds_with_nulls", + "ints_beside_nan_inf_subnormal", + "nullable_ints_beside_nullable_nan_inf_subnormal", + "decimal", + ], +) +def test_parallel_mode_stores_what_the_json_route_stores( + temp_db, monkeypatch, bridge, case +): + rows, typed = CASES[case] + _make(temp_db, "Old") + _make(temp_db, "New") + with monkeypatch.context() as old: + _old_route(old) + bridge.clear() + assert temp_db.insert_many("Old", rows, parallel=True) == len(rows) + assert "insertManyJsonParallel" in bridge + bridge.clear() + assert temp_db.insert_many("New", rows, parallel=True) == len(rows) + assert bridge[0] == ( + "insertManyColumnsParallel" if typed else "insertManyJsonParallel" + ) + assert _stored(temp_db, "New") == _stored(temp_db, "Old") + + +def test_a_caller_transaction_is_left_to_the_caller(temp_db, bridge): + _make(temp_db, "T") + with pytest.raises(RuntimeError): + with temp_db.transaction(): + temp_db.insert_many("T", _one_column(list(range(30))), commit_every=10) + raise RuntimeError("roll back") + assert bridge == ["insertManyColumns"] + assert temp_db.query("sql", "SELECT count(*) AS c FROM T").first().get("c") == 0 + + +@pytest.mark.parametrize( + "values, expected", + [ + ([1, 2, 3], ("int", False)), + ([1, None], ("int", True)), + ([2**63], None), + ([-(2**63) - 1, None], None), + ([1, 2.0], None), + ([True, False], ("bool", False)), + ([True, 1], None), + ([None, None], ("null", True)), + (["a", None], ("str", True)), + ([1.5, math.nan], ("float", False)), + ([dt.date(2020, 1, 2)], ("date", False)), + ([dt.datetime(2020, 1, 2)], ("datetime", False)), + ([dt.datetime(2020, 1, 2), None], ("datetime", True)), + ([dt.datetime(2020, 1, 2, tzinfo=dt.timezone.utc)], None), + ([dt.date(2020, 1, 2), dt.datetime(2020, 1, 2)], None), + ([decimal.Decimal(1)], None), + ([b"x"], None), + ([[1, 2]], None), + ([{"a": 1}], None), + ([np.int64(1)], None), + ([np.float64(1)], None), + ], +) +def test_which_columns_qualify(values, expected): + assert core._typed_kind(values) == expected + + +@pytest.mark.parametrize( + "chunk", + [ + [], + [{}, {}], + [{1: 1}], + [{"a": 1}, [("a", 2)]], + [{"a": 1, "b": 2}, {"b": 2, "a": 1}], + [{"a": 1}, {"a": 1, "b": 2}], + [{"a": 1}, {"b": 1}], + [{"a": 1}, {"a": 2.5}], + [{"a": [1]}, {"a": [2]}], + ], + ids=[ + "empty", + "no_keys", + "non_str_key", + "not_a_dict", + "key_order", + "missing_key", + "other_key", + "mixed_kind", + "list", + ], +) +def test_which_chunks_qualify(chunk): + assert core._typed_chunk_columns(chunk) is None