Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/modules/ROOT/pages/how-to/bulk-loads.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ Because the two packages have different defaults, the same program can keep diff

On both packages, `insert_many` keeps rows with `datetime`, `date`, `Decimal`, `bytes`, numpy scalar, or pandas `NA` and `NaT` values on the bulk path. On the `arcadedb` (native) package, a row with a value the bulk text cannot carry (NaN or infinity, an integer beyond 64 bits, a date-time or `Decimal` inside a list or dict, or a `datetime.time`) goes in with its own statement, in the order given.

On the `arcadedb-jvm` package, a chunk of 10,000 rows crosses as typed columns, with no JSON text, when every row is a `dict` with the same keys in the same order and each column holds `None` and values of one type: `int` within 64 bits, `float` (NaN and infinity included), `bool`, `str`, `date`, or a `datetime` without a time zone. The stored values are the same as the JSON text stores: a whole number is an `Integer` when it fits an int and a `Long` beyond, and `None` is a null property. A row with a NaN, an infinity, or a subnormal `float` stores its whole numbers as `Long`, which is what the per-row path stores for such a row. Any other chunk keeps the JSON text, as before: a row that lacks a key or lists the keys in another order, a column that mixes types (an `int` beside a `float`), a `Decimal`, `bytes`, numpy or pandas value, a `datetime` with a time zone, a list, tuple or dict, and a float vector for a property declared `ARRAY_OF_FLOATS`. A chunk is judged as a whole, so one such row sends its 10,000 rows down the JSON route. The `arcadedb` (native) package already sends the rows of one shape through its columnar entry.

`insert_many` and `create_vertices` check their rows while they write the JSON text, in one pass over the rows, not in a loop before it. The same rows fail the check as before, and they take the same fallback path as before. The check raises no error to your code.

[[columnar-loads]]
Expand Down
181 changes: 176 additions & 5 deletions src/arcadedb/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import itertools
import operator
from collections.abc import Mapping
from datetime import date
from datetime import date, datetime
from decimal import Decimal
from os import PathLike
from typing import TYPE_CHECKING, Any, Callable, List, Optional
Expand Down Expand Up @@ -36,6 +36,8 @@
from .results import ResultSet
from .transactions import TransactionContext
from .type_conversion import (
_INT64_MAX,
_INT64_MIN,
_is_numpy_bool,
bulk_row_ok,
convert_python_to_java,
Expand Down Expand Up @@ -336,6 +338,131 @@ def _column_to_java(name, values):
return jpype.JArray(jpype.JObject)(converted), len(converted)


# insert_many: the kinds of column a chunk can cross as typed columns (see _typed_chunk_columns). The exact types only: a subclass
# (pandas.Timestamp, numpy.float64) and every other type (Decimal, bytes, list, dict, tuple) keep the JSON text.
_TYPED_KINDS = {
int: "int",
float: "float",
bool: "bool",
str: "str",
date: "date",
datetime: "datetime",
}
_DICT_ONLY = frozenset((dict,))
_STR_ONLY = frozenset((str,))
_NO_ZONE = frozenset((None,))
_EPOCH_ORDINAL = date(1970, 1, 1).toordinal()
_EPOCH_DATE = date(1970, 1, 1)
_EPOCH_DATETIME = datetime(1970, 1, 1)


def _typed_kind(values):
"""``(kind, nullable)`` of one column of an insert_many chunk when it can cross as a typed column, else None.

The column holds ``None`` and values of ONE exact type of ``_TYPED_KINDS`` (``"null"`` when there is nothing but ``None``).
A mixed column (an int beside a float, a bool beside an int) would store each value as its own type on the JSON route, which one
typed array cannot do; an int beyond 64 bits, a date-time with a zone and a type the table does not list keep the JSON route.
"""
kinds = set(map(type, values))
nullable = _NONE_TYPE in kinds
kinds.discard(_NONE_TYPE)
if not kinds:
return "null", True
if len(kinds) != 1:
return None
kind = _TYPED_KINDS.get(next(iter(kinds)))
if kind == "int":
present = [v for v in values if v is not None] if nullable else values
if min(present) < _INT64_MIN or max(present) > _INT64_MAX:
return None
elif kind == "datetime":
present = [v for v in values if v is not None] if nullable else values
if set(map(operator.attrgetter("tzinfo"), present)) != _NO_ZONE:
return None
return (kind, nullable) if kind else None


def _typed_java_column(name, values, kind, nullable):
"""One column of an insert_many chunk as the Java column ``DocumentBatcher.insertManyColumns`` takes, or None.

A whole column of int, float or bool without a null is one numpy array and one buffer copy (``long[]``, ``double[]``,
``boolean[]``); a column of str and None is one ``StringColumn``; date and naive date-time columns are one ``TemporalColumn``
(a ``long[]`` of epoch days or microseconds, and one ``boolean[]`` when there is a null); a numeric column with a null converts
per element into an ``Object[]``, as ``insert_columns`` does, and a column of nothing but None is an ``Object[]`` of nulls.
"""
n = len(values)
if kind == "null":
return jpype.JArray(jpype.JObject)([None] * n)
if kind == "str":
return _string_column(values)
if kind == "date" or kind == "datetime":
mask = None
if nullable:
mask = jpype.JArray(jpype.JBoolean)(
_np.fromiter(
map(operator.is_, values, itertools.repeat(None)),
dtype=_np.bool_,
count=n,
)
)
filler = _EPOCH_DATE if kind == "date" else _EPOCH_DATETIME
values = [filler if v is None else v for v in values]
temporal = _java_class("com.arcadedb.python.TemporalColumn")
if kind == "date":
ticks = (
_np.fromiter(map(date.toordinal, values), dtype=_np.int64, count=n)
- _EPOCH_ORDINAL
)
return temporal.dates(jpype.JArray(jpype.JLong)(ticks), 1, mask)
ticks = _np.array(values, dtype="datetime64[us]").astype(_np.int64)
return temporal.dateTimes(jpype.JArray(jpype.JLong)(ticks), 1_000_000, mask)
if nullable:
return _column_to_java(name, values)[0]
dtype = {"int": _np.int64, "float": _np.float64, "bool": _np.bool_}[kind]
try:
array = _np.fromiter(values, dtype=dtype, count=n)
except (OverflowError, ValueError, TypeError):
return None
return _column_to_java(name, array)[0]


def _typed_chunk_columns(chunk):
"""``(names, columns)`` for one chunk of insert_many rows that can cross as typed columns, else None (the chunk keeps the JSON text).

The chunk qualifies when every row is a ``dict`` with the same str keys in the same order (a row that lacks a key, has another
one, or lists them in another order is a different shape, and a record keeps its properties in the order they were set), and every
column holds one kind of value (``_typed_kind``). Whether a column qualifies is decided from the types alone before any bridge
array is built, so a chunk that cannot cross this way costs the census and nothing else. The result is stored as the JSON text
stores it: ``DocumentBatcher.insertManyColumns`` stores a whole number as an Integer when it fits an int, as the text parser
does, and ``None`` is a null property, never a missing one. A row with a NaN, an infinity or a subnormal float is the exception: the
text cannot carry it, so the JSON route inserts that row on the per-row path, which stores its whole numbers as Long, and the
bridge stores them as Long for that row too.
"""
if _np is None or not chunk or set(map(type, chunk)) != _DICT_ONLY:
return None
names = tuple(chunk[0])
if not names or set(map(type, names)) != _STR_ONLY:
return None
if len(set(map(tuple, chunk))) != 1:
return None
lists = []
shapes = []
for name in names:
values = list(map(operator.itemgetter(name), chunk))
shape = _typed_kind(values)
if shape is None:
return None
lists.append(values)
shapes.append(shape)
columns = []
for name, values, (kind, nullable) in zip(names, lists, shapes):
column = _typed_java_column(name, values, kind, nullable)
if column is None:
return None
columns.append(column)
return names, columns


def _wrap_java_record(java_record, database=None):
"""Wrap a Java record in the matching Python class (Vertex/Edge/Document)."""
if java_record is None:
Expand Down Expand Up @@ -765,11 +892,18 @@ def insert_many(
) -> int:
"""Bulk-insert documents with one FFI crossing per chunk of rows.

The iterable is read 10,000 rows at a time; each chunk is serialized
to one JSON string and looped Java-side (``DocumentBatcher``),
avoiding the per-row JNI cost that caps ``new_document``-loop ingest.
The iterable is read 10,000 rows at a time; each chunk crosses once and
is looped Java-side (``DocumentBatcher``), avoiding the per-row JNI cost
that caps ``new_document``-loop ingest. A chunk whose rows are dicts with
the same keys in the same order, and whose columns each hold ``None`` and
values of one type (``int`` within 64 bits, ``float``, ``bool``, ``str``,
``date``, or a ``datetime`` without a zone), crosses as typed columns, with
no text (``DocumentBatcher.insertManyColumns``); a whole number is stored
as an Integer when it fits an int and as a Long beyond, which is what the
JSON text stored (a row with a NaN, an infinity or a subnormal ``float``
stores its whole numbers as Long, as the per-row path always did). Any other chunk is serialized to one JSON string.
Memory stays bounded by the chunk whatever the row count, so a
generator of any length can be loaded (#294). A value that sits directly
generator of any length can be loaded (#294). On the JSON route, a value that sits directly
in a row may be JSON-representable (str/int/float/bool/None, nested
lists/dicts), a ``datetime`` (also ``pandas.Timestamp`` and
``numpy.datetime64``), ``date``, ``Decimal``, ``bytes``/``bytearray``,
Expand Down Expand Up @@ -944,6 +1078,12 @@ def _insert_many_chunk(self, type_name, chunk):
still go in bulk, and every row is inserted in the order given. Dates, date-times, Decimals, bytes, numpy scalars and
pandas NA and NaT sitting directly in a row cross inside the text as tagged values (see json_bulk_dumps_tagged).
"""
# A chunk of rows with the same keys whose columns each hold one kind of value crosses as typed columns, with no text
# (see _typed_chunk_columns); every other chunk keeps the JSON text below, as it was.
typed = _typed_chunk_columns(chunk)
if typed is not None:
self._insert_many_typed(type_name, typed, len(chunk))
return
# Float vectors for ARRAY_OF_FLOATS properties cross as one binary matrix
# beside the JSON, not as text (see split_float_vectors).
split = split_float_vectors(self._java_db, type_name, chunk)
Expand All @@ -954,6 +1094,21 @@ def _insert_many_chunk(self, type_name, chunk):
return
self._insert_many_bulk(type_name, payload, split)

def _insert_many_typed(self, type_name, typed, n):
"""One chunk of insert_many as typed columns inside the open transaction (commitEvery 0: the transaction stays with insert_many)."""
names, columns = typed
try:
_java_class("com.arcadedb.python.DocumentBatcher").insertManyColumns(
self._java_db,
type_name,
jpype.JArray(jpype.JString)(names),
jpype.JArray(jpype.JObject)(columns),
n,
0,
)
except Exception as e:
raise ArcadeDBError(f"Failed to bulk-insert into '{type_name}': {e}") from e

def _insert_many_runs(self, type_name, chunk):
"""A chunk with a row the JSON text cannot carry: runs of carriable rows go in bulk, the other rows one by one, in order."""
skip = declared_float_vectors(self._java_db, type_name)
Expand Down Expand Up @@ -1013,6 +1168,22 @@ def _insert_many_parallel(self, type_name, first, rows):
chunk = first
try:
while chunk:
typed = _typed_chunk_columns(chunk)
if typed is not None:
try:
failures = batcher.insertManyColumnsParallel(
self._java_db,
type_name,
jpype.JArray(jpype.JString)(typed[0]),
jpype.JArray(jpype.JObject)(typed[1]),
len(chunk),
)
except Exception as e:
raise self._parallel_load_failed(type_name, e) from e
chunk_failures.append(failures)
n += len(chunk)
chunk = list(itertools.islice(rows, _INSERT_MANY_CHUNK))
continue
bad = []
split = split_float_vectors(self._java_db, type_name, chunk)
try:
Expand Down
82 changes: 75 additions & 7 deletions src/java/com/arcadedb/python/DocumentBatcher.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
* the async executor's parallel bucket writers (insertManyJsonParallel; the
* Python insert_many wrapper waits for completion itself, then reads the
* failures the writers reported). insertColumns takes whole columns instead of
* JSON (Database.insert_columns). Float vectors for ARRAY_OF_FLOATS properties cross as binary matrices (FloatVectors), beside
* JSON (Database.insert_columns); insertManyColumns is the same for a chunk of insert_many rows whose columns each hold one kind of
* value, and stores every number as the JSON text route did (see narrowed and carriedByText). Float vectors for ARRAY_OF_FLOATS properties cross as binary matrices (FloatVectors), beside
* the JSON or as a column. The boxDoubles/boxLongs
* helpers below serve AsyncExecutor.append_samples' numpy fast path.
*
Expand Down Expand Up @@ -95,11 +96,28 @@ public static long insertManyJson(final Database db, final String typeName, fina
*/
public static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns,
final int n, final int commitEvery) {
return insertColumns(db, typeName, names, columns, n, commitEvery, false);
}

/**
* The columns of one insert_many chunk (every row has the same keys in the same order and each column holds one kind of value): as
* {@link #insertColumns}, except that a whole number is stored as the JSON text route stored it, an Integer when it fits an int and a
* Long beyond, so the type of a schemaless property does not depend on which route carried the chunk (see {@link #narrowed}). A row
* the text could not have carried (a NaN, an infinity or a subnormal double) keeps every whole number a Long, as the per-row path
* stores it (see {@link #carriedByText}).
*/
public static long insertManyColumns(final Database db, final String typeName, final String[] names, final Object[] columns,
final int n, final int commitEvery) {
return insertColumns(db, typeName, names, columns, n, commitEvery, true);
}

private static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns,
final int n, final int commitEvery, final boolean narrowIntegers) {
checkColumns(names, columns, n);
final boolean vertex = n > 0 && isVertexType(db, typeName);
writeChunks(db, n, commitEvery, i -> {
final MutableDocument doc = newRecord(db, typeName, vertex);
fillRow(doc, names, columns, i);
fillRow(doc, names, columns, i, narrowIntegers);
doc.save();
});
return n;
Expand Down Expand Up @@ -180,13 +198,24 @@ private static void backoffBeforeRetry(final int attempt) {
*/
public static AsyncFailures insertColumnsParallel(final Database db, final String typeName, final String[] names,
final Object[] columns, final int n) {
return insertColumnsParallel(db, typeName, names, columns, n, false);
}

/** The parallel twin of {@link #insertManyColumns}. */
public static AsyncFailures insertManyColumnsParallel(final Database db, final String typeName, final String[] names,
final Object[] columns, final int n) {
return insertColumnsParallel(db, typeName, names, columns, n, true);
}

private static AsyncFailures insertColumnsParallel(final Database db, final String typeName, final String[] names,
final Object[] columns, final int n, final boolean narrowIntegers) {
checkColumns(names, columns, n);
final AsyncFailures failures = new AsyncFailures();
final ErrorCallback onError = failures::record;
final boolean vertex = n > 0 && isVertexType(db, typeName);
for (int i = 0; i < n; i++) {
final MutableDocument doc = newRecord(db, typeName, vertex);
fillRow(doc, names, columns, i);
fillRow(doc, names, columns, i, narrowIntegers);
db.async().createRecord(doc, null, onError);
}
return failures;
Expand Down Expand Up @@ -239,12 +268,48 @@ else if (col instanceof TemporalColumn t)
}
}

private static void fillRow(final MutableDocument doc, final String[] names, final Object[] columns, final int i) {
/**
* The value a JSON number token becomes in the engine (JSONObject.parseNumber): an Integer when a whole number fits an int, a Long
* beyond. The columns of a numpy long[] hold every whole number as a long, which would store a Long where the text route stored an
* Integer, so the insert_many columns narrow it back. Written with returns, not a conditional: {@code c ? (int) v : v} would promote
* both arms to long.
*/
private static Object narrowed(final long v) {
if (v >= Integer.MIN_VALUE && v <= Integer.MAX_VALUE)
return (int) v;
return v;
}

/**
* Whether the JSON text route could have carried row i. The text cannot carry a NaN, an infinity or a subnormal double (the engine's
* parser reads 5e-324 as a BigDecimal), so Python sends a row that has one down the per-row path, which stores every whole number of
* that row as a Long: the same value the text would have made an Integer. insert_many columns keep that for such a row, so what a
* record stores does not depend on which route carried its chunk. Only a double[] and an Object[] (a column with a null) can hold one.
*/
private static boolean carriedByText(final Object[] columns, final int i) {
for (final Object col : columns) {
if (col instanceof double[] a) {
if (!textDouble(a[i]))
return false;
} else if (col instanceof Object[] o && o[i] instanceof Double d && !textDouble(d))
return false;
}
return true;
}

/** A double the JSON text keeps as a double: finite, and zero or at least the smallest normal value (Double.MIN_NORMAL). */
private static boolean textDouble(final double v) {
return Double.isFinite(v) && (v == 0.0 || Math.abs(v) >= Double.MIN_NORMAL);
}

private static void fillRow(final MutableDocument doc, final String[] names, final Object[] columns, final int i,
final boolean narrowIntegers) {
final boolean narrow = narrowIntegers && carriedByText(columns, i);
for (int c = 0; c < names.length; c++) {
final Object col = columns[c];
final Object v;
Object v;
if (col instanceof long[] a)
v = a[i];
v = narrow ? narrowed(a[i]) : (Object) a[i];
else if (col instanceof double[] a)
v = a[i];
else if (col instanceof boolean[] a)
Expand All @@ -255,8 +320,11 @@ else if (col instanceof StringColumn s)
v = s.get(i);
else if (col instanceof TemporalColumn t)
v = t.get(i);
else
else {
v = ((Object[]) col)[i];
if (narrow && v instanceof Long l)
v = narrowed(l);
}
doc.set(names[c], v);
}
}
Expand Down
Loading
Loading