Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
c1a7838
Test: a TypedRows batch with no tagged value is decoded without an ob…
tae898 Nov 13, 2026
650c3ee
JVM to_list and iter_dicts: parse a TypedRows batch with no tag in it…
tae898 Nov 13, 2026
04fd969
Tests first: timestamp and date columns of insert_arrow and insert_da…
tae898 Nov 13, 2026
ef38280
JVM insert_arrow, insert_dataframe, insert_columns: a timestamp or da…
tae898 Nov 13, 2026
a17daae
Tests: a native transaction crossing is one foreign call, with no rep…
tae898 Nov 13, 2026
452e93c
Native transactions: arcadedb_n_tx returns the state as one integer, …
tae898 Nov 13, 2026
3db17be
Test: save() of a saved record with auto-transaction on spends two cr…
tae898 Nov 13, 2026
e8e0c45
Native: save() of a saved record does not ask for the transaction sta…
tae898 Nov 13, 2026
55b427a
Tests: the nesting depth the native mark reports is the engine's, and…
tae898 Nov 13, 2026
1fe1160
Tests: native GraphBatch.create_vertices sends columns and reads RIDs…
tae898 Nov 14, 2026
c8d2b87
Native: GraphBatch.create_vertices sends a chunk as columns and gets …
tae898 Nov 14, 2026
0966a23
Tests: fix the stand-in and three expectations of the native GraphBat…
tae898 Nov 14, 2026
c0b3aae
Native: create_vertices stores float32 array vectors after the other …
tae898 Nov 14, 2026
4f5e5fb
Tests: three native files that pinned the JSON route of create_vertic…
tae898 Nov 14, 2026
013da60
Native create_vertices: a chunk of fewer than 64 rows keeps the JSON …
tae898 Nov 15, 2026
1fd3ac5
Native create_vertices: the comment of the 64 row floor names the mic…
tae898 Nov 15, 2026
ae0623b
Tests: a def in the floor test, not an assigned lambda (ruff E731 of …
tae898 Nov 15, 2026
2b2eed1
Tests: nosec B404 and B603 on the subprocess of the floor-from-enviro…
tae898 Nov 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions native/src/main/java/org/humemai/nativelib/ArcadeNative.java
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Every function returns a
* Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Most functions return a
* malloc'd UTF-8 JSON text that the caller releases with arcadedb_free: {"ok":true,...} or
* {"ok":false,"error":msg,"cls":java class,"chain":[[class,msg],..],"retry":"need_retry"|"duplicated_key"|null,"retry_ms":n}.
* arcadedb_n_cmd and arcadedb_n_tx return a long (-1 for an error, whose text in that form arcadedb_n_last_error returns).
* Databases, result sets and servers are long handles. Any thread attached to the isolate may call any function; a
* transaction belongs to the (Java) thread that began it, as in the embedded API.
*/
Expand Down Expand Up @@ -184,9 +185,15 @@ public static CCharPointer info(final IsolateThread thread, final long h) {
}
}

/** op 0 begin, 1 commit, 2 rollback, 3 mark. Every reply carries {"active":b,"depth":n,"commits":n}. */
/**
* op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one non-negative long: bit 0 is
* isTransactionActive, bits 1 to 31 the nesting depth, bits 32 to 62 the low 31 bits of the commit count of the thread's
* transaction context (callers compare two counts for equality). -1 on error: the error text is then read with
* arcadedb_n_last_error on the same thread, as for arcadedb_n_cmd. No reply text, so no malloc, no arcadedb_free and no JSON
* decode in the caller.
*/
@CEntryPoint(name = "arcadedb_n_tx")
public static CCharPointer tx(final IsolateThread thread, final long h, final int op) {
public static long tx(final IsolateThread thread, final long h, final int op) {
try {
final Database db = db(h);
switch (op) {
Expand All @@ -196,14 +203,15 @@ public static CCharPointer tx(final IsolateThread thread, final long h, final in
default -> {
}
}
return out(txState(db));
return txState(db);
} catch (final Throwable t) {
return err(t);
LAST_ERROR.set(errorJson(t));
return -1;
}
}

private static String txState(final Database db) {
int depth = 0;
private static long txState(final Database db) {
long depth = 0;
long commits = 0;
if (db instanceof LocalDatabase local)
depth = local.getNestedTransactions();
Expand All @@ -212,7 +220,7 @@ private static String txState(final Database db) {
if (ctx != null)
commits = ctx.getCommitCount();
}
return "{\"ok\":true,\"active\":" + db.isTransactionActive() + ",\"depth\":" + depth + ",\"commits\":" + commits + "}";
return (db.isTransactionActive() ? 1L : 0L) | ((depth & 0x7FFFFFFFL) << 1) | ((commits & 0x7FFFFFFFL) << 32);
}

// ---- statements
Expand Down
270 changes: 172 additions & 98 deletions native/src/main/java/org/humemai/nativelib/BulkOps.java

Large diffs are not rendered by default.

36 changes: 36 additions & 0 deletions native/src/main/java/org/humemai/nativelib/NativeExtras.java
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,42 @@ public static CCharPointer gbBin(final IsolateThread thread, final long h, final
return gbImpl(h, op, argsJson, buf.rawValue(), bufFloats);
}

/**
* create_vertices from columns, by pointer, as arcadedb_n_insert_colp takes them (see BulkOps.Table): header JSON {"n":rows,"cols":[[name,
* kind, dim],..],"last":[names]} (the columns stored after the others, see BulkOps.vertexRows) and an address table of nspec longs (BulkOps.SLOTS per column). The rows are built from the columns (BulkOps.vertexRows:
* the objects, and the property order, that the JSON rows of arcadedb_n_gb give) and go to the same GraphBatch.createVertices(type,
* Object[][]), so the transaction, the retries and the commit are the engine's and nothing differs from the JSON route but the transport.
* The RIDs do not come back as text: rids points at ridLongs int64 slots of the caller's (at least 2 per row), and row i gets bucket id
* at slot 2i and position at slot 2i + 1. Returns {"ok":true,"n":rows}. The caller keeps every buffer alive until the call returns.
*/
@CEntryPoint(name = "arcadedb_n_gb_colp")
public static CCharPointer gbColp(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header,
final CCharPointer spec, final int nspec, final CCharPointer rids, final long ridLongs) {
try {
final BatchEntry entry = BATCHES.get(h);
if (entry == null)
throw new IllegalArgumentException("unknown GraphBatch " + h);
if (nspec < 0 || nspec % BulkOps.SLOTS != 0)
throw new IllegalArgumentException("address table of " + nspec + " entries is not a multiple of " + BulkOps.SLOTS);
final long[] table = new long[nspec];
for (int i = 0; i < nspec; i++)
table[i] = ((Pointer) spec).readLong(WordFactory.signed(8L * i));
final Map<String, Object> head = args(header);
final BulkOps.Table columns = BulkOps.Table.of(head, table);
if (rids.isNull() || ridLongs < 2L * columns.n)
throw new IllegalArgumentException("the RID buffer holds " + ridLongs + " int64 slots, " + 2L * columns.n + " needed");
final RID[] created = entry.batch().createVertices(ArcadeNative.str(type), BulkOps.vertexRows(columns, head.get("last")));
final Pointer out = (Pointer) rids;
for (int i = 0; i < created.length; i++) {
out.writeLong(WordFactory.signed(16L * i), created[i].getBucketId());
out.writeLong(WordFactory.signed(16L * i + 8L), created[i].getPosition());
}
return ArcadeNative.out("{\"ok\":true,\"n\":" + created.length + "}");
} catch (final Throwable t) {
return ArcadeNative.err(t);
}
}

@SuppressWarnings("unchecked")
private static CCharPointer gbImpl(final long h, final CCharPointer op, final CCharPointer argsJson, final long buf,
final long bufFloats) {
Expand Down
55 changes: 55 additions & 0 deletions src/arcadedb/_native/bulk.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,58 @@ def insert_rows_columnar(db, type_name, chunk):
rows, cols = packed
_send(db, type_name, cols, rows, 0, f"Failed to bulk-insert into '{type_name}'")
return True


def rids_of(out):
"""The RID texts (``#bucket:position``, the ``RID.toString`` text) of an int64 array of ``(bucket, position)`` pairs, in row order."""
pairs = out.reshape(-1, 2)
return list(map("#{}:{}".format, pairs[:, 0].tolist(), pairs[:, 1].tolist()))


def create_vertices_columns(gb, type_name, rows, declared, prefix):
"""One ``GraphBatch.create_vertices`` chunk (a list of dicts) through ``arcadedb_n_gb_colp``: the rows cross as columns by address and
the RIDs come back in an int64 buffer, one crossing and one transaction, as the JSON route has it. Returns the list of RID texts, or None
when the chunk needs the JSON route (nothing was sent then): rows that do not all have the same keys in the same order, a value no column
holds (a nested list or map, a mixed-type column, a set, a NaT, a numpy scalar, an integer past 64 bits, a lone surrogate), an empty chunk or a
chunk of fewer rows than ``lib.GB_COLP_MIN_ROWS`` (64 by default: a handful of rows does not repay the fixed cost of the columnar entry).

``declared``: a :class:`columnar.Declared` of the type, read once by the caller for all chunks of a call.
"""
if not (lib.GB_COLP and _usable()) or len(rows) < lib.GB_COLP_MIN_ROWS:
return None # a chunk below the floor keeps the JSON route of post3 (lib.GB_COLP_MIN_ROWS), as does a chunk the columns cannot carry
columns = columnar.rows_to_columns(rows, same_order=True)
if columns is None:
return None
packed = columnar.pack(
columns, declared, exact_nanos=True
) # the JSON route keeps the nanoseconds of a Timestamp
if packed is None:
return None
count, cols = packed
# a column of float32 arrays is stored after the other properties of a row: the old route took those vectors out of the JSON rows and
# appended them (NativeExtras.withVectors); the other columns keep the order of a HashMap of their names, as the JSON rows do
last = [
c.name
for c in cols
if c.kind == "f" and isinstance(columns[c.name][0], columnar._np.ndarray)
]
spec = columnar.spec_of(cols)
out = columnar._np.empty(2 * count, dtype=columnar._np.int64)
from .database import _call

reply = _call(
prefix,
"arcadedb_n_gb_colp",
gb._gb,
lib.b(type_name),
lib.b(columnar.header_of(count, 0, cols, last=last)),
spec,
len(spec),
out.ctypes.data,
out.size,
) # `cols` and `out` hold the memory the addresses point into until the call returns
if reply["n"] != count:
raise ArcadeDBError(
f"{prefix}: the library created {reply['n']} vertices for {count} rows"
)
return rids_of(out)
30 changes: 23 additions & 7 deletions src/arcadedb/_native/columnar.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ def _epoch_micros(value):
return seconds * 1_000_000 + value.microsecond


def _list_column(name, values, declared):
def _list_column(name, values, declared, exact_nanos=False):
"""A list or tuple of Python values as a column: one type (None allowed) or a list of equal-length float arrays; else None."""
n = len(values)
types = set(map(type, values))
Expand Down Expand Up @@ -255,6 +255,12 @@ def _list_column(name, values, declared):
if issubclass(only, datetime):
if only.__name__ == "NaTType":
return None # the JSON path stores it as a null
if (
exact_nanos
and only is not datetime
and any(getattr(v, "nanosecond", 0) for v in values if v is not None)
):
return None # a pandas Timestamp with nanoseconds, for a caller whose JSON route keeps them (insert_many's keeps microseconds)
try:
ticks = [0 if v is None else _epoch_micros(v) for v in values]
except (OverflowError, ValueError):
Expand Down Expand Up @@ -407,7 +413,7 @@ def _arrow_column(name, arr, declared):
return None


def pack(columns, declared):
def pack(columns, declared, exact_nanos=False):
"""``(rows, [Packed])`` for a dict of name -> column, or None when any column cannot be laid out or the columns differ in length.

A column is a numpy array (1-D ints, floats and bools, or a 2-D float32 matrix), a list or tuple of Python values of one type (None
Expand All @@ -425,14 +431,14 @@ def pack(columns, declared):
elif isinstance(values, _np.ndarray):
if values.dtype.kind in "OUS":
col = (
_list_column(name, values.tolist(), declared)
_list_column(name, values.tolist(), declared, exact_nanos)
if values.ndim == 1
else None
)
else:
col = _numpy_column(name, values, declared)
elif isinstance(values, (list, tuple)) and values:
col = _list_column(name, values, declared)
col = _list_column(name, values, declared, exact_nanos)
else:
col = None # a pandas Series, a generator, an empty list: the row path handles them
if col is None:
Expand All @@ -454,7 +460,7 @@ def spec_of(packed):
return (ctypes.c_int64 * len(flat))(*flat)


def header_of(rows, commit_every, packed, fid=None):
def header_of(rows, commit_every, packed, fid=None, last=()):
"""The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column; with a failure token ``fid`` it also says
``"parallel": true``, which hands the records to the async executor (the image then ignores the commit interval and counts the
failed records on that token).
Expand All @@ -466,14 +472,22 @@ def header_of(rows, commit_every, packed, fid=None):
"commit": min(int(commit_every), 2**31 - 1),
"cols": [[c.name, c.kind, c.dim] for c in packed],
}
if (
last
): # create_vertices: the columns the old route appended to the end of each row (float32 array vectors)
header["last"] = list(last)
if fid is not None:
header["parallel"] = True
header["fid"] = fid
return json.dumps(header)


def rows_to_columns(rows):
"""A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key)."""
def rows_to_columns(rows, same_order=False):
"""A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key).

``same_order``: every row must also list its keys in the order of the first row (a row with the same keys in another order gives
None), for a caller whose stored property order follows the key order of its rows.
"""
if not rows:
return None
try:
Expand All @@ -482,6 +496,8 @@ def rows_to_columns(rows):
return None
if set(map(len, rows)) != {len(keys)}:
return None
if same_order and not all(map(keys.__eq__, map(list, rows))):
return None
return {key: [row[key] for row in rows] for key in keys}
except (
KeyError,
Expand Down
Loading
Loading