Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
c1a7838
Test: a TypedRows batch with no tagged value is decoded without an ob…
tae898 Nov 13, 2026
650c3ee
JVM to_list and iter_dicts: parse a TypedRows batch with no tag in it…
tae898 Nov 13, 2026
04fd969
Tests first: timestamp and date columns of insert_arrow and insert_da…
tae898 Nov 13, 2026
ef38280
JVM insert_arrow, insert_dataframe, insert_columns: a timestamp or da…
tae898 Nov 13, 2026
a17daae
Tests: a native transaction crossing is one foreign call, with no rep…
tae898 Nov 13, 2026
452e93c
Native transactions: arcadedb_n_tx returns the state as one integer, …
tae898 Nov 13, 2026
3db17be
Test: save() of a saved record with auto-transaction on spends two cr…
tae898 Nov 13, 2026
e8e0c45
Native: save() of a saved record does not ask for the transaction sta…
tae898 Nov 13, 2026
55b427a
Tests: the nesting depth the native mark reports is the engine's, and…
tae898 Nov 13, 2026
6f44004
Tests: native async create_record hands records over in batches and e…
tae898 Nov 14, 2026
fd65da7
Native: async create_record hands records to the library as one colum…
tae898 Nov 14, 2026
ec9ebcf
Tests: the async create_record tests import numpy at the top, so a mi…
tae898 Nov 14, 2026
89c44d9
Tests: the type dump of the create_record parity test uses literal st…
tae898 Nov 14, 2026
fa35ac4
Tests: the native create_record tests do not close the database's asy…
tae898 Nov 14, 2026
9b39edf
Native: a collected create_record is handed over after 50 ms with no …
tae898 Nov 14, 2026
f15a1f9
Tests: the native async create_record parity test compares the stored…
tae898 Nov 14, 2026
6574a54
Native async create_record: the first 32 records of an executor go on…
tae898 Nov 15, 2026
72d8acb
Native async create_record: a hand-over of fewer than 64 collected re…
tae898 Nov 15, 2026
7854838
Native async create_record: the first 96 records of an executor go on…
tae898 Nov 15, 2026
b77f887
Native async create_record: the first 128 records of an executor go o…
tae898 Nov 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions native/src/main/java/org/humemai/nativelib/ArcadeNative.java
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Every function returns a
* Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Most functions return a
* malloc'd UTF-8 JSON text that the caller releases with arcadedb_free: {"ok":true,...} or
* {"ok":false,"error":msg,"cls":java class,"chain":[[class,msg],..],"retry":"need_retry"|"duplicated_key"|null,"retry_ms":n}.
* arcadedb_n_cmd and arcadedb_n_tx return a long (-1 for an error, whose text in that form arcadedb_n_last_error returns).
* Databases, result sets and servers are long handles. Any thread attached to the isolate may call any function; a
* transaction belongs to the (Java) thread that began it, as in the embedded API.
*/
Expand Down Expand Up @@ -184,9 +185,15 @@ public static CCharPointer info(final IsolateThread thread, final long h) {
}
}

/** op 0 begin, 1 commit, 2 rollback, 3 mark. Every reply carries {"active":b,"depth":n,"commits":n}. */
/**
* op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one non-negative long: bit 0 is
* isTransactionActive, bits 1 to 31 the nesting depth, bits 32 to 62 the low 31 bits of the commit count of the thread's
* transaction context (callers compare two counts for equality). -1 on error: the error text is then read with
* arcadedb_n_last_error on the same thread, as for arcadedb_n_cmd. No reply text, so no malloc, no arcadedb_free and no JSON
* decode in the caller.
*/
@CEntryPoint(name = "arcadedb_n_tx")
public static CCharPointer tx(final IsolateThread thread, final long h, final int op) {
public static long tx(final IsolateThread thread, final long h, final int op) {
try {
final Database db = db(h);
switch (op) {
Expand All @@ -196,14 +203,15 @@ public static CCharPointer tx(final IsolateThread thread, final long h, final in
default -> {
}
}
return out(txState(db));
return txState(db);
} catch (final Throwable t) {
return err(t);
LAST_ERROR.set(errorJson(t));
return -1;
}
}

private static String txState(final Database db) {
int depth = 0;
private static long txState(final Database db) {
long depth = 0;
long commits = 0;
if (db instanceof LocalDatabase local)
depth = local.getNestedTransactions();
Expand All @@ -212,7 +220,7 @@ private static String txState(final Database db) {
if (ctx != null)
commits = ctx.getCommitCount();
}
return "{\"ok\":true,\"active\":" + db.isTransactionActive() + ",\"depth\":" + depth + ",\"commits\":" + commits + "}";
return (db.isTransactionActive() ? 1L : 0L) | ((depth & 0x7FFFFFFFL) << 1) | ((commits & 0x7FFFFFFFL) << 32);
}

// ---- statements
Expand Down
16 changes: 15 additions & 1 deletion native/src/main/java/org/humemai/nativelib/BulkOps.java
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,17 @@ static String insertColumnsReply(final Database db, final String type, final Str
return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, NativeExtras.FailureToken.of(fid.longValue()).onError) + "}";
}

/**
* The records of a frame for the async writers, each with its own callbacks (the "create_records" operation of arcadedb_n_async_bin):
* as the "parallel" form above, but record i goes to {@link NativeExtras.RecordHooks#submit} instead of one shared error callback. The
* header is the one of arcadedb_n_insert_colp ("n", "commit", "cols"); the frame is checked before the first record is handed over.
* A failure after that is thrown as an IllegalStateException "after handing k of n rows", and hooks.handed is k.
*/
static long handColumns(final Database db, final String type, final Map<String, Object> header, final long[] spec,
final NativeExtras.RecordHooks hooks) {
return insertColumns(db, type, header, spec, hooks);
}

/**
* header: {"n":rows,"commit":every,"cols":[[name,kind,dim],..]}. Kinds: q (int64), d (float64), b (bool byte), i (int32), e (float32),
* f (float32 matrix, dim floats per row), s (UTF-8 string: int32 offsets[n+1] in the data slot, the bytes they index in the aux slot, as
Expand Down Expand Up @@ -182,7 +193,10 @@ private static long insertColumns(final Database db, final String type, final Ma
for (; handed < n; handed++) {
final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type);
setRow(doc, handed, names, kinds, dims, data, nulls, blobs, scratch);
async.createRecord(doc, null, onError);
if (onError instanceof NativeExtras.RecordHooks hooks)
hooks.submit(async, doc, handed); // an async create_record batch: each record has its own callbacks
else
async.createRecord(doc, null, onError);
}
} catch (final Throwable t) {
// the records handed over before the failure are the writers' now, whatever this call does
Expand Down
111 changes: 100 additions & 11 deletions native/src/main/java/org/humemai/nativelib/NativeExtras.java
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,78 @@ void drop() {
}
}

/**
* The callback of a record that an async writer stored: Python gets the stored record (its RID included) under the callback id, or
* nothing when the record has no callback.
*/
private static com.arcadedb.database.async.NewRecordCallback recordCallback(final long cb) {
if (cb == 0)
return null;
return rec -> {
try {
final StringBuilder sb = new StringBuilder("\"rec\":");
NativeCodec.append(sb, rec);
fire("record callback", cb, event("record", sb.toString()));
} catch (final Throwable e) {
swallowed("record callback", e); // the writer goes on
}
};
}

/**
* The callbacks of the records of one "create_records" batch (callback id and error callback id per record, 0 for none). It is the
* error callback that BulkOps gets for the frame; BulkOps hands record i to {@link #submit} when it is one of these, so that every
* record has its own callbacks, as a "create_record" call has.
*/
static final class RecordHooks implements ErrorCallback {
private final long[] cbs;
private final long[] errs;
/** The records handed to the writers so far: after a failure, the number of records that the writers hold. */
int handed;

RecordHooks(final long[] cbs, final long[] errs) {
this.cbs = cbs;
this.errs = errs;
}

/** Not used: every record is handed over with its own error callback. */
@Override
public void call(final Throwable t) {
swallowed("create_records error callback", t);
}

void submit(final DatabaseAsyncExecutor ex, final MutableDocument doc, final int i) {
final com.arcadedb.database.async.NewRecordCallback ok = recordCallback(cbs[i]);
if (errs[i] == 0)
ex.createRecord(doc, ok);
else
ex.createRecord(doc, ok, errorCallback(errs[i]));
handed = i + 1;
}
}

/** The address table of arcadedb_n_async_bin as a long[] of BulkOps.SLOTS entries per column (the layout of arcadedb_n_insert_colp). */
private static long[] columnTable(final long table, final int nTable) {
if (table == 0 || nTable <= 0 || nTable % BulkOps.SLOTS != 0)
throw new IllegalArgumentException("address table of " + nTable + " entries is not a multiple of " + BulkOps.SLOTS);
final Pointer t = WordFactory.pointer(table);
final long[] spec = new long[nTable];
for (int i = 0; i < nTable; i++)
spec[i] = t.readLong(WordFactory.signed(8L * i));
return spec;
}

@SuppressWarnings("unchecked")
private static long[] callbackIds(final Object list, final int n, final String what) {
final List<Object> in = (List<Object>) list;
if (in == null || in.size() != n)
throw new IllegalArgumentException("\"" + what + "\" needs one entry per record (" + n + ")");
final long[] out = new long[n];
for (int i = 0; i < n; i++)
out[i] = ((Number) in.get(i)).longValue();
return out;
}

private static String replyRows(final ResultSet rs) {
final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":0,");
NativeCodec.appendBatch(sb, rs, Integer.MAX_VALUE);
Expand Down Expand Up @@ -673,8 +745,8 @@ private static boolean isVertexType(final Database db, final String typeName) {
}

/**
* ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | command / query {lang, text, params,
* cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}.
* ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | create_records {type, kind, n, commit,
* cols, cbs, errs} (arcadedb_n_async_bin only: a frame) | command / query {lang, text, params, cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}.
*/
@CEntryPoint(name = "arcadedb_n_async")
public static CCharPointer async(final IsolateThread thread, final long dbh, final CCharPointer op, final CCharPointer argsJson) {
Expand Down Expand Up @@ -781,20 +853,37 @@ private static CCharPointer asyncImpl(final long dbh, final CCharPointer op, fin
final MutableDocument doc = "v".equals(a.get("kind")) ? db.newVertex(type) : db.newDocument(type);
for (final Map.Entry<String, Object> e : ((Map<String, Object>) a.get("props")).entrySet())
doc.set(e.getKey(), e.getValue());
final com.arcadedb.database.async.NewRecordCallback ok = cb == 0 ? null : rec -> {
try {
final StringBuilder sb = new StringBuilder("\"rec\":");
NativeCodec.append(sb, rec);
fire("record callback", cb, event("record", sb.toString()));
} catch (final Throwable e) {
swallowed("record callback", e); // the writer goes on
}
};
final com.arcadedb.database.async.NewRecordCallback ok = recordCallback(cb);
if (errCb == 0)
ex.createRecord(doc, ok);
else
ex.createRecord(doc, ok, errorCallback(errCb));
}
case "create_records" -> {
// a frame of records of one type and kind, each with its own callbacks (cbs, errs: callback ids, 0 for none), through BulkOps' column
// reader. The frame and the type are checked before the first record is handed over (an exception then: nothing was queued). A
// record that fails while the batch is handed over ends the call with {"ok":true,"n":k,"err":..}: k records are the writers'
// now and the record k is refused with that error, so the caller can give it to its error callback and send the rest again.
final String type = (String) a.get("type");
final boolean vertex = "v".equals(a.get("kind"));
final Object schemaType = db.getSchema().getType(type);
if (schemaType instanceof EdgeType)
throw new IllegalArgumentException("Cannot bulk-insert into '" + type
+ "' because it is an edge type; load the vertices, then create the edges (graph_batch().new_edges() or CREATE EDGE)");
if (vertex != (schemaType instanceof VertexType))
throw new IllegalArgumentException("'" + type + "' is " + (vertex ? "not a vertex type" : "a vertex type")
+ ", but its records were created as " + (vertex ? "vertices" : "documents"));
final int n = intOf(a.get("n"), -1);
final RecordHooks hooks = new RecordHooks(callbackIds(a.get("cbs"), n, "cbs"), callbackIds(a.get("errs"), n, "errs"));
try {
final long handed = BulkOps.handColumns(db, type, a, columnTable(table, nTable), hooks);
return ArcadeNative.out("{\"ok\":true,\"n\":" + handed + "}");
} catch (final IllegalStateException e) {
if (e.getMessage() == null || !e.getMessage().startsWith("after handing ") || e.getCause() == null)
throw e;
return ArcadeNative.out("{\"ok\":true,\"n\":" + hooks.handed + "," + errorBody(e.getCause()) + "}");
}
}
case "command", "query" -> {
final boolean command = "command".equals(ArcadeNative.str(op));
final com.arcadedb.database.async.AsyncResultsetCallback callback = cb == 0 && errCb == 0 ? null : new com.arcadedb.database.async.AsyncResultsetCallback() {
Expand Down
80 changes: 34 additions & 46 deletions src/arcadedb/_native/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,10 @@ def _call(prefix: str, name: str, *args) -> dict:
return reply


# Every successful ``arcadedb_n_tx`` reply is ``{"ok":true,"active":<bool>,"depth":<n>,"commits":<n>}`` (``ArcadeNative.txState``). A caller
# that reads none of the state (begin, commit, rollback), or only ``active``, tells success and that bool from the text it already holds
# instead of decoding a JSON object. An error reply is ``{"ok":false,...}``: it fails the prefix test and is decoded, as before.
_TX_OK = '{"ok":true,'
_TX_ACTIVE = '{"ok":true,"active":true,'
_TX_IDLE = '{"ok":true,"active":false,'
# ``arcadedb_n_tx`` answers with the transaction state after the call as one int (``ArcadeNative.tx``, no reply text to copy, free or
# decode): bit 0 is ``active``, bits 1 to 31 the nesting ``depth``, bits 32 to 62 the ``commits`` of the transaction context (its low 31
# bits: callers compare two of them for equality). Callers read the fields inline: ``state & 1``, ``(state >> 1) & 0x7FFFFFFF``,
# ``state >> 32``. -1 is an error: its text, the ``{"ok":false,...}`` reply of every other entry point, is ``lib.last_error()``.


def _first_row(database, reply: dict):
Expand Down Expand Up @@ -793,73 +791,63 @@ def _command_first_row(self, language: str, command: str, *args) -> NativeResult

# -- transactions

def _tx_text(self, op: int, prefix: str) -> str:
"""One ``arcadedb_n_tx`` crossing: the reply text of a success (not decoded), an ArcadeDBError for a failure."""
def _tx_state(self, op: int, prefix: str) -> int:
"""One ``arcadedb_n_tx`` crossing, one foreign call: the transaction state after the operation as an int (its layout is
described at the top of this module), an ArcadeDBError for a failure.
"""
if self._closed:
raise ArcadeDBError("Database is closed")
if (op == 1 or op == 2) and lib.CLOSE_BATCH:
lib.flush_closes()
text = lib.call("arcadedb_n_tx", self._h, op)
if text.startswith(_TX_OK):
return text
reply = json.loads(text)
if not reply.get("ok"):
raise _error(prefix, reply)
return text

def _tx(self, op: int, prefix: str) -> dict:
"""The decoded state reply: for a caller that reads ``depth`` or ``commits``."""
return json.loads(self._tx_text(op, prefix))
state = lib.tx(self._h, op)
if state < 0:
raise _error(prefix, json.loads(lib.last_error()))
return state

def begin(self):
self._tx_text(0, "Failed to begin transaction")
self._tx_state(0, "Failed to begin transaction")

def _begin_state(self) -> dict:
"""begin(), and the state right after it: the reply of ``arcadedb_n_tx`` carries ``active``, ``depth`` and ``commits``."""
return self._tx(0, "Failed to begin transaction")
def _begin_state(self) -> int:
"""begin(), and the state right after it: the int of ``arcadedb_n_tx`` carries ``active``, ``depth`` and ``commits``."""
return self._tx_state(0, "Failed to begin transaction")

def commit(self):
self._tx_text(1, "Failed to commit transaction")
self._tx_state(1, "Failed to commit transaction")

def rollback(self):
self._tx_text(2, "Failed to rollback transaction")
self._tx_state(2, "Failed to rollback transaction")

def _rollback_state(self) -> dict:
"""rollback(), and the state right after it (the reply carries it, as begin's does): the unwind reads it instead of asking."""
return self._tx(2, "Failed to rollback transaction")
def _rollback_state(self) -> int:
"""rollback(), and the state right after it (the int carries it, as begin's does): the unwind reads it instead of asking."""
return self._tx_state(2, "Failed to rollback transaction")

def transaction(self) -> TransactionContext:
return TransactionContext(self)

def is_transaction_active(self) -> bool:
text = self._tx_text(3, "Failed to check transaction status")
if text.startswith(_TX_ACTIVE):
return True
if text.startswith(_TX_IDLE):
return False
return json.loads(text)["active"]
return bool(self._tx_state(3, "Failed to check transaction status") & 1)

def _transaction_stack_mark(self):
state = self._tx(3, "Failed to read the transaction state")
return state["active"], state["depth"]
state = self._tx_state(3, "Failed to read the transaction state")
return bool(state & 1), (state >> 1) & 0x7FFFFFFF

def _transaction_commit_mark(self):
"""The commit count of the current transaction (the witness that a BATCH boundary published part of its work)."""
return self._tx(3, "Failed to read the transaction state")["commits"]
return self._tx_state(3, "Failed to read the transaction state") >> 32

def _unwind_opened_since(self, mark):
"""Roll back every level opened since ``mark``, and no other; returns ``(state, first_error)``.

``state`` is the transaction state after the last crossing (every rollback reply carries it, so no read sits between two
``state`` is the transaction state int after the last crossing (every rollback carries it, so no read sits between two
steps), or None when that crossing failed. A level the caller held at the mark is theirs, as in the JPype ``Database``.
"""
was_active, depth_before = mark
first_error = None
state = self._tx(3, "Failed to read the transaction state")
for _ in range(state["depth"] - depth_before):
state = self._tx_state(3, "Failed to read the transaction state")
for _ in range(((state >> 1) & 0x7FFFFFFF) - depth_before):
if state is None:
state = self._tx(3, "Failed to check transaction status")
if not state["active"]:
state = self._tx_state(3, "Failed to check transaction status")
if not state & 1:
break
try:
state = self._rollback_state()
Expand All @@ -868,8 +856,8 @@ def _unwind_opened_since(self, mark):
first_error = first_error or e
if not was_active:
if state is None:
state = self._tx(3, "Failed to check transaction status")
if state["active"]:
state = self._tx_state(3, "Failed to check transaction status")
if state & 1:
try:
state = self._rollback_state()
except Exception as e:
Expand All @@ -893,7 +881,7 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005):
mark = self._transaction_stack_mark()
commits_at_start = None
try:
commits_at_start = self._begin_state()["commits"]
commits_at_start = self._begin_state() >> 32
result = fn()
self.commit()
return result
Expand All @@ -919,7 +907,7 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005):
# retry, as before); the raise sits outside that try, or its own except would swallow it.
try:
commits_now = (
state["commits"]
state >> 32
if state is not None
else self._transaction_commit_mark()
)
Expand Down
Loading
Loading