From 7f350cd034b137ab169111fea760429682a14504 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 15:32:07 +0900 Subject: [PATCH 1/4] Native: copy reply text out of the image in bulk, not one byte per loop turn out() and the float32-tail reply wrote the reply with p.write(i, b[i]) in a Java loop. Both now fill a direct ByteBuffer view of the malloc'd block (Unsafe.copyMemory); the bytes written are the same. New test: replies of one, several and many batches, multi-byte text, megabyte rows and vectors round-trip exactly. Co-Authored-By: Claude Sonnet 5.5 --- .../tests/test_native_result_transport.py | 93 +++++++++++++++++++ .../org/humemai/nativelib/ArcadeNative.java | 20 ++-- 2 files changed, 102 insertions(+), 11 deletions(-) create mode 100644 bindings/python/tests/test_native_result_transport.py diff --git a/bindings/python/tests/test_native_result_transport.py b/bindings/python/tests/test_native_result_transport.py new file mode 100644 index 00000000000..4cd023e4ba2 --- /dev/null +++ b/bindings/python/tests/test_native_result_transport.py @@ -0,0 +1,93 @@ +"""Query results across the native boundary: the reply text (and the float32 tail of a reply with vectors) the image hands to Python +is copied out of the image in bulk, not one byte at a time. Every value has to come back exactly as it went in, for replies of +every size: tiny, several batches (arcadedb_n_exec returns the first 256 rows, arcadedb_n_next the following 2048 at a time) and +single rows of about a megabyte of multi-byte text. The tests hold on both backends; only the native one has this copy. +""" + +import struct + +DIM = 16 # 8 or more floats per vector: they travel in the float32 tail of the first batch, as JSON numbers in the later ones + +TEXTS = [ + "", + "plain", + "café 日本語 \U0001f600 end", # 2-, 3- and 4-byte UTF-8 sequences + 'quote " backslash \\ slash / newline \n tab \t nul-free \u0001 control', + "x" * 300, +] + + +def f32(x): + """The double a float32 value widens to (what the engine returns for an ARRAY_OF_FLOATS element).""" + return struct.unpack("f", struct.pack("f", x))[0] + + +def _rows(n): + rows = [] + for i in range(n): + rows.append( + { + "id": i, + "big": (1 << 62) - i if i % 2 else -(1 << 62) + i, + "val": (i + 0.5) / 7.0, + "txt": TEXTS[i % len(TEXTS)], + "vec": [f32((i * DIM + k) / 1000.0 - 3.0) for k in range(DIM)], + } + ) + return rows + + +def _typed_db(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE R") + for p, t in ( + ("id", "INTEGER"), + ("big", "LONG"), + ("val", "DOUBLE"), + ("txt", "STRING"), + ("vec", "ARRAY_OF_FLOATS"), + ): + temp_db.command("sql", f"CREATE PROPERTY R.{p} {t}") + return temp_db + + +def test_a_reply_of_several_batches_round_trips_exactly(temp_db): + db = _typed_db(temp_db) + rows = _rows( + 5000 + ) # 256 in the first reply (vectors in the float tail), then 2048 + 2048 + 648 (vectors as numbers) + db.insert_many("R", rows) + got = db.query("sql", "SELECT id, big, val, txt, vec FROM R ORDER BY id").to_list() + assert len(got) == len(rows) + for want, have in zip(rows, got): + assert have["id"] == want["id"] + assert have["big"] == want["big"] + assert have["val"] == want["val"] + assert have["txt"] == want["txt"] + assert list(have["vec"]) == want["vec"] + + +def test_a_small_reply_with_a_vector_round_trips_exactly(temp_db): + db = _typed_db(temp_db) + rows = _rows(3) + db.insert_many("R", rows) + got = db.query("sql", "SELECT id, vec, txt FROM R ORDER BY id").to_list() + assert [list(r["vec"]) for r in got] == [r["vec"] for r in rows] + assert [r["txt"] for r in got] == [r["txt"] for r in rows] + + +def test_rows_of_about_a_megabyte_of_multibyte_text_round_trip(temp_db): + db = _typed_db(temp_db) + unit = "aé日\U0001f600" # 1 + 2 + 3 + 4 bytes + rows = [ + {"id": i, "txt": unit * (250_000 + i)} for i in range(4) + ] # ~1.2 MB of UTF-8 per row + db.insert_many("R", rows) + got = db.query("sql", "SELECT id, txt FROM R ORDER BY id").to_list() + assert [(r["id"], r["txt"]) for r in got] == [(r["id"], r["txt"]) for r in rows] + + +def test_empty_and_one_row_replies(temp_db): + db = _typed_db(temp_db) + assert db.query("sql", "SELECT FROM R").to_list() == [] + assert db.query("sql", "SELECT '' AS s").to_list() == [{"s": ""}] + assert db.query("sql", "SELECT 1 AS x").first().get("x") == 1 diff --git a/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java index 5758d865936..e43d608b3c3 100644 --- a/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -499,14 +499,13 @@ private static CCharPointer replyOf(final ResultSet rs, final int max, final int return out(reply); final byte[] b = (reply.substring(0, reply.length() - 1) + ",\"bin\":" + sink.n + "}").getBytes(StandardCharsets.UTF_8); final int off = (b.length + 1 + 3) & ~3; - final CCharPointer p = UnmanagedMemory.malloc(off + 4 * sink.n); - for (int i = 0; i < b.length; i++) - p.write(i, b[i]); - for (int i = b.length; i < off; i++) - p.write(i, (byte) 0); - final CFloatPointer fp = (CFloatPointer) ((org.graalvm.word.Pointer) p).add(off); - for (int i = 0; i < sink.n; i++) - fp.write(i, sink.a[i]); + final CCharPointer p = UnmanagedMemory.malloc(off + 4L * sink.n); + // bulk copies (Unsafe.copyMemory), not one p.write per byte: the text, the zero padding up to the 4-byte boundary, then the float32 tail in native order + final java.nio.ByteBuffer bb = CTypeConversion.asByteBuffer(p, (int) (off + 4L * sink.n)); + bb.put(b, 0, b.length); + while (bb.position() < off) + bb.put((byte) 0); + bb.order(java.nio.ByteOrder.nativeOrder()).asFloatBuffer().put(sink.a, 0, sink.n); return p; } @@ -683,9 +682,8 @@ static Chars chars(final CCharPointer p) { static CCharPointer out(final String s) { final byte[] b = s.getBytes(StandardCharsets.UTF_8); final CCharPointer p = UnmanagedMemory.malloc(b.length + 1); - for (int i = 0; i < b.length; i++) - p.write(i, b[i]); - p.write(b.length, (byte) 0); + // one bulk copy (Unsafe.copyMemory) and the terminating zero; this was a Java loop of p.write(i, b[i]), one byte per iteration + CTypeConversion.asByteBuffer(p, b.length + 1).put(b, 0, b.length).put((byte) 0); return p; } } From c4c1fb41cb75743e957d82807c6652310be3a205 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 16:29:09 +0900 Subject: [PATCH 2/4] Native: malloc takes an int here; keep the reply size arithmetic in int The previous commit widened off + 4 * sink.n to long, which UnmanagedMemory.malloc does not accept (found by the first image build). Co-Authored-By: Claude Sonnet 5.5 --- .../src/main/java/org/humemai/nativelib/ArcadeNative.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java index e43d608b3c3..fa27f95035b 100644 --- a/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -499,7 +499,7 @@ private static CCharPointer replyOf(final ResultSet rs, final int max, final int return out(reply); final byte[] b = (reply.substring(0, reply.length() - 1) + ",\"bin\":" + sink.n + "}").getBytes(StandardCharsets.UTF_8); final int off = (b.length + 1 + 3) & ~3; - final CCharPointer p = UnmanagedMemory.malloc(off + 4L * sink.n); + final CCharPointer p = UnmanagedMemory.malloc(off + 4 * sink.n); // bulk copies (Unsafe.copyMemory), not one p.write per byte: the text, the zero padding up to the 4-byte boundary, then the float32 tail in native order final java.nio.ByteBuffer bb = CTypeConversion.asByteBuffer(p, (int) (off + 4L * sink.n)); bb.put(b, 0, b.length); From e956f0a67cc0d3add68785c932320c8cd676e538 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sun, 25 Oct 2026 16:34:03 +0900 Subject: [PATCH 3/4] Test: vectors are bit-exact on the first and on later batches, with short, empty and missing vectors Passes against the current library (vectors of later batches travel as number text); it must keep passing when arcadedb_n_next carries them in a float32 tail. Co-Authored-By: Claude Sonnet 5.5 --- .../tests/test_native_result_transport.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/bindings/python/tests/test_native_result_transport.py b/bindings/python/tests/test_native_result_transport.py index 4cd023e4ba2..12edbfcb33d 100644 --- a/bindings/python/tests/test_native_result_transport.py +++ b/bindings/python/tests/test_native_result_transport.py @@ -91,3 +91,83 @@ def test_empty_and_one_row_replies(temp_db): assert db.query("sql", "SELECT FROM R").to_list() == [] assert db.query("sql", "SELECT '' AS s").to_list() == [{"s": ""}] assert db.query("sql", "SELECT 1 AS x").first().get("x") == 1 + + +# ---- vectors on every batch (first reply and arcadedb_n_next batches), bit-exact, with short, empty and missing vectors + +SPECIAL = [ + float("inf"), + float("-inf"), + -0.0, + 0.0, + 1e-45, + 3.4028235e38, + 0.1, + -123456.789, +] # 8 values: a float32 tail on the binary path + + +def _bits(x): + """The 8 bytes of a double as hex (any NaN is 'nan'): bit-exact comparison, so -0.0 differs from 0.0.""" + return "nan" if x != x else struct.pack(" Date: Sun, 25 Oct 2026 16:35:04 +0900 Subject: [PATCH 4/4] Native: arcadedb_n_next_bin, later result batches carry vectors in the float32 tail arcadedb_n_exec flag 4 already sends float[] values of 8 or more elements in a binary tail, but only for the first reply (256 rows). Every later batch (arcadedb_n_next, 2048 rows) sent them as number text, which Python parses with json.loads. New entry point arcadedb_n_next_bin encodes a batch with the same sink; the tail code of replyOf is now one helper (withFloatTail) used by both. lib.py uses it when the library has it (ARCADEDB_NATIVE_NEXT_BIN=0 and ARCADEDB_NATIVE_BIN_REPLY=0 turn it off); arcadedb_n_next is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .../src/arcadedb_embedded/_native/database.py | 7 ++++- .../src/arcadedb_embedded/_native/lib.py | 9 +++++- .../org/humemai/nativelib/ArcadeNative.java | 31 ++++++++++++++++++- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/bindings/python/src/arcadedb_embedded/_native/database.py b/bindings/python/src/arcadedb_embedded/_native/database.py index 68fe220b894..3396106bf38 100644 --- a/bindings/python/src/arcadedb_embedded/_native/database.py +++ b/bindings/python/src/arcadedb_embedded/_native/database.py @@ -199,7 +199,12 @@ def _fill(self) -> bool: if not self._more: return False try: - reply = loads(lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH)) + if lib.NEXT_BIN and _BINARY_REPLY: + reply = loads( + *lib.call_bin("arcadedb_n_next_bin", self._rs, _NEXT_BATCH) + ) + else: + reply = loads(lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH)) except ArcadeDBError: raise if not reply.get("ok"): diff --git a/bindings/python/src/arcadedb_embedded/_native/lib.py b/bindings/python/src/arcadedb_embedded/_native/lib.py index 8ac33145143..4cf692f45c7 100644 --- a/bindings/python/src/arcadedb_embedded/_native/lib.py +++ b/bindings/python/src/arcadedb_embedded/_native/lib.py @@ -28,6 +28,8 @@ INSERT_ROWS = os.environ.get("ARCADEDB_NATIVE_INSERT_ROWS", "1") != "0" # ARCADEDB_NATIVE_EXEC_MANY: the rows of an insert_many chunk that JSON text cannot carry unchanged go in one crossing (arcadedb_n_exec_many), not one call each EXEC_MANY = os.environ.get("ARCADEDB_NATIVE_EXEC_MANY", "1") != "0" +# ARCADEDB_NATIVE_NEXT_BIN: the batches after the first read through arcadedb_n_next_bin, float arrays of 8 or more elements in a float32 tail (as flag 4 of arcadedb_n_exec does for the first batch), not as number text +NEXT_BIN = os.environ.get("ARCADEDB_NATIVE_NEXT_BIN", "1") != "0" _lock = threading.Lock() _lib = None @@ -159,6 +161,7 @@ class _Params(ctypes.Structure): "arcadedb_n_exec_bin": [P, L, I, S, S, S, I, S, L], "arcadedb_n_insert_cols": [P, L, S, S, S, L], "arcadedb_n_next": [P, L, I], + "arcadedb_n_next_bin": [P, L, I], "arcadedb_n_rs_close": [P, L], "arcadedb_n_cmd": [P, L, I, S, S, S, S, L, I], "arcadedb_n_last_error": [P], @@ -195,7 +198,7 @@ class _Params(ctypes.Structure): ) else ctypes.c_longlong if name == "arcadedb_n_cmd" else ctypes.c_void_p ) - global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY + global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN if not hasattr( lib, "arcadedb_n_exec_bin" ): # a library built before the binary path @@ -208,6 +211,10 @@ class _Params(ctypes.Structure): INSERT_ROWS = False if not hasattr(lib, "arcadedb_n_exec_many"): EXEC_MANY = False + if not hasattr( + lib, "arcadedb_n_next_bin" + ): # a library built before the binary tail of later batches + NEXT_BIN = False _isolate = isolate _creator = threading.get_ident() _local.thread = _Thread(thread, owned=False) diff --git a/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java index fa27f95035b..3e0232ce41c 100644 --- a/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -290,6 +290,27 @@ public static CCharPointer next(final IsolateThread thread, final long rsId, fin } } + /** + * arcadedb_n_next with the float32 tail of arcadedb_n_exec flag 4: float[] values of 8 or more elements of the batch come back in a binary tail + * ({"\u0001":["B",offset,count]} in the rows, ,"bin":total at the end of the JSON, the floats after its NUL), not as number text. Offsets are + * per reply. A reply without such a value is the same text arcadedb_n_next returns. + */ + @CEntryPoint(name = "arcadedb_n_next_bin") + public static CCharPointer nextBin(final IsolateThread thread, final long rsId, final int max) { + ResultSet rs = null; + try { + rs = RESULT_SETS.get(rsId); + if (rs == null) + throw new IllegalArgumentException("unknown result set " + rsId); + final ResultSet cursor = rs; + return withFloatTail(() -> batchOf(rsId, cursor, max)); + } catch (final Throwable t) { + RESULT_SETS.remove(rsId); + closeQuietly(rs); + return err(t); + } + } + @CEntryPoint(name = "arcadedb_n_rs_close") public static CCharPointer rsClose(final IsolateThread thread, final long rsId) { closeQuietly(RESULT_SETS.remove(rsId)); @@ -487,11 +508,19 @@ private static CCharPointer replyOf(final ResultSet rs, final int max, final int } if ((flags & 4) == 0) return out(batch(rs, max)); + return withFloatTail(() -> batch(rs, max)); + } + + /** + * Runs encode with the float32 sink set (float[] values of 8 or more elements go to the tail of the reply as {"\u0001":["B",off,n]}) and returns + * the reply: out(text) when no vector was met, else the text with ,"bin":total, a NUL, padding to 4 bytes and the floats. + */ + private static CCharPointer withFloatTail(final java.util.function.Supplier encode) { final NativeCodec.FloatSink sink = new NativeCodec.FloatSink(); NativeCodec.SINK.set(sink); final String reply; try { - reply = batch(rs, max); + reply = encode.get(); } finally { NativeCodec.SINK.remove(); }