Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion bindings/python/src/arcadedb_embedded/_native/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,12 @@ def _fill(self) -> bool:
if not self._more:
return False
try:
reply = loads(lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH))
if lib.NEXT_BIN and _BINARY_REPLY:
reply = loads(
*lib.call_bin("arcadedb_n_next_bin", self._rs, _NEXT_BATCH)
)
else:
reply = loads(lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH))
except ArcadeDBError:
raise
if not reply.get("ok"):
Expand Down
9 changes: 8 additions & 1 deletion bindings/python/src/arcadedb_embedded/_native/lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
INSERT_ROWS = os.environ.get("ARCADEDB_NATIVE_INSERT_ROWS", "1") != "0"
# ARCADEDB_NATIVE_EXEC_MANY: the rows of an insert_many chunk that JSON text cannot carry unchanged go in one crossing (arcadedb_n_exec_many), not one call each
EXEC_MANY = os.environ.get("ARCADEDB_NATIVE_EXEC_MANY", "1") != "0"
# ARCADEDB_NATIVE_NEXT_BIN: the batches after the first read through arcadedb_n_next_bin, float arrays of 8 or more elements in a float32 tail (as flag 4 of arcadedb_n_exec does for the first batch), not as number text
NEXT_BIN = os.environ.get("ARCADEDB_NATIVE_NEXT_BIN", "1") != "0"

_lock = threading.Lock()
_lib = None
Expand Down Expand Up @@ -159,6 +161,7 @@ class _Params(ctypes.Structure):
"arcadedb_n_exec_bin": [P, L, I, S, S, S, I, S, L],
"arcadedb_n_insert_cols": [P, L, S, S, S, L],
"arcadedb_n_next": [P, L, I],
"arcadedb_n_next_bin": [P, L, I],
"arcadedb_n_rs_close": [P, L],
"arcadedb_n_cmd": [P, L, I, S, S, S, S, L, I],
"arcadedb_n_last_error": [P],
Expand Down Expand Up @@ -195,7 +198,7 @@ class _Params(ctypes.Structure):
)
else ctypes.c_longlong if name == "arcadedb_n_cmd" else ctypes.c_void_p
)
global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY
global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN
if not hasattr(
lib, "arcadedb_n_exec_bin"
): # a library built before the binary path
Expand All @@ -208,6 +211,10 @@ class _Params(ctypes.Structure):
INSERT_ROWS = False
if not hasattr(lib, "arcadedb_n_exec_many"):
EXEC_MANY = False
if not hasattr(
lib, "arcadedb_n_next_bin"
): # a library built before the binary tail of later batches
NEXT_BIN = False
_isolate = isolate
_creator = threading.get_ident()
_local.thread = _Thread(thread, owned=False)
Expand Down
173 changes: 173 additions & 0 deletions bindings/python/tests/test_native_result_transport.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
"""Query results across the native boundary: the reply text (and the float32 tail of a reply with vectors) the image hands to Python
is copied out of the image in bulk, not one byte at a time. Every value has to come back exactly as it went in, for replies of
every size: tiny, several batches (arcadedb_n_exec returns the first 256 rows, arcadedb_n_next the following 2048 at a time) and
single rows of about a megabyte of multi-byte text. The tests hold on both backends; only the native one has this copy.
"""

import struct

DIM = 16 # 8 or more floats per vector: they travel in the float32 tail of the first batch, as JSON numbers in the later ones

TEXTS = [
"",
"plain",
"café 日本語 \U0001f600 end", # 2-, 3- and 4-byte UTF-8 sequences
'quote " backslash \\ slash / newline \n tab \t nul-free \u0001 control',
"x" * 300,
]


def f32(x):
"""The double a float32 value widens to (what the engine returns for an ARRAY_OF_FLOATS element)."""
return struct.unpack("f", struct.pack("f", x))[0]


def _rows(n):
rows = []
for i in range(n):
rows.append(
{
"id": i,
"big": (1 << 62) - i if i % 2 else -(1 << 62) + i,
"val": (i + 0.5) / 7.0,
"txt": TEXTS[i % len(TEXTS)],
"vec": [f32((i * DIM + k) / 1000.0 - 3.0) for k in range(DIM)],
}
)
return rows


def _typed_db(temp_db):
temp_db.command("sql", "CREATE DOCUMENT TYPE R")
for p, t in (
("id", "INTEGER"),
("big", "LONG"),
("val", "DOUBLE"),
("txt", "STRING"),
("vec", "ARRAY_OF_FLOATS"),
):
temp_db.command("sql", f"CREATE PROPERTY R.{p} {t}")
return temp_db


def test_a_reply_of_several_batches_round_trips_exactly(temp_db):
db = _typed_db(temp_db)
rows = _rows(
5000
) # 256 in the first reply (vectors in the float tail), then 2048 + 2048 + 648 (vectors as numbers)
db.insert_many("R", rows)
got = db.query("sql", "SELECT id, big, val, txt, vec FROM R ORDER BY id").to_list()
assert len(got) == len(rows)
for want, have in zip(rows, got):
assert have["id"] == want["id"]
assert have["big"] == want["big"]
assert have["val"] == want["val"]
assert have["txt"] == want["txt"]
assert list(have["vec"]) == want["vec"]


def test_a_small_reply_with_a_vector_round_trips_exactly(temp_db):
db = _typed_db(temp_db)
rows = _rows(3)
db.insert_many("R", rows)
got = db.query("sql", "SELECT id, vec, txt FROM R ORDER BY id").to_list()
assert [list(r["vec"]) for r in got] == [r["vec"] for r in rows]
assert [r["txt"] for r in got] == [r["txt"] for r in rows]


def test_rows_of_about_a_megabyte_of_multibyte_text_round_trip(temp_db):
db = _typed_db(temp_db)
unit = "aé日\U0001f600" # 1 + 2 + 3 + 4 bytes
rows = [
{"id": i, "txt": unit * (250_000 + i)} for i in range(4)
] # ~1.2 MB of UTF-8 per row
db.insert_many("R", rows)
got = db.query("sql", "SELECT id, txt FROM R ORDER BY id").to_list()
assert [(r["id"], r["txt"]) for r in got] == [(r["id"], r["txt"]) for r in rows]


def test_empty_and_one_row_replies(temp_db):
db = _typed_db(temp_db)
assert db.query("sql", "SELECT FROM R").to_list() == []
assert db.query("sql", "SELECT '' AS s").to_list() == [{"s": ""}]
assert db.query("sql", "SELECT 1 AS x").first().get("x") == 1


# ---- vectors on every batch (first reply and arcadedb_n_next batches), bit-exact, with short, empty and missing vectors

SPECIAL = [
float("inf"),
float("-inf"),
-0.0,
0.0,
1e-45,
3.4028235e38,
0.1,
-123456.789,
] # 8 values: a float32 tail on the binary path


def _bits(x):
"""The 8 bytes of a double as hex (any NaN is 'nan'): bit-exact comparison, so -0.0 differs from 0.0."""
return "nan" if x != x else struct.pack("<d", x).hex()


def _vector_rows(n):
rows, want = [], {}
for i in range(n):
kind = i % 6
if kind == 0:
vec = [f32(((i * 7 + k) % 1000) / 37.0 - 5.0) for k in range(32)]
elif kind == 1:
vec = [f32(x) for x in SPECIAL]
elif kind == 2:
vec = []
elif kind == 3:
vec = [
f32(0.1),
f32(-2.5),
f32(1e-45),
] # fewer than 8 floats: number text on every path
elif kind == 4:
vec = [
f32((i + k) / 11.0) for k in range(8)
] # exactly 8: the smallest vector that travels as a float32 tail
else:
vec = None # no vec property at all
row = {"id": i}
if vec is not None:
row["vec"] = vec
rows.append(row)
want[i] = vec
return rows, want


def test_vectors_are_bit_exact_on_the_first_and_on_later_batches(temp_db):
db = _typed_db(temp_db)
rows, want = _vector_rows(3000) # 256 rows in the first reply, then 2048 + 696
db.insert_many("R", rows)
got = db.query("sql", "SELECT id, vec FROM R ORDER BY id").to_list()
assert [r["id"] for r in got] == list(range(3000))
for r in got:
expect = want[r["id"]]
if expect is None:
assert r.get("vec") is None, r["id"]
else:
assert [_bits(x) for x in r["vec"]] == [_bits(x) for x in expect], r["id"]


def test_vectors_in_a_cursor_read_row_by_row_are_bit_exact(temp_db):
db = _typed_db(temp_db)
rows, want = _vector_rows(700)
db.insert_many("R", rows)
seen = 0
for r in db.query("sql", "SELECT id, vec FROM R ORDER BY id"):
expect = want[r.get("id")]
vec = r.get("vec")
assert (
(vec is None)
if expect is None
else [_bits(x) for x in vec] == [_bits(x) for x in expect]
)
seen += 1
assert seen == 700
49 changes: 38 additions & 11 deletions native-spike/src/main/java/org/humemai/nativelib/ArcadeNative.java
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,27 @@ public static CCharPointer next(final IsolateThread thread, final long rsId, fin
}
}

/**
* arcadedb_n_next with the float32 tail of arcadedb_n_exec flag 4: float[] values of 8 or more elements of the batch come back in a binary tail
* ({"\u0001":["B",offset,count]} in the rows, ,"bin":total at the end of the JSON, the floats after its NUL), not as number text. Offsets are
* per reply. A reply without such a value is the same text arcadedb_n_next returns.
*/
@CEntryPoint(name = "arcadedb_n_next_bin")
public static CCharPointer nextBin(final IsolateThread thread, final long rsId, final int max) {
ResultSet rs = null;
try {
rs = RESULT_SETS.get(rsId);
if (rs == null)
throw new IllegalArgumentException("unknown result set " + rsId);
final ResultSet cursor = rs;
return withFloatTail(() -> batchOf(rsId, cursor, max));
} catch (final Throwable t) {
RESULT_SETS.remove(rsId);
closeQuietly(rs);
return err(t);
}
}

@CEntryPoint(name = "arcadedb_n_rs_close")
public static CCharPointer rsClose(final IsolateThread thread, final long rsId) {
closeQuietly(RESULT_SETS.remove(rsId));
Expand Down Expand Up @@ -487,11 +508,19 @@ private static CCharPointer replyOf(final ResultSet rs, final int max, final int
}
if ((flags & 4) == 0)
return out(batch(rs, max));
return withFloatTail(() -> batch(rs, max));
}

/**
* Runs encode with the float32 sink set (float[] values of 8 or more elements go to the tail of the reply as {"\u0001":["B",off,n]}) and returns
* the reply: out(text) when no vector was met, else the text with ,"bin":total, a NUL, padding to 4 bytes and the floats.
*/
private static CCharPointer withFloatTail(final java.util.function.Supplier<String> encode) {
final NativeCodec.FloatSink sink = new NativeCodec.FloatSink();
NativeCodec.SINK.set(sink);
final String reply;
try {
reply = batch(rs, max);
reply = encode.get();
} finally {
NativeCodec.SINK.remove();
}
Expand All @@ -500,13 +529,12 @@ private static CCharPointer replyOf(final ResultSet rs, final int max, final int
final byte[] b = (reply.substring(0, reply.length() - 1) + ",\"bin\":" + sink.n + "}").getBytes(StandardCharsets.UTF_8);
final int off = (b.length + 1 + 3) & ~3;
final CCharPointer p = UnmanagedMemory.malloc(off + 4 * sink.n);
for (int i = 0; i < b.length; i++)
p.write(i, b[i]);
for (int i = b.length; i < off; i++)
p.write(i, (byte) 0);
final CFloatPointer fp = (CFloatPointer) ((org.graalvm.word.Pointer) p).add(off);
for (int i = 0; i < sink.n; i++)
fp.write(i, sink.a[i]);
// bulk copies (Unsafe.copyMemory), not one p.write per byte: the text, the zero padding up to the 4-byte boundary, then the float32 tail in native order
final java.nio.ByteBuffer bb = CTypeConversion.asByteBuffer(p, (int) (off + 4L * sink.n));
bb.put(b, 0, b.length);
while (bb.position() < off)
bb.put((byte) 0);
bb.order(java.nio.ByteOrder.nativeOrder()).asFloatBuffer().put(sink.a, 0, sink.n);
return p;
}

Expand Down Expand Up @@ -683,9 +711,8 @@ static Chars chars(final CCharPointer p) {
static CCharPointer out(final String s) {
final byte[] b = s.getBytes(StandardCharsets.UTF_8);
final CCharPointer p = UnmanagedMemory.malloc(b.length + 1);
for (int i = 0; i < b.length; i++)
p.write(i, b[i]);
p.write(b.length, (byte) 0);
// one bulk copy (Unsafe.copyMemory) and the terminating zero; this was a Java loop of p.write(i, b[i]), one byte per iteration
CTypeConversion.asByteBuffer(p, b.length + 1).put(b, 0, b.length).put((byte) 0);
return p;
}
}
Loading