diff --git a/packages/flow/openadapt_flow/runtime/effects/db_snapshot.py b/packages/flow/openadapt_flow/runtime/effects/db_snapshot.py index 0140412fdc..25b52aea07 100644 --- a/packages/flow/openadapt_flow/runtime/effects/db_snapshot.py +++ b/packages/flow/openadapt_flow/runtime/effects/db_snapshot.py @@ -46,8 +46,10 @@ its stored text), and numbers parse as ``Decimal``. On PostgreSQL each column is read as text by a rule for its type (:func:`_pg_rule`), in a session pinned to UTC and to exact float output; a type without a rule - makes the table unreadable. :func:`canonical` gives each value one exact - text form. + makes the table unreadable. On SQLite each column is read as the driver + returns it (a ``REAL`` as a double, a ``BLOB`` as its hex), with its storage + class (:class:`SqliteValue`), which the row MAC keeps. :func:`canonical` + gives each value one exact text form. Read-only: SQLite files open with ``mode=ro``; a driver connection should use an account with SELECT (and, on MySQL, TRIGGER and SHOW VIEW; on PostgreSQL, @@ -61,6 +63,7 @@ import datetime import hashlib import json +import math import os import re import secrets @@ -203,6 +206,38 @@ class StoredNumber(Decimal): stored: str +class SqliteValue: + """A SQLite value's storage class. A column with no type, ``BLOB`` or + ``NUMERIC`` can hold ``1`` (an integer), ``1.0`` (a real), ``'1'`` (a + text) and ``x'31'`` (a blob), and ``1`` and ``'1'`` have one text form. + The value still compares and prints by its text, so selectors and cell + MACs are as before; :func:`row_hash` adds the class, so a change of class + alone still shows. An integer stays a plain ``int``.""" + + storage: str = "" + + +class SqliteText(SqliteValue, str): + storage = "text" + + +class SqliteBlob(SqliteValue, str): + """A blob as the upper-case hex of its bytes, as SQLite's ``hex()``.""" + + storage = "blob" + + +class SqliteReal(SqliteValue, Decimal): + storage = "real" + + +def _plain(number: Decimal) -> Decimal: + """A finite number in plain notation (``10``, not ``1E+1``), still exact, + so ``str()`` matches the text a person writes and :func:`canonical`.""" + + return Decimal(canonical(number)) + + def _stored_number(text: str) -> StoredNumber: number = Decimal(text) exact = Context(prec=max(len(number.as_tuple().digits), 1)) @@ -276,7 +311,13 @@ def row_hash( skip = set(ignore) body = json.dumps( - sorted((c, canonical(v)) for c, v in row.items() if c not in skip), + sorted( + (c, canonical(v), v.storage) + if isinstance(v, SqliteValue) + else (c, canonical(v)) + for c, v in row.items() + if c not in skip + ), separators=(",", ":"), ) return _mac(key, f"r\x1f{table}\x1f{body}") @@ -764,27 +805,18 @@ def _mysql_encoding(data_type: str) -> str: def _cell(dialect: Dialect, column: str, how: str = "") -> str: q = _quote(dialect, column) - if dialect == "mysql": - if how == "hex": - return f"HEX({q})" - if how == "double": - return f"CAST({q} AS DOUBLE)" - if how == "text": - return f"CONCAT({q})" - return q - # SQLite's json_object prints a REAL with 15 digits, so two doubles can - # print the same. Printed with 17 they can't; json() keeps it a number. - return ( - f"CASE WHEN typeof({q}) = 'blob' THEN hex({q}) " - f"WHEN typeof({q}) = 'real' THEN json(printf('%!.17g', {q})) ELSE {q} END" - ) + if how == "hex": + return f"HEX({q})" + if how == "double": + return f"CAST({q} AS DOUBLE)" + if how == "text": + return f"CONCAT({q})" + return q def _row_json(dialect: Dialect, d: _Def) -> list[str]: - obj = "JSON_OBJECT" if dialect == "mysql" else "json_object" return [ - obj - + "(" + "JSON_OBJECT(" + ", ".join( f"{_literal(c)}, {_cell(dialect, c, d.encode.get(c, ''))}" for c in d.columns[i : i + _CHUNK] @@ -809,15 +841,56 @@ def _read_rows( return out +def _read_rows_sqlite( + s: _Session, name: str, d: _Def, where: str = "" +) -> list[dict[str, Any]]: + """A SQLite table's rows, each column as the driver returns it. + + Not through ``json_object``: before SQLite 3.53 its text for a double is + not exact at the ends of the range (above about 1e120, below about + 1e-90), and it loses the storage class. + """ + + cells = ", ".join(_quote("sqlite", c) for c in d.columns) + return [ + {c: _sqlite_value(name, c, v) for c, v in zip(d.columns, r)} + for r in s.rows(f"SELECT {cells} FROM {_quote('sqlite', name)} {where}") + ] + + +def _sqlite_value(table: str, column: str, value: Any) -> Any: + if value is None or type(value) is int: + return value + if type(value) is float: + if not math.isfinite(value): + return SqliteReal(repr(value)) + # The shortest text that reads back as the same double, in plain + # notation: 0.3, not 0.29999999999999999; 10, not 10.0. + return SqliteReal(canonical(Decimal(repr(value)))) + if type(value) is str: + return SqliteText(value) + if type(value) is bytes: + return SqliteBlob(value.hex().upper()) + # A connection with converters (detect_types) or a text_factory returns + # other types, whose storage class can't be told: unreadable. + raise SnapshotError( + f"{table}.{column}: the driver returned a {type(value).__name__}; " + "connect without converters or a text_factory" + ) + + def _float32_text(value: Decimal) -> Decimal: """The shortest decimal that reads back as the same 32-bit float.""" single = struct.unpack("f", struct.pack("f", float(value)))[0] + if not math.isfinite(single): + return Decimal(repr(single)) for digits in range(1, 10): text = f"{single:.{digits}g}" if struct.unpack("f", struct.pack("f", float(text)))[0] == single: - return Decimal(text) - return Decimal(repr(single)) # pragma: no cover - 9 digits always do + # %g prints 10 as 1e+01: plain, so a selector "10" matches. + return _plain(Decimal(text)) + return _plain(Decimal(repr(single))) # pragma: no cover - 9 digits do def _plain_numbers(dialect: Dialect, d: _Def, row: dict[str, Any]) -> dict[str, Any]: @@ -826,8 +899,6 @@ def _plain_numbers(dialect: Dialect, d: _Def, row: dict[str, Any]) -> dict[str, A row's numbers parse as ``Decimal`` so no digit is lost; the plain form is what selectors and parameters compare with (``str()``): - - a SQLite ``REAL`` (read with 17 digits) as the shortest text that reads - back as the same double (``0.3``, not ``0.29999999999999999``); - a MySQL ``FLOAT`` (read widened to ``DOUBLE``) as the shortest text that reads back as the same 32-bit float (``1.1``, not ``1.100000023841858``); @@ -840,9 +911,7 @@ def _plain_numbers(dialect: Dialect, d: _Def, row: dict[str, Any]) -> dict[str, for column, value in row.items(): if not isinstance(value, Decimal) or not value.is_finite(): continue - if dialect == "sqlite": - row[column] = Decimal(repr(float(value))) - elif d.encode.get(column) == "double": + if d.encode.get(column) == "double": row[column] = _float32_text(value) else: exact = Context(prec=max(len(value.as_tuple().digits), 1)) @@ -1059,7 +1128,7 @@ def _pg_value(rule: str, text: Optional[str]) -> Any: if rule == "float4": return _float32_text(number) if rule == "float8": - return Decimal(repr(float(number))) + return _plain(Decimal(repr(float(number)))) exact = Context(prec=max(len(number.as_tuple().digits), 1)) return Decimal(format(number.normalize(exact), "f")) return str(text) @@ -1735,6 +1804,8 @@ def rel(t: str) -> str: read = [] elif dialect == "postgresql": read = _read_rows_pg(s, d) + elif dialect == "sqlite": + read = _read_rows_sqlite(s, t, d) else: read = _read_rows(s, t, d) for row in read: diff --git a/packages/flow/tests/test_db_snapshot_column_types.py b/packages/flow/tests/test_db_snapshot_column_types.py index 2f222b9c11..6b57daf203 100644 --- a/packages/flow/tests/test_db_snapshot_column_types.py +++ b/packages/flow/tests/test_db_snapshot_column_types.py @@ -172,7 +172,14 @@ def test_mysql_types_are_read_by_a_declared_rule(data_type: str, how: str) -> No ("c_varchar", "VARCHAR(20)", "'a '", "'a'"), ("c_text", "TEXT", "'x'", "'X'"), ("c_longtext", "LONGTEXT", '\'{"b": 1, "a": 2}\'', '\'{"a": 2, "b": 1}\''), - ("c_json", "JSON", "'{\"b\": 1.10}'", "'{\"b\": 1.1}'"), + # MySQL 8 stores JSON parsed (1.10 and 1.1 are one value there), MariaDB + # as text: two doubles one step apart differ on both. + ( + "c_json", + "JSON", + "'{\"b\": 0.30000000000000004}'", + "'{\"b\": 0.3}'", + ), ("c_enum", "ENUM('x','y')", "'x'", "'y'"), ("c_set", "SET('a','b')", "'a,b'", "'a'"), ("c_binary", "BINARY(2)", "x'00ff'", "x'00fe'"), diff --git a/packages/flow/tests/test_db_snapshot_exact_values.py b/packages/flow/tests/test_db_snapshot_exact_values.py index c8e1413b1c..269348079d 100644 --- a/packages/flow/tests/test_db_snapshot_exact_values.py +++ b/packages/flow/tests/test_db_snapshot_exact_values.py @@ -14,15 +14,23 @@ from __future__ import annotations import json +import math import os import secrets import sqlite3 +from decimal import Decimal from pathlib import Path from typing import Any import pytest -from openadapt_flow.runtime.effects.db_snapshot import canonical, diff, take_snapshot +from openadapt_flow.runtime.effects import db_snapshot +from openadapt_flow.runtime.effects.db_snapshot import ( + SnapshotError, + canonical, + diff, + take_snapshot, +) from openadapt_flow.runtime.effects.effect import record_matches KEY = b"k" * 32 @@ -70,21 +78,116 @@ def test_sqlite_a_real_reads_as_people_write_it(tmp_path: Path) -> None: assert record_matches(row, {"v": "0.3"}) +@pytest.mark.parametrize("column", ["", "BLOB", "NUMERIC"]) @pytest.mark.parametrize( "before, after", - [("1", "'1'"), ("x'AB'", "'AB'")], - ids=["integer to text", "blob to text"], + [("1", "'1'"), ("x'AB'", "'AB'"), ("1", "1.0"), ("x'31'", "'31'")], + ids=["integer to text", "blob to text", "integer to real", "blob to its hex"], ) -@pytest.mark.xfail( - strict=True, - reason="limit: in a SQLite column with no type, a value that changes type " - "but not its text (1 to '1', x'AB' to 'AB') reads the same", +def test_sqlite_a_storage_class_change_shows( + tmp_path: Path, column: str, before: str, after: str +) -> None: + path = tmp_path / "t.sqlite" + con = sqlite3.connect(path) + con.execute(f"CREATE TABLE t (id INTEGER PRIMARY KEY, v {column})") + con.execute(f"INSERT INTO t VALUES (1, {before}), (2, {after})") + stored = con.execute("SELECT typeof(v) FROM t ORDER BY id").fetchall() + con.close() + (tmp_path / "case").mkdir() + moved, _ = _sqlite_case(tmp_path / "case", column, before, after) + if stored[0] == stored[1]: + # NUMERIC stores '1' and 1.0 as the integer 1: nothing moved. + assert column == "NUMERIC" and not moved + else: + assert (moved["t"].changed, moved["t"].changed_columns) == ( + 1, + ["(columns not compared)"], + ) + + +@pytest.mark.parametrize( + "after, text", + [("1", "1"), ("'1'", "1"), ("x'AB'", "AB"), ("1.5", "1.5")], + ids=["integer", "text", "blob", "real"], ) -def test_sqlite_a_type_change_in_an_untyped_column( - tmp_path: Path, before: str, after: str +def test_sqlite_a_storage_class_keeps_selectors_and_cell_macs( + tmp_path: Path, after: str, text: str ) -> None: - moved, _ = _sqlite_case(tmp_path, "", before, after) - assert moved + # The class is in the row MAC only: a selector and a cell MAC compare by + # the value's text, as before. + _, b = _sqlite_case(tmp_path, "", "NULL", after) + (row,) = b.tables["t"].values.values() + assert str(row["v"]) == text + assert record_matches(row, {"v": text}) + assert canonical(row["v"]) == text + + +@pytest.mark.parametrize( + "literal, text", + [("10.0", "10"), ("1e16", "10000000000000000"), ("1500.0", "1500")], +) +def test_sqlite_a_round_real_reads_in_plain_notation( + tmp_path: Path, literal: str, text: str +) -> None: + _, b = _sqlite_case(tmp_path, "REAL", "NULL", literal) + (row,) = b.tables["t"].values.values() + assert str(row["v"]) == text + assert record_matches(row, {"v": text}) + + +@pytest.mark.parametrize( + "value", [1.0000000000000002e300, 3e-300, 1.7976931348623157e308] +) +def test_sqlite_adjacent_doubles_at_the_ends_of_the_range_differ( + tmp_path: Path, value: float +) -> None: + # json_object's 17-digit text was not exact here before SQLite 3.53; the + # snapshot reads the double itself. + path = tmp_path / "t.sqlite" + con = sqlite3.connect(path) + con.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v REAL)") + con.execute("INSERT INTO t VALUES (1, ?)", (value,)) + con.commit() + + def connect() -> Any: + return sqlite3.connect(f"file:{path}?mode=ro", uri=True) + + a = take_snapshot(connect, "sqlite", key=KEY) + con.execute("UPDATE t SET v = ?", (math.nextafter(value, 0.0),)) + con.commit() + con.close() + b = take_snapshot(connect, "sqlite", key=KEY, before=a) + moved = diff(a, b, key=KEY) + assert (moved["t"].changed, moved["t"].changed_columns) == (1, ["v"]) + (row,) = b.tables["t"].values.values() + assert float(row["v"]) == math.nextafter(value, 0.0) + + +def test_sqlite_a_connection_with_converters_is_unreadable(tmp_path: Path) -> None: + path = tmp_path / "t.sqlite" + con = sqlite3.connect(path) + con.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v DATE)") + con.execute("INSERT INTO t VALUES (1, '2026-01-02')") + con.commit() + con.close() + + def connect() -> Any: + return sqlite3.connect( + f"file:{path}?mode=ro", uri=True, detect_types=sqlite3.PARSE_DECLTYPES + ) + + with pytest.raises(SnapshotError, match="converters"): + take_snapshot(connect, "sqlite", key=KEY) + + +def test_float32_text_is_plain() -> None: + # MySQL 8 FLOAT, read widened to DOUBLE: %g printed 10 as 1e+01. + assert str(db_snapshot._float32_text(Decimal("10"))) == "10" + assert str(db_snapshot._float32_text(Decimal("1500"))) == "1500" + assert str(db_snapshot._float32_text(Decimal("1.100000023841858"))) == "1.1" + assert str(db_snapshot._float32_text(Decimal("1E+20"))) == "100000000000000000000" + assert str(db_snapshot._pg_value("float8", "1e+16")) == "10000000000000000" + assert str(db_snapshot._pg_value("float4", "10")) == "10" def test_the_text_nul_null_is_not_null() -> None: