Hash SQLite snapshot cells with their storage class

packages/flow/openadapt_flow/runtime/effects/db_snapshot.py

@@ -46,8 +46,10 @@
   its stored text), and numbers parse as ``Decimal``. On PostgreSQL each
   column is read as text by a rule for its type (:func:`_pg_rule`), in a
   session pinned to UTC and to exact float output; a type without a rule
-  makes the table unreadable. :func:`canonical` gives each value one exact
-  text form.
+  makes the table unreadable. On SQLite each column is read as the driver
+  returns it (a ``REAL`` as a double, a ``BLOB`` as its hex), with its storage
+  class (:class:`SqliteValue`), which the row MAC keeps. :func:`canonical`
+  gives each value one exact text form.
 
 Read-only: SQLite files open with ``mode=ro``; a driver connection should use
 an account with SELECT (and, on MySQL, TRIGGER and SHOW VIEW; on PostgreSQL,
@@ -61,6 +63,7 @@
 import datetime
 import hashlib
 import json
+import math
 import os
 import re
 import secrets
@@ -203,6 +206,38 @@ class StoredNumber(Decimal):
     stored: str
 
 
+class SqliteValue:
+    """A SQLite value's storage class. A column with no type, ``BLOB`` or
+    ``NUMERIC`` can hold ``1`` (an integer), ``1.0`` (a real), ``'1'`` (a
+    text) and ``x'31'`` (a blob), and ``1`` and ``'1'`` have one text form.
+    The value still compares and prints by its text, so selectors and cell
+    MACs are as before; :func:`row_hash` adds the class, so a change of class
+    alone still shows. An integer stays a plain ``int``."""
+
+    storage: str = ""
+
+
+class SqliteText(SqliteValue, str):
+    storage = "text"
+
+
+class SqliteBlob(SqliteValue, str):
+    """A blob as the upper-case hex of its bytes, as SQLite's ``hex()``."""
+
+    storage = "blob"
+
+
+class SqliteReal(SqliteValue, Decimal):
+    storage = "real"
+
+
+def _plain(number: Decimal) -> Decimal:
+    """A finite number in plain notation (``10``, not ``1E+1``), still exact,
+    so ``str()`` matches the text a person writes and :func:`canonical`."""
+
+    return Decimal(canonical(number))
+
+
 def _stored_number(text: str) -> StoredNumber:
     number = Decimal(text)
     exact = Context(prec=max(len(number.as_tuple().digits), 1))
@@ -276,7 +311,13 @@ def row_hash(
 
     skip = set(ignore)
     body = json.dumps(
-        sorted((c, canonical(v)) for c, v in row.items() if c not in skip),
+        sorted(
+            (c, canonical(v), v.storage)
+            if isinstance(v, SqliteValue)
+            else (c, canonical(v))
+            for c, v in row.items()
+            if c not in skip
+        ),
         separators=(",", ":"),
     )
     return _mac(key, f"r\x1f{table}\x1f{body}")
@@ -764,27 +805,18 @@ def _mysql_encoding(data_type: str) -> str:
 
 def _cell(dialect: Dialect, column: str, how: str = "") -> str:
     q = _quote(dialect, column)
-    if dialect == "mysql":
-        if how == "hex":
-            return f"HEX({q})"
-        if how == "double":
-            return f"CAST({q} AS DOUBLE)"
-        if how == "text":
-            return f"CONCAT({q})"
-        return q
-    # SQLite's json_object prints a REAL with 15 digits, so two doubles can
-    # print the same. Printed with 17 they can't; json() keeps it a number.
-    return (
-        f"CASE WHEN typeof({q}) = 'blob' THEN hex({q}) "
-        f"WHEN typeof({q}) = 'real' THEN json(printf('%!.17g', {q})) ELSE {q} END"
-    )
+    if how == "hex":
+        return f"HEX({q})"
+    if how == "double":
+        return f"CAST({q} AS DOUBLE)"
+    if how == "text":
+        return f"CONCAT({q})"
+    return q
 
 
 def _row_json(dialect: Dialect, d: _Def) -> list[str]:
-    obj = "JSON_OBJECT" if dialect == "mysql" else "json_object"
     return [
-        obj
-        + "("
+        "JSON_OBJECT("
         + ", ".join(
             f"{_literal(c)}, {_cell(dialect, c, d.encode.get(c, ''))}"
             for c in d.columns[i : i + _CHUNK]
@@ -809,15 +841,56 @@ def _read_rows(
     return out
 
 
+def _read_rows_sqlite(
+    s: _Session, name: str, d: _Def, where: str = ""
+) -> list[dict[str, Any]]:
+    """A SQLite table's rows, each column as the driver returns it.
+
+    Not through ``json_object``: before SQLite 3.53 its text for a double is
+    not exact at the ends of the range (above about 1e120, below about
+    1e-90), and it loses the storage class.
+    """
+
+    cells = ", ".join(_quote("sqlite", c) for c in d.columns)
+    return [
+        {c: _sqlite_value(name, c, v) for c, v in zip(d.columns, r)}
+        for r in s.rows(f"SELECT {cells} FROM {_quote('sqlite', name)} {where}")
+    ]
+
+
+def _sqlite_value(table: str, column: str, value: Any) -> Any:
+    if value is None or type(value) is int:
+        return value
+    if type(value) is float:
+        if not math.isfinite(value):
+            return SqliteReal(repr(value))
+        # The shortest text that reads back as the same double, in plain
+        # notation: 0.3, not 0.29999999999999999; 10, not 10.0.
+        return SqliteReal(canonical(Decimal(repr(value))))
+    if type(value) is str:
+        return SqliteText(value)
+    if type(value) is bytes:
+        return SqliteBlob(value.hex().upper())
+    # A connection with converters (detect_types) or a text_factory returns
+    # other types, whose storage class can't be told: unreadable.
+    raise SnapshotError(
+        f"{table}.{column}: the driver returned a {type(value).__name__}; "
+        "connect without converters or a text_factory"
+    )
+
+
 def _float32_text(value: Decimal) -> Decimal:
     """The shortest decimal that reads back as the same 32-bit float."""
 
     single = struct.unpack("f", struct.pack("f", float(value)))[0]
+    if not math.isfinite(single):
+        return Decimal(repr(single))
     for digits in range(1, 10):
         text = f"{single:.{digits}g}"
         if struct.unpack("f", struct.pack("f", float(text)))[0] == single:
-            return Decimal(text)
-    return Decimal(repr(single))  # pragma: no cover - 9 digits always do
+            # %g prints 10 as 1e+01: plain, so a selector "10" matches.
+            return _plain(Decimal(text))
+    return _plain(Decimal(repr(single)))  # pragma: no cover - 9 digits do
 
 
 def _plain_numbers(dialect: Dialect, d: _Def, row: dict[str, Any]) -> dict[str, Any]:
@@ -826,8 +899,6 @@ def _plain_numbers(dialect: Dialect, d: _Def, row: dict[str, Any]) -> dict[str,
     A row's numbers parse as ``Decimal`` so no digit is lost; the plain form
     is what selectors and parameters compare with (``str()``):
 
-    - a SQLite ``REAL`` (read with 17 digits) as the shortest text that reads
-      back as the same double (``0.3``, not ``0.29999999999999999``);
     - a MySQL ``FLOAT`` (read widened to ``DOUBLE``) as the shortest text that
       reads back as the same 32-bit float (``1.1``, not
       ``1.100000023841858``);
@@ -840,9 +911,7 @@ def _plain_numbers(dialect: Dialect, d: _Def, row: dict[str, Any]) -> dict[str,
     for column, value in row.items():
         if not isinstance(value, Decimal) or not value.is_finite():
             continue
-        if dialect == "sqlite":
-            row[column] = Decimal(repr(float(value)))
-        elif d.encode.get(column) == "double":
+        if d.encode.get(column) == "double":
             row[column] = _float32_text(value)
         else:
             exact = Context(prec=max(len(value.as_tuple().digits), 1))
@@ -1059,7 +1128,7 @@ def _pg_value(rule: str, text: Optional[str]) -> Any:
         if rule == "float4":
             return _float32_text(number)
         if rule == "float8":
-            return Decimal(repr(float(number)))
+            return _plain(Decimal(repr(float(number))))
         exact = Context(prec=max(len(number.as_tuple().digits), 1))
         return Decimal(format(number.normalize(exact), "f"))
     return str(text)
@@ -1735,6 +1804,8 @@ def rel(t: str) -> str:
             read = []
         elif dialect == "postgresql":
             read = _read_rows_pg(s, d)
+        elif dialect == "sqlite":
+            read = _read_rows_sqlite(s, t, d)
         else:
             read = _read_rows(s, t, d)
         for row in read:

packages/flow/tests/test_db_snapshot_column_types.py

@@ -172,7 +172,14 @@ def test_mysql_types_are_read_by_a_declared_rule(data_type: str, how: str) -> No
     ("c_varchar", "VARCHAR(20)", "'a '", "'a'"),
     ("c_text", "TEXT", "'x'", "'X'"),
     ("c_longtext", "LONGTEXT", '\'{"b": 1, "a": 2}\'', '\'{"a": 2, "b": 1}\''),
-    ("c_json", "JSON", "'{\"b\": 1.10}'", "'{\"b\": 1.1}'"),
+    # MySQL 8 stores JSON parsed (1.10 and 1.1 are one value there), MariaDB
+    # as text: two doubles one step apart differ on both.
+    (
+        "c_json",
+        "JSON",
+        "'{\"b\": 0.30000000000000004}'",
+        "'{\"b\": 0.3}'",
+    ),
     ("c_enum", "ENUM('x','y')", "'x'", "'y'"),
     ("c_set", "SET('a','b')", "'a,b'", "'a'"),
     ("c_binary", "BINARY(2)", "x'00ff'", "x'00fe'"),

packages/flow/tests/test_db_snapshot_exact_values.py

@@ -14,15 +14,23 @@
 from __future__ import annotations
 
 import json
+import math
 import os
 import secrets
 import sqlite3
+from decimal import Decimal
 from pathlib import Path
 from typing import Any
 
 import pytest
 
-from openadapt_flow.runtime.effects.db_snapshot import canonical, diff, take_snapshot
+from openadapt_flow.runtime.effects import db_snapshot
+from openadapt_flow.runtime.effects.db_snapshot import (
+    SnapshotError,
+    canonical,
+    diff,
+    take_snapshot,
+)
 from openadapt_flow.runtime.effects.effect import record_matches
 
 KEY = b"k" * 32
@@ -70,21 +78,116 @@ def test_sqlite_a_real_reads_as_people_write_it(tmp_path: Path) -> None:
     assert record_matches(row, {"v": "0.3"})
 
 
+@pytest.mark.parametrize("column", ["", "BLOB", "NUMERIC"])
 @pytest.mark.parametrize(
     "before, after",
-    [("1", "'1'"), ("x'AB'", "'AB'")],
-    ids=["integer to text", "blob to text"],
+    [("1", "'1'"), ("x'AB'", "'AB'"), ("1", "1.0"), ("x'31'", "'31'")],
+    ids=["integer to text", "blob to text", "integer to real", "blob to its hex"],
 )
-@pytest.mark.xfail(
-    strict=True,
-    reason="limit: in a SQLite column with no type, a value that changes type "
-    "but not its text (1 to '1', x'AB' to 'AB') reads the same",
+def test_sqlite_a_storage_class_change_shows(
+    tmp_path: Path, column: str, before: str, after: str
+) -> None:
+    path = tmp_path / "t.sqlite"
+    con = sqlite3.connect(path)
+    con.execute(f"CREATE TABLE t (id INTEGER PRIMARY KEY, v {column})")
+    con.execute(f"INSERT INTO t VALUES (1, {before}), (2, {after})")
+    stored = con.execute("SELECT typeof(v) FROM t ORDER BY id").fetchall()
+    con.close()
+    (tmp_path / "case").mkdir()
+    moved, _ = _sqlite_case(tmp_path / "case", column, before, after)
+    if stored[0] == stored[1]:
+        # NUMERIC stores '1' and 1.0 as the integer 1: nothing moved.
+        assert column == "NUMERIC" and not moved
+    else:
+        assert (moved["t"].changed, moved["t"].changed_columns) == (
+            1,
+            ["(columns not compared)"],
+        )
+
+
+@pytest.mark.parametrize(
+    "after, text",
+    [("1", "1"), ("'1'", "1"), ("x'AB'", "AB"), ("1.5", "1.5")],
+    ids=["integer", "text", "blob", "real"],
 )
-def test_sqlite_a_type_change_in_an_untyped_column(
-    tmp_path: Path, before: str, after: str
+def test_sqlite_a_storage_class_keeps_selectors_and_cell_macs(
+    tmp_path: Path, after: str, text: str
 ) -> None:
-    moved, _ = _sqlite_case(tmp_path, "", before, after)
-    assert moved
+    # The class is in the row MAC only: a selector and a cell MAC compare by
+    # the value's text, as before.
+    _, b = _sqlite_case(tmp_path, "", "NULL", after)
+    (row,) = b.tables["t"].values.values()
+    assert str(row["v"]) == text
+    assert record_matches(row, {"v": text})
+    assert canonical(row["v"]) == text
+
+
+@pytest.mark.parametrize(
+    "literal, text",
+    [("10.0", "10"), ("1e16", "10000000000000000"), ("1500.0", "1500")],
+)
+def test_sqlite_a_round_real_reads_in_plain_notation(
+    tmp_path: Path, literal: str, text: str
+) -> None:
+    _, b = _sqlite_case(tmp_path, "REAL", "NULL", literal)
+    (row,) = b.tables["t"].values.values()
+    assert str(row["v"]) == text
+    assert record_matches(row, {"v": text})
+
+
+@pytest.mark.parametrize(
+    "value", [1.0000000000000002e300, 3e-300, 1.7976931348623157e308]
+)
+def test_sqlite_adjacent_doubles_at_the_ends_of_the_range_differ(
+    tmp_path: Path, value: float
+) -> None:
+    # json_object's 17-digit text was not exact here before SQLite 3.53; the
+    # snapshot reads the double itself.
+    path = tmp_path / "t.sqlite"
+    con = sqlite3.connect(path)
+    con.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v REAL)")
+    con.execute("INSERT INTO t VALUES (1, ?)", (value,))
+    con.commit()
+
+    def connect() -> Any:
+        return sqlite3.connect(f"file:{path}?mode=ro", uri=True)
+
+    a = take_snapshot(connect, "sqlite", key=KEY)
+    con.execute("UPDATE t SET v = ?", (math.nextafter(value, 0.0),))
+    con.commit()
+    con.close()
+    b = take_snapshot(connect, "sqlite", key=KEY, before=a)
+    moved = diff(a, b, key=KEY)
+    assert (moved["t"].changed, moved["t"].changed_columns) == (1, ["v"])
+    (row,) = b.tables["t"].values.values()
+    assert float(row["v"]) == math.nextafter(value, 0.0)
+
+
+def test_sqlite_a_connection_with_converters_is_unreadable(tmp_path: Path) -> None:
+    path = tmp_path / "t.sqlite"
+    con = sqlite3.connect(path)
+    con.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v DATE)")
+    con.execute("INSERT INTO t VALUES (1, '2026-01-02')")
+    con.commit()
+    con.close()
+
+    def connect() -> Any:
+        return sqlite3.connect(
+            f"file:{path}?mode=ro", uri=True, detect_types=sqlite3.PARSE_DECLTYPES
+        )
+
+    with pytest.raises(SnapshotError, match="converters"):
+        take_snapshot(connect, "sqlite", key=KEY)
+
+
+def test_float32_text_is_plain() -> None:
+    # MySQL 8 FLOAT, read widened to DOUBLE: %g printed 10 as 1e+01.
+    assert str(db_snapshot._float32_text(Decimal("10"))) == "10"
+    assert str(db_snapshot._float32_text(Decimal("1500"))) == "1500"
+    assert str(db_snapshot._float32_text(Decimal("1.100000023841858"))) == "1.1"
+    assert str(db_snapshot._float32_text(Decimal("1E+20"))) == "100000000000000000000"
+    assert str(db_snapshot._pg_value("float8", "1e+16")) == "10000000000000000"
+    assert str(db_snapshot._pg_value("float4", "10")) == "10"
 
 
 def test_the_text_nul_null_is_not_null() -> None: