diff --git a/docs/changes/newsfragments/8724.new b/docs/changes/newsfragments/8724.new new file mode 100644 index 000000000000..4c246984c208 --- /dev/null +++ b/docs/changes/newsfragments/8724.new @@ -0,0 +1,6 @@ +Snapshots can be stored losslessly using versioned, checksummed Zstandard +level 1 compression by setting ``dataset.snapshot_compression`` to ``"zstd"``. +Compressed and uncompressed snapshots are always readable, and snapshot APIs and +NetCDF exports return the complete JSON. Uncompressed snapshots, the default, +can be read by all QCoDeS versions; compressed snapshots can only be read by +QCoDeS 0.64.0 and later. No database upgrade is required. diff --git a/docs/dataset/dataset_design.rst b/docs/dataset/dataset_design.rst index a9e00082d3a1..38e91fddf471 100644 --- a/docs/dataset/dataset_design.rst +++ b/docs/dataset/dataset_design.rst @@ -76,6 +76,55 @@ SQLite database. This is not an intrinsic limitation of the dataset and measurement layer. It is possible that at a future state support for writing to a different backend will be added. +.. _sec:design_snapshot_storage: + +Snapshot storage +================ + +The ``runs.snapshot`` column holds either JSON TEXT or a versioned, compressed +BLOB. Both are read transparently, independent of the current write policy; +SQL NULL continues to mean no snapshot. The write policy is controlled by:: + + qcodes.config.dataset.snapshot_compression = "zstd" + +The default, ``"none"``, writes uncompressed JSON TEXT, which all QCoDeS +versions can read. ``"zstd"`` writes lossless Zstandard level 1 BLOBs, which +can only be read by QCoDeS 0.64.0 and later. The policy +applies to each new write, including overwrites, database copies and imported +metadata; it never rewrites existing snapshots, so one database may contain +both representations. No database schema change or upgrade is involved; +SQLite already allows BLOB values in this column. + +The BLOB format is the ASCII bytes ``QCoDeS.snapshot``, a NUL byte, a version +byte (currently ``0x01``), then exactly one Zstandard frame with a checksum. +The frame contains the original JSON string encoded as UTF-8. Compression +does not parse or re-encode JSON, drop fields, deduplicate values, or change +NumPy/custom encoder, complex number, uncertainty or non-finite float semantics. +Unknown versions, truncated or corrupt frames, checksum failures and invalid +UTF-8 raise exceptions; there is no fallback to an empty or legacy snapshot. +JSON parsing remains the responsibility of the decoded snapshot API, so +malformed JSON can still be stored/read raw and raises when decoded. + +``DataSet.snapshot`` returns complete decoded contents; ``snapshot_raw``, +``_snapshot_raw``, ``get_metadata("snapshot")`` and low-level raw run attributes +return JSON strings, never compressed bytes. Xarray and NetCDF exports also +contain JSON strings. In-memory datasets retain JSON strings and use this write +policy when persisting snapshot metadata. Results backends and foreground or +background result writers do not change the snapshot format or startup order. +CPython 3.14 and newer use ``compression.zstd``; earlier supported Python +versions use the declared ``backports.zstd`` runtime dependency. + +Compressed snapshots are opt-in because QCoDeS versions before 0.64.0 cannot +decode them. Those versions still open the database, and runs with +uncompressed snapshots are unaffected. For runs written with ``"zstd"``, they +can still read the measured data and other metadata, but accessing +the snapshot, exporting to xarray or NetCDF and extracting the run into another +database fail; ``get_metadata("snapshot")`` returns the raw bytes; and runs +without a results table, such as those written by ``DataSetInMem``, cannot be +loaded at all. Direct SQL consumers must handle BLOBs explicitly rather than +passing them to SQLite JSON functions. Exporting to NetCDF with QCoDeS 0.64.0 or +later retains the snapshot as JSON for interchange with earlier versions. + .. _sec:design_split_storage: Split Raw Data Storage diff --git a/pyproject.toml b/pyproject.toml index 700f2bde59db..6f8be6cfe0b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ license = "MIT" readme = "README.rst" requires-python = ">=3.12" dependencies = [ + "backports.zstd>=1.0.0; python_version < '3.14'", "broadbean>=0.11.0", "h5netcdf>=1.3.0", "h5py>=3.10.0", diff --git a/src/qcodes/configuration/qcodesrc.json b/src/qcodes/configuration/qcodesrc.json index 7f9083b1da1e..e08d2091caff 100644 --- a/src/qcodes/configuration/qcodesrc.json +++ b/src/qcodes/configuration/qcodesrc.json @@ -65,6 +65,7 @@ "sample": 0 }, "dataset": { + "snapshot_compression": "none", "write_in_background": false, "write_period": 5.0, "use_threads": false, diff --git a/src/qcodes/configuration/qcodesrc_schema.json b/src/qcodes/configuration/qcodesrc_schema.json index 0ec9d64eaac8..6ab354bcfcfa 100644 --- a/src/qcodes/configuration/qcodesrc_schema.json +++ b/src/qcodes/configuration/qcodesrc_schema.json @@ -303,6 +303,12 @@ "dataset": { "type": "object", "properties": { + "snapshot_compression": { + "type": "string", + "enum": ["none", "zstd"], + "default": "none", + "description": "Snapshot write policy: uncompressed JSON TEXT, readable by all QCoDeS versions, or lossless Zstandard level 1 BLOBs, readable by QCoDeS 0.64.0 and later. Reads support both independently of this setting." + }, "write_in_background": { "type": "boolean", "default": false, diff --git a/src/qcodes/dataset/data_set.py b/src/qcodes/dataset/data_set.py index 94f30e9541ca..48abcf90897a 100644 --- a/src/qcodes/dataset/data_set.py +++ b/src/qcodes/dataset/data_set.py @@ -84,6 +84,7 @@ insert_many_values, select_one_where, ) +from qcodes.dataset.sqlite.snapshot_storage import decode_snapshot from qcodes.utils import ( NumpyJSONEncoder, QCoDeSDeprecationWarning, @@ -497,11 +498,9 @@ def snapshot(self) -> dict[str, Any] | None: @property def _snapshot_raw(self) -> str | None: """Snapshot of the run as a JSON-formatted string (or None)""" - snapshot_raw = select_one_where( - self.conn, "runs", "snapshot", "run_id", self.run_id - ) - assert isinstance(snapshot_raw, (str, type(None))) - return snapshot_raw + stored = select_one_where(self.conn, "runs", "snapshot", "run_id", self.run_id) + assert isinstance(stored, (str, bytes, type(None))) + return decode_snapshot(stored) @property def snapshot_raw(self) -> str | None: diff --git a/src/qcodes/dataset/sqlite/db_overview.py b/src/qcodes/dataset/sqlite/db_overview.py index 088bdae9ed3f..4c5ab57e7aba 100644 --- a/src/qcodes/dataset/sqlite/db_overview.py +++ b/src/qcodes/dataset/sqlite/db_overview.py @@ -28,6 +28,7 @@ conn_from_dbpath_or_conn, ) from qcodes.dataset.sqlite.query_helpers import is_column_in_table +from qcodes.dataset.sqlite.snapshot_storage import decode_snapshot if TYPE_CHECKING: from collections.abc import Sequence @@ -283,6 +284,8 @@ def get_db_overview( extra = { col: row[n_fixed + i] for i, col in enumerate(valid_extra_columns) } + if "snapshot" in extra: + extra["snapshot"] = decode_snapshot(extra["snapshot"]) # The keys of ``extra`` are only known at runtime (they are the # user-supplied ``extra_columns``), so they cannot be part of # the closed ``RunOverviewDict`` definition. diff --git a/src/qcodes/dataset/sqlite/queries.py b/src/qcodes/dataset/sqlite/queries.py index e9dda262e425..bdcfb69db897 100644 --- a/src/qcodes/dataset/sqlite/queries.py +++ b/src/qcodes/dataset/sqlite/queries.py @@ -49,6 +49,7 @@ sql_placeholder_string, update_where, ) +from qcodes.dataset.sqlite.snapshot_storage import decode_snapshot, encode_snapshot from qcodes.parameters import ParamSpecBase from qcodes.utils import list_of_data_to_maybe_ragged_nd_array @@ -1761,7 +1762,7 @@ def create_run( ) if metadata: add_data_to_dynamic_columns(conn, run_id, metadata) - if snapshot_raw: + if snapshot_raw is not None: add_data_to_dynamic_columns(conn, run_id, {"snapshot": snapshot_raw}) _update_experiment_run_counter(conn, exp_id, run_counter) if create_run_table: @@ -1838,6 +1839,9 @@ def get_data_by_tag_and_table_name( data = None else: raise + if tag == "snapshot": + assert isinstance(data, (str, bytes, type(None))) + return decode_snapshot(data) return data @@ -1960,6 +1964,14 @@ def update_columns( """ validate_dynamic_column_data(data) + if table_name == "runs" and "snapshot" in data: + snapshot = data["snapshot"] + if not isinstance(snapshot, str): + raise TypeError("Snapshot must be a raw JSON string") + data = { + **data, + "snapshot": encode_snapshot(snapshot, config.dataset.snapshot_compression), + } update_where(conn, table_name, "rowid", row_id, **data) @@ -2307,7 +2319,7 @@ def get_raw_run_attributes( assert isinstance(name, str) rawsnapshot = select_one_where(conn, "runs", "snapshot", "guid", guid) - assert isinstance(rawsnapshot, (str, type(None))) + assert isinstance(rawsnapshot, (str, bytes, type(None))) output: RawRunAttributesDict = { "run_id": run_id, "experiment": experiment, @@ -2320,7 +2332,7 @@ def get_raw_run_attributes( "metadata": get_metadata_from_run_id(conn, run_id), "parent_dataset_links": get_parent_dataset_links(conn, run_id), "run_description": get_run_description(conn, run_id), - "snapshot": rawsnapshot, + "snapshot": decode_snapshot(rawsnapshot), } return output diff --git a/src/qcodes/dataset/sqlite/snapshot_storage.py b/src/qcodes/dataset/sqlite/snapshot_storage.py new file mode 100644 index 000000000000..cd49a0716409 --- /dev/null +++ b/src/qcodes/dataset/sqlite/snapshot_storage.py @@ -0,0 +1,66 @@ +"""Versioned, lossless storage of snapshot JSON strings in SQLite.""" + +from __future__ import annotations + +import sys + +if sys.version_info >= (3, 14): + from compression import zstd +else: + from backports import zstd + +_MAGIC = b"QCoDeS.snapshot\x00" +_VERSION_1 = _MAGIC + b"\x01" + + +def encode_snapshot(snapshot: str, compression: str) -> str | bytes: + """Encode raw JSON without parsing or changing its encoder semantics. + + Args: + snapshot: The original JSON string, including any whitespace. + compression: ``zstd`` for version 1 storage, or ``none`` for legacy TEXT. + + Returns: + The original string or a versioned BLOB containing one checksummed frame. + + Raises: + ValueError: If the write policy is unsupported. + + """ + if compression == "none": + return snapshot + if compression != "zstd": + raise ValueError(f"Unsupported snapshot compression: {compression!r}") + return _VERSION_1 + zstd.compress( + snapshot.encode("utf-8"), + options={ + zstd.CompressionParameter.compression_level: 1, + zstd.CompressionParameter.checksum_flag: 1, + }, + ) + + +def decode_snapshot(snapshot: str | bytes | None) -> str | None: + """Return original JSON from legacy TEXT, versioned BLOB, or SQL NULL. + + Args: + snapshot: The value stored in the snapshot column. + + Returns: + The original JSON string, or None for SQL NULL. + + Raises: + ValueError: If the format is unknown, truncated, or has trailing data. + zstd.ZstdError: If the compressed frame or its checksum is corrupt. + UnicodeDecodeError: If the decompressed bytes are not UTF-8. + + """ + if snapshot is None or isinstance(snapshot, str): + return snapshot + if not isinstance(snapshot, bytes) or not snapshot.startswith(_VERSION_1): + raise ValueError("Unsupported snapshot storage format or version") + decompressor = zstd.ZstdDecompressor() + raw = decompressor.decompress(snapshot[len(_VERSION_1) :]) + if not decompressor.eof or decompressor.unused_data: + raise ValueError("Incomplete snapshot frame or trailing data") + return raw.decode("utf-8") diff --git a/tests/dataset/test_db_overview.py b/tests/dataset/test_db_overview.py index 5e4fdaf63bf2..c6f358c6e8bd 100644 --- a/tests/dataset/test_db_overview.py +++ b/tests/dataset/test_db_overview.py @@ -7,6 +7,7 @@ import pytest +import qcodes as qc from qcodes.dataset import ( get_db_overview, load_or_create_experiment, @@ -155,6 +156,24 @@ def test_get_db_overview_extra_columns(db_conn: AtomicConnection) -> None: assert "does_not_exist" not in overview[1] +def test_get_db_overview_decodes_snapshot_extra_column( + db_conn: AtomicConnection, +) -> None: + exp = load_or_create_experiment("exp", sample_name="sample", conn=db_conn) + plain = new_data_set("plain", exp_id=exp.exp_id, conn=db_conn) + plain.add_snapshot('{"plain": true}') + qc.config.dataset.snapshot_compression = "zstd" + compressed = new_data_set("compressed", exp_id=exp.exp_id, conn=db_conn) + compressed.add_snapshot('{"compressed": true}') + new_data_set("no_snapshot", exp_id=exp.exp_id, conn=db_conn) + + overview = get_db_overview(conn=db_conn, extra_columns=["snapshot"]) + + assert overview[1]["snapshot"] == '{"plain": true}' # type: ignore[typeddict-item] + assert overview[2]["snapshot"] == '{"compressed": true}' # type: ignore[typeddict-item] + assert overview[3]["snapshot"] is None # type: ignore[typeddict-item] + + def test_get_db_overview_accepts_connection(db_conn: AtomicConnection) -> None: _create_runs(db_conn, n_runs=1) diff --git a/tests/dataset/test_snapshot_storage.py b/tests/dataset/test_snapshot_storage.py new file mode 100644 index 000000000000..b7abb469d3b8 --- /dev/null +++ b/tests/dataset/test_snapshot_storage.py @@ -0,0 +1,395 @@ +from __future__ import annotations + +import json +import math +from contextlib import closing +from typing import TYPE_CHECKING + +import numpy as np +import pytest +import uncertainties # type: ignore[import-untyped] + +import qcodes as qc +from qcodes.dataset import load_by_id, new_data_set, new_experiment +from qcodes.dataset.data_set import DataSet +from qcodes.dataset.data_set_in_memory import DataSetInMem, load_from_netcdf +from qcodes.dataset.database_extract_runs import extract_runs_into_db +from qcodes.dataset.descriptions.dependencies import InterDependencies_ +from qcodes.dataset.guids import generate_guid +from qcodes.dataset.measurements import Measurement +from qcodes.dataset.sqlite.connection import atomic +from qcodes.dataset.sqlite.database import connect +from qcodes.dataset.sqlite.db_upgrades.version import get_user_version +from qcodes.dataset.sqlite.queries import ( + add_data_to_dynamic_columns, + create_run, + get_raw_run_attributes, +) +from qcodes.dataset.sqlite.snapshot_storage import ( + decode_snapshot, + encode_snapshot, + zstd, +) +from qcodes.parameters import ParamSpecBase +from qcodes.utils import NumpyJSONEncoder + +if TYPE_CHECKING: + from pathlib import Path + + from qcodes.dataset.experiment_container import Experiment + +PREFIX = b"QCoDeS.snapshot\x00\x01" + + +def stored_snapshot(dataset: DataSet) -> str | bytes | None: + return dataset.conn.execute( + "SELECT snapshot FROM runs WHERE run_id = ?", (dataset.run_id,) + ).fetchone()[0] + + +@pytest.mark.parametrize("compression", ["none", "zstd"]) +@pytest.mark.parametrize( + "raw", + [ + ' { "full": [1, 2], "unicode": "\\u00e6" }\n', + '{"text": "\u00e6"}', + "null", + "", + "{", + ], +) +def test_codec_preserves_raw_strings(raw: str, compression: str) -> None: + stored = encode_snapshot(raw, compression) + assert decode_snapshot(stored) == raw + assert isinstance(stored, bytes if compression == "zstd" else str) + assert decode_snapshot(None) is None + + +@pytest.mark.parametrize("compression", ["none", "zstd"]) +def test_write_read_and_reopen(dataset: DataSet, compression: str) -> None: + qc.config.dataset.snapshot_compression = compression + assert dataset.snapshot is None + assert dataset.snapshot_raw is None + assert dataset.get_metadata("snapshot") is None + raw = ' { "instruments": {"any_name": {"values": [1, 2]}} }\n' + dataset.add_snapshot(raw) + stored = stored_snapshot(dataset) + if compression == "zstd": + assert isinstance(stored, bytes) + assert stored.startswith(PREFIX) + else: + assert stored == raw + + qc.config.dataset.snapshot_compression = "none" if compression == "zstd" else "zstd" + assert dataset.path_to_db is not None + with closing(connect(dataset.path_to_db)) as conn: + loaded = load_by_id(dataset.run_id, conn=conn) + assert isinstance(loaded, DataSet) + assert loaded.snapshot == json.loads(raw) + assert loaded.snapshot_raw == loaded._snapshot_raw == raw + assert loaded.get_metadata("snapshot") == raw + attrs = get_raw_run_attributes(conn, dataset.guid) + assert attrs is not None + assert attrs["snapshot"] == raw + assert attrs["metadata"] == {} + assert loaded.metadata == {} + + +def test_overwrite_null_and_history( + dataset: DataSet, caplog: pytest.LogCaptureFixture +) -> None: + qc.config.dataset.snapshot_compression = "zstd" + dataset.add_snapshot("null") + assert dataset.snapshot is None + assert dataset.snapshot_raw == "null" + dataset.add_snapshot('{"nested": {"values": [1, 2]}}') + historical = new_data_set("historical") + historical.add_snapshot(dataset.snapshot_raw or "") + dataset.add_snapshot('{"ignored": true}') + assert "already has a snapshot" in caplog.text + assert dataset.snapshot == {"nested": {"values": [1, 2]}} + decoded = dataset.snapshot + assert decoded is not None + decoded["nested"]["values"].append(3) + assert dataset.snapshot == historical.snapshot == {"nested": {"values": [1, 2]}} + qc.config.dataset.snapshot_compression = "none" + dataset.add_snapshot('{"new": true}', overwrite=True) + assert stored_snapshot(dataset) == '{"new": true}' + assert historical.snapshot == {"nested": {"values": [1, 2]}} + + +@pytest.mark.parametrize("raw", ["", "{"]) +def test_malformed_json_semantics(dataset: DataSet, raw: str) -> None: + dataset.add_snapshot(raw) + assert dataset.snapshot_raw == dataset.get_metadata("snapshot") == raw + with pytest.raises(json.JSONDecodeError): + _ = dataset.snapshot + with pytest.raises(json.JSONDecodeError): + dataset.add_snapshot("{}", overwrite=True) + + +@pytest.mark.parametrize( + "stored", + [ + b"unknown", + b"QCoDeS.snapshot\x00\x02" + zstd.compress(b"{}"), + PREFIX, + PREFIX + zstd.compress(b"{}")[:-1], + PREFIX + zstd.compress(b"{}") + b"extra", + PREFIX + zstd.compress(b"{}") + zstd.compress(b"{}"), + PREFIX + zstd.compress(b"\xff"), + PREFIX + b"not a zstandard frame", + ], +) +def test_corrupt_storage_propagates(dataset: DataSet, stored: bytes) -> None: + dataset.conn.execute( + "UPDATE runs SET snapshot = ? WHERE run_id = ?", (stored, dataset.run_id) + ) + dataset.conn.commit() + errors = (ValueError, zstd.ZstdError, UnicodeDecodeError) + with pytest.raises(errors): + _ = dataset.snapshot + with pytest.raises(errors): + _ = dataset.snapshot_raw + with pytest.raises(errors): + dataset.get_metadata("snapshot") + with pytest.raises(errors): + get_raw_run_attributes(dataset.conn, dataset.guid) + with pytest.raises(errors): + dataset.add_snapshot("{}", overwrite=True) + + +def test_checksum_failure() -> None: + stored = encode_snapshot('{"data": [1, 2, 3]}', "zstd") + assert isinstance(stored, bytes) + damaged = stored[:-1] + bytes([stored[-1] ^ 1]) + with pytest.raises(zstd.ZstdError): + decode_snapshot(damaged) + + +def test_invalid_policy_rolls_back_create_run(experiment: Experiment) -> None: + qc.config.dataset.snapshot_compression = "unknown" + count = experiment.conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] + with pytest.raises(RuntimeError, match="Rolling back") as exc: + create_run( + experiment.conn, + experiment.exp_id, + "invalid", + generate_guid(), + snapshot_raw="{}", + ) + assert isinstance(exc.value.__cause__, ValueError) + assert experiment.conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] == count + assert ( + experiment.conn.execute( + "SELECT run_counter FROM experiments WHERE exp_id = ?", (experiment.exp_id,) + ).fetchone()[0] + == 0 + ) + + +@pytest.mark.parametrize("raw", [None, "null", "", '{"complete": [1, 2]}']) +def test_create_run(experiment: Experiment, raw: str | None) -> None: + _, run_id, _ = create_run( + experiment.conn, + experiment.exp_id, + "created", + generate_guid(), + snapshot_raw=raw, + ) + dataset = DataSet(run_id=run_id, conn=experiment.conn) + assert dataset.snapshot_raw == raw + + +@pytest.mark.parametrize("compression", ["none", "zstd"]) +@pytest.mark.usefixtures("empty_temp_db") +def test_snapshot_read_by_run_id_with_shared_table_name(compression: str) -> None: + qc.config.dataset.raw_data_backend = "sqlite_per_dataset_db" + qc.config.dataset.snapshot_compression = compression + new_experiment("shared", "sample", format_string="results") + first = new_data_set("first") + second = new_data_set("second") + assert first.table_name == second.table_name + first.add_snapshot('{"run": 1}') + second.add_snapshot('{"run": 2}') + assert first.snapshot == {"run": 1} + assert second.snapshot == {"run": 2} + assert first.path_to_db is not None + with closing(connect(first.path_to_db, read_only=True)) as conn: + loaded = load_by_id(second.run_id, conn=conn) + assert loaded.snapshot == {"run": 2} + + +def test_default_writes_uncompressed_text(dataset: DataSet) -> None: + raw = '{"default": true}' + dataset.add_snapshot(raw) + assert stored_snapshot(dataset) == raw + + +def test_mixed_storage_without_schema_change(dataset: DataSet) -> None: + assert dataset.path_to_db is not None + raw = '{"legacy": true}' + dataset.add_snapshot(raw) + qc.config.dataset.snapshot_compression = "zstd" + compressed = new_data_set("compressed", conn=dataset.conn) + compressed.add_snapshot('{"compressed": true}') + assert isinstance(stored_snapshot(compressed), bytes) + assert stored_snapshot(dataset) == raw + assert get_user_version(dataset.conn) == 9 + qc.config.dataset.snapshot_compression = "none" + with closing(connect(dataset.path_to_db)) as conn: + assert get_user_version(conn) == 9 + assert DataSet(run_id=dataset.run_id, conn=conn).snapshot_raw == raw + assert DataSet(run_id=compressed.run_id, conn=conn).snapshot == { + "compressed": True + } + + +class CustomEncoded: + def _JSONEncoder(self) -> dict[str, list[int]]: + return {"custom": [42]} + + +def test_prepare_preserves_encoder_types(dataset: DataSet) -> None: + qc.config.dataset.snapshot_compression = "zstd" + snapshot = { + "array": np.array([1.0, 2.0]), + "int": np.int64(2), + "bool": np.bool_(True), + "complex": 1 + 2j, + "uncertain": uncertainties.ufloat(1, 0.1), + "custom": CustomEncoded(), + "nan": float("nan"), + "inf": float("inf"), + "minus_inf": -float("inf"), + "negative_zero": -0.0, + "large_integer": 2**100, + } + dataset.prepare( + snapshot=snapshot, + interdeps=InterDependencies_(standalones=(ParamSpecBase("p", "numeric"),)), + ) + expected_raw = json.dumps(snapshot, cls=NumpyJSONEncoder) + assert dataset.snapshot_raw == expected_raw + decoded = dataset.snapshot + assert decoded is not None + assert math.isnan(decoded.pop("nan")) + expected = json.loads(expected_raw) + expected.pop("nan") + assert decoded == expected + assert math.copysign(1, decoded["negative_zero"]) == -1 + dataset.mark_completed() + + +@pytest.mark.parametrize( + "source_policy,target_policy", [("zstd", "none"), ("none", "zstd")] +) +def test_extract_reencodes_complete_snapshot( + dataset: DataSet, tmp_path: Path, source_policy: str, target_policy: str +) -> None: + qc.config.dataset.snapshot_compression = source_policy + raw = ' {"full": [1, 2], "nested": {"name": "anything"}}\n' + dataset.add_snapshot(raw) + dataset.mark_started() + dataset.mark_completed() + qc.config.dataset.snapshot_compression = target_policy + target = tmp_path / "copy.db" + assert dataset.path_to_db is not None + extract_runs_into_db(dataset.path_to_db, target, dataset.run_id) + with closing(connect(target)) as conn: + copied = DataSet(run_id=1, conn=conn) + assert copied.snapshot_raw == raw + assert copied.snapshot == dataset.snapshot + assert copied.metadata == {} + assert isinstance( + stored_snapshot(copied), bytes if target_policy == "zstd" else str + ) + + +def test_netcdf_export_import_and_metadata_db( + scalar_dataset: DataSet, tmp_path: Path +) -> None: + raw = ' {"full": {"values": [1, 2, 3]}}\n' + qc.config.dataset.snapshot_compression = "zstd" + scalar_dataset.add_snapshot(raw) + assert scalar_dataset.to_xarray_dataset().attrs["snapshot"] == raw + scalar_dataset.export("netcdf", path=tmp_path) + imported = load_from_netcdf(scalar_dataset.export_info.export_paths["nc"]) + assert imported._snapshot_raw == raw + assert imported.snapshot == scalar_dataset.snapshot + assert "snapshot" not in imported.metadata + target = tmp_path / "imported.db" + imported.write_metadata_to_db(path_to_db=target) + with closing(connect(target)) as conn: + attrs = get_raw_run_attributes(conn, imported.guid) + assert attrs is not None + assert attrs["snapshot"] == raw + assert "snapshot" not in attrs["metadata"] + loaded = DataSetInMem._load_from_db(conn, imported.guid) + assert loaded._snapshot_raw == raw + assert loaded.snapshot == scalar_dataset.snapshot + loaded.add_snapshot('{"updated": true}', overwrite=True) + attrs = get_raw_run_attributes(conn, imported.guid) + assert attrs is not None + assert attrs["snapshot"] == '{"updated": true}' + stored = conn.execute("SELECT snapshot FROM runs").fetchone()[0] + assert isinstance(stored, bytes) + + +def test_metadata_snapshot_write(dataset: DataSet) -> None: + raw = '{"legacy_import": true}' + qc.config.dataset.snapshot_compression = "zstd" + with atomic(dataset.conn): + add_data_to_dynamic_columns(dataset.conn, dataset.run_id, {"snapshot": raw}) + assert isinstance(stored_snapshot(dataset), bytes) + assert dataset.get_metadata("snapshot") == raw + + +def test_in_memory_snapshot_overwrites( + experiment: Experiment, caplog: pytest.LogCaptureFixture +) -> None: + qc.config.dataset.snapshot_compression = "zstd" + dataset = DataSetInMem._create_new_run(name="in-memory") + assert dataset.snapshot is None + assert dataset._snapshot_raw is None + dataset.add_snapshot("null") + assert dataset.snapshot is None + assert dataset._snapshot_raw == "null" + raw = '{"full": {"values": [1, 2]}}' + dataset.add_snapshot(raw) + dataset.add_snapshot('{"ignored": true}') + assert "already has a snapshot" in caplog.text + assert dataset._snapshot_raw == raw + decoded = dataset.snapshot + assert decoded is not None + decoded["full"]["values"].append(3) + assert dataset.snapshot == {"full": {"values": [1, 2]}} + dataset.add_snapshot('{"updated": true}', overwrite=True) + attrs = get_raw_run_attributes(experiment.conn, dataset.guid) + assert attrs is not None + assert dataset._snapshot_raw == attrs["snapshot"] == '{"updated": true}' + assert isinstance( + experiment.conn.execute( + "SELECT snapshot FROM runs WHERE guid = ?", (dataset.guid,) + ).fetchone()[0], + bytes, + ) + + +@pytest.mark.parametrize("background", [False, True]) +@pytest.mark.parametrize("backend", ["sqlite_main_db", "sqlite_per_dataset_db"]) +def test_measurement_writers_and_backends( + experiment: Experiment, background: bool, backend: str +) -> None: + qc.config.dataset.raw_data_backend = backend + qc.config.dataset.snapshot_compression = "zstd" + measurement = Measurement(exp=experiment) + measurement.register_custom_parameter("p") + with measurement.run(write_in_background=background) as saver: + saver.add_result(("p", 1)) + dataset = saver.dataset + assert isinstance(dataset, DataSet) + assert isinstance(stored_snapshot(dataset), bytes) + assert dataset.snapshot is not None + assert dataset.get_parameter_data()["p"]["p"].tolist() == [1] + assert dataset.completed