From 7c935a032ecd018f83f43b0b2c05be3d72aa1b6d Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Fri, 6 Nov 2026 15:39:19 +0100 Subject: [PATCH 1/5] Add lossless Zstandard snapshot compression Store snapshots as versioned, checksummed Zstandard frames while preserving JSON APIs and legacy TEXT reads. Add a configurable write policy, schema version gate, conditional backport dependency, documentation and regression coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/changes/newsfragments/8724.new | 6 + docs/dataset/dataset_design.rst | 44 ++ pyproject.toml | 1 + src/qcodes/configuration/qcodesrc.json | 1 + src/qcodes/configuration/qcodesrc_schema.json | 6 + src/qcodes/dataset/data_set.py | 4 +- .../dataset/sqlite/db_upgrades/__init__.py | 12 + src/qcodes/dataset/sqlite/queries.py | 21 +- src/qcodes/dataset/sqlite/snapshot_storage.py | 66 +++ .../test_database_creation_and_upgrading.py | 2 +- tests/dataset/test_snapshot_storage.py | 377 ++++++++++++++++++ 11 files changed, 534 insertions(+), 6 deletions(-) create mode 100644 docs/changes/newsfragments/8724.new create mode 100644 src/qcodes/dataset/sqlite/snapshot_storage.py create mode 100644 tests/dataset/test_snapshot_storage.py diff --git a/docs/changes/newsfragments/8724.new b/docs/changes/newsfragments/8724.new new file mode 100644 index 000000000000..f8d93457d688 --- /dev/null +++ b/docs/changes/newsfragments/8724.new @@ -0,0 +1,6 @@ +Snapshots are stored losslessly using versioned, checksummed Zstandard level 1 +compression by default. JSON snapshot APIs and NetCDF exports retain complete +contents, and legacy uncompressed snapshots remain readable. Set +``dataset.snapshot_compression`` to ``"none"`` for uncompressed writes. +Database schema version 10 prevents older QCoDeS versions from opening upgraded +databases; direct SQL consumers must account for the new BLOB representation. diff --git a/docs/dataset/dataset_design.rst b/docs/dataset/dataset_design.rst index a9e00082d3a1..b6e1cbfceda4 100644 --- a/docs/dataset/dataset_design.rst +++ b/docs/dataset/dataset_design.rst @@ -76,6 +76,50 @@ SQLite database. This is not an intrinsic limitation of the dataset and measurement layer. It is possible that at a future state support for writing to a different backend will be added. +.. _sec:design_snapshot_storage: + +Snapshot storage +================ + +Database schema version 10 permits either legacy JSON TEXT or a versioned BLOB +in ``runs.snapshot``. New snapshot writes default to lossless Zstandard level 1. +To write legacy uncompressed TEXT instead, set:: + + qcodes.config.dataset.snapshot_compression = "none" + +The other supported value is ``"zstd"`` (the default). This policy applies to +each new write, including overwrites, database copies and imported metadata; +it does not rewrite existing snapshots. Small snapshots are compressed too, +even when the framing overhead makes them larger. Reads recognize both formats +regardless of the current write policy. SQL NULL continues to mean no snapshot. + +The BLOB format is the ASCII bytes ``QCoDeS.snapshot``, a NUL byte, a version +byte (currently ``0x01``), then exactly one Zstandard frame with a checksum. +The frame contains the original JSON string encoded as UTF-8. Compression +does not parse or re-encode JSON, drop fields, deduplicate values, or change +NumPy/custom encoder, complex number, uncertainty or non-finite float semantics. +Unknown versions, truncated or corrupt frames, checksum failures and invalid +UTF-8 raise exceptions; there is no fallback to an empty or legacy snapshot. +JSON parsing remains the responsibility of the decoded snapshot API, so +malformed JSON can still be stored/read raw and raises when decoded. + +``DataSet.snapshot`` returns complete decoded contents; ``snapshot_raw``, +``_snapshot_raw``, ``get_metadata("snapshot")`` and low-level raw run attributes +return JSON strings, never compressed bytes. Xarray and NetCDF exports also +contain JSON strings. In-memory datasets retain JSON strings and use this write +policy when persisting snapshot metadata. Results backends and foreground or +background result writers do not change the snapshot format or startup order. +CPython 3.14 and newer use ``compression.zstd``; earlier supported Python +versions use the declared ``backports.zstd`` runtime dependency. + +Upgrading a database to version 10 leaves existing TEXT snapshots untouched. +Older QCoDeS versions cannot open the upgraded database, even if uncompressed +writes are selected. Direct SQL consumers must handle BLOBs explicitly rather +than passing them to JSON functions. For interchange with older software, +export to NetCDF (which retains JSON snapshots), or explicitly migrate decoded +JSON into a separately created compatible database using appropriate tooling. +Changing ``PRAGMA user_version`` is not a supported downgrade. + .. _sec:design_split_storage: Split Raw Data Storage diff --git a/pyproject.toml b/pyproject.toml index 700f2bde59db..6f8be6cfe0b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ license = "MIT" readme = "README.rst" requires-python = ">=3.12" dependencies = [ + "backports.zstd>=1.0.0; python_version < '3.14'", "broadbean>=0.11.0", "h5netcdf>=1.3.0", "h5py>=3.10.0", diff --git a/src/qcodes/configuration/qcodesrc.json b/src/qcodes/configuration/qcodesrc.json index 7f9083b1da1e..97c3a835b5fb 100644 --- a/src/qcodes/configuration/qcodesrc.json +++ b/src/qcodes/configuration/qcodesrc.json @@ -65,6 +65,7 @@ "sample": 0 }, "dataset": { + "snapshot_compression": "zstd", "write_in_background": false, "write_period": 5.0, "use_threads": false, diff --git a/src/qcodes/configuration/qcodesrc_schema.json b/src/qcodes/configuration/qcodesrc_schema.json index 0ec9d64eaac8..0359cdf25f62 100644 --- a/src/qcodes/configuration/qcodesrc_schema.json +++ b/src/qcodes/configuration/qcodesrc_schema.json @@ -303,6 +303,12 @@ "dataset": { "type": "object", "properties": { + "snapshot_compression": { + "type": "string", + "enum": ["none", "zstd"], + "default": "zstd", + "description": "Snapshot write policy: lossless Zstandard level 1 BLOBs or legacy uncompressed JSON TEXT. Reads support both independently of this setting." + }, "write_in_background": { "type": "boolean", "default": false, diff --git a/src/qcodes/dataset/data_set.py b/src/qcodes/dataset/data_set.py index 94f30e9541ca..45f0d802d664 100644 --- a/src/qcodes/dataset/data_set.py +++ b/src/qcodes/dataset/data_set.py @@ -497,8 +497,8 @@ def snapshot(self) -> dict[str, Any] | None: @property def _snapshot_raw(self) -> str | None: """Snapshot of the run as a JSON-formatted string (or None)""" - snapshot_raw = select_one_where( - self.conn, "runs", "snapshot", "run_id", self.run_id + snapshot_raw = get_data_by_tag_and_table_name( + self.conn, "snapshot", self.table_name ) assert isinstance(snapshot_raw, (str, type(None))) return snapshot_raw diff --git a/src/qcodes/dataset/sqlite/db_upgrades/__init__.py b/src/qcodes/dataset/sqlite/db_upgrades/__init__.py index c8e9d9b866ce..990ef5bdb09a 100644 --- a/src/qcodes/dataset/sqlite/db_upgrades/__init__.py +++ b/src/qcodes/dataset/sqlite/db_upgrades/__init__.py @@ -404,3 +404,15 @@ def perform_db_upgrade_8_to_9( transaction(connection, _IX_runs_captured_run_id) else: raise RuntimeError(f"found {n_run_tables} runs tables expected 1") + + +@upgrader +def perform_db_upgrade_9_to_10( + conn: AtomicConnection, show_progress_bar: bool = True +) -> None: + """Allow versioned compressed BLOBs alongside TEXT in ``runs.snapshot``. + + SQLite already supports both storage classes in this column. Existing + snapshots are deliberately not rewritten. The version gate prevents older + QCoDeS from treating compressed snapshots as JSON. + """ diff --git a/src/qcodes/dataset/sqlite/queries.py b/src/qcodes/dataset/sqlite/queries.py index e9dda262e425..3737a19f22ae 100644 --- a/src/qcodes/dataset/sqlite/queries.py +++ b/src/qcodes/dataset/sqlite/queries.py @@ -34,6 +34,7 @@ atomic_transaction, transaction, ) +from qcodes.dataset.sqlite.db_upgrades.version import get_user_version from qcodes.dataset.sqlite.query_helpers import ( VALUE, VALUES, @@ -49,6 +50,7 @@ sql_placeholder_string, update_where, ) +from qcodes.dataset.sqlite.snapshot_storage import decode_snapshot, encode_snapshot from qcodes.parameters import ParamSpecBase from qcodes.utils import list_of_data_to_maybe_ragged_nd_array @@ -1761,7 +1763,7 @@ def create_run( ) if metadata: add_data_to_dynamic_columns(conn, run_id, metadata) - if snapshot_raw: + if snapshot_raw is not None: add_data_to_dynamic_columns(conn, run_id, {"snapshot": snapshot_raw}) _update_experiment_run_counter(conn, exp_id, run_counter) if create_run_table: @@ -1838,6 +1840,9 @@ def get_data_by_tag_and_table_name( data = None else: raise + if tag == "snapshot": + assert isinstance(data, (str, bytes, type(None))) + return decode_snapshot(data) return data @@ -1960,6 +1965,16 @@ def update_columns( """ validate_dynamic_column_data(data) + if table_name == "runs" and "snapshot" in data: + snapshot = data["snapshot"] + if not isinstance(snapshot, str): + raise TypeError("Snapshot must be a raw JSON string") + compression = config.dataset.snapshot_compression + if compression == "zstd" and get_user_version(conn) < 10: + raise RuntimeError( + "Compressed snapshots require database schema version 10" + ) + data = {**data, "snapshot": encode_snapshot(snapshot, compression)} update_where(conn, table_name, "rowid", row_id, **data) @@ -2307,7 +2322,7 @@ def get_raw_run_attributes( assert isinstance(name, str) rawsnapshot = select_one_where(conn, "runs", "snapshot", "guid", guid) - assert isinstance(rawsnapshot, (str, type(None))) + assert isinstance(rawsnapshot, (str, bytes, type(None))) output: RawRunAttributesDict = { "run_id": run_id, "experiment": experiment, @@ -2320,7 +2335,7 @@ def get_raw_run_attributes( "metadata": get_metadata_from_run_id(conn, run_id), "parent_dataset_links": get_parent_dataset_links(conn, run_id), "run_description": get_run_description(conn, run_id), - "snapshot": rawsnapshot, + "snapshot": decode_snapshot(rawsnapshot), } return output diff --git a/src/qcodes/dataset/sqlite/snapshot_storage.py b/src/qcodes/dataset/sqlite/snapshot_storage.py new file mode 100644 index 000000000000..cd49a0716409 --- /dev/null +++ b/src/qcodes/dataset/sqlite/snapshot_storage.py @@ -0,0 +1,66 @@ +"""Versioned, lossless storage of snapshot JSON strings in SQLite.""" + +from __future__ import annotations + +import sys + +if sys.version_info >= (3, 14): + from compression import zstd +else: + from backports import zstd + +_MAGIC = b"QCoDeS.snapshot\x00" +_VERSION_1 = _MAGIC + b"\x01" + + +def encode_snapshot(snapshot: str, compression: str) -> str | bytes: + """Encode raw JSON without parsing or changing its encoder semantics. + + Args: + snapshot: The original JSON string, including any whitespace. + compression: ``zstd`` for version 1 storage, or ``none`` for legacy TEXT. + + Returns: + The original string or a versioned BLOB containing one checksummed frame. + + Raises: + ValueError: If the write policy is unsupported. + + """ + if compression == "none": + return snapshot + if compression != "zstd": + raise ValueError(f"Unsupported snapshot compression: {compression!r}") + return _VERSION_1 + zstd.compress( + snapshot.encode("utf-8"), + options={ + zstd.CompressionParameter.compression_level: 1, + zstd.CompressionParameter.checksum_flag: 1, + }, + ) + + +def decode_snapshot(snapshot: str | bytes | None) -> str | None: + """Return original JSON from legacy TEXT, versioned BLOB, or SQL NULL. + + Args: + snapshot: The value stored in the snapshot column. + + Returns: + The original JSON string, or None for SQL NULL. + + Raises: + ValueError: If the format is unknown, truncated, or has trailing data. + zstd.ZstdError: If the compressed frame or its checksum is corrupt. + UnicodeDecodeError: If the decompressed bytes are not UTF-8. + + """ + if snapshot is None or isinstance(snapshot, str): + return snapshot + if not isinstance(snapshot, bytes) or not snapshot.startswith(_VERSION_1): + raise ValueError("Unsupported snapshot storage format or version") + decompressor = zstd.ZstdDecompressor() + raw = decompressor.decompress(snapshot[len(_VERSION_1) :]) + if not decompressor.eof or decompressor.unused_data: + raise ValueError("Incomplete snapshot frame or trailing data") + return raw.decode("utf-8") diff --git a/tests/dataset/test_database_creation_and_upgrading.py b/tests/dataset/test_database_creation_and_upgrading.py index 0c85107e5a4a..fa4ca000c811 100644 --- a/tests/dataset/test_database_creation_and_upgrading.py +++ b/tests/dataset/test_database_creation_and_upgrading.py @@ -1009,7 +1009,7 @@ def test_cannot_connect_to_newer_db() -> None: def test_latest_available_version() -> None: - assert _latest_available_version() == 9 + assert _latest_available_version() == 10 @pytest.mark.parametrize("version", VERSIONS[:-1]) diff --git a/tests/dataset/test_snapshot_storage.py b/tests/dataset/test_snapshot_storage.py new file mode 100644 index 000000000000..dacd8d670990 --- /dev/null +++ b/tests/dataset/test_snapshot_storage.py @@ -0,0 +1,377 @@ +from __future__ import annotations + +import json +import math +from contextlib import closing +from typing import TYPE_CHECKING + +import numpy as np +import pytest +import uncertainties # type: ignore[import-untyped] + +import qcodes as qc +from qcodes.dataset import load_by_id, new_data_set +from qcodes.dataset.data_set import DataSet +from qcodes.dataset.data_set_in_memory import DataSetInMem, load_from_netcdf +from qcodes.dataset.database_extract_runs import extract_runs_into_db +from qcodes.dataset.descriptions.dependencies import InterDependencies_ +from qcodes.dataset.guids import generate_guid +from qcodes.dataset.measurements import Measurement +from qcodes.dataset.sqlite.connection import atomic +from qcodes.dataset.sqlite.database import connect +from qcodes.dataset.sqlite.db_upgrades.version import get_user_version, set_user_version +from qcodes.dataset.sqlite.queries import ( + add_data_to_dynamic_columns, + create_run, + get_raw_run_attributes, +) +from qcodes.dataset.sqlite.snapshot_storage import ( + decode_snapshot, + encode_snapshot, + zstd, +) +from qcodes.parameters import ParamSpecBase +from qcodes.utils import NumpyJSONEncoder + +if TYPE_CHECKING: + from pathlib import Path + + from qcodes.dataset.experiment_container import Experiment + +PREFIX = b"QCoDeS.snapshot\x00\x01" + + +def stored_snapshot(dataset: DataSet) -> str | bytes | None: + return dataset.conn.execute( + "SELECT snapshot FROM runs WHERE run_id = ?", (dataset.run_id,) + ).fetchone()[0] + + +@pytest.mark.parametrize("compression", ["none", "zstd"]) +@pytest.mark.parametrize( + "raw", + [ + ' { "full": [1, 2], "unicode": "\\u00e6" }\n', + '{"text": "\u00e6"}', + "null", + "", + "{", + ], +) +def test_codec_preserves_raw_strings(raw: str, compression: str) -> None: + stored = encode_snapshot(raw, compression) + assert decode_snapshot(stored) == raw + assert isinstance(stored, bytes if compression == "zstd" else str) + assert decode_snapshot(None) is None + + +@pytest.mark.parametrize("compression", ["none", "zstd"]) +def test_write_read_and_reopen(dataset: DataSet, compression: str) -> None: + qc.config.dataset.snapshot_compression = compression + assert dataset.snapshot is None + assert dataset.snapshot_raw is None + assert dataset.get_metadata("snapshot") is None + raw = ' { "instruments": {"any_name": {"values": [1, 2]}} }\n' + dataset.add_snapshot(raw) + stored = stored_snapshot(dataset) + if compression == "zstd": + assert isinstance(stored, bytes) + assert stored.startswith(PREFIX) + else: + assert stored == raw + + qc.config.dataset.snapshot_compression = "none" if compression == "zstd" else "zstd" + assert dataset.path_to_db is not None + with closing(connect(dataset.path_to_db)) as conn: + loaded = load_by_id(dataset.run_id, conn=conn) + assert isinstance(loaded, DataSet) + assert loaded.snapshot == json.loads(raw) + assert loaded.snapshot_raw == loaded._snapshot_raw == raw + assert loaded.get_metadata("snapshot") == raw + attrs = get_raw_run_attributes(conn, dataset.guid) + assert attrs is not None + assert attrs["snapshot"] == raw + assert attrs["metadata"] == {} + assert loaded.metadata == {} + + +def test_overwrite_null_and_history( + dataset: DataSet, caplog: pytest.LogCaptureFixture +) -> None: + dataset.add_snapshot("null") + assert dataset.snapshot is None + assert dataset.snapshot_raw == "null" + dataset.add_snapshot('{"nested": {"values": [1, 2]}}') + historical = new_data_set("historical") + historical.add_snapshot(dataset.snapshot_raw or "") + dataset.add_snapshot('{"ignored": true}') + assert "already has a snapshot" in caplog.text + assert dataset.snapshot == {"nested": {"values": [1, 2]}} + decoded = dataset.snapshot + assert decoded is not None + decoded["nested"]["values"].append(3) + assert dataset.snapshot == historical.snapshot == {"nested": {"values": [1, 2]}} + qc.config.dataset.snapshot_compression = "none" + dataset.add_snapshot('{"new": true}', overwrite=True) + assert stored_snapshot(dataset) == '{"new": true}' + assert historical.snapshot == {"nested": {"values": [1, 2]}} + + +@pytest.mark.parametrize("raw", ["", "{"]) +def test_malformed_json_semantics(dataset: DataSet, raw: str) -> None: + dataset.add_snapshot(raw) + assert dataset.snapshot_raw == dataset.get_metadata("snapshot") == raw + with pytest.raises(json.JSONDecodeError): + _ = dataset.snapshot + with pytest.raises(json.JSONDecodeError): + dataset.add_snapshot("{}", overwrite=True) + + +@pytest.mark.parametrize( + "stored", + [ + b"unknown", + b"QCoDeS.snapshot\x00\x02" + zstd.compress(b"{}"), + PREFIX, + PREFIX + zstd.compress(b"{}")[:-1], + PREFIX + zstd.compress(b"{}") + b"extra", + PREFIX + zstd.compress(b"{}") + zstd.compress(b"{}"), + PREFIX + zstd.compress(b"\xff"), + PREFIX + b"not a zstandard frame", + ], +) +def test_corrupt_storage_propagates(dataset: DataSet, stored: bytes) -> None: + dataset.conn.execute( + "UPDATE runs SET snapshot = ? WHERE run_id = ?", (stored, dataset.run_id) + ) + dataset.conn.commit() + errors = (ValueError, zstd.ZstdError, UnicodeDecodeError) + with pytest.raises(errors): + _ = dataset.snapshot + with pytest.raises(errors): + _ = dataset.snapshot_raw + with pytest.raises(errors): + dataset.get_metadata("snapshot") + with pytest.raises(errors): + get_raw_run_attributes(dataset.conn, dataset.guid) + with pytest.raises(errors): + dataset.add_snapshot("{}", overwrite=True) + + +def test_checksum_failure() -> None: + stored = encode_snapshot('{"data": [1, 2, 3]}', "zstd") + assert isinstance(stored, bytes) + damaged = stored[:-1] + bytes([stored[-1] ^ 1]) + with pytest.raises(zstd.ZstdError): + decode_snapshot(damaged) + + +def test_invalid_policy_rolls_back_create_run(experiment: Experiment) -> None: + qc.config.dataset.snapshot_compression = "unknown" + count = experiment.conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] + with pytest.raises(RuntimeError, match="Rolling back") as exc: + create_run( + experiment.conn, + experiment.exp_id, + "invalid", + generate_guid(), + snapshot_raw="{}", + ) + assert isinstance(exc.value.__cause__, ValueError) + assert experiment.conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] == count + assert ( + experiment.conn.execute( + "SELECT run_counter FROM experiments WHERE exp_id = ?", (experiment.exp_id,) + ).fetchone()[0] + == 0 + ) + + +@pytest.mark.parametrize("raw", [None, "null", "", '{"complete": [1, 2]}']) +def test_create_run(experiment: Experiment, raw: str | None) -> None: + _, run_id, _ = create_run( + experiment.conn, + experiment.exp_id, + "created", + generate_guid(), + snapshot_raw=raw, + ) + dataset = DataSet(run_id=run_id, conn=experiment.conn) + assert dataset.snapshot_raw == raw + + +def test_legacy_upgrade_and_mixed_storage(dataset: DataSet) -> None: + assert dataset.path_to_db is not None + raw = '{"legacy": true}' + qc.config.dataset.snapshot_compression = "none" + dataset.add_snapshot(raw) + set_user_version(dataset.conn, 9) + with closing(connect(dataset.path_to_db)) as conn: + assert get_user_version(conn) == 10 + legacy = DataSet(run_id=dataset.run_id, conn=conn) + assert stored_snapshot(legacy) == raw + assert legacy.snapshot == {"legacy": True} + qc.config.dataset.snapshot_compression = "zstd" + compressed = new_data_set("compressed", conn=conn) + compressed.add_snapshot('{"compressed": true}') + assert isinstance(stored_snapshot(compressed), bytes) + assert stored_snapshot(legacy) == raw + assert compressed.snapshot == {"compressed": True} + with closing(connect(dataset.path_to_db)) as conn: + assert DataSet(run_id=dataset.run_id, conn=conn).snapshot_raw == raw + assert DataSet(run_id=compressed.run_id, conn=conn).snapshot == { + "compressed": True + } + + +def test_compressed_writes_require_version_10(dataset: DataSet) -> None: + set_user_version(dataset.conn, 9) + with pytest.raises(RuntimeError, match="Rolling back") as exc: + dataset.add_snapshot("{}") + assert "schema version 10" in str(exc.value.__cause__) + assert stored_snapshot(dataset) is None + + +class CustomEncoded: + def _JSONEncoder(self) -> dict[str, list[int]]: + return {"custom": [42]} + + +def test_prepare_preserves_encoder_types(dataset: DataSet) -> None: + snapshot = { + "array": np.array([1.0, 2.0]), + "int": np.int64(2), + "bool": np.bool_(True), + "complex": 1 + 2j, + "uncertain": uncertainties.ufloat(1, 0.1), + "custom": CustomEncoded(), + "nan": float("nan"), + "inf": float("inf"), + "minus_inf": -float("inf"), + "negative_zero": -0.0, + "large_integer": 2**100, + } + dataset.prepare( + snapshot=snapshot, + interdeps=InterDependencies_(standalones=(ParamSpecBase("p", "numeric"),)), + ) + expected_raw = json.dumps(snapshot, cls=NumpyJSONEncoder) + assert dataset.snapshot_raw == expected_raw + decoded = dataset.snapshot + assert decoded is not None + assert math.isnan(decoded.pop("nan")) + expected = json.loads(expected_raw) + expected.pop("nan") + assert decoded == expected + assert math.copysign(1, decoded["negative_zero"]) == -1 + dataset.mark_completed() + + +@pytest.mark.parametrize( + "source_policy,target_policy", [("zstd", "none"), ("none", "zstd")] +) +def test_extract_reencodes_complete_snapshot( + dataset: DataSet, tmp_path: Path, source_policy: str, target_policy: str +) -> None: + qc.config.dataset.snapshot_compression = source_policy + raw = ' {"full": [1, 2], "nested": {"name": "anything"}}\n' + dataset.add_snapshot(raw) + dataset.mark_started() + dataset.mark_completed() + qc.config.dataset.snapshot_compression = target_policy + target = tmp_path / "copy.db" + assert dataset.path_to_db is not None + extract_runs_into_db(dataset.path_to_db, target, dataset.run_id) + with closing(connect(target)) as conn: + copied = DataSet(run_id=1, conn=conn) + assert copied.snapshot_raw == raw + assert copied.snapshot == dataset.snapshot + assert copied.metadata == {} + assert isinstance( + stored_snapshot(copied), bytes if target_policy == "zstd" else str + ) + + +def test_netcdf_export_import_and_metadata_db( + scalar_dataset: DataSet, tmp_path: Path +) -> None: + raw = ' {"full": {"values": [1, 2, 3]}}\n' + scalar_dataset.add_snapshot(raw) + assert scalar_dataset.to_xarray_dataset().attrs["snapshot"] == raw + scalar_dataset.export("netcdf", path=tmp_path) + imported = load_from_netcdf(scalar_dataset.export_info.export_paths["nc"]) + assert imported._snapshot_raw == raw + assert imported.snapshot == scalar_dataset.snapshot + assert "snapshot" not in imported.metadata + target = tmp_path / "imported.db" + imported.write_metadata_to_db(path_to_db=target) + with closing(connect(target)) as conn: + attrs = get_raw_run_attributes(conn, imported.guid) + assert attrs is not None + assert attrs["snapshot"] == raw + assert "snapshot" not in attrs["metadata"] + loaded = DataSetInMem._load_from_db(conn, imported.guid) + assert loaded._snapshot_raw == raw + assert loaded.snapshot == scalar_dataset.snapshot + loaded.add_snapshot('{"updated": true}', overwrite=True) + attrs = get_raw_run_attributes(conn, imported.guid) + assert attrs is not None + assert attrs["snapshot"] == '{"updated": true}' + stored = conn.execute("SELECT snapshot FROM runs").fetchone()[0] + assert isinstance(stored, bytes) + + +def test_metadata_snapshot_write(dataset: DataSet) -> None: + raw = '{"legacy_import": true}' + with atomic(dataset.conn): + add_data_to_dynamic_columns(dataset.conn, dataset.run_id, {"snapshot": raw}) + assert isinstance(stored_snapshot(dataset), bytes) + assert dataset.get_metadata("snapshot") == raw + + +def test_in_memory_snapshot_overwrites( + experiment: Experiment, caplog: pytest.LogCaptureFixture +) -> None: + dataset = DataSetInMem._create_new_run(name="in-memory") + assert dataset.snapshot is None + assert dataset._snapshot_raw is None + dataset.add_snapshot("null") + assert dataset.snapshot is None + assert dataset._snapshot_raw == "null" + raw = '{"full": {"values": [1, 2]}}' + dataset.add_snapshot(raw) + dataset.add_snapshot('{"ignored": true}') + assert "already has a snapshot" in caplog.text + assert dataset._snapshot_raw == raw + decoded = dataset.snapshot + assert decoded is not None + decoded["full"]["values"].append(3) + assert dataset.snapshot == {"full": {"values": [1, 2]}} + dataset.add_snapshot('{"updated": true}', overwrite=True) + attrs = get_raw_run_attributes(experiment.conn, dataset.guid) + assert attrs is not None + assert dataset._snapshot_raw == attrs["snapshot"] == '{"updated": true}' + assert isinstance( + experiment.conn.execute( + "SELECT snapshot FROM runs WHERE guid = ?", (dataset.guid,) + ).fetchone()[0], + bytes, + ) + + +@pytest.mark.parametrize("background", [False, True]) +@pytest.mark.parametrize("backend", ["sqlite_main_db", "sqlite_per_dataset_db"]) +def test_measurement_writers_and_backends( + experiment: Experiment, background: bool, backend: str +) -> None: + qc.config.dataset.raw_data_backend = backend + measurement = Measurement(exp=experiment) + measurement.register_custom_parameter("p") + with measurement.run(write_in_background=background) as saver: + saver.add_result(("p", 1)) + dataset = saver.dataset + assert isinstance(dataset, DataSet) + assert isinstance(stored_snapshot(dataset), bytes) + assert dataset.snapshot is not None + assert dataset.get_parameter_data()["p"]["p"].tolist() == [1] + assert dataset.completed From b092ab1f49edc308e917ecffc703e3831875f86a Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 16 Nov 2026 10:44:54 +0100 Subject: [PATCH 2/5] Make snapshot compression opt-in without a database upgrade The version 10 upgrade only changed user_version, since SQLite already accepts BLOBs in runs.snapshot, and it prevented older QCoDeS from opening upgraded databases at all. Remove it and default dataset.snapshot_compression to none so new snapshots stay readable by released QCoDeS. Compressed snapshots remain readable in all cases and can be enabled explicitly; the default can be switched once a release supports reading them. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/changes/newsfragments/8724.new | 12 ++--- docs/dataset/dataset_design.rst | 36 ++++++++------- src/qcodes/configuration/qcodesrc.json | 2 +- src/qcodes/configuration/qcodesrc_schema.json | 4 +- .../dataset/sqlite/db_upgrades/__init__.py | 12 ----- src/qcodes/dataset/sqlite/queries.py | 11 ++--- .../test_database_creation_and_upgrading.py | 2 +- tests/dataset/test_snapshot_storage.py | 45 +++++++++---------- 8 files changed, 56 insertions(+), 68 deletions(-) diff --git a/docs/changes/newsfragments/8724.new b/docs/changes/newsfragments/8724.new index f8d93457d688..8c97d07b8884 100644 --- a/docs/changes/newsfragments/8724.new +++ b/docs/changes/newsfragments/8724.new @@ -1,6 +1,6 @@ -Snapshots are stored losslessly using versioned, checksummed Zstandard level 1 -compression by default. JSON snapshot APIs and NetCDF exports retain complete -contents, and legacy uncompressed snapshots remain readable. Set -``dataset.snapshot_compression`` to ``"none"`` for uncompressed writes. -Database schema version 10 prevents older QCoDeS versions from opening upgraded -databases; direct SQL consumers must account for the new BLOB representation. +Snapshots can be stored losslessly using versioned, checksummed Zstandard +level 1 compression by setting ``dataset.snapshot_compression`` to ``"zstd"``. +Compressed and uncompressed snapshots are always readable, and snapshot APIs and +NetCDF exports return the complete JSON. Compression is not enabled by default +because QCoDeS versions without this support cannot read compressed snapshots. +No database upgrade is required. diff --git a/docs/dataset/dataset_design.rst b/docs/dataset/dataset_design.rst index b6e1cbfceda4..a13fb9f4eecb 100644 --- a/docs/dataset/dataset_design.rst +++ b/docs/dataset/dataset_design.rst @@ -81,17 +81,18 @@ to a different backend will be added. Snapshot storage ================ -Database schema version 10 permits either legacy JSON TEXT or a versioned BLOB -in ``runs.snapshot``. New snapshot writes default to lossless Zstandard level 1. -To write legacy uncompressed TEXT instead, set:: +The ``runs.snapshot`` column holds either JSON TEXT or a versioned, compressed +BLOB. Both are read transparently, independent of the current write policy; +SQL NULL continues to mean no snapshot. The write policy is controlled by:: - qcodes.config.dataset.snapshot_compression = "none" + qcodes.config.dataset.snapshot_compression = "zstd" -The other supported value is ``"zstd"`` (the default). This policy applies to -each new write, including overwrites, database copies and imported metadata; -it does not rewrite existing snapshots. Small snapshots are compressed too, -even when the framing overhead makes them larger. Reads recognize both formats -regardless of the current write policy. SQL NULL continues to mean no snapshot. +The default, ``"none"``, writes uncompressed JSON TEXT as previous QCoDeS +versions do. ``"zstd"`` writes lossless Zstandard level 1 BLOBs. The policy +applies to each new write, including overwrites, database copies and imported +metadata; it never rewrites existing snapshots, so one database may contain +both representations. No database schema change or upgrade is involved; +SQLite already allows BLOB values in this column. The BLOB format is the ASCII bytes ``QCoDeS.snapshot``, a NUL byte, a version byte (currently ``0x01``), then exactly one Zstandard frame with a checksum. @@ -112,13 +113,16 @@ background result writers do not change the snapshot format or startup order. CPython 3.14 and newer use ``compression.zstd``; earlier supported Python versions use the declared ``backports.zstd`` runtime dependency. -Upgrading a database to version 10 leaves existing TEXT snapshots untouched. -Older QCoDeS versions cannot open the upgraded database, even if uncompressed -writes are selected. Direct SQL consumers must handle BLOBs explicitly rather -than passing them to JSON functions. For interchange with older software, -export to NetCDF (which retains JSON snapshots), or explicitly migrate decoded -JSON into a separately created compatible database using appropriate tooling. -Changing ``PRAGMA user_version`` is not a supported downgrade. +Compressed snapshots are opt-in because QCoDeS versions without this support +cannot decode them. Such versions still open the database, and runs with +uncompressed snapshots are unaffected. For runs written with ``"zstd"``, older +versions can still read the measured data and other metadata, but accessing +the snapshot, exporting to xarray or NetCDF and extracting the run into another +database fail; ``get_metadata("snapshot")`` returns the raw bytes; and runs +without a results table, such as those written by ``DataSetInMem``, cannot be +loaded at all. Direct SQL consumers must handle BLOBs explicitly rather than +passing them to SQLite JSON functions. Exporting to NetCDF with a compatible +QCoDeS version retains the snapshot as JSON for interchange with older software. .. _sec:design_split_storage: diff --git a/src/qcodes/configuration/qcodesrc.json b/src/qcodes/configuration/qcodesrc.json index 97c3a835b5fb..e08d2091caff 100644 --- a/src/qcodes/configuration/qcodesrc.json +++ b/src/qcodes/configuration/qcodesrc.json @@ -65,7 +65,7 @@ "sample": 0 }, "dataset": { - "snapshot_compression": "zstd", + "snapshot_compression": "none", "write_in_background": false, "write_period": 5.0, "use_threads": false, diff --git a/src/qcodes/configuration/qcodesrc_schema.json b/src/qcodes/configuration/qcodesrc_schema.json index 0359cdf25f62..98770378e0d3 100644 --- a/src/qcodes/configuration/qcodesrc_schema.json +++ b/src/qcodes/configuration/qcodesrc_schema.json @@ -306,8 +306,8 @@ "snapshot_compression": { "type": "string", "enum": ["none", "zstd"], - "default": "zstd", - "description": "Snapshot write policy: lossless Zstandard level 1 BLOBs or legacy uncompressed JSON TEXT. Reads support both independently of this setting." + "default": "none", + "description": "Snapshot write policy: uncompressed JSON TEXT or lossless Zstandard level 1 BLOBs. Reads support both independently of this setting. QCoDeS versions without compressed snapshot support cannot read runs written with 'zstd'." }, "write_in_background": { "type": "boolean", diff --git a/src/qcodes/dataset/sqlite/db_upgrades/__init__.py b/src/qcodes/dataset/sqlite/db_upgrades/__init__.py index 990ef5bdb09a..c8e9d9b866ce 100644 --- a/src/qcodes/dataset/sqlite/db_upgrades/__init__.py +++ b/src/qcodes/dataset/sqlite/db_upgrades/__init__.py @@ -404,15 +404,3 @@ def perform_db_upgrade_8_to_9( transaction(connection, _IX_runs_captured_run_id) else: raise RuntimeError(f"found {n_run_tables} runs tables expected 1") - - -@upgrader -def perform_db_upgrade_9_to_10( - conn: AtomicConnection, show_progress_bar: bool = True -) -> None: - """Allow versioned compressed BLOBs alongside TEXT in ``runs.snapshot``. - - SQLite already supports both storage classes in this column. Existing - snapshots are deliberately not rewritten. The version gate prevents older - QCoDeS from treating compressed snapshots as JSON. - """ diff --git a/src/qcodes/dataset/sqlite/queries.py b/src/qcodes/dataset/sqlite/queries.py index 3737a19f22ae..bdcfb69db897 100644 --- a/src/qcodes/dataset/sqlite/queries.py +++ b/src/qcodes/dataset/sqlite/queries.py @@ -34,7 +34,6 @@ atomic_transaction, transaction, ) -from qcodes.dataset.sqlite.db_upgrades.version import get_user_version from qcodes.dataset.sqlite.query_helpers import ( VALUE, VALUES, @@ -1969,12 +1968,10 @@ def update_columns( snapshot = data["snapshot"] if not isinstance(snapshot, str): raise TypeError("Snapshot must be a raw JSON string") - compression = config.dataset.snapshot_compression - if compression == "zstd" and get_user_version(conn) < 10: - raise RuntimeError( - "Compressed snapshots require database schema version 10" - ) - data = {**data, "snapshot": encode_snapshot(snapshot, compression)} + data = { + **data, + "snapshot": encode_snapshot(snapshot, config.dataset.snapshot_compression), + } update_where(conn, table_name, "rowid", row_id, **data) diff --git a/tests/dataset/test_database_creation_and_upgrading.py b/tests/dataset/test_database_creation_and_upgrading.py index fa4ca000c811..0c85107e5a4a 100644 --- a/tests/dataset/test_database_creation_and_upgrading.py +++ b/tests/dataset/test_database_creation_and_upgrading.py @@ -1009,7 +1009,7 @@ def test_cannot_connect_to_newer_db() -> None: def test_latest_available_version() -> None: - assert _latest_available_version() == 10 + assert _latest_available_version() == 9 @pytest.mark.parametrize("version", VERSIONS[:-1]) diff --git a/tests/dataset/test_snapshot_storage.py b/tests/dataset/test_snapshot_storage.py index dacd8d670990..d4973fe3fc9f 100644 --- a/tests/dataset/test_snapshot_storage.py +++ b/tests/dataset/test_snapshot_storage.py @@ -19,7 +19,7 @@ from qcodes.dataset.measurements import Measurement from qcodes.dataset.sqlite.connection import atomic from qcodes.dataset.sqlite.database import connect -from qcodes.dataset.sqlite.db_upgrades.version import get_user_version, set_user_version +from qcodes.dataset.sqlite.db_upgrades.version import get_user_version from qcodes.dataset.sqlite.queries import ( add_data_to_dynamic_columns, create_run, @@ -98,6 +98,7 @@ def test_write_read_and_reopen(dataset: DataSet, compression: str) -> None: def test_overwrite_null_and_history( dataset: DataSet, caplog: pytest.LogCaptureFixture ) -> None: + qc.config.dataset.snapshot_compression = "zstd" dataset.add_snapshot("null") assert dataset.snapshot is None assert dataset.snapshot_raw == "null" @@ -200,44 +201,38 @@ def test_create_run(experiment: Experiment, raw: str | None) -> None: assert dataset.snapshot_raw == raw -def test_legacy_upgrade_and_mixed_storage(dataset: DataSet) -> None: +def test_default_writes_uncompressed_text(dataset: DataSet) -> None: + raw = '{"default": true}' + dataset.add_snapshot(raw) + assert stored_snapshot(dataset) == raw + + +def test_mixed_storage_without_schema_change(dataset: DataSet) -> None: assert dataset.path_to_db is not None raw = '{"legacy": true}' - qc.config.dataset.snapshot_compression = "none" dataset.add_snapshot(raw) - set_user_version(dataset.conn, 9) - with closing(connect(dataset.path_to_db)) as conn: - assert get_user_version(conn) == 10 - legacy = DataSet(run_id=dataset.run_id, conn=conn) - assert stored_snapshot(legacy) == raw - assert legacy.snapshot == {"legacy": True} - qc.config.dataset.snapshot_compression = "zstd" - compressed = new_data_set("compressed", conn=conn) - compressed.add_snapshot('{"compressed": true}') - assert isinstance(stored_snapshot(compressed), bytes) - assert stored_snapshot(legacy) == raw - assert compressed.snapshot == {"compressed": True} + qc.config.dataset.snapshot_compression = "zstd" + compressed = new_data_set("compressed", conn=dataset.conn) + compressed.add_snapshot('{"compressed": true}') + assert isinstance(stored_snapshot(compressed), bytes) + assert stored_snapshot(dataset) == raw + assert get_user_version(dataset.conn) == 9 + qc.config.dataset.snapshot_compression = "none" with closing(connect(dataset.path_to_db)) as conn: + assert get_user_version(conn) == 9 assert DataSet(run_id=dataset.run_id, conn=conn).snapshot_raw == raw assert DataSet(run_id=compressed.run_id, conn=conn).snapshot == { "compressed": True } -def test_compressed_writes_require_version_10(dataset: DataSet) -> None: - set_user_version(dataset.conn, 9) - with pytest.raises(RuntimeError, match="Rolling back") as exc: - dataset.add_snapshot("{}") - assert "schema version 10" in str(exc.value.__cause__) - assert stored_snapshot(dataset) is None - - class CustomEncoded: def _JSONEncoder(self) -> dict[str, list[int]]: return {"custom": [42]} def test_prepare_preserves_encoder_types(dataset: DataSet) -> None: + qc.config.dataset.snapshot_compression = "zstd" snapshot = { "array": np.array([1.0, 2.0]), "int": np.int64(2), @@ -296,6 +291,7 @@ def test_netcdf_export_import_and_metadata_db( scalar_dataset: DataSet, tmp_path: Path ) -> None: raw = ' {"full": {"values": [1, 2, 3]}}\n' + qc.config.dataset.snapshot_compression = "zstd" scalar_dataset.add_snapshot(raw) assert scalar_dataset.to_xarray_dataset().attrs["snapshot"] == raw scalar_dataset.export("netcdf", path=tmp_path) @@ -323,6 +319,7 @@ def test_netcdf_export_import_and_metadata_db( def test_metadata_snapshot_write(dataset: DataSet) -> None: raw = '{"legacy_import": true}' + qc.config.dataset.snapshot_compression = "zstd" with atomic(dataset.conn): add_data_to_dynamic_columns(dataset.conn, dataset.run_id, {"snapshot": raw}) assert isinstance(stored_snapshot(dataset), bytes) @@ -332,6 +329,7 @@ def test_metadata_snapshot_write(dataset: DataSet) -> None: def test_in_memory_snapshot_overwrites( experiment: Experiment, caplog: pytest.LogCaptureFixture ) -> None: + qc.config.dataset.snapshot_compression = "zstd" dataset = DataSetInMem._create_new_run(name="in-memory") assert dataset.snapshot is None assert dataset._snapshot_raw is None @@ -365,6 +363,7 @@ def test_measurement_writers_and_backends( experiment: Experiment, background: bool, backend: str ) -> None: qc.config.dataset.raw_data_backend = backend + qc.config.dataset.snapshot_compression = "zstd" measurement = Measurement(exp=experiment) measurement.register_custom_parameter("p") with measurement.run(write_in_background=background) as saver: From 3c4bfb886ed091442e393fa69d27c0ee9a393b33 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 16 Nov 2026 10:54:24 +0100 Subject: [PATCH 3/5] Read snapshots by run ID Runs can share a result table name, for example with split raw data storage and a constant format string, so looking up the snapshot by table name raised 'Expected only one row'. Select by run ID, as before, and decode the stored value. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/qcodes/dataset/data_set.py | 9 ++++----- tests/dataset/test_snapshot_storage.py | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/qcodes/dataset/data_set.py b/src/qcodes/dataset/data_set.py index 45f0d802d664..48abcf90897a 100644 --- a/src/qcodes/dataset/data_set.py +++ b/src/qcodes/dataset/data_set.py @@ -84,6 +84,7 @@ insert_many_values, select_one_where, ) +from qcodes.dataset.sqlite.snapshot_storage import decode_snapshot from qcodes.utils import ( NumpyJSONEncoder, QCoDeSDeprecationWarning, @@ -497,11 +498,9 @@ def snapshot(self) -> dict[str, Any] | None: @property def _snapshot_raw(self) -> str | None: """Snapshot of the run as a JSON-formatted string (or None)""" - snapshot_raw = get_data_by_tag_and_table_name( - self.conn, "snapshot", self.table_name - ) - assert isinstance(snapshot_raw, (str, type(None))) - return snapshot_raw + stored = select_one_where(self.conn, "runs", "snapshot", "run_id", self.run_id) + assert isinstance(stored, (str, bytes, type(None))) + return decode_snapshot(stored) @property def snapshot_raw(self) -> str | None: diff --git a/tests/dataset/test_snapshot_storage.py b/tests/dataset/test_snapshot_storage.py index d4973fe3fc9f..b7abb469d3b8 100644 --- a/tests/dataset/test_snapshot_storage.py +++ b/tests/dataset/test_snapshot_storage.py @@ -10,7 +10,7 @@ import uncertainties # type: ignore[import-untyped] import qcodes as qc -from qcodes.dataset import load_by_id, new_data_set +from qcodes.dataset import load_by_id, new_data_set, new_experiment from qcodes.dataset.data_set import DataSet from qcodes.dataset.data_set_in_memory import DataSetInMem, load_from_netcdf from qcodes.dataset.database_extract_runs import extract_runs_into_db @@ -201,6 +201,25 @@ def test_create_run(experiment: Experiment, raw: str | None) -> None: assert dataset.snapshot_raw == raw +@pytest.mark.parametrize("compression", ["none", "zstd"]) +@pytest.mark.usefixtures("empty_temp_db") +def test_snapshot_read_by_run_id_with_shared_table_name(compression: str) -> None: + qc.config.dataset.raw_data_backend = "sqlite_per_dataset_db" + qc.config.dataset.snapshot_compression = compression + new_experiment("shared", "sample", format_string="results") + first = new_data_set("first") + second = new_data_set("second") + assert first.table_name == second.table_name + first.add_snapshot('{"run": 1}') + second.add_snapshot('{"run": 2}') + assert first.snapshot == {"run": 1} + assert second.snapshot == {"run": 2} + assert first.path_to_db is not None + with closing(connect(first.path_to_db, read_only=True)) as conn: + loaded = load_by_id(second.run_id, conn=conn) + assert loaded.snapshot == {"run": 2} + + def test_default_writes_uncompressed_text(dataset: DataSet) -> None: raw = '{"default": true}' dataset.add_snapshot(raw) From 635c70341f2dc52bcc2acd545789b841b21b4063 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 16 Nov 2026 11:41:04 +0100 Subject: [PATCH 4/5] Decode snapshots requested from get_db_overview get_db_overview copies extra_columns straight from the runs table, so requesting the snapshot column returned the compressed BLOB for runs written with snapshot_compression set to zstd. Decode it like every other snapshot read. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/qcodes/dataset/sqlite/db_overview.py | 3 +++ tests/dataset/test_db_overview.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/qcodes/dataset/sqlite/db_overview.py b/src/qcodes/dataset/sqlite/db_overview.py index 088bdae9ed3f..4c5ab57e7aba 100644 --- a/src/qcodes/dataset/sqlite/db_overview.py +++ b/src/qcodes/dataset/sqlite/db_overview.py @@ -28,6 +28,7 @@ conn_from_dbpath_or_conn, ) from qcodes.dataset.sqlite.query_helpers import is_column_in_table +from qcodes.dataset.sqlite.snapshot_storage import decode_snapshot if TYPE_CHECKING: from collections.abc import Sequence @@ -283,6 +284,8 @@ def get_db_overview( extra = { col: row[n_fixed + i] for i, col in enumerate(valid_extra_columns) } + if "snapshot" in extra: + extra["snapshot"] = decode_snapshot(extra["snapshot"]) # The keys of ``extra`` are only known at runtime (they are the # user-supplied ``extra_columns``), so they cannot be part of # the closed ``RunOverviewDict`` definition. diff --git a/tests/dataset/test_db_overview.py b/tests/dataset/test_db_overview.py index 5e4fdaf63bf2..c6f358c6e8bd 100644 --- a/tests/dataset/test_db_overview.py +++ b/tests/dataset/test_db_overview.py @@ -7,6 +7,7 @@ import pytest +import qcodes as qc from qcodes.dataset import ( get_db_overview, load_or_create_experiment, @@ -155,6 +156,24 @@ def test_get_db_overview_extra_columns(db_conn: AtomicConnection) -> None: assert "does_not_exist" not in overview[1] +def test_get_db_overview_decodes_snapshot_extra_column( + db_conn: AtomicConnection, +) -> None: + exp = load_or_create_experiment("exp", sample_name="sample", conn=db_conn) + plain = new_data_set("plain", exp_id=exp.exp_id, conn=db_conn) + plain.add_snapshot('{"plain": true}') + qc.config.dataset.snapshot_compression = "zstd" + compressed = new_data_set("compressed", exp_id=exp.exp_id, conn=db_conn) + compressed.add_snapshot('{"compressed": true}') + new_data_set("no_snapshot", exp_id=exp.exp_id, conn=db_conn) + + overview = get_db_overview(conn=db_conn, extra_columns=["snapshot"]) + + assert overview[1]["snapshot"] == '{"plain": true}' # type: ignore[typeddict-item] + assert overview[2]["snapshot"] == '{"compressed": true}' # type: ignore[typeddict-item] + assert overview[3]["snapshot"] is None # type: ignore[typeddict-item] + + def test_get_db_overview_accepts_connection(db_conn: AtomicConnection) -> None: _create_runs(db_conn, n_runs=1) From 9582151ffffc0da583bc6a5d5c7c4d243945bc20 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 16 Nov 2026 11:41:06 +0100 Subject: [PATCH 5/5] State which QCoDeS versions read each snapshot format Uncompressed snapshots are readable by all QCoDeS versions, compressed snapshots only by QCoDeS 0.64.0 and later. Say so explicitly instead of referring to versions without compressed snapshot support. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/changes/newsfragments/8724.new | 6 +++--- docs/dataset/dataset_design.rst | 17 +++++++++-------- src/qcodes/configuration/qcodesrc_schema.json | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/changes/newsfragments/8724.new b/docs/changes/newsfragments/8724.new index 8c97d07b8884..4c246984c208 100644 --- a/docs/changes/newsfragments/8724.new +++ b/docs/changes/newsfragments/8724.new @@ -1,6 +1,6 @@ Snapshots can be stored losslessly using versioned, checksummed Zstandard level 1 compression by setting ``dataset.snapshot_compression`` to ``"zstd"``. Compressed and uncompressed snapshots are always readable, and snapshot APIs and -NetCDF exports return the complete JSON. Compression is not enabled by default -because QCoDeS versions without this support cannot read compressed snapshots. -No database upgrade is required. +NetCDF exports return the complete JSON. Uncompressed snapshots, the default, +can be read by all QCoDeS versions; compressed snapshots can only be read by +QCoDeS 0.64.0 and later. No database upgrade is required. diff --git a/docs/dataset/dataset_design.rst b/docs/dataset/dataset_design.rst index a13fb9f4eecb..38e91fddf471 100644 --- a/docs/dataset/dataset_design.rst +++ b/docs/dataset/dataset_design.rst @@ -87,8 +87,9 @@ SQL NULL continues to mean no snapshot. The write policy is controlled by:: qcodes.config.dataset.snapshot_compression = "zstd" -The default, ``"none"``, writes uncompressed JSON TEXT as previous QCoDeS -versions do. ``"zstd"`` writes lossless Zstandard level 1 BLOBs. The policy +The default, ``"none"``, writes uncompressed JSON TEXT, which all QCoDeS +versions can read. ``"zstd"`` writes lossless Zstandard level 1 BLOBs, which +can only be read by QCoDeS 0.64.0 and later. The policy applies to each new write, including overwrites, database copies and imported metadata; it never rewrites existing snapshots, so one database may contain both representations. No database schema change or upgrade is involved; @@ -113,16 +114,16 @@ background result writers do not change the snapshot format or startup order. CPython 3.14 and newer use ``compression.zstd``; earlier supported Python versions use the declared ``backports.zstd`` runtime dependency. -Compressed snapshots are opt-in because QCoDeS versions without this support -cannot decode them. Such versions still open the database, and runs with -uncompressed snapshots are unaffected. For runs written with ``"zstd"``, older -versions can still read the measured data and other metadata, but accessing +Compressed snapshots are opt-in because QCoDeS versions before 0.64.0 cannot +decode them. Those versions still open the database, and runs with +uncompressed snapshots are unaffected. For runs written with ``"zstd"``, they +can still read the measured data and other metadata, but accessing the snapshot, exporting to xarray or NetCDF and extracting the run into another database fail; ``get_metadata("snapshot")`` returns the raw bytes; and runs without a results table, such as those written by ``DataSetInMem``, cannot be loaded at all. Direct SQL consumers must handle BLOBs explicitly rather than -passing them to SQLite JSON functions. Exporting to NetCDF with a compatible -QCoDeS version retains the snapshot as JSON for interchange with older software. +passing them to SQLite JSON functions. Exporting to NetCDF with QCoDeS 0.64.0 or +later retains the snapshot as JSON for interchange with earlier versions. .. _sec:design_split_storage: diff --git a/src/qcodes/configuration/qcodesrc_schema.json b/src/qcodes/configuration/qcodesrc_schema.json index 98770378e0d3..6ab354bcfcfa 100644 --- a/src/qcodes/configuration/qcodesrc_schema.json +++ b/src/qcodes/configuration/qcodesrc_schema.json @@ -307,7 +307,7 @@ "type": "string", "enum": ["none", "zstd"], "default": "none", - "description": "Snapshot write policy: uncompressed JSON TEXT or lossless Zstandard level 1 BLOBs. Reads support both independently of this setting. QCoDeS versions without compressed snapshot support cannot read runs written with 'zstd'." + "description": "Snapshot write policy: uncompressed JSON TEXT, readable by all QCoDeS versions, or lossless Zstandard level 1 BLOBs, readable by QCoDeS 0.64.0 and later. Reads support both independently of this setting." }, "write_in_background": { "type": "boolean",