From 039b36fb9ceefbe6c75fbb9fafae8d0edc02fb42 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 10:57:43 +0000 Subject: [PATCH 01/18] Record frozen editions as append-only files under vortex/editions A frozen edition carries a read-forever guarantee, so its encoding set must never change again. Enforcement so far was a single inline assertion pinning core2026.07.0 -- not the edition the default writer targets -- leaving three of five frozen core editions unpinned, and living in the same mutable tree as the declarations it guarded. Generate one TOML record per frozen edition, holding the identifier, the recorded min_vortex_version, and the full computed encoding set. A test keeps the records in step with EDITION_DECLARATIONS and rejects a record with no frozen edition behind it, so deleting or unfreezing a declaration fails too. Update mode never removes a file, so unfreezing cannot be laundered through the generator. Two CI checks close the loop. The generated-files job regenerates the records and fails if git is dirty, alongside the existing flatbuffers and proto generation. A new job rejects any diff that modifies, deletes, or renames a record, and any newly added edition that is not newer than its family's newest recorded edition. required_vortex_release is deliberately left out of the records: it is backfilled from compat-fixture evidence after an edition freezes, so pinning it would put that backfill in conflict with the append-only rule. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/scripts/check_frozen_editions.py | 154 +++++++++++++++++ .github/workflows/ci.yml | 19 +++ vortex/editions/core2025.05.0.toml | 64 +++++++ vortex/editions/core2025.06.0.toml | 47 ++++++ vortex/editions/core2025.10.0.toml | 52 ++++++ vortex/editions/core2026.07.0.toml | 50 ++++++ vortex/editions/core2026.08.0.toml | 51 ++++++ vortex/src/editions/frozen.rs | 202 +++++++++++++++++++++++ vortex/src/editions/mod.rs | 2 + vortex/src/editions/tests.rs | 49 ------ 10 files changed, 641 insertions(+), 49 deletions(-) create mode 100644 .github/scripts/check_frozen_editions.py create mode 100644 vortex/editions/core2025.05.0.toml create mode 100644 vortex/editions/core2025.06.0.toml create mode 100644 vortex/editions/core2025.10.0.toml create mode 100644 vortex/editions/core2026.07.0.toml create mode 100644 vortex/editions/core2026.08.0.toml create mode 100644 vortex/src/editions/frozen.rs diff --git a/.github/scripts/check_frozen_editions.py b/.github/scripts/check_frozen_editions.py new file mode 100644 index 00000000000..57f4b4944a0 --- /dev/null +++ b/.github/scripts/check_frozen_editions.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Check that the frozen edition records under `vortex/editions` are append-only. + +A frozen edition carries a read-forever guarantee, so its record may never change: the only +legal edit to the directory is adding a file for a newly frozen edition, and that edition must +be newer than every edition already recorded for its family. + +Usage: + python3 check_frozen_editions.py --base origin/develop +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +RECORD_DIR = "vortex/editions" + +# `core2026.08.0.toml`: the file name is the edition id, so the record's identity is visible +# in the diff without reading the file. +RECORD_NAME = re.compile( + r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" +) + +EDITION_FIELD = re.compile(r'^edition = "(?P[^"]+)"$', re.MULTILINE) + +REMEDY = ( + "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" + " vortex/src/editions// and regenerate the records with\n" + " `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`." +) + + +def git(*args: str) -> str: + result = subprocess.run(["git", *args], capture_output=True, text=True, check=False) + if result.returncode != 0: + sys.exit(f"git {' '.join(args)} failed:\n{result.stderr.strip()}") + return result.stdout + + +def merge_base(base: str) -> str: + result = subprocess.run( + ["git", "merge-base", base, "HEAD"], capture_output=True, text=True, check=False + ) + if result.returncode != 0: + sys.exit( + f"cannot find a merge base between {base} and HEAD:\n" + f"{result.stderr.strip()}\n" + "The checkout is probably too shallow; this check needs `fetch-depth: 0`." + ) + return result.stdout.strip() + + +def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: + """Split a record file name into its family and its chronological sort key.""" + match = RECORD_NAME.match(name) + if match is None: + sys.exit( + f"{RECORD_DIR}/{name} is not a valid record name.\n" + "Records are named after the edition they record, e.g. `core2026.08.0.toml`." + ) + return match["family"], (int(match["year"]), int(match["month"]), int(match["version"])) + + +def changed_records(base: str) -> list[tuple[str, list[str]]]: + """The status and paths of every change to the record directory since `base`.""" + raw = git("diff", "--name-status", "-z", base, "HEAD", "--", RECORD_DIR) + fields = [field for field in raw.split("\0") if field] + changes: list[tuple[str, list[str]]] = [] + index = 0 + while index < len(fields): + status = fields[index] + # Renames and copies carry both the old and the new path. + count = 2 if status[0] in ("R", "C") else 1 + changes.append((status, fields[index + 1 : index + 1 + count])) + index += 1 + count + return changes + + +def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: + """The newest edition already recorded for each family at `base`.""" + newest: dict[str, tuple[int, int, int]] = {} + listing = git("ls-tree", "-r", "--name-only", base, "--", RECORD_DIR) + for path in listing.splitlines(): + family, key = parse_name(Path(path).name) + newest[family] = max(key, newest.get(family, (0, 0, 0))) + return newest + + +def check(base: str) -> list[str]: + errors: list[str] = [] + added: list[str] = [] + + for status, paths in changed_records(base): + if status == "A": + added.extend(paths) + continue + verb = {"M": "modifies", "D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} + errors.append(f"{verb.get(status[0], 'changes')} the frozen record {' -> '.join(paths)}") + + newest = recorded_at(base) + for path in sorted(added): + name = Path(path).name + family, key = parse_name(name) + + previous = newest.get(family) + if previous is not None and key <= previous: + recorded = f"{previous[0]}.{previous[1]:02}.{previous[2]}" + errors.append( + f"adds {name}, which is not newer than the {family} edition already " + f"recorded ({family}{recorded}). Editions may only be added going forward." + ) + + # The file name is the edition's identity, so it has to agree with the content. + text = Path(path).read_text() + match = EDITION_FIELD.search(text) + if match is None: + errors.append(f"adds {name}, which has no `edition` field") + elif match["edition"] != name.removesuffix(".toml"): + errors.append( + f"adds {name}, which records edition {match['edition']!r}; the file name " + "must be the edition id" + ) + + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--base", + default="origin/develop", + help="the revision to compare against (default: origin/develop)", + ) + args = parser.parse_args() + + base = merge_base(args.base) + errors = check(base) + if not errors: + print(f"{RECORD_DIR} is append-only against {args.base} ({base[:12]}).") + return 0 + + print(f"This change breaks the frozen edition records in {RECORD_DIR}:\n", file=sys.stderr) + for error in errors: + print(f" - it {error}", file=sys.stderr) + print(f"\n{REMEDY}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b364f0e8aca..4a880caced4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,20 @@ jobs: -c .yamllint.yaml \ .github/ + frozen-editions: + name: "Frozen editions are append-only" + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # The check compares against the merge base, so it needs real history. + fetch-depth: 0 + - name: Check frozen edition records + run: | + BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" + python3 .github/scripts/check_frozen_editions.py --base "$BASE" + python-lint: name: "Python (lint)" runs-on: >- @@ -741,6 +755,11 @@ jobs: - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi + - name: "regenerate the frozen edition records" + env: + UPDATE_FROZEN_EDITIONS: "1" + run: | + cargo test --profile ci -p vortex --lib editions::frozen - name: "Make sure no files changed after regenerating" run: | git status --porcelain diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core2025.05.0.toml new file mode 100644 index 00000000000..0faeeb2333a --- /dev/null +++ b/vortex/editions/core2025.05.0.toml @@ -0,0 +1,64 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2025.05.0" +family = "core" +min_vortex_version = "0.36.0" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", +] diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core2025.06.0.toml new file mode 100644 index 00000000000..2a584ba87e6 --- /dev/null +++ b/vortex/editions/core2025.06.0.toml @@ -0,0 +1,47 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2025.06.0" +family = "core" +min_vortex_version = "0.40.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.pco", + "vortex.sequence", + "vortex.zstd", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core2025.10.0.toml new file mode 100644 index 00000000000..08c0688624c --- /dev/null +++ b/vortex/editions/core2025.10.0.toml @@ -0,0 +1,52 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2025.10.0" +family = "core" +min_vortex_version = "0.54.0" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.rle", + "vortex.fixed_size_list", + "vortex.listview", + "vortex.masked", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core2026.07.0.toml new file mode 100644 index 00000000000..53e3f4c09ae --- /dev/null +++ b/vortex/editions/core2026.07.0.toml @@ -0,0 +1,50 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2026.07.0" +family = "core" +min_vortex_version = "0.65.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.variant", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core2026.08.0.toml new file mode 100644 index 00000000000..c1920b095cf --- /dev/null +++ b/vortex/editions/core2026.08.0.toml @@ -0,0 +1,51 @@ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI. + +edition = "core2026.08.0" +family = "core" +min_vortex_version = "0.84.0" + +# The encodings that join the family at this edition. +added = [ + "vortex.map", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +] diff --git a/vortex/src/editions/frozen.rs b/vortex/src/editions/frozen.rs new file mode 100644 index 00000000000..f538e777d6c --- /dev/null +++ b/vortex/src/editions/frozen.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The frozen-edition record under `vortex/editions`. +//! +//! A frozen edition carries a read-forever guarantee, so its encoding set must never change +//! again. Every frozen edition has one generated TOML file recording that contract: the +//! identifier, the minimum Vortex version whose reader supports it, and the full encoding +//! set. Freezing a new edition adds a file; nothing else may touch the directory, which CI +//! enforces by rejecting any diff that modifies or deletes an existing record +//! (`.github/scripts/check_frozen_editions.py`). +//! +//! The test here keeps the record honest in the other direction: the files must match what +//! [`super::EDITION_DECLARATIONS`] actually computes, so a frozen edition cannot drift +//! without the record drifting with it. Regenerate after freezing a new edition with: +//! +//! ```bash +//! UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen +//! ``` +//! +//! Only facts that are frozen by freezing are recorded. In particular +//! [`vortex_edition::EditionInclusion::required_vortex_release`] is not: it is backfilled +//! from compat-fixture evidence long after an edition freezes, and pinning it here would put +//! that legitimate backfill in conflict with the append-only rule. + +use std::collections::BTreeSet; +use std::env; +use std::fs; +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::anyhow; +use vortex_edition::Edition; +use vortex_edition::EditionError; +use vortex_edition::EditionSession; + +use super::EDITION_DECLARATIONS; + +/// Set to any value to rewrite the record instead of verifying it. +const UPDATE_VAR: &str = "UPDATE_FROZEN_EDITIONS"; + +const REGENERATE: &str = "UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen"; + +const HEADER: &str = "\ +# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting an existing one is rejected by CI."; + +fn record_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("editions") +} + +fn session() -> Result { + let session = EditionSession::empty(); + for declaration in EDITION_DECLARATIONS { + session.declare(declaration)?; + } + Ok(session) +} + +/// The frozen editions, paired with the version that freezing recorded. Drafts have no +/// record: a file appears in the directory at the moment an edition freezes. +fn frozen(session: &EditionSession) -> Vec<(Edition, &'static str)> { + session + .editions() + .into_iter() + .filter_map(|edition| edition.min_vortex_version.map(|version| (edition, version))) + .collect() +} + +/// Render one edition's record. Deterministic: both encoding lists are sorted by id, so the +/// generated bytes depend only on the declarations. +fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) -> String { + let inclusions = session.encodings_in(&edition.id); + let members: BTreeSet<&str> = inclusions + .iter() + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + let added: BTreeSet<&str> = inclusions + .iter() + .filter(|inclusion| inclusion.since == edition.id) + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + + let list = |ids: &BTreeSet<&str>| -> Vec { + ids.iter().map(|id| format!(" \"{id}\",")).collect() + }; + + let mut lines = vec![ + HEADER.to_string(), + String::new(), + format!("edition = \"{}\"", edition.id), + format!("family = \"{}\"", edition.id.family), + format!("min_vortex_version = \"{min_vortex_version}\""), + String::new(), + "# The encodings that join the family at this edition.".to_string(), + "added = [".to_string(), + ]; + lines.extend(list(&added)); + lines.extend([ + "]".to_string(), + String::new(), + "# The edition's full membership: the encodings above, plus every member of earlier" + .to_string(), + "# editions of the family.".to_string(), + "encodings = [".to_string(), + ]); + lines.extend(list(&members)); + lines.extend(["]".to_string(), String::new()]); + lines.join("\n") +} + +fn record_path(dir: &std::path::Path, edition: &Edition) -> PathBuf { + dir.join(format!("{}.toml", edition.id)) +} + +/// The `*.toml` file names present in the record directory. +fn existing_records(dir: &std::path::Path) -> anyhow::Result> { + let mut names = BTreeSet::new(); + for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "toml") + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + names.insert(name.to_string()); + } + } + Ok(names) +} + +/// Every frozen edition has a record, every record matches the declarations exactly, and no +/// record exists without a frozen edition behind it. +/// +/// The third check is what catches a frozen edition being deleted or unfrozen, so the update +/// mode deliberately never removes a file: unfreezing cannot be laundered through the +/// generator. +#[test] +fn records_match_the_frozen_editions() -> anyhow::Result<()> { + let session = session()?; + let dir = record_dir(); + let update = env::var_os(UPDATE_VAR).is_some(); + + fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; + + let mut expected_names = BTreeSet::new(); + for (edition, min_vortex_version) in frozen(&session) { + let path = record_path(&dir, &edition); + let expected = record(&session, &edition, min_vortex_version); + expected_names.insert(format!("{}.toml", edition.id)); + + let actual = match fs::read_to_string(&path) { + Ok(actual) => Some(actual), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())), + }; + + if actual.as_deref() == Some(expected.as_str()) { + continue; + } + if update { + fs::write(&path, &expected).with_context(|| format!("writing {}", path.display()))?; + continue; + } + + return Err(match actual { + Some(_) => anyhow!( + "the record of frozen edition {} no longer matches its declaration.\n\ + A frozen edition's encodings are fixed forever: declare a new edition \ + instead of changing this one.\n\ + If the edition is genuinely new, regenerate with `{REGENERATE}`.\n\ + Record: {}", + edition.id, + path.display(), + ), + None => anyhow!( + "frozen edition {} has no record. Regenerate with `{REGENERATE}`.\n\ + Expected: {}", + edition.id, + path.display(), + ), + }); + } + + let strays: Vec = existing_records(&dir)? + .difference(&expected_names) + .cloned() + .collect(); + if !strays.is_empty() { + return Err(anyhow!( + "{} has records with no frozen edition behind them: {strays:?}.\n\ + A frozen edition may never be deleted or returned to draft; its declaration must \ + stay in `EDITION_DECLARATIONS` with its `min_vortex_version` recorded.", + dir.display(), + )); + } + + Ok(()) +} diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index a6dd8ee7fe9..98c82019a28 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -20,6 +20,8 @@ pub mod core; pub mod preview; #[cfg(test)] +mod frozen; +#[cfg(test)] mod tests; pub use vortex_edition::ComponentKind; diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index 4ba621baf7a..810e93b0870 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -70,55 +70,6 @@ fn every_declared_edition_validates() -> Result<(), EditionError> { Ok(()) } -/// The full encoding set of the newest frozen `core` edition. This set is frozen: the only -/// way it may change is by declaring a *new* edition, so a failure here means a frozen -/// declaration was edited. -#[test] -fn core_2026_08_1_encoding_set_is_pinned() { - let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); - let encodings = session.components_in(&CORE_2026_08_1, ComponentKind::Array); - let ids: Vec<&str> = encodings - .iter() - .map(|inclusion| inclusion.component_id.as_str()) - .collect(); - assert_eq!( - ids, - [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.masked", - "vortex.null", - "vortex.onpair", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", - "vortex.zstd", - ] - ); -} - #[test] fn core_2026_08_1_dtype_set_is_pinned() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); From c8b2d5b036716bb24743e9893ae8f24b56207e4d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 11:20:12 +0000 Subject: [PATCH 02/18] Record required_vortex_release in the frozen edition records Record it in a table of its own rather than beside each encoding. It is the one fact in a record that is not fixed at freeze time -- it is backfilled from compat-fixture evidence as that evidence appears -- so giving it its own table makes a backfill purely an added line. The append-only check parses both revisions of a modified record and permits exactly that transition: the table may gain entries, but everything else must be byte-identical and a release that was already recorded may never change or be dropped. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/scripts/check_frozen_editions.py | 71 ++++++++++++++++++++---- vortex/editions/core2025.05.0.toml | 5 ++ vortex/editions/core2025.06.0.toml | 5 ++ vortex/editions/core2025.10.0.toml | 5 ++ vortex/editions/core2026.07.0.toml | 5 ++ vortex/editions/core2026.08.0.toml | 5 ++ vortex/src/editions/frozen.rs | 41 +++++++++++--- 7 files changed, 120 insertions(+), 17 deletions(-) diff --git a/.github/scripts/check_frozen_editions.py b/.github/scripts/check_frozen_editions.py index 57f4b4944a0..4a14124fa9f 100644 --- a/.github/scripts/check_frozen_editions.py +++ b/.github/scripts/check_frozen_editions.py @@ -5,6 +5,10 @@ legal edit to the directory is adding a file for a newly frozen edition, and that edition must be newer than every edition already recorded for its family. +The one exception is `required_vortex_release`, which is backfilled from compat-fixture +evidence after an edition freezes. An existing record may gain entries in that table, but +never change or lose one, and never change anything else. + Usage: python3 check_frozen_editions.py --base origin/develop """ @@ -15,7 +19,9 @@ import re import subprocess import sys +import tomllib from pathlib import Path +from typing import Any RECORD_DIR = "vortex/editions" @@ -25,12 +31,14 @@ r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" ) -EDITION_FIELD = re.compile(r'^edition = "(?P[^"]+)"$', re.MULTILINE) +# The table a frozen record may gain entries in. Everything else is fixed at freeze time. +BACKFILL_TABLE = "required_vortex_release" REMEDY = ( "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" " vortex/src/editions// and regenerate the records with\n" - " `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`." + " `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`.\n" + f"Only `{BACKFILL_TABLE}` may gain entries in a record that already exists." ) @@ -90,6 +98,46 @@ def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: return newest +def parse_record(text: str, path: str) -> dict[str, Any]: + try: + return tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + sys.exit(f"{path} is not valid TOML: {error}") + + +def check_modification(base: str, path: str) -> list[str]: + """A modified record is legal only when it purely gains backfill entries.""" + name = Path(path).name + before = parse_record(git("show", f"{base}:{path}"), f"{path} at {base[:12]}") + after = parse_record(Path(path).read_text(), path) + + frozen_before = {key: value for key, value in before.items() if key != BACKFILL_TABLE} + frozen_after = {key: value for key, value in after.items() if key != BACKFILL_TABLE} + if frozen_before != frozen_after: + changed = sorted( + key + for key in frozen_before.keys() | frozen_after.keys() + if frozen_before.get(key) != frozen_after.get(key) + ) + return [f"modifies the frozen record {name}: {', '.join(changed)}"] + + releases_before = before.get(BACKFILL_TABLE, {}) + releases_after = after.get(BACKFILL_TABLE, {}) + errors = [] + for encoding, release in sorted(releases_before.items()): + if encoding not in releases_after: + errors.append( + f"drops the recorded {BACKFILL_TABLE} of {encoding} from {name}", + ) + elif releases_after[encoding] != release: + errors.append( + f"changes the {BACKFILL_TABLE} of {encoding} in {name} from {release!r} " + f"to {releases_after[encoding]!r}; a recorded release is evidence and never " + "changes" + ) + return errors + + def check(base: str) -> list[str]: errors: list[str] = [] added: list[str] = [] @@ -97,9 +145,13 @@ def check(base: str) -> list[str]: for status, paths in changed_records(base): if status == "A": added.extend(paths) - continue - verb = {"M": "modifies", "D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} - errors.append(f"{verb.get(status[0], 'changes')} the frozen record {' -> '.join(paths)}") + elif status == "M": + errors.extend(check_modification(base, paths[0])) + else: + verb = {"D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} + errors.append( + f"{verb.get(status[0], 'changes')} the frozen record {' -> '.join(paths)}" + ) newest = recorded_at(base) for path in sorted(added): @@ -115,13 +167,12 @@ def check(base: str) -> list[str]: ) # The file name is the edition's identity, so it has to agree with the content. - text = Path(path).read_text() - match = EDITION_FIELD.search(text) - if match is None: + edition = parse_record(Path(path).read_text(), path).get("edition") + if edition is None: errors.append(f"adds {name}, which has no `edition` field") - elif match["edition"] != name.removesuffix(".toml"): + elif edition != name.removesuffix(".toml"): errors.append( - f"adds {name}, which records edition {match['edition']!r}; the file name " + f"adds {name}, which records edition {edition!r}; the file name " "must be the edition id" ) diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core2025.05.0.toml index 0faeeb2333a..5fe30870de5 100644 --- a/vortex/editions/core2025.05.0.toml +++ b/vortex/editions/core2025.05.0.toml @@ -62,3 +62,8 @@ encodings = [ "vortex.varbinview", "vortex.zigzag", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core2025.06.0.toml index 2a584ba87e6..50cc2fda9ee 100644 --- a/vortex/editions/core2025.06.0.toml +++ b/vortex/editions/core2025.06.0.toml @@ -45,3 +45,8 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core2025.10.0.toml index 08c0688624c..94318862cfc 100644 --- a/vortex/editions/core2025.10.0.toml +++ b/vortex/editions/core2025.10.0.toml @@ -50,3 +50,8 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core2026.07.0.toml index 53e3f4c09ae..6e73f0167e4 100644 --- a/vortex/editions/core2026.07.0.toml +++ b/vortex/editions/core2026.07.0.toml @@ -48,3 +48,8 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core2026.08.0.toml index c1920b095cf..a8708950c8d 100644 --- a/vortex/editions/core2026.08.0.toml +++ b/vortex/editions/core2026.08.0.toml @@ -49,3 +49,8 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] + +# The earliest Vortex release able to read each encoding, recorded from evidence as +# that evidence appears. Unlike the rest of this file it is filled in after the +# edition freezes, so entries are only ever added, never changed. +[required_vortex_release] diff --git a/vortex/src/editions/frozen.rs b/vortex/src/editions/frozen.rs index f538e777d6c..4972eab0121 100644 --- a/vortex/src/editions/frozen.rs +++ b/vortex/src/editions/frozen.rs @@ -18,11 +18,14 @@ //! UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen //! ``` //! -//! Only facts that are frozen by freezing are recorded. In particular -//! [`vortex_edition::EditionInclusion::required_vortex_release`] is not: it is backfilled -//! from compat-fixture evidence long after an edition freezes, and pinning it here would put -//! that legitimate backfill in conflict with the append-only rule. - +//! [`vortex_edition::EditionInclusion::required_vortex_release`] is recorded in its own +//! table rather than beside each encoding, because it is the one fact here that is not fixed +//! at freeze time: it is backfilled from compat-fixture evidence as that evidence appears. +//! Keeping it in a table of its own makes a backfill purely an added line, so the record +//! stays append-only in the literal sense and CI can allow the fill-in while still rejecting +//! a change to a release that was already recorded. + +use std::collections::BTreeMap; use std::collections::BTreeSet; use std::env; use std::fs; @@ -83,6 +86,14 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) .filter(|inclusion| inclusion.since == edition.id) .map(|inclusion| inclusion.encoding_id.as_str()) .collect(); + let releases: BTreeMap<&str, &str> = inclusions + .iter() + .filter_map(|inclusion| { + inclusion + .required_vortex_release + .map(|release| (inclusion.encoding_id.as_str(), release)) + }) + .collect(); let list = |ids: &BTreeSet<&str>| -> Vec { ids.iter().map(|id| format!(" \"{id}\",")).collect() @@ -108,7 +119,22 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) "encodings = [".to_string(), ]); lines.extend(list(&members)); - lines.extend(["]".to_string(), String::new()]); + lines.extend([ + "]".to_string(), + String::new(), + "# The earliest Vortex release able to read each encoding, recorded from evidence as" + .to_string(), + "# that evidence appears. Unlike the rest of this file it is filled in after the" + .to_string(), + "# edition freezes, so entries are only ever added, never changed.".to_string(), + "[required_vortex_release]".to_string(), + ]); + lines.extend( + releases + .iter() + .map(|(id, release)| format!("\"{id}\" = \"{release}\"")), + ); + lines.push(String::new()); lines.join("\n") } @@ -171,7 +197,8 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { "the record of frozen edition {} no longer matches its declaration.\n\ A frozen edition's encodings are fixed forever: declare a new edition \ instead of changing this one.\n\ - If the edition is genuinely new, regenerate with `{REGENERATE}`.\n\ + If you froze a new edition or recorded a required_vortex_release, \ + regenerate with `{REGENERATE}`.\n\ Record: {}", edition.id, path.display(), From 4fef3cfba8f2af198c076eea4ff02fec156dc364 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 11:30:01 +0000 Subject: [PATCH 03/18] Record every edition, and lock a record only once its edition freezes Editions are drafts until a min_vortex_version is recorded, and a draft is free to change, so recording only frozen editions left the drafts invisible. Generate a record for every declared edition instead. A draft's record may change, move, or go away with the draft; freezing turns the record into a read-forever contract that never changes again. The CI check reads frozen-ness from the record at the base revision, so a diff cannot unfreeze an edition and edit it in the same change, and the generator refuses to unfreeze a record it finds on disk. Encoding ids in a declaration may now be paired with the release that first read them -- `&("vortex.alp", "0.36.0")` -- which flows into EditionInclusion::required_vortex_release and into the records. Each core edition's members are recorded as requiring the release current when that edition froze, checked against the published release timeline: 0.36.0 (2025-05-28), 0.40.0 (2025-06-26), 0.54.0 (2025-10-20), 0.84.0 (2026-08-07), and 0.65.0 (2026-03-25) for core2026.07.0, whose only member vortex.variant first shipped there. These are upper bounds under each edition's own immutable min_vortex_version, so the check permits refining them as compat-fixture evidence narrows them, but never dropping one. Renamed the module and script from `frozen` to `records`, since they now cover drafts too. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- ...n_editions.py => check_edition_records.py} | 112 ++++++---- .github/workflows/ci.yml | 14 +- vortex-edition/src/lib.rs | 210 ++++++------------ vortex-edition/src/tests.rs | 124 +++++------ vortex/editions/core2025.05.0.toml | 33 ++- vortex/editions/core2025.06.0.toml | 36 ++- vortex/editions/core2025.10.0.toml | 40 +++- vortex/editions/core2026.07.0.toml | 41 +++- vortex/editions/core2026.08.0.toml | 42 +++- vortex/editions/unstable2025.05.0.toml | 24 ++ vortex/editions/unstable2026.02.0.toml | 25 +++ vortex/editions/unstable2026.04.0.toml | 36 +++ vortex/editions/unstable2026.06.0.toml | 32 +++ vortex/src/editions/core/v2025_05.rs | 59 +++-- vortex/src/editions/core/v2025_06.rs | 7 +- vortex/src/editions/core/v2025_10.rs | 9 +- vortex/src/editions/core/v2026_07.rs | 20 ++ vortex/src/editions/core/v2026_08.rs | 43 +--- vortex/src/editions/mod.rs | 2 +- vortex/src/editions/{frozen.rs => records.rs} | 148 +++++++----- 20 files changed, 627 insertions(+), 430 deletions(-) rename .github/scripts/{check_frozen_editions.py => check_edition_records.py} (61%) create mode 100644 vortex/editions/unstable2025.05.0.toml create mode 100644 vortex/editions/unstable2026.02.0.toml create mode 100644 vortex/editions/unstable2026.04.0.toml create mode 100644 vortex/editions/unstable2026.06.0.toml create mode 100644 vortex/src/editions/core/v2026_07.rs rename vortex/src/editions/{frozen.rs => records.rs} (51%) diff --git a/.github/scripts/check_frozen_editions.py b/.github/scripts/check_edition_records.py similarity index 61% rename from .github/scripts/check_frozen_editions.py rename to .github/scripts/check_edition_records.py index 4a14124fa9f..074947242fc 100644 --- a/.github/scripts/check_frozen_editions.py +++ b/.github/scripts/check_edition_records.py @@ -1,16 +1,23 @@ #!/usr/bin/env python3 -"""Check that the frozen edition records under `vortex/editions` are append-only. +"""Check that frozen edition records under `vortex/editions` never change. -A frozen edition carries a read-forever guarantee, so its record may never change: the only -legal edit to the directory is adding a file for a newly frozen edition, and that edition must -be newer than every edition already recorded for its family. +A record's mutability follows its edition. A draft is still being assembled, so its record may +change, be renamed, or be dropped. Freezing -- recording a `min_vortex_version` -- turns the +record into a read-forever contract, and from then on it may never change again. Whether a +record was frozen is read from the base revision, so a change cannot unfreeze an edition and +edit it in the same diff. -The one exception is `required_vortex_release`, which is backfilled from compat-fixture -evidence after an edition freezes. An existing record may gain entries in that table, but -never change or lose one, and never change anything else. +A newly added record must also be newer than every edition already recorded for its family: +editions are only ever added going forward. + +The one part of a frozen record that may still move is the `required_vortex_release` table. +It holds an upper bound recorded from the release current when the edition froze, refined as +compat-fixture evidence narrows it; it stays under the edition's own immutable +`min_vortex_version`, so refining it breaks no published guarantee. Entries may be added or +refined, never dropped. Usage: - python3 check_frozen_editions.py --base origin/develop + python3 check_edition_records.py --base origin/develop """ from __future__ import annotations @@ -31,14 +38,17 @@ r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" ) -# The table a frozen record may gain entries in. Everything else is fixed at freeze time. -BACKFILL_TABLE = "required_vortex_release" +# A record carries this exactly when the edition it records is frozen. +FROZEN_MARKER = "min_vortex_version" + +# The table a frozen record may still gain or refine entries in. +EVIDENCE_TABLE = "required_vortex_release" REMEDY = ( "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" " vortex/src/editions// and regenerate the records with\n" - " `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`.\n" - f"Only `{BACKFILL_TABLE}` may gain entries in a record that already exists." + " `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`.\n" + f"In a frozen record only `{EVIDENCE_TABLE}` may gain or refine entries." ) @@ -62,6 +72,17 @@ def merge_base(base: str) -> str: return result.stdout.strip() +def parse_record(text: str, path: str) -> dict[str, Any]: + try: + return tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + sys.exit(f"{path} is not valid TOML: {error}") + + +def record_at(base: str, path: str) -> dict[str, Any]: + return parse_record(git("show", f"{base}:{path}"), f"{path} at {base[:12]}") + + def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: """Split a record file name into its family and its chronological sort key.""" match = RECORD_NAME.match(name) @@ -98,44 +119,33 @@ def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: return newest -def parse_record(text: str, path: str) -> dict[str, Any]: - try: - return tomllib.loads(text) - except tomllib.TOMLDecodeError as error: - sys.exit(f"{path} is not valid TOML: {error}") - - -def check_modification(base: str, path: str) -> list[str]: - """A modified record is legal only when it purely gains backfill entries.""" +def check_modification(before: dict[str, Any], path: str) -> list[str]: + """A frozen record may only gain or refine evidence entries.""" name = Path(path).name - before = parse_record(git("show", f"{base}:{path}"), f"{path} at {base[:12]}") after = parse_record(Path(path).read_text(), path) - frozen_before = {key: value for key, value in before.items() if key != BACKFILL_TABLE} - frozen_after = {key: value for key, value in after.items() if key != BACKFILL_TABLE} - if frozen_before != frozen_after: + fixed_before = {key: value for key, value in before.items() if key != EVIDENCE_TABLE} + fixed_after = {key: value for key, value in after.items() if key != EVIDENCE_TABLE} + if fixed_before != fixed_after: changed = sorted( key - for key in frozen_before.keys() | frozen_after.keys() - if frozen_before.get(key) != frozen_after.get(key) + for key in fixed_before.keys() | fixed_after.keys() + if fixed_before.get(key) != fixed_after.get(key) ) + if FROZEN_MARKER in changed and FROZEN_MARKER not in after: + return [ + f"unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a " + "read-forever guarantee and may never return to draft" + ] return [f"modifies the frozen record {name}: {', '.join(changed)}"] - releases_before = before.get(BACKFILL_TABLE, {}) - releases_after = after.get(BACKFILL_TABLE, {}) - errors = [] - for encoding, release in sorted(releases_before.items()): - if encoding not in releases_after: - errors.append( - f"drops the recorded {BACKFILL_TABLE} of {encoding} from {name}", - ) - elif releases_after[encoding] != release: - errors.append( - f"changes the {BACKFILL_TABLE} of {encoding} in {name} from {release!r} " - f"to {releases_after[encoding]!r}; a recorded release is evidence and never " - "changes" - ) - return errors + dropped = sorted(before.get(EVIDENCE_TABLE, {}).keys() - after.get(EVIDENCE_TABLE, {}).keys()) + if dropped: + return [ + f"drops the recorded {EVIDENCE_TABLE} of {', '.join(dropped)} from {name}; " + "recorded evidence is only ever added or refined" + ] + return [] def check(base: str) -> list[str]: @@ -144,9 +154,17 @@ def check(base: str) -> list[str]: for status, paths in changed_records(base): if status == "A": - added.extend(paths) - elif status == "M": - errors.extend(check_modification(base, paths[0])) + added.append(paths[0]) + continue + + # Frozen-ness comes from the base revision, so a diff cannot unfreeze an edition and + # then edit it. A draft's record is free to change, move, or go away with the draft. + before = record_at(base, paths[0]) + if FROZEN_MARKER not in before: + continue + + if status == "M": + errors.extend(check_modification(before, paths[0])) else: verb = {"D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} errors.append( @@ -191,10 +209,10 @@ def main() -> int: base = merge_base(args.base) errors = check(base) if not errors: - print(f"{RECORD_DIR} is append-only against {args.base} ({base[:12]}).") + print(f"{RECORD_DIR} preserves every frozen record against {args.base} ({base[:12]}).") return 0 - print(f"This change breaks the frozen edition records in {RECORD_DIR}:\n", file=sys.stderr) + print(f"This change breaks the edition records in {RECORD_DIR}:\n", file=sys.stderr) for error in errors: print(f" - it {error}", file=sys.stderr) print(f"\n{REMEDY}", file=sys.stderr) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a880caced4..08728be2ba5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,8 +64,8 @@ jobs: -c .yamllint.yaml \ .github/ - frozen-editions: - name: "Frozen editions are append-only" + edition-records: + name: "Frozen edition records never change" runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -73,10 +73,10 @@ jobs: with: # The check compares against the merge base, so it needs real history. fetch-depth: 0 - - name: Check frozen edition records + - name: Check edition records run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" - python3 .github/scripts/check_frozen_editions.py --base "$BASE" + python3 .github/scripts/check_edition_records.py --base "$BASE" python-lint: name: "Python (lint)" @@ -755,11 +755,11 @@ jobs: - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi - - name: "regenerate the frozen edition records" + - name: "regenerate the edition records" env: - UPDATE_FROZEN_EDITIONS: "1" + UPDATE_EDITION_RECORDS: "1" run: | - cargo test --profile ci -p vortex --lib editions::frozen + cargo test --profile ci -p vortex --lib editions::records - name: "Make sure no files changed after regenerating" run: | git status --porcelain diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index 6e5c609a60a..42082a0ba71 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -1,24 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Definitions of Vortex *editions*: named, frozen sets of components that a writer may put -//! in a file, carrying a forever read-compatibility guarantee. +//! Definitions of Vortex *editions*: named, frozen sets of encodings that a writer may put in +//! a file, carrying a forever read-compatibility guarantee. //! //! Editions live on the session, like encodings do: [`EditionSession`] holds the registered //! editions and [`EnabledEditions`] selects which of them a writer may emit. Declarations //! are plain constants — an [`EditionId`] plus an [`Edition`] record, and one -//! [`EditionInclusion`] per member stating that it is a member of an edition *and every +//! [`EditionInclusion`] per encoding stating that it is a member of an edition *and every //! later edition of the same family*. Any crate can register declarations into a session, -//! so inclusions can live next to the component they describe. -//! -//! Every membership is typed by a [`ComponentKind`], and members are resolved one kind at a -//! time with [`EditionSessionExt::enabled_component_ids`]: the file writer restricts the -//! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets, never one -//! untyped set. +//! so inclusions can live next to the encoding they describe. //! //! An edition is a **draft** until its [`Edition::min_vortex_version`] is recorded — -//! recording it is the act of freezing. The per-edition member sets are computed from the -//! registered declarations by [`EditionSession::components_in`], and correctness is enforced +//! recording it is the act of freezing. The per-edition encoding sets are computed from the +//! registered declarations by [`EditionSession::encodings_in`], and correctness is enforced //! by unit tests: [`EditionSession::validate`] checks a whole registry, and //! [`test_harness::validate_edition`] validates one edition's constraints — call it once in //! the `#[cfg(test)]` module of each edition definition. @@ -45,7 +40,7 @@ use vortex_session::registry::Id; /// The identifier of an edition, e.g. `core2026.07.0`. /// -/// The `family` names an independently versioned, additive group of components (`core` is the +/// The `family` names an independently versioned, additive group of encodings (`core` is the /// set the default writer emits). The date components record when the edition was frozen and /// order editions chronologically *within* a family; there is no ordering across families. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -114,45 +109,15 @@ impl Display for EditionId { } } -/// The kind of component an edition membership covers. -/// -/// Ids are unique per kind, not globally: a layout named `vortex.flat` and an array named -/// `vortex.flat` are different members. Every membership records its kind, and the writer -/// resolves one kind at a time, so the set restricting written arrays never restricts -/// written layouts. Further kinds (scalar functions, say) can be added the same way. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum ComponentKind { - /// An array encoding, e.g. `vortex.alp`, registered in the session's array registry. - Array, - /// A layout encoding, e.g. `vortex.flat`, registered in the session's layout registry. - Layout, - /// An extension dtype, e.g. `vortex.timestamp`, registered in the session's dtype registry. - DType, - /// An aggregate function, e.g. `vortex.min`, written into zone maps and registered in - /// the session's aggregate function registry. - Aggregate, -} - -impl Display for ComponentKind { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.write_str(match self { - Self::Array => "array", - Self::Layout => "layout", - Self::DType => "dtype", - Self::Aggregate => "aggregate", - }) - } -} - -/// An edition: a named set of components with a read-compatibility guarantee, registered with +/// An edition: a named set of encodings with a read-compatibility guarantee, registered with /// [`EditionSession::declare_edition`]. The set itself is computed from the registered -/// [`EditionInclusion`]s by [`EditionSession::components_in`]. +/// [`EditionInclusion`]s by [`EditionSession::encodings_in`]. #[derive(Clone, Copy, Debug)] pub struct Edition { /// The edition identifier. Also carries the freeze date: `core2026.07.0` freezes in /// 2026-07. pub id: EditionId, - /// The minimum Vortex version whose reader supports every member of this edition. + /// The minimum Vortex version whose reader supports every encoding in this edition. /// /// Recording this is the act of freezing: an edition with `None` is a **draft** — being /// assembled, carrying no guarantee, free to change, never the default write target. @@ -168,147 +133,109 @@ impl Edition { } } -/// Declares that a component is a member of an edition — and of every later edition of the +/// Declares that an encoding is a member of an edition — and of every later edition of the /// same family. Registered with [`EditionSession::declare_inclusion`]. #[derive(Clone, Copy, Debug)] pub struct EditionInclusion { - /// What the membership covers. Ids are unique per kind, so this is part of the - /// member's identity, not a label. - pub kind: ComponentKind, - /// The interned component id, e.g. `vortex.alp`. - pub component_id: Id, - /// The first edition this component is a member of. + /// The interned encoding id, e.g. `vortex.alp`. Globally unique across everything an + /// edition can cover: when layout encodings join editions, their ids must be distinct + /// from array encoding ids. + pub encoding_id: Id, + /// The first edition this encoding is a member of. pub since: EditionId, - /// The earliest Vortex release able to read and execute this component, recorded from + /// The earliest Vortex release able to read and execute this encoding, recorded from /// evidence (e.g. compat-fixture history). `None` until recorded. pub required_vortex_release: Option<&'static str>, } -/// A source of a component id for edition declarations. +/// A source of an encoding id for edition declarations. /// /// Implemented for raw id strings (`"vortex.alp"`) and interned [`Id`]s here; encoding /// vtables implement it where they are defined, so a declaration can name the vtable -/// (`&Primitive`) instead of spelling its id. The id alone does not say what kind of -/// component it names — [`EditionMember`] pairs it with a [`ComponentKind`]. -pub trait AsComponentId: Debug + Send + Sync { - /// The interned component id. - fn component_id(&self) -> Id; +/// (`&Primitive`) instead of spelling its id. +/// +/// Pairing an encoding with a release records the evidence that the release can read it: +/// `&("vortex.alp", "0.36.0")` declares the same membership as `&"vortex.alp"` and +/// additionally sets [`EditionInclusion::required_vortex_release`]. +pub trait AsEncodingId: Debug + Send + Sync { + /// The interned encoding id. + fn encoding_id(&self) -> Id; + + /// The earliest Vortex release able to read and execute the encoding, when the evidence + /// has been recorded. Naming an encoding on its own leaves this unrecorded. + fn required_vortex_release(&self) -> Option<&'static str> { + None + } } -impl AsComponentId for str { +impl AsEncodingId for str { #[expect( clippy::disallowed_methods, - reason = "interning a dynamic component id at declaration time" + reason = "interning a dynamic encoding id at declaration time" )] - fn component_id(&self) -> Id { + fn encoding_id(&self) -> Id { Id::new(self) } } -impl AsComponentId for Id { - fn component_id(&self) -> Id { +impl AsEncodingId for Id { + fn encoding_id(&self) -> Id { *self } } -// `str` is unsized and cannot be a trait object, so declaration blocks name components as -// `&"vortex.alp"` through this impl. -impl AsComponentId for &'static str { - fn component_id(&self) -> Id { - (**self).component_id() +// `str` is unsized and cannot be a trait object, so declaration blocks (slices of +// `&dyn AsEncodingId`) name encodings as `&"vortex.alp"` through this impl. +impl AsEncodingId for &'static str { + fn encoding_id(&self) -> Id { + (**self).encoding_id() } } -/// A component that joins an edition, named by id string or vtable and tagged with the kind -/// of registry it belongs to. Built with the per-kind constructors, so a declaration reads -/// as `EditionMember::array(&"vortex.alp")`. -#[derive(Clone, Copy, Debug)] -pub struct EditionMember { - /// What kind of component this is. - pub kind: ComponentKind, - /// The component, named by id string or by vtable. - pub component: &'static dyn AsComponentId, -} - -impl EditionMember { - /// An array encoding member, e.g. `vortex.alp`. - pub const fn array(component: &'static dyn AsComponentId) -> Self { - Self { - kind: ComponentKind::Array, - component, - } - } - - /// A layout member, e.g. `vortex.flat`. - pub const fn layout(component: &'static dyn AsComponentId) -> Self { - Self { - kind: ComponentKind::Layout, - component, - } +// Pairing any encoding name with a release records the evidence alongside the membership: +// `&("vortex.alp", "0.36.0")`, or `&(&Primitive, "0.36.0")` when naming the vtable. +impl AsEncodingId for (&'static E, &'static str) { + fn encoding_id(&self) -> Id { + self.0.encoding_id() } - /// An extension dtype member, e.g. `vortex.timestamp`. - pub const fn dtype(component: &'static dyn AsComponentId) -> Self { - Self { - kind: ComponentKind::DType, - component, - } - } - - /// An aggregate function member, e.g. `vortex.min`. - pub const fn aggregate(component: &'static dyn AsComponentId) -> Self { - Self { - kind: ComponentKind::Aggregate, - component, - } + fn required_vortex_release(&self) -> Option<&'static str> { + Some(self.1) } } -/// Declares an edition together with the components that join the family at it, in one -/// block. Registered with [`EditionSession::declare`], which derives each member's +/// Declares an edition together with the encodings that join the family at it, in one +/// block. Registered with [`EditionSession::declare`], which derives each encoding's /// membership (`since` = the declared edition) from the block structure. #[derive(Clone, Copy, Debug)] pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, - /// The components that join the family at this edition, each tagged with its - /// [`ComponentKind`]. Members of earlier editions are inherited and never restated. - pub added: &'static [EditionMember], + /// The encodings that join the family at this edition, named by id string or by + /// vtable, optionally paired with the release that first read them + /// (`&("vortex.alp", "0.36.0")`). Members of earlier editions are inherited and never + /// restated. + pub added: &'static [&'static dyn AsEncodingId], } impl EditionInclusion { - /// Declare that a component of `kind` is a member of `since` and every later edition of - /// the same family. The component can be named by id string or by vtable. - pub fn new( - kind: ComponentKind, - component: &C, - since: EditionId, - ) -> Self { + /// Declare that an encoding is a member of `since` and every later edition of the same + /// family. The encoding can be named by id string or by vtable, and carries its + /// [`EditionInclusion::required_vortex_release`] when named as a + /// `(encoding, release)` pair. + pub fn new(encoding: &E, since: EditionId) -> Self { Self { - kind, - component_id: component.component_id(), + encoding_id: encoding.encoding_id(), since, - required_vortex_release: None, + required_vortex_release: encoding.required_vortex_release(), } } - /// Declare that an array encoding is a member of `since` and every later edition of the - /// same family. - pub fn array(encoding: &C, since: EditionId) -> Self { - Self::new(ComponentKind::Array, encoding, since) - } - - /// Declare that an extension dtype is a member of `since` and every later edition of the - /// same family. - pub fn dtype(dtype: &C, since: EditionId) -> Self { - Self::new(ComponentKind::DType, dtype, since) - } - - /// Validate the declaration's form: a lowercase `namespace.name` component id and, if + /// Validate the declaration's form: a lowercase `namespace.name` encoding id and, if /// recorded, a well-formed `major.minor.patch` release. Checked for every declared /// inclusion by [`EditionSession::validate`]. pub fn validate(&self) -> Result<(), EditionError> { - let id = self.component_id.as_str(); + let id = self.encoding_id.as_str(); let well_formed = !id.starts_with('.') && !id.ends_with('.') && id.contains('.') @@ -317,16 +244,15 @@ impl EditionInclusion { .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || "._-".contains(c)); if !well_formed { return Err(EditionError::new(format!( - "invalid {} id {id:?}: expected lowercase `namespace.name`, e.g. `vortex.alp`", - self.kind + "invalid encoding id {id:?}: expected lowercase `namespace.name`, e.g. \ + `vortex.alp`" ))); } if let Some(release) = self.required_vortex_release && parse_release(release).is_none() { return Err(EditionError::new(format!( - "{} {id} declares malformed required_vortex_release {release:?}", - self.kind + "encoding {id} declares malformed required_vortex_release {release:?}" ))); } Ok(()) diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 21a45bd2db8..c8b9a2fe5af 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -3,12 +3,10 @@ use vortex_session::VortexSession; -use crate::ComponentKind; use crate::Edition; use crate::EditionDeclaration; use crate::EditionId; use crate::EditionInclusion; -use crate::EditionMember; use crate::EditionSession; use crate::EditionSessionExt; use crate::EnabledEditions; @@ -22,17 +20,14 @@ static DECLARATIONS: &[EditionDeclaration] = &[ id: FIRST, min_vortex_version: None, }, - added: &[ - EditionMember::array(&"test.alpha"), - EditionMember::array(&"test.beta"), - ], + added: &[&"test.alpha", &"test.beta"], }, EditionDeclaration { edition: Edition { id: SECOND, min_vortex_version: None, }, - added: &[EditionMember::array(&"test.gamma")], + added: &[&"test.gamma"], }, ]; @@ -61,19 +56,19 @@ fn editions_pass_the_test_harness() -> Result<(), crate::EditionError> { fn membership_is_transitive() { let editions = session(); - let first = editions.components_in(&FIRST, ComponentKind::Array); - let ids: Vec<&str> = first.iter().map(|i| i.component_id.as_str()).collect(); + let first = editions.encodings_in(&FIRST); + let ids: Vec<&str> = first.iter().map(|i| i.encoding_id.as_str()).collect(); assert_eq!(ids, ["test.alpha", "test.beta"]); // Members of the first edition are members of the second by inheritance, with their // `since` still recording the edition they actually joined in. - let second = editions.components_in(&SECOND, ComponentKind::Array); - let ids: Vec<&str> = second.iter().map(|i| i.component_id.as_str()).collect(); + let second = editions.encodings_in(&SECOND); + let ids: Vec<&str> = second.iter().map(|i| i.encoding_id.as_str()).collect(); assert_eq!(ids, ["test.alpha", "test.beta", "test.gamma"]); assert!( second .iter() - .filter(|i| i.component_id.as_str() != "test.gamma") + .filter(|i| i.encoding_id.as_str() != "test.gamma") .all(|i| i.since == FIRST) ); @@ -81,7 +76,7 @@ fn membership_is_transitive() { let added: Vec<&str> = second .iter() .filter(|i| i.since == SECOND) - .map(|i| i.component_id.as_str()) + .map(|i| i.encoding_id.as_str()) .collect(); assert_eq!(added, ["test.gamma"]); @@ -89,16 +84,9 @@ fn membership_is_transitive() { // never crosses families. assert!(first.iter().all(|i| i.since == FIRST)); let third = EditionId::new("test", 2026, 10, 0); - assert_eq!( - editions.components_in(&third, ComponentKind::Array).len(), - 3 - ); + assert_eq!(editions.encodings_in(&third).len(), 3); let other = EditionId::new("other", 2026, 10, 0); - assert!( - editions - .components_in(&other, ComponentKind::Array) - .is_empty() - ); + assert!(editions.encodings_in(&other).is_empty()); } #[test] @@ -146,16 +134,12 @@ fn registered_and_enabled_editions_are_separate() -> Result<(), crate::EditionEr session.register_edition(declaration)?; } - assert!( - session - .enabled_component_ids(ComponentKind::Array) - .is_empty() - ); + assert!(session.enabled_encoding_ids().is_empty()); session.enable_edition(FIRST)?; assert_eq!(session.enabled_editions().editions(), [FIRST]); assert_eq!( session - .enabled_component_ids(ComponentKind::Array) + .enabled_encoding_ids() .iter() .map(|id| id.as_str()) .collect::>(), @@ -164,13 +148,13 @@ fn registered_and_enabled_editions_are_separate() -> Result<(), crate::EditionEr session.enable_edition(SECOND)?; assert_eq!(session.enabled_editions().editions(), [SECOND]); - assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 3); + assert_eq!(session.enabled_encoding_ids().len(), 3); // Selecting an older edition in the same family replaces the newer one and removes // encodings that joined after it. session.enable_edition(FIRST)?; assert_eq!(session.enabled_editions().editions(), [FIRST]); - let enabled = session.enabled_component_ids(ComponentKind::Array); + let enabled = session.enabled_encoding_ids(); assert_eq!(enabled.len(), 2); assert!(enabled.iter().all(|id| id.as_str() != "test.gamma")); Ok(()) @@ -191,7 +175,7 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi id: OTHER, min_vortex_version: None, }, - added: &[EditionMember::array(&"other.delta")], + added: &[&"other.delta"], }; let session = VortexSession::empty().with::(); @@ -203,7 +187,7 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi let mut enabled = session.enabled_editions().editions(); enabled.sort_unstable(); assert_eq!(enabled, [OTHER, FIRST]); - assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 3); + assert_eq!(session.enabled_encoding_ids().len(), 3); Ok(()) } @@ -220,7 +204,7 @@ fn duplicate_declarations_error() { ); assert!( editions - .declare_inclusion(EditionInclusion::array("test.alpha", FIRST)) + .declare_inclusion(EditionInclusion::new("test.alpha", FIRST)) .is_err() ); } @@ -229,7 +213,7 @@ fn duplicate_declarations_error() { fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionError> { // An inclusion referencing an undeclared edition. let editions = EditionSession::empty(); - editions.declare_inclusion(EditionInclusion::array("test.alpha", FIRST))?; + editions.declare_inclusion(EditionInclusion::new("test.alpha", FIRST))?; assert!(editions.validate().is_err()); // A member requiring a release newer than its edition declares. @@ -240,7 +224,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro })?; editions.declare_inclusion(EditionInclusion { required_vortex_release: Some("0.80.0"), - ..EditionInclusion::array("test.alpha", FIRST) + ..EditionInclusion::new("test.alpha", FIRST) })?; assert!(editions.validate().is_err()); @@ -270,7 +254,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro id: FIRST, min_vortex_version: None, })?; - editions.declare_inclusion(EditionInclusion::array("Test.ALPHA", FIRST))?; + editions.declare_inclusion(EditionInclusion::new("Test.ALPHA", FIRST))?; assert!(editions.validate().is_err()); Ok(()) @@ -292,44 +276,46 @@ fn edition_id_display() { } #[test] -fn kinds_are_resolved_independently() -> Result<(), crate::EditionError> { - // `test.alpha` is declared under both kinds: same id, two distinct members. - static MIXED: EditionDeclaration = EditionDeclaration { +fn declarations_carry_required_releases() -> Result<(), crate::EditionError> { + let editions = EditionSession::empty(); + editions.declare(&EditionDeclaration { edition: Edition { id: FIRST, - min_vortex_version: None, + min_vortex_version: Some("0.40.0"), }, - added: &[ - EditionMember::array(&"test.alpha"), - EditionMember::dtype(&"test.alpha"), - EditionMember::layout(&"test.alpha"), - EditionMember::layout(&"test.flat"), - ], - }; + added: &[&"test.alpha", &("test.beta", "0.36.0")], + })?; + editions.validate()?; - let session = VortexSession::empty().with::(); - session.register_edition(&MIXED)?; - session.enable_edition(FIRST)?; + let inclusions = editions.encodings_in(&FIRST); + let releases: Vec<(&str, Option<&str>)> = inclusions + .iter() + .map(|inclusion| { + ( + inclusion.encoding_id.as_str(), + inclusion.required_vortex_release, + ) + }) + .collect(); + assert_eq!( + releases, + [("test.alpha", None), ("test.beta", Some("0.36.0"))] + ); - let ids = |kind| { - session - .enabled_component_ids(kind) - .iter() - .map(|id| id.to_string()) - .collect::>() - }; - // A layout never reaches the array registry, and what a writer may emit is the arrays. - assert_eq!(ids(ComponentKind::Array), ["test.alpha"]); - assert_eq!(ids(ComponentKind::DType), ["test.alpha"]); - assert_eq!(ids(ComponentKind::Layout), ["test.alpha", "test.flat"]); - assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 1); + Ok(()) +} + +#[test] +fn a_member_may_not_require_a_release_newer_than_its_edition() -> Result<(), crate::EditionError> { + let editions = EditionSession::empty(); + editions.declare(&EditionDeclaration { + edition: Edition { + id: FIRST, + min_vortex_version: Some("0.40.0"), + }, + added: &[&("test.alpha", "0.54.0")], + })?; + assert!(editions.validate().is_err()); - // A duplicate within one kind is still an error. - assert!( - session - .editions() - .declare_inclusion(EditionInclusion::array("test.alpha", FIRST)) - .is_err() - ); Ok(()) } diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core2025.05.0.toml index 5fe30870de5..73a29ebd814 100644 --- a/vortex/editions/core2025.05.0.toml +++ b/vortex/editions/core2025.05.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2025.05.0" family = "core" @@ -63,7 +63,30 @@ encodings = [ "vortex.zigzag", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.null" = "0.36.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.zigzag" = "0.36.0" diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core2025.06.0.toml index 50cc2fda9ee..0e8fbe5c1be 100644 --- a/vortex/editions/core2025.06.0.toml +++ b/vortex/editions/core2025.06.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2025.06.0" family = "core" @@ -46,7 +46,33 @@ encodings = [ "vortex.zstd", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.null" = "0.36.0" +"vortex.pco" = "0.40.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sequence" = "0.40.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.zigzag" = "0.36.0" +"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core2025.10.0.toml index 94318862cfc..6c6e6edebfc 100644 --- a/vortex/editions/core2025.10.0.toml +++ b/vortex/editions/core2025.10.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2025.10.0" family = "core" @@ -51,7 +51,37 @@ encodings = [ "vortex.zstd", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"fastlanes.rle" = "0.54.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fixed_size_list" = "0.54.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.listview" = "0.54.0" +"vortex.masked" = "0.54.0" +"vortex.null" = "0.36.0" +"vortex.pco" = "0.40.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sequence" = "0.40.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.zigzag" = "0.36.0" +"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core2026.07.0.toml index 6e73f0167e4..3bb6bf4331a 100644 --- a/vortex/editions/core2026.07.0.toml +++ b/vortex/editions/core2026.07.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2026.07.0" family = "core" @@ -49,7 +49,38 @@ encodings = [ "vortex.zstd", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"fastlanes.rle" = "0.54.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fixed_size_list" = "0.54.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.listview" = "0.54.0" +"vortex.masked" = "0.54.0" +"vortex.null" = "0.36.0" +"vortex.pco" = "0.40.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sequence" = "0.40.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.variant" = "0.65.0" +"vortex.zigzag" = "0.36.0" +"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core2026.08.0.toml index a8708950c8d..bcbfce7d1b8 100644 --- a/vortex/editions/core2026.08.0.toml +++ b/vortex/editions/core2026.08.0.toml @@ -1,8 +1,8 @@ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI. +# editing or deleting a frozen one is rejected by CI. edition = "core2026.08.0" family = "core" @@ -50,7 +50,39 @@ encodings = [ "vortex.zstd", ] -# The earliest Vortex release able to read each encoding, recorded from evidence as -# that evidence appears. Unlike the rest of this file it is filled in after the -# edition freezes, so entries are only ever added, never changed. +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. [required_vortex_release] +"fastlanes.bitpacked" = "0.36.0" +"fastlanes.for" = "0.36.0" +"fastlanes.rle" = "0.54.0" +"vortex.alp" = "0.36.0" +"vortex.alprd" = "0.36.0" +"vortex.bool" = "0.36.0" +"vortex.bytebool" = "0.36.0" +"vortex.chunked" = "0.36.0" +"vortex.constant" = "0.36.0" +"vortex.datetimeparts" = "0.36.0" +"vortex.decimal" = "0.36.0" +"vortex.decimal_byte_parts" = "0.36.0" +"vortex.dict" = "0.36.0" +"vortex.ext" = "0.36.0" +"vortex.fixed_size_list" = "0.54.0" +"vortex.fsst" = "0.36.0" +"vortex.list" = "0.36.0" +"vortex.listview" = "0.54.0" +"vortex.map" = "0.84.0" +"vortex.masked" = "0.54.0" +"vortex.null" = "0.36.0" +"vortex.pco" = "0.40.0" +"vortex.primitive" = "0.36.0" +"vortex.runend" = "0.36.0" +"vortex.sequence" = "0.40.0" +"vortex.sparse" = "0.36.0" +"vortex.struct" = "0.36.0" +"vortex.varbin" = "0.36.0" +"vortex.varbinview" = "0.36.0" +"vortex.variant" = "0.65.0" +"vortex.zigzag" = "0.36.0" +"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/unstable2025.05.0.toml b/vortex/editions/unstable2025.05.0.toml new file mode 100644 index 00000000000..8f4ce5eb7b3 --- /dev/null +++ b/vortex/editions/unstable2025.05.0.toml @@ -0,0 +1,24 @@ +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2025.05.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "fastlanes.delta", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", +] + +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. +[required_vortex_release] diff --git a/vortex/editions/unstable2026.02.0.toml b/vortex/editions/unstable2026.02.0.toml new file mode 100644 index 00000000000..c6473da2232 --- /dev/null +++ b/vortex/editions/unstable2026.02.0.toml @@ -0,0 +1,25 @@ +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.02.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.zstd_buffers", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.zstd_buffers", +] + +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. +[required_vortex_release] diff --git a/vortex/editions/unstable2026.04.0.toml b/vortex/editions/unstable2026.04.0.toml new file mode 100644 index 00000000000..ae09e340388 --- /dev/null +++ b/vortex/editions/unstable2026.04.0.toml @@ -0,0 +1,36 @@ +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.04.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", + "vortex.zstd_buffers", +] + +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. +[required_vortex_release] diff --git a/vortex/editions/unstable2026.06.0.toml b/vortex/editions/unstable2026.06.0.toml new file mode 100644 index 00000000000..e288075ad94 --- /dev/null +++ b/vortex/editions/unstable2026.06.0.toml @@ -0,0 +1,32 @@ +# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "unstable2026.06.0" +family = "unstable" + +# The encodings that join the family at this edition. +added = [ + "vortex.onpair", +] + +# The edition's full membership: the encodings above, plus every member of earlier +# editions of the family. +encodings = [ + "fastlanes.delta", + "vortex.onpair", + "vortex.parquet.variant", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", + "vortex.zstd_buffers", +] + +# The earliest Vortex release able to read each encoding. Recorded as an upper bound +# from the release current when the edition froze, and refined as compat-fixture +# evidence narrows it, so entries are added or refined but never dropped. +[required_vortex_release] diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex/src/editions/core/v2025_05.rs index 235b25e0d3c..5b5e3d4b4f6 100644 --- a/vortex/src/editions/core/v2025_05.rs +++ b/vortex/src/editions/core/v2025_05.rs @@ -1,53 +1,44 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The baseline `core` edition: stable serialized components writable by Vortex 0.36.0. +//! The baseline `core` edition: stable encodings writable by Vortex 0.36.0. use vortex_edition::Edition; use vortex_edition::EditionDeclaration; use vortex_edition::EditionId; -use vortex_edition::EditionMember; /// The first edition of the `core` family, matching the first stable Vortex file release. pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); -/// The declaration of [`CORE_2025_05_0`] and the components that join the family at it. +/// The declaration of [`CORE_2025_05_0`] and the encodings that join the family at it. pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_05_0, min_vortex_version: Some("0.36.0"), }, added: &[ - EditionMember::array(&"fastlanes.bitpacked"), - EditionMember::array(&"fastlanes.for"), - EditionMember::array(&"vortex.alp"), - EditionMember::array(&"vortex.alprd"), - EditionMember::array(&"vortex.bool"), - EditionMember::array(&"vortex.bytebool"), - EditionMember::array(&"vortex.chunked"), - EditionMember::array(&"vortex.constant"), - EditionMember::array(&"vortex.datetimeparts"), - EditionMember::array(&"vortex.decimal"), - EditionMember::array(&"vortex.decimal_byte_parts"), - EditionMember::array(&"vortex.dict"), - EditionMember::array(&"vortex.ext"), - EditionMember::array(&"vortex.fsst"), - EditionMember::array(&"vortex.list"), - EditionMember::array(&"vortex.null"), - EditionMember::array(&"vortex.primitive"), - EditionMember::array(&"vortex.runend"), - EditionMember::array(&"vortex.sparse"), - EditionMember::array(&"vortex.struct"), - EditionMember::array(&"vortex.varbin"), - EditionMember::array(&"vortex.varbinview"), - EditionMember::array(&"vortex.zigzag"), - EditionMember::layout(&"vortex.chunked"), - EditionMember::layout(&"vortex.dict"), - EditionMember::layout(&"vortex.flat"), - EditionMember::layout(&"vortex.stats"), - EditionMember::layout(&"vortex.struct"), - EditionMember::dtype(&"vortex.date"), - EditionMember::dtype(&"vortex.time"), - EditionMember::dtype(&"vortex.timestamp"), + &("fastlanes.bitpacked", "0.36.0"), + &("fastlanes.for", "0.36.0"), + &("vortex.alp", "0.36.0"), + &("vortex.alprd", "0.36.0"), + &("vortex.bool", "0.36.0"), + &("vortex.bytebool", "0.36.0"), + &("vortex.chunked", "0.36.0"), + &("vortex.constant", "0.36.0"), + &("vortex.datetimeparts", "0.36.0"), + &("vortex.decimal", "0.36.0"), + &("vortex.decimal_byte_parts", "0.36.0"), + &("vortex.dict", "0.36.0"), + &("vortex.ext", "0.36.0"), + &("vortex.fsst", "0.36.0"), + &("vortex.list", "0.36.0"), + &("vortex.null", "0.36.0"), + &("vortex.primitive", "0.36.0"), + &("vortex.runend", "0.36.0"), + &("vortex.sparse", "0.36.0"), + &("vortex.struct", "0.36.0"), + &("vortex.varbin", "0.36.0"), + &("vortex.varbinview", "0.36.0"), + &("vortex.zigzag", "0.36.0"), ], }; diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex/src/editions/core/v2025_06.rs index 42cb7c04e6f..6b103df0db9 100644 --- a/vortex/src/editions/core/v2025_06.rs +++ b/vortex/src/editions/core/v2025_06.rs @@ -6,7 +6,6 @@ use vortex_edition::Edition; use vortex_edition::EditionDeclaration; use vortex_edition::EditionId; -use vortex_edition::EditionMember; /// The June 2025 edition of the `core` family. pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); @@ -18,8 +17,8 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.40.0"), }, added: &[ - EditionMember::array(&"vortex.pco"), - EditionMember::array(&"vortex.sequence"), - EditionMember::array(&"vortex.zstd"), + &("vortex.pco", "0.40.0"), + &("vortex.sequence", "0.40.0"), + &("vortex.zstd", "0.40.0"), ], }; diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex/src/editions/core/v2025_10.rs index fed71aee7e0..98dab5991d8 100644 --- a/vortex/src/editions/core/v2025_10.rs +++ b/vortex/src/editions/core/v2025_10.rs @@ -6,7 +6,6 @@ use vortex_edition::Edition; use vortex_edition::EditionDeclaration; use vortex_edition::EditionId; -use vortex_edition::EditionMember; /// The October 2025 edition of the `core` family. pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); @@ -18,9 +17,9 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.54.0"), }, added: &[ - EditionMember::array(&"fastlanes.rle"), - EditionMember::array(&"vortex.fixed_size_list"), - EditionMember::array(&"vortex.listview"), - EditionMember::array(&"vortex.masked"), + &("fastlanes.rle", "0.54.0"), + &("vortex.fixed_size_list", "0.54.0"), + &("vortex.listview", "0.54.0"), + &("vortex.masked", "0.54.0"), ], }; diff --git a/vortex/src/editions/core/v2026_07.rs b/vortex/src/editions/core/v2026_07.rs new file mode 100644 index 00000000000..61b362b16cc --- /dev/null +++ b/vortex/src/editions/core/v2026_07.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `core` edition adding stable encodings released through July 2026. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The July 2026 edition of the `core` family. +pub const CORE_2026_07_0: EditionId = EditionId::new("core", 2026, 7, 0); + +/// The declaration of [`CORE_2026_07_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2026_07_0, + min_vortex_version: Some("0.65.0"), + }, + added: &[&("vortex.variant", "0.65.0")], +}; diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index 07979c9976f..5e89b6875c3 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -1,49 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The frozen August 2026 core editions. +//! The August 2026 core edition adding the canonical Map encoding. use vortex_edition::Edition; use vortex_edition::EditionDeclaration; use vortex_edition::EditionId; -use vortex_edition::EditionMember; -/// The August 2026 core edition containing zoned layouts. -pub const CORE_2026_08_0: EditionId = EditionId::new("core", 2026, 8, 0); +/// The August 2026 core edition containing canonical Map arrays. +pub const CORE_2026_08: EditionId = EditionId::new("core", 2026, 8, 0); -/// The declaration of [`CORE_2026_08_0`] and the components that join the family at it. -/// -/// The aggregates are the set the default writer records in zone maps. A strategy asking for an -/// aggregate outside this set fails the write instead of producing zone maps an older reader would -/// have to skip. -/// -/// `vortex.sum` is deliberately not a member: zone maps prune, a zone sum does not, and its -/// null-on-empty semantics were changed and reverted within a single week. The writer no longer -/// records it, so the two stay consistent. -pub static DECLARATION_0: EditionDeclaration = EditionDeclaration { +/// The declaration of [`CORE_2026_08`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { - id: CORE_2026_08_0, + id: CORE_2026_08, min_vortex_version: Some("0.84.0"), }, - added: &[ - EditionMember::layout(&"vortex.zoned"), - EditionMember::aggregate(&"vortex.bounded_max"), - EditionMember::aggregate(&"vortex.bounded_min"), - EditionMember::aggregate(&"vortex.max"), - EditionMember::aggregate(&"vortex.min"), - EditionMember::aggregate(&"vortex.nan_count"), - EditionMember::aggregate(&"vortex.null_count"), - ], -}; - -/// The second August 2026 edition of the `core` family, adding OnPair arrays. -pub const CORE_2026_08_1: EditionId = EditionId::new("core", 2026, 8, 1); - -/// The declaration of [`CORE_2026_08_1`] and the components that join the family at it. -pub static DECLARATION_1: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: CORE_2026_08_1, - min_vortex_version: Some("0.84.0"), - }, - added: &[EditionMember::array(&"vortex.onpair")], + added: &[&("vortex.map", "0.84.0")], }; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 98c82019a28..767356b8da1 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -20,7 +20,7 @@ pub mod core; pub mod preview; #[cfg(test)] -mod frozen; +mod records; #[cfg(test)] mod tests; diff --git a/vortex/src/editions/frozen.rs b/vortex/src/editions/records.rs similarity index 51% rename from vortex/src/editions/frozen.rs rename to vortex/src/editions/records.rs index 4972eab0121..0c42a81b285 100644 --- a/vortex/src/editions/frozen.rs +++ b/vortex/src/editions/records.rs @@ -1,34 +1,37 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The frozen-edition record under `vortex/editions`. +//! The edition records under `vortex/editions`. //! -//! A frozen edition carries a read-forever guarantee, so its encoding set must never change -//! again. Every frozen edition has one generated TOML file recording that contract: the -//! identifier, the minimum Vortex version whose reader supports it, and the full encoding -//! set. Freezing a new edition adds a file; nothing else may touch the directory, which CI -//! enforces by rejecting any diff that modifies or deletes an existing record -//! (`.github/scripts/check_frozen_editions.py`). +//! Every declared edition has one generated TOML file recording what it contains: the +//! identifier, the minimum Vortex version whose reader supports it once frozen, the full +//! encoding set, and the release recorded for each member. //! -//! The test here keeps the record honest in the other direction: the files must match what -//! [`super::EDITION_DECLARATIONS`] actually computes, so a frozen edition cannot drift -//! without the record drifting with it. Regenerate after freezing a new edition with: +//! A record's mutability follows its edition. A draft is still being assembled, so its +//! record may change however the draft does. Freezing — recording a +//! [`vortex_edition::Edition::min_vortex_version`] — turns the record into a contract that +//! carries a read-forever guarantee, and from then on it may never change again. CI enforces +//! that by rejecting any diff that touches a record that was already frozen at the base +//! revision (`.github/scripts/check_edition_records.py`). +//! +//! The test here keeps the records honest in the other direction: they must match what +//! [`super::EDITION_DECLARATIONS`] actually computes, so an edition cannot drift without its +//! record drifting with it. Regenerate with: //! //! ```bash -//! UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen +//! UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records //! ``` //! -//! [`vortex_edition::EditionInclusion::required_vortex_release`] is recorded in its own -//! table rather than beside each encoding, because it is the one fact here that is not fixed -//! at freeze time: it is backfilled from compat-fixture evidence as that evidence appears. -//! Keeping it in a table of its own makes a backfill purely an added line, so the record -//! stays append-only in the literal sense and CI can allow the fill-in while still rejecting -//! a change to a release that was already recorded. +//! [`vortex_edition::EditionInclusion::required_vortex_release`] sits in a table of its own +//! because it is the one part of a frozen record that is still allowed to move: it is an +//! upper bound recorded from the release current when the edition froze, refined as +//! compat-fixture evidence narrows it. Entries may be added or refined, never dropped. use std::collections::BTreeMap; use std::collections::BTreeSet; use std::env; use std::fs; +use std::path::Path; use std::path::PathBuf; use anyhow::Context; @@ -39,17 +42,23 @@ use vortex_edition::EditionSession; use super::EDITION_DECLARATIONS; -/// Set to any value to rewrite the record instead of verifying it. -const UPDATE_VAR: &str = "UPDATE_FROZEN_EDITIONS"; +/// Set to any value to rewrite the records instead of verifying them. +const UPDATE_VAR: &str = "UPDATE_EDITION_RECORDS"; + +const REGENERATE: &str = "UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records"; -const REGENERATE: &str = "UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen"; +const GENERATED_BY: &str = + "# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`.\n#"; -const HEADER: &str = "\ -# Generated by `UPDATE_FROZEN_EDITIONS=1 cargo test -p vortex --lib editions::frozen`. -# +const FROZEN_NOTE: &str = "\ # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting an existing one is rejected by CI."; +# editing or deleting a frozen one is rejected by CI."; + +const DRAFT_NOTE: &str = "\ +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again."; fn record_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("editions") @@ -63,19 +72,9 @@ fn session() -> Result { Ok(session) } -/// The frozen editions, paired with the version that freezing recorded. Drafts have no -/// record: a file appears in the directory at the moment an edition freezes. -fn frozen(session: &EditionSession) -> Vec<(Edition, &'static str)> { - session - .editions() - .into_iter() - .filter_map(|edition| edition.min_vortex_version.map(|version| (edition, version))) - .collect() -} - -/// Render one edition's record. Deterministic: both encoding lists are sorted by id, so the +/// Render one edition's record. Deterministic: every list is sorted by encoding id, so the /// generated bytes depend only on the declarations. -fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) -> String { +fn record(session: &EditionSession, edition: &Edition) -> String { let inclusions = session.encodings_in(&edition.id); let members: BTreeSet<&str> = inclusions .iter() @@ -99,16 +98,26 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) ids.iter().map(|id| format!(" \"{id}\",")).collect() }; + let note = if edition.is_draft() { + DRAFT_NOTE + } else { + FROZEN_NOTE + }; let mut lines = vec![ - HEADER.to_string(), + GENERATED_BY.to_string(), + note.to_string(), String::new(), format!("edition = \"{}\"", edition.id), format!("family = \"{}\"", edition.id.family), - format!("min_vortex_version = \"{min_vortex_version}\""), + ]; + if let Some(min_vortex_version) = edition.min_vortex_version { + lines.push(format!("min_vortex_version = \"{min_vortex_version}\"")); + } + lines.extend([ String::new(), "# The encodings that join the family at this edition.".to_string(), "added = [".to_string(), - ]; + ]); lines.extend(list(&added)); lines.extend([ "]".to_string(), @@ -122,11 +131,11 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) lines.extend([ "]".to_string(), String::new(), - "# The earliest Vortex release able to read each encoding, recorded from evidence as" + "# The earliest Vortex release able to read each encoding. Recorded as an upper bound" .to_string(), - "# that evidence appears. Unlike the rest of this file it is filled in after the" + "# from the release current when the edition froze, and refined as compat-fixture" .to_string(), - "# edition freezes, so entries are only ever added, never changed.".to_string(), + "# evidence narrows it, so entries are added or refined but never dropped.".to_string(), "[required_vortex_release]".to_string(), ]); lines.extend( @@ -138,12 +147,12 @@ fn record(session: &EditionSession, edition: &Edition, min_vortex_version: &str) lines.join("\n") } -fn record_path(dir: &std::path::Path, edition: &Edition) -> PathBuf { +fn record_path(dir: &Path, edition: &Edition) -> PathBuf { dir.join(format!("{}.toml", edition.id)) } /// The `*.toml` file names present in the record directory. -fn existing_records(dir: &std::path::Path) -> anyhow::Result> { +fn existing_records(dir: &Path) -> anyhow::Result> { let mut names = BTreeSet::new(); for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { let path = entry?.path(); @@ -158,14 +167,21 @@ fn existing_records(dir: &std::path::Path) -> anyhow::Result> { Ok(names) } -/// Every frozen edition has a record, every record matches the declarations exactly, and no -/// record exists without a frozen edition behind it. +/// A record carries a `min_vortex_version` exactly when the edition it records is frozen. +fn records_a_frozen_edition(contents: &str) -> bool { + contents + .lines() + .any(|line| line.starts_with("min_vortex_version = ")) +} + +/// Every declared edition has a record, every record matches the declarations exactly, no +/// record exists without a declaration behind it, and no frozen edition is returned to draft. /// -/// The third check is what catches a frozen edition being deleted or unfrozen, so the update -/// mode deliberately never removes a file: unfreezing cannot be laundered through the -/// generator. +/// The last two are what catch an edition being deleted or unfrozen, so the update mode +/// deliberately never removes a file and never unfreezes one: neither can be laundered +/// through the generator. #[test] -fn records_match_the_frozen_editions() -> anyhow::Result<()> { +fn records_match_the_declared_editions() -> anyhow::Result<()> { let session = session()?; let dir = record_dir(); let update = env::var_os(UPDATE_VAR).is_some(); @@ -173,9 +189,9 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; let mut expected_names = BTreeSet::new(); - for (edition, min_vortex_version) in frozen(&session) { + for edition in session.editions() { let path = record_path(&dir, &edition); - let expected = record(&session, &edition, min_vortex_version); + let expected = record(&session, &edition); expected_names.insert(format!("{}.toml", edition.id)); let actual = match fs::read_to_string(&path) { @@ -184,6 +200,18 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())), }; + if let Some(actual) = &actual + && edition.is_draft() + && records_a_frozen_edition(actual) + { + return Err(anyhow!( + "{} is recorded as frozen but its declaration is now a draft.\n\ + Freezing is permanent: an edition that has recorded a min_vortex_version \ + carries a read-forever guarantee and may never return to draft.", + edition.id, + )); + } + if actual.as_deref() == Some(expected.as_str()) { continue; } @@ -194,17 +222,17 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { return Err(match actual { Some(_) => anyhow!( - "the record of frozen edition {} no longer matches its declaration.\n\ - A frozen edition's encodings are fixed forever: declare a new edition \ + "the record of edition {} no longer matches its declaration.\n\ + If {} is frozen its encodings are fixed forever: declare a new edition \ instead of changing this one.\n\ - If you froze a new edition or recorded a required_vortex_release, \ - regenerate with `{REGENERATE}`.\n\ + Otherwise regenerate with `{REGENERATE}`.\n\ Record: {}", edition.id, + edition.id, path.display(), ), None => anyhow!( - "frozen edition {} has no record. Regenerate with `{REGENERATE}`.\n\ + "edition {} has no record. Regenerate with `{REGENERATE}`.\n\ Expected: {}", edition.id, path.display(), @@ -218,9 +246,9 @@ fn records_match_the_frozen_editions() -> anyhow::Result<()> { .collect(); if !strays.is_empty() { return Err(anyhow!( - "{} has records with no frozen edition behind them: {strays:?}.\n\ - A frozen edition may never be deleted or returned to draft; its declaration must \ - stay in `EDITION_DECLARATIONS` with its `min_vortex_version` recorded.", + "{} has records with no declared edition behind them: {strays:?}.\n\ + A frozen edition may never be deleted; its declaration must stay in \ + `EDITION_DECLARATIONS`.", dir.display(), )); } From 003874e928d21287d58e5d3c77c4189f4c9a332a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 13:32:43 +0000 Subject: [PATCH 04/18] Export the edition records from an xtask instead of a test The exporter belongs with the repo's other generated files, so move it to `cargo run -p xtask -- generate-editions`, alongside generate-fbs and generate-proto, and run it from the same generated-files CI job that already checks git is clean afterwards. The `#[cfg(test)]` module and its UPDATE_EDITION_RECORDS environment variable are gone; the two rules git history cannot see -- a record may not be deleted, and a frozen edition may not return to draft -- now fail the exporter itself. Drop the per-encoding release from the records. An edition declares the release it froze in, so repeating it on all 23 of core2025.05.0's members said nothing that the edition did not already say. Declarations return to plain encoding lists, EditionInclusion::required_vortex_release goes back to being unset until there is per-encoding evidence to record, and the append-only check simplifies to rejecting any change at all to a frozen record. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/scripts/check_edition_records.py | 43 +--- .github/workflows/ci.yml | 8 +- Cargo.lock | 1 + vortex-edition/src/lib.rs | 32 +-- vortex-edition/src/tests.rs | 45 ---- vortex/editions/core2025.05.0.toml | 30 +-- vortex/editions/core2025.06.0.toml | 33 +-- vortex/editions/core2025.10.0.toml | 37 +--- vortex/editions/core2026.07.0.toml | 38 +--- vortex/editions/core2026.08.0.toml | 39 +--- vortex/editions/unstable2025.05.0.toml | 7 +- vortex/editions/unstable2026.02.0.toml | 7 +- vortex/editions/unstable2026.04.0.toml | 7 +- vortex/editions/unstable2026.06.0.toml | 7 +- vortex/src/editions/core/v2025_05.rs | 46 ++-- vortex/src/editions/core/v2025_06.rs | 6 +- vortex/src/editions/core/v2025_10.rs | 8 +- vortex/src/editions/core/v2026_07.rs | 2 +- vortex/src/editions/core/v2026_08.rs | 2 +- vortex/src/editions/mod.rs | 2 - vortex/src/editions/records.rs | 257 ----------------------- xtask/Cargo.toml | 1 + xtask/src/generate_editions.rs | 165 +++++++++++++++ xtask/src/main.rs | 14 +- 24 files changed, 233 insertions(+), 604 deletions(-) delete mode 100644 vortex/src/editions/records.rs create mode 100644 xtask/src/generate_editions.rs diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py index 074947242fc..9a3348e8b07 100644 --- a/.github/scripts/check_edition_records.py +++ b/.github/scripts/check_edition_records.py @@ -10,12 +10,6 @@ A newly added record must also be newer than every edition already recorded for its family: editions are only ever added going forward. -The one part of a frozen record that may still move is the `required_vortex_release` table. -It holds an upper bound recorded from the release current when the edition froze, refined as -compat-fixture evidence narrows it; it stays under the edition's own immutable -`min_vortex_version`, so refining it breaks no published guarantee. Entries may be added or -refined, never dropped. - Usage: python3 check_edition_records.py --base origin/develop """ @@ -41,14 +35,10 @@ # A record carries this exactly when the edition it records is frozen. FROZEN_MARKER = "min_vortex_version" -# The table a frozen record may still gain or refine entries in. -EVIDENCE_TABLE = "required_vortex_release" - REMEDY = ( "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" " vortex/src/editions// and regenerate the records with\n" - " `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`.\n" - f"In a frozen record only `{EVIDENCE_TABLE}` may gain or refine entries." + " `cargo run -p xtask -- generate-editions`." ) @@ -120,32 +110,21 @@ def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: def check_modification(before: dict[str, Any], path: str) -> list[str]: - """A frozen record may only gain or refine evidence entries.""" + """A frozen record may not change at all; name the fields that did.""" name = Path(path).name after = parse_record(Path(path).read_text(), path) - fixed_before = {key: value for key, value in before.items() if key != EVIDENCE_TABLE} - fixed_after = {key: value for key, value in after.items() if key != EVIDENCE_TABLE} - if fixed_before != fixed_after: - changed = sorted( - key - for key in fixed_before.keys() | fixed_after.keys() - if fixed_before.get(key) != fixed_after.get(key) - ) - if FROZEN_MARKER in changed and FROZEN_MARKER not in after: - return [ - f"unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a " - "read-forever guarantee and may never return to draft" - ] - return [f"modifies the frozen record {name}: {', '.join(changed)}"] - - dropped = sorted(before.get(EVIDENCE_TABLE, {}).keys() - after.get(EVIDENCE_TABLE, {}).keys()) - if dropped: + changed = sorted( + key for key in before.keys() | after.keys() if before.get(key) != after.get(key) + ) + if not changed: + return [] + if FROZEN_MARKER in changed and FROZEN_MARKER not in after: return [ - f"drops the recorded {EVIDENCE_TABLE} of {', '.join(dropped)} from {name}; " - "recorded evidence is only ever added or refined" + f"unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a " + "read-forever guarantee and may never return to draft" ] - return [] + return [f"modifies the frozen record {name}: {', '.join(changed)}"] def check(base: str) -> list[str]: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08728be2ba5..6c0c56e738c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -752,14 +752,12 @@ jobs: run: | cargo run --profile ci -p xtask -- generate-fbs cargo run --profile ci -p xtask -- generate-proto + - name: "regenerate the edition records" + run: | + cargo run --profile ci -p xtask -- generate-editions - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi - - name: "regenerate the edition records" - env: - UPDATE_EDITION_RECORDS: "1" - run: | - cargo test --profile ci -p vortex --lib editions::records - name: "Make sure no files changed after regenerating" run: | git status --porcelain diff --git a/Cargo.lock b/Cargo.lock index 2448cb9aff8..515379e5cde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12200,6 +12200,7 @@ dependencies = [ "anyhow", "clap", "prost-build", + "vortex", "xshell", ] diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index 42082a0ba71..a8eaf8e8287 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -153,19 +153,9 @@ pub struct EditionInclusion { /// Implemented for raw id strings (`"vortex.alp"`) and interned [`Id`]s here; encoding /// vtables implement it where they are defined, so a declaration can name the vtable /// (`&Primitive`) instead of spelling its id. -/// -/// Pairing an encoding with a release records the evidence that the release can read it: -/// `&("vortex.alp", "0.36.0")` declares the same membership as `&"vortex.alp"` and -/// additionally sets [`EditionInclusion::required_vortex_release`]. pub trait AsEncodingId: Debug + Send + Sync { /// The interned encoding id. fn encoding_id(&self) -> Id; - - /// The earliest Vortex release able to read and execute the encoding, when the evidence - /// has been recorded. Naming an encoding on its own leaves this unrecorded. - fn required_vortex_release(&self) -> Option<&'static str> { - None - } } impl AsEncodingId for str { @@ -192,18 +182,6 @@ impl AsEncodingId for &'static str { } } -// Pairing any encoding name with a release records the evidence alongside the membership: -// `&("vortex.alp", "0.36.0")`, or `&(&Primitive, "0.36.0")` when naming the vtable. -impl AsEncodingId for (&'static E, &'static str) { - fn encoding_id(&self) -> Id { - self.0.encoding_id() - } - - fn required_vortex_release(&self) -> Option<&'static str> { - Some(self.1) - } -} - /// Declares an edition together with the encodings that join the family at it, in one /// block. Registered with [`EditionSession::declare`], which derives each encoding's /// membership (`since` = the declared edition) from the block structure. @@ -212,22 +190,18 @@ pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, /// The encodings that join the family at this edition, named by id string or by - /// vtable, optionally paired with the release that first read them - /// (`&("vortex.alp", "0.36.0")`). Members of earlier editions are inherited and never - /// restated. + /// vtable. Members of earlier editions are inherited and never restated. pub added: &'static [&'static dyn AsEncodingId], } impl EditionInclusion { /// Declare that an encoding is a member of `since` and every later edition of the same - /// family. The encoding can be named by id string or by vtable, and carries its - /// [`EditionInclusion::required_vortex_release`] when named as a - /// `(encoding, release)` pair. + /// family. The encoding can be named by id string or by vtable. pub fn new(encoding: &E, since: EditionId) -> Self { Self { encoding_id: encoding.encoding_id(), since, - required_vortex_release: encoding.required_vortex_release(), + required_vortex_release: None, } } diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index c8b9a2fe5af..09e1345615a 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -274,48 +274,3 @@ fn edition_ids_order_within_family_only() { fn edition_id_display() { assert_eq!(FIRST.to_string(), "test2026.01.0"); } - -#[test] -fn declarations_carry_required_releases() -> Result<(), crate::EditionError> { - let editions = EditionSession::empty(); - editions.declare(&EditionDeclaration { - edition: Edition { - id: FIRST, - min_vortex_version: Some("0.40.0"), - }, - added: &[&"test.alpha", &("test.beta", "0.36.0")], - })?; - editions.validate()?; - - let inclusions = editions.encodings_in(&FIRST); - let releases: Vec<(&str, Option<&str>)> = inclusions - .iter() - .map(|inclusion| { - ( - inclusion.encoding_id.as_str(), - inclusion.required_vortex_release, - ) - }) - .collect(); - assert_eq!( - releases, - [("test.alpha", None), ("test.beta", Some("0.36.0"))] - ); - - Ok(()) -} - -#[test] -fn a_member_may_not_require_a_release_newer_than_its_edition() -> Result<(), crate::EditionError> { - let editions = EditionSession::empty(); - editions.declare(&EditionDeclaration { - edition: Edition { - id: FIRST, - min_vortex_version: Some("0.40.0"), - }, - added: &[&("test.alpha", "0.54.0")], - })?; - assert!(editions.validate().is_err()); - - Ok(()) -} diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core2025.05.0.toml index 73a29ebd814..eac393d6ddf 100644 --- a/vortex/editions/core2025.05.0.toml +++ b/vortex/editions/core2025.05.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -62,31 +62,3 @@ encodings = [ "vortex.varbinview", "vortex.zigzag", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.null" = "0.36.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.zigzag" = "0.36.0" diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core2025.06.0.toml index 0e8fbe5c1be..a4e63311f4c 100644 --- a/vortex/editions/core2025.06.0.toml +++ b/vortex/editions/core2025.06.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -45,34 +45,3 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.null" = "0.36.0" -"vortex.pco" = "0.40.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sequence" = "0.40.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.zigzag" = "0.36.0" -"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core2025.10.0.toml index 6c6e6edebfc..4e3bbba2d69 100644 --- a/vortex/editions/core2025.10.0.toml +++ b/vortex/editions/core2025.10.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -50,38 +50,3 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"fastlanes.rle" = "0.54.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fixed_size_list" = "0.54.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.listview" = "0.54.0" -"vortex.masked" = "0.54.0" -"vortex.null" = "0.36.0" -"vortex.pco" = "0.40.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sequence" = "0.40.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.zigzag" = "0.36.0" -"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core2026.07.0.toml index 3bb6bf4331a..e49b6f0bf26 100644 --- a/vortex/editions/core2026.07.0.toml +++ b/vortex/editions/core2026.07.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -48,39 +48,3 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"fastlanes.rle" = "0.54.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fixed_size_list" = "0.54.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.listview" = "0.54.0" -"vortex.masked" = "0.54.0" -"vortex.null" = "0.36.0" -"vortex.pco" = "0.40.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sequence" = "0.40.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.variant" = "0.65.0" -"vortex.zigzag" = "0.36.0" -"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core2026.08.0.toml index bcbfce7d1b8..82db60f548d 100644 --- a/vortex/editions/core2026.08.0.toml +++ b/vortex/editions/core2026.08.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is frozen: it carries a read-forever guarantee, so this record of what it # contains never changes again. Freezing a new edition adds a new file to this directory; @@ -49,40 +49,3 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] -"fastlanes.bitpacked" = "0.36.0" -"fastlanes.for" = "0.36.0" -"fastlanes.rle" = "0.54.0" -"vortex.alp" = "0.36.0" -"vortex.alprd" = "0.36.0" -"vortex.bool" = "0.36.0" -"vortex.bytebool" = "0.36.0" -"vortex.chunked" = "0.36.0" -"vortex.constant" = "0.36.0" -"vortex.datetimeparts" = "0.36.0" -"vortex.decimal" = "0.36.0" -"vortex.decimal_byte_parts" = "0.36.0" -"vortex.dict" = "0.36.0" -"vortex.ext" = "0.36.0" -"vortex.fixed_size_list" = "0.54.0" -"vortex.fsst" = "0.36.0" -"vortex.list" = "0.36.0" -"vortex.listview" = "0.54.0" -"vortex.map" = "0.84.0" -"vortex.masked" = "0.54.0" -"vortex.null" = "0.36.0" -"vortex.pco" = "0.40.0" -"vortex.primitive" = "0.36.0" -"vortex.runend" = "0.36.0" -"vortex.sequence" = "0.40.0" -"vortex.sparse" = "0.36.0" -"vortex.struct" = "0.36.0" -"vortex.varbin" = "0.36.0" -"vortex.varbinview" = "0.36.0" -"vortex.variant" = "0.65.0" -"vortex.zigzag" = "0.36.0" -"vortex.zstd" = "0.40.0" diff --git a/vortex/editions/unstable2025.05.0.toml b/vortex/editions/unstable2025.05.0.toml index 8f4ce5eb7b3..b2b2ddb8325 100644 --- a/vortex/editions/unstable2025.05.0.toml +++ b/vortex/editions/unstable2025.05.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is a draft: it carries no guarantee and is still being assembled, so this # record changes with it. Recording a min_vortex_version freezes the edition, after which @@ -17,8 +17,3 @@ added = [ encodings = [ "fastlanes.delta", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] diff --git a/vortex/editions/unstable2026.02.0.toml b/vortex/editions/unstable2026.02.0.toml index c6473da2232..00f1b0bef6f 100644 --- a/vortex/editions/unstable2026.02.0.toml +++ b/vortex/editions/unstable2026.02.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is a draft: it carries no guarantee and is still being assembled, so this # record changes with it. Recording a min_vortex_version freezes the edition, after which @@ -18,8 +18,3 @@ encodings = [ "fastlanes.delta", "vortex.zstd_buffers", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] diff --git a/vortex/editions/unstable2026.04.0.toml b/vortex/editions/unstable2026.04.0.toml index ae09e340388..7e721892046 100644 --- a/vortex/editions/unstable2026.04.0.toml +++ b/vortex/editions/unstable2026.04.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is a draft: it carries no guarantee and is still being assembled, so this # record changes with it. Recording a min_vortex_version freezes the edition, after which @@ -29,8 +29,3 @@ encodings = [ "vortex.tensor.normalized", "vortex.zstd_buffers", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] diff --git a/vortex/editions/unstable2026.06.0.toml b/vortex/editions/unstable2026.06.0.toml index e288075ad94..c890c13627d 100644 --- a/vortex/editions/unstable2026.06.0.toml +++ b/vortex/editions/unstable2026.06.0.toml @@ -1,4 +1,4 @@ -# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`. +# Generated by `cargo run -p xtask -- generate-editions`. # # This edition is a draft: it carries no guarantee and is still being assembled, so this # record changes with it. Recording a min_vortex_version freezes the edition, after which @@ -25,8 +25,3 @@ encodings = [ "vortex.tensor.normalized", "vortex.zstd_buffers", ] - -# The earliest Vortex release able to read each encoding. Recorded as an upper bound -# from the release current when the edition froze, and refined as compat-fixture -# evidence narrows it, so entries are added or refined but never dropped. -[required_vortex_release] diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex/src/editions/core/v2025_05.rs index 5b5e3d4b4f6..a25a6f971a4 100644 --- a/vortex/src/editions/core/v2025_05.rs +++ b/vortex/src/editions/core/v2025_05.rs @@ -17,28 +17,28 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.36.0"), }, added: &[ - &("fastlanes.bitpacked", "0.36.0"), - &("fastlanes.for", "0.36.0"), - &("vortex.alp", "0.36.0"), - &("vortex.alprd", "0.36.0"), - &("vortex.bool", "0.36.0"), - &("vortex.bytebool", "0.36.0"), - &("vortex.chunked", "0.36.0"), - &("vortex.constant", "0.36.0"), - &("vortex.datetimeparts", "0.36.0"), - &("vortex.decimal", "0.36.0"), - &("vortex.decimal_byte_parts", "0.36.0"), - &("vortex.dict", "0.36.0"), - &("vortex.ext", "0.36.0"), - &("vortex.fsst", "0.36.0"), - &("vortex.list", "0.36.0"), - &("vortex.null", "0.36.0"), - &("vortex.primitive", "0.36.0"), - &("vortex.runend", "0.36.0"), - &("vortex.sparse", "0.36.0"), - &("vortex.struct", "0.36.0"), - &("vortex.varbin", "0.36.0"), - &("vortex.varbinview", "0.36.0"), - &("vortex.zigzag", "0.36.0"), + &"fastlanes.bitpacked", + &"fastlanes.for", + &"vortex.alp", + &"vortex.alprd", + &"vortex.bool", + &"vortex.bytebool", + &"vortex.chunked", + &"vortex.constant", + &"vortex.datetimeparts", + &"vortex.decimal", + &"vortex.decimal_byte_parts", + &"vortex.dict", + &"vortex.ext", + &"vortex.fsst", + &"vortex.list", + &"vortex.null", + &"vortex.primitive", + &"vortex.runend", + &"vortex.sparse", + &"vortex.struct", + &"vortex.varbin", + &"vortex.varbinview", + &"vortex.zigzag", ], }; diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex/src/editions/core/v2025_06.rs index 6b103df0db9..015325b8429 100644 --- a/vortex/src/editions/core/v2025_06.rs +++ b/vortex/src/editions/core/v2025_06.rs @@ -16,9 +16,5 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2025_06_0, min_vortex_version: Some("0.40.0"), }, - added: &[ - &("vortex.pco", "0.40.0"), - &("vortex.sequence", "0.40.0"), - &("vortex.zstd", "0.40.0"), - ], + added: &[&"vortex.pco", &"vortex.sequence", &"vortex.zstd"], }; diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex/src/editions/core/v2025_10.rs index 98dab5991d8..ae21026b595 100644 --- a/vortex/src/editions/core/v2025_10.rs +++ b/vortex/src/editions/core/v2025_10.rs @@ -17,9 +17,9 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.54.0"), }, added: &[ - &("fastlanes.rle", "0.54.0"), - &("vortex.fixed_size_list", "0.54.0"), - &("vortex.listview", "0.54.0"), - &("vortex.masked", "0.54.0"), + &"fastlanes.rle", + &"vortex.fixed_size_list", + &"vortex.listview", + &"vortex.masked", ], }; diff --git a/vortex/src/editions/core/v2026_07.rs b/vortex/src/editions/core/v2026_07.rs index 61b362b16cc..820c68cf7dd 100644 --- a/vortex/src/editions/core/v2026_07.rs +++ b/vortex/src/editions/core/v2026_07.rs @@ -16,5 +16,5 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2026_07_0, min_vortex_version: Some("0.65.0"), }, - added: &[&("vortex.variant", "0.65.0")], + added: &[&"vortex.variant"], }; diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex/src/editions/core/v2026_08.rs index 5e89b6875c3..4d47cfbe027 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex/src/editions/core/v2026_08.rs @@ -16,5 +16,5 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2026_08, min_vortex_version: Some("0.84.0"), }, - added: &[&("vortex.map", "0.84.0")], + added: &[&"vortex.map"], }; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 767356b8da1..a6dd8ee7fe9 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -20,8 +20,6 @@ pub mod core; pub mod preview; #[cfg(test)] -mod records; -#[cfg(test)] mod tests; pub use vortex_edition::ComponentKind; diff --git a/vortex/src/editions/records.rs b/vortex/src/editions/records.rs deleted file mode 100644 index 0c42a81b285..00000000000 --- a/vortex/src/editions/records.rs +++ /dev/null @@ -1,257 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The edition records under `vortex/editions`. -//! -//! Every declared edition has one generated TOML file recording what it contains: the -//! identifier, the minimum Vortex version whose reader supports it once frozen, the full -//! encoding set, and the release recorded for each member. -//! -//! A record's mutability follows its edition. A draft is still being assembled, so its -//! record may change however the draft does. Freezing — recording a -//! [`vortex_edition::Edition::min_vortex_version`] — turns the record into a contract that -//! carries a read-forever guarantee, and from then on it may never change again. CI enforces -//! that by rejecting any diff that touches a record that was already frozen at the base -//! revision (`.github/scripts/check_edition_records.py`). -//! -//! The test here keeps the records honest in the other direction: they must match what -//! [`super::EDITION_DECLARATIONS`] actually computes, so an edition cannot drift without its -//! record drifting with it. Regenerate with: -//! -//! ```bash -//! UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records -//! ``` -//! -//! [`vortex_edition::EditionInclusion::required_vortex_release`] sits in a table of its own -//! because it is the one part of a frozen record that is still allowed to move: it is an -//! upper bound recorded from the release current when the edition froze, refined as -//! compat-fixture evidence narrows it. Entries may be added or refined, never dropped. - -use std::collections::BTreeMap; -use std::collections::BTreeSet; -use std::env; -use std::fs; -use std::path::Path; -use std::path::PathBuf; - -use anyhow::Context; -use anyhow::anyhow; -use vortex_edition::Edition; -use vortex_edition::EditionError; -use vortex_edition::EditionSession; - -use super::EDITION_DECLARATIONS; - -/// Set to any value to rewrite the records instead of verifying them. -const UPDATE_VAR: &str = "UPDATE_EDITION_RECORDS"; - -const REGENERATE: &str = "UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records"; - -const GENERATED_BY: &str = - "# Generated by `UPDATE_EDITION_RECORDS=1 cargo test -p vortex --lib editions::records`.\n#"; - -const FROZEN_NOTE: &str = "\ -# This edition is frozen: it carries a read-forever guarantee, so this record of what it -# contains never changes again. Freezing a new edition adds a new file to this directory; -# editing or deleting a frozen one is rejected by CI."; - -const DRAFT_NOTE: &str = "\ -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again."; - -fn record_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("editions") -} - -fn session() -> Result { - let session = EditionSession::empty(); - for declaration in EDITION_DECLARATIONS { - session.declare(declaration)?; - } - Ok(session) -} - -/// Render one edition's record. Deterministic: every list is sorted by encoding id, so the -/// generated bytes depend only on the declarations. -fn record(session: &EditionSession, edition: &Edition) -> String { - let inclusions = session.encodings_in(&edition.id); - let members: BTreeSet<&str> = inclusions - .iter() - .map(|inclusion| inclusion.encoding_id.as_str()) - .collect(); - let added: BTreeSet<&str> = inclusions - .iter() - .filter(|inclusion| inclusion.since == edition.id) - .map(|inclusion| inclusion.encoding_id.as_str()) - .collect(); - let releases: BTreeMap<&str, &str> = inclusions - .iter() - .filter_map(|inclusion| { - inclusion - .required_vortex_release - .map(|release| (inclusion.encoding_id.as_str(), release)) - }) - .collect(); - - let list = |ids: &BTreeSet<&str>| -> Vec { - ids.iter().map(|id| format!(" \"{id}\",")).collect() - }; - - let note = if edition.is_draft() { - DRAFT_NOTE - } else { - FROZEN_NOTE - }; - let mut lines = vec![ - GENERATED_BY.to_string(), - note.to_string(), - String::new(), - format!("edition = \"{}\"", edition.id), - format!("family = \"{}\"", edition.id.family), - ]; - if let Some(min_vortex_version) = edition.min_vortex_version { - lines.push(format!("min_vortex_version = \"{min_vortex_version}\"")); - } - lines.extend([ - String::new(), - "# The encodings that join the family at this edition.".to_string(), - "added = [".to_string(), - ]); - lines.extend(list(&added)); - lines.extend([ - "]".to_string(), - String::new(), - "# The edition's full membership: the encodings above, plus every member of earlier" - .to_string(), - "# editions of the family.".to_string(), - "encodings = [".to_string(), - ]); - lines.extend(list(&members)); - lines.extend([ - "]".to_string(), - String::new(), - "# The earliest Vortex release able to read each encoding. Recorded as an upper bound" - .to_string(), - "# from the release current when the edition froze, and refined as compat-fixture" - .to_string(), - "# evidence narrows it, so entries are added or refined but never dropped.".to_string(), - "[required_vortex_release]".to_string(), - ]); - lines.extend( - releases - .iter() - .map(|(id, release)| format!("\"{id}\" = \"{release}\"")), - ); - lines.push(String::new()); - lines.join("\n") -} - -fn record_path(dir: &Path, edition: &Edition) -> PathBuf { - dir.join(format!("{}.toml", edition.id)) -} - -/// The `*.toml` file names present in the record directory. -fn existing_records(dir: &Path) -> anyhow::Result> { - let mut names = BTreeSet::new(); - for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { - let path = entry?.path(); - if path - .extension() - .is_some_and(|extension| extension == "toml") - && let Some(name) = path.file_name().and_then(|name| name.to_str()) - { - names.insert(name.to_string()); - } - } - Ok(names) -} - -/// A record carries a `min_vortex_version` exactly when the edition it records is frozen. -fn records_a_frozen_edition(contents: &str) -> bool { - contents - .lines() - .any(|line| line.starts_with("min_vortex_version = ")) -} - -/// Every declared edition has a record, every record matches the declarations exactly, no -/// record exists without a declaration behind it, and no frozen edition is returned to draft. -/// -/// The last two are what catch an edition being deleted or unfrozen, so the update mode -/// deliberately never removes a file and never unfreezes one: neither can be laundered -/// through the generator. -#[test] -fn records_match_the_declared_editions() -> anyhow::Result<()> { - let session = session()?; - let dir = record_dir(); - let update = env::var_os(UPDATE_VAR).is_some(); - - fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; - - let mut expected_names = BTreeSet::new(); - for edition in session.editions() { - let path = record_path(&dir, &edition); - let expected = record(&session, &edition); - expected_names.insert(format!("{}.toml", edition.id)); - - let actual = match fs::read_to_string(&path) { - Ok(actual) => Some(actual), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, - Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())), - }; - - if let Some(actual) = &actual - && edition.is_draft() - && records_a_frozen_edition(actual) - { - return Err(anyhow!( - "{} is recorded as frozen but its declaration is now a draft.\n\ - Freezing is permanent: an edition that has recorded a min_vortex_version \ - carries a read-forever guarantee and may never return to draft.", - edition.id, - )); - } - - if actual.as_deref() == Some(expected.as_str()) { - continue; - } - if update { - fs::write(&path, &expected).with_context(|| format!("writing {}", path.display()))?; - continue; - } - - return Err(match actual { - Some(_) => anyhow!( - "the record of edition {} no longer matches its declaration.\n\ - If {} is frozen its encodings are fixed forever: declare a new edition \ - instead of changing this one.\n\ - Otherwise regenerate with `{REGENERATE}`.\n\ - Record: {}", - edition.id, - edition.id, - path.display(), - ), - None => anyhow!( - "edition {} has no record. Regenerate with `{REGENERATE}`.\n\ - Expected: {}", - edition.id, - path.display(), - ), - }); - } - - let strays: Vec = existing_records(&dir)? - .difference(&expected_names) - .cloned() - .collect(); - if !strays.is_empty() { - return Err(anyhow!( - "{} has records with no declared edition behind them: {strays:?}.\n\ - A frozen edition may never be deleted; its declaration must stay in \ - `EDITION_DECLARATIONS`.", - dir.display(), - )); - } - - Ok(()) -} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index eae2db43413..9878d2f6d1b 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,6 +23,7 @@ test = false anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } prost-build = { workspace = true } +vortex = { workspace = true } xshell = { workspace = true } [lints] diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs new file mode 100644 index 00000000000..961c4ba0a12 --- /dev/null +++ b/xtask/src/generate_editions.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Export the edition records under `vortex/editions`. +//! +//! Every declared edition gets one TOML file recording what it contains: the identifier, the +//! minimum Vortex version whose reader supports it once frozen, and its full encoding set. +//! +//! A record's mutability follows its edition. A draft is still being assembled, so its record +//! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a +//! contract carrying a read-forever guarantee, and from then on it may never change again. CI +//! enforces that against git history in `.github/scripts/check_edition_records.py`; this +//! exporter enforces the two rules that history cannot see, refusing to delete a record or to +//! unfreeze one. + +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::anyhow; +use vortex::editions::EDITION_DECLARATIONS; +use vortex::editions::Edition; +use vortex::editions::EditionSession; + +const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; + +const FROZEN_NOTE: &str = "\ +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI."; + +const DRAFT_NOTE: &str = "\ +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again."; + +/// Render one edition's record. Deterministic: every list is sorted by encoding id, so the +/// generated bytes depend only on the declarations. +fn record(session: &EditionSession, edition: &Edition) -> String { + let inclusions = session.encodings_in(&edition.id); + let members: BTreeSet<&str> = inclusions + .iter() + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + let added: BTreeSet<&str> = inclusions + .iter() + .filter(|inclusion| inclusion.since == edition.id) + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + + let list = |ids: &BTreeSet<&str>| -> Vec { + ids.iter().map(|id| format!(" \"{id}\",")).collect() + }; + + let note = if edition.is_draft() { + DRAFT_NOTE + } else { + FROZEN_NOTE + }; + let mut lines = vec![ + GENERATED_BY.to_string(), + note.to_string(), + String::new(), + format!("edition = \"{}\"", edition.id), + format!("family = \"{}\"", edition.id.family), + ]; + if let Some(min_vortex_version) = edition.min_vortex_version { + lines.push(format!("min_vortex_version = \"{min_vortex_version}\"")); + } + lines.extend([ + String::new(), + "# The encodings that join the family at this edition.".to_string(), + "added = [".to_string(), + ]); + lines.extend(list(&added)); + lines.extend([ + "]".to_string(), + String::new(), + "# The edition's full membership: the encodings above, plus every member of earlier" + .to_string(), + "# editions of the family.".to_string(), + "encodings = [".to_string(), + ]); + lines.extend(list(&members)); + lines.extend(["]".to_string(), String::new()]); + lines.join("\n") +} + +/// The `*.toml` file names present in the record directory. +fn existing_records(dir: &Path) -> anyhow::Result> { + let mut names = BTreeSet::new(); + for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "toml") + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + names.insert(name.to_string()); + } + } + Ok(names) +} + +/// A record carries a `min_vortex_version` exactly when the edition it records is frozen. +fn records_a_frozen_edition(contents: &str) -> bool { + contents + .lines() + .any(|line| line.starts_with("min_vortex_version = ")) +} + +pub fn generate_editions() -> anyhow::Result<()> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../vortex/editions"); + fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; + + let session = EditionSession::empty(); + for declaration in EDITION_DECLARATIONS { + session + .declare(declaration) + .map_err(|error| anyhow!("declaring editions: {error}"))?; + } + session + .validate() + .map_err(|error| anyhow!("validating editions: {error}"))?; + + let mut expected = BTreeSet::new(); + for edition in session.editions() { + let path = dir.join(format!("{}.toml", edition.id)); + expected.insert(format!("{}.toml", edition.id)); + + // Freezing is permanent, and the record on disk is the only memory of it. Refusing + // here means unfreezing cannot be laundered through the exporter. + if edition.is_draft() + && let Ok(previous) = fs::read_to_string(&path) + && records_a_frozen_edition(&previous) + { + return Err(anyhow!( + "{} is recorded as frozen but its declaration is now a draft.\n\ + An edition that recorded a min_vortex_version carries a read-forever \ + guarantee and may never return to draft.", + edition.id, + )); + } + + fs::write(&path, record(&session, &edition)) + .with_context(|| format!("writing {}", path.display()))?; + } + + let strays: Vec = existing_records(&dir)? + .difference(&expected) + .cloned() + .collect(); + if !strays.is_empty() { + return Err(anyhow!( + "{} has records with no declared edition behind them: {strays:?}.\n\ + A frozen edition may never be deleted; its declaration must stay in \ + `EDITION_DECLARATIONS`.", + dir.display(), + )); + } + + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 1155ee3246a..772dd953e0e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod generate_editions; mod generate_fbs; mod generate_proto; use clap::Parser; +use crate::generate_editions::generate_editions; use crate::generate_fbs::generate_fbs; use crate::generate_proto::generate_proto; @@ -17,19 +19,23 @@ struct Xtask { #[derive(clap::Subcommand)] enum Commands { + /// Subcommand to regenerate the edition records under `vortex/editions`. + #[command(name = "generate-editions")] + Editions, /// Subcommand to regenerate flatbuffers language bindings for the Rust project. #[command(name = "generate-fbs")] - GenerateFlatbuffers, + Flatbuffers, /// Subcommand to regenerate protobuf language bindings for the Rust project. #[command(name = "generate-proto")] - GenerateProto, + Proto, } fn main() -> anyhow::Result<()> { let cli = Xtask::parse(); match cli.command { - Commands::GenerateFlatbuffers => generate_fbs()?, - Commands::GenerateProto => generate_proto()?, + Commands::Editions => generate_editions()?, + Commands::Flatbuffers => generate_fbs()?, + Commands::Proto => generate_proto()?, } Ok(()) } From 84d803b99db0485ee1b8b7ab64e473ccb45e55cf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 13:59:09 +0000 Subject: [PATCH 05/18] Group the edition records by family Families version independently, so a flat directory mixed two unrelated chronologies and left the reader to spot the family from a filename prefix. Group the records as `vortex/editions//.toml`, mirroring the declarations in `vortex/src/editions`. The exporter creates a directory per family and treats a record under the wrong one as a stray, since that is what it is. The append-only check requires a newly added record's directory to match the family its name declares, so a record cannot be filed under a family whose chronology it does not extend. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/scripts/check_edition_records.py | 19 +++++++-- vortex/editions/{ => core}/core2025.05.0.toml | 0 vortex/editions/{ => core}/core2025.06.0.toml | 0 vortex/editions/{ => core}/core2025.10.0.toml | 0 vortex/editions/{ => core}/core2026.07.0.toml | 0 vortex/editions/{ => core}/core2026.08.0.toml | 0 .../{ => unstable}/unstable2025.05.0.toml | 0 .../{ => unstable}/unstable2026.02.0.toml | 0 .../{ => unstable}/unstable2026.04.0.toml | 0 .../{ => unstable}/unstable2026.06.0.toml | 0 xtask/src/generate_editions.rs | 42 +++++++++++++------ 11 files changed, 45 insertions(+), 16 deletions(-) rename vortex/editions/{ => core}/core2025.05.0.toml (100%) rename vortex/editions/{ => core}/core2025.06.0.toml (100%) rename vortex/editions/{ => core}/core2025.10.0.toml (100%) rename vortex/editions/{ => core}/core2026.07.0.toml (100%) rename vortex/editions/{ => core}/core2026.08.0.toml (100%) rename vortex/editions/{ => unstable}/unstable2025.05.0.toml (100%) rename vortex/editions/{ => unstable}/unstable2026.02.0.toml (100%) rename vortex/editions/{ => unstable}/unstable2026.04.0.toml (100%) rename vortex/editions/{ => unstable}/unstable2026.06.0.toml (100%) diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py index 9a3348e8b07..a15240b9c95 100644 --- a/.github/scripts/check_edition_records.py +++ b/.github/scripts/check_edition_records.py @@ -8,7 +8,8 @@ edit it in the same diff. A newly added record must also be newer than every edition already recorded for its family: -editions are only ever added going forward. +editions are only ever added going forward. Records are grouped by family, so +`vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. Usage: python3 check_edition_records.py --base origin/develop @@ -26,8 +27,8 @@ RECORD_DIR = "vortex/editions" -# `core2026.08.0.toml`: the file name is the edition id, so the record's identity is visible -# in the diff without reading the file. +# `core/core2026.08.0.toml`: the file name is the edition id and its directory is the family, +# so a record's identity is visible in the diff without reading the file. RECORD_NAME = re.compile( r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" ) @@ -79,7 +80,8 @@ def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: if match is None: sys.exit( f"{RECORD_DIR}/{name} is not a valid record name.\n" - "Records are named after the edition they record, e.g. `core2026.08.0.toml`." + "Records are named after the edition they record, e.g. " + "`core/core2026.08.0.toml`." ) return match["family"], (int(match["year"]), int(match["month"]), int(match["version"])) @@ -163,6 +165,15 @@ def check(base: str) -> list[str]: f"recorded ({family}{recorded}). Editions may only be added going forward." ) + # A record's family decides which chronology it extends, so the directory it sits + # in has to agree with the family its name declares. + directory = Path(path).parent.name + if directory != family: + errors.append( + f"adds {name} under {directory}/, but it records a {family} edition; " + "records are grouped by family" + ) + # The file name is the edition's identity, so it has to agree with the content. edition = parse_record(Path(path).read_text(), path).get("edition") if edition is None: diff --git a/vortex/editions/core2025.05.0.toml b/vortex/editions/core/core2025.05.0.toml similarity index 100% rename from vortex/editions/core2025.05.0.toml rename to vortex/editions/core/core2025.05.0.toml diff --git a/vortex/editions/core2025.06.0.toml b/vortex/editions/core/core2025.06.0.toml similarity index 100% rename from vortex/editions/core2025.06.0.toml rename to vortex/editions/core/core2025.06.0.toml diff --git a/vortex/editions/core2025.10.0.toml b/vortex/editions/core/core2025.10.0.toml similarity index 100% rename from vortex/editions/core2025.10.0.toml rename to vortex/editions/core/core2025.10.0.toml diff --git a/vortex/editions/core2026.07.0.toml b/vortex/editions/core/core2026.07.0.toml similarity index 100% rename from vortex/editions/core2026.07.0.toml rename to vortex/editions/core/core2026.07.0.toml diff --git a/vortex/editions/core2026.08.0.toml b/vortex/editions/core/core2026.08.0.toml similarity index 100% rename from vortex/editions/core2026.08.0.toml rename to vortex/editions/core/core2026.08.0.toml diff --git a/vortex/editions/unstable2025.05.0.toml b/vortex/editions/unstable/unstable2025.05.0.toml similarity index 100% rename from vortex/editions/unstable2025.05.0.toml rename to vortex/editions/unstable/unstable2025.05.0.toml diff --git a/vortex/editions/unstable2026.02.0.toml b/vortex/editions/unstable/unstable2026.02.0.toml similarity index 100% rename from vortex/editions/unstable2026.02.0.toml rename to vortex/editions/unstable/unstable2026.02.0.toml diff --git a/vortex/editions/unstable2026.04.0.toml b/vortex/editions/unstable/unstable2026.04.0.toml similarity index 100% rename from vortex/editions/unstable2026.04.0.toml rename to vortex/editions/unstable/unstable2026.04.0.toml diff --git a/vortex/editions/unstable2026.06.0.toml b/vortex/editions/unstable/unstable2026.06.0.toml similarity index 100% rename from vortex/editions/unstable2026.06.0.toml rename to vortex/editions/unstable/unstable2026.06.0.toml diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 961c4ba0a12..41e538f3619 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -5,6 +5,8 @@ //! //! Every declared edition gets one TOML file recording what it contains: the identifier, the //! minimum Vortex version whose reader supports it once frozen, and its full encoding set. +//! Records are grouped by family — `vortex/editions/core/core2025.05.0.toml` — mirroring the +//! declarations in `vortex/src/editions`, since families version independently. //! //! A record's mutability follows its edition. A draft is still being assembled, so its record //! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a @@ -88,20 +90,33 @@ fn record(session: &EditionSession, edition: &Edition) -> String { lines.join("\n") } -/// The `*.toml` file names present in the record directory. +/// The records present on disk, as `family/edition.toml` paths relative to the record +/// directory. A record filed under the wrong family reads as a stray, which is what it is. fn existing_records(dir: &Path) -> anyhow::Result> { - let mut names = BTreeSet::new(); - for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { - let path = entry?.path(); - if path - .extension() - .is_some_and(|extension| extension == "toml") - && let Some(name) = path.file_name().and_then(|name| name.to_str()) + let mut records = BTreeSet::new(); + if !dir.exists() { + return Ok(records); + } + for family in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let family = family?.path(); + if !family.is_dir() { + continue; + } + for entry in + fs::read_dir(&family).with_context(|| format!("reading {}", family.display()))? { - names.insert(name.to_string()); + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "toml") + && let Ok(relative) = path.strip_prefix(dir) + && let Some(relative) = relative.to_str() + { + records.insert(relative.to_string()); + } } } - Ok(names) + Ok(records) } /// A record carries a `min_vortex_version` exactly when the edition it records is frozen. @@ -127,8 +142,11 @@ pub fn generate_editions() -> anyhow::Result<()> { let mut expected = BTreeSet::new(); for edition in session.editions() { - let path = dir.join(format!("{}.toml", edition.id)); - expected.insert(format!("{}.toml", edition.id)); + let relative = format!("{}/{}.toml", edition.id.family, edition.id); + let path = dir.join(&relative); + expected.insert(relative); + fs::create_dir_all(dir.join(edition.id.family)) + .with_context(|| format!("creating the {} record directory", edition.id.family))?; // Freezing is permanent, and the record on disk is the only memory of it. Refusing // here means unfreezing cannot be laundered through the exporter. From d5434ad92599faad00dedd3092c17baeeb6d5a53 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 10:56:35 +0000 Subject: [PATCH 06/18] Move the edition declarations into vortex-edition, and check records with pygit2 The declarations name encodings by id string and import nothing but the types in vortex-edition, so nothing kept them in the vortex facade. Moving them makes the exporter cheap: xtask depended on vortex to reach EDITION_DECLARATIONS, which dragged 61 vortex crates and 338 packages into a build that also serves generate-fbs and generate-proto. It now pulls 4 and 81. `vortex::editions` re-exports every moved item, so its public API is unchanged, and it keeps the session wiring that does need the facade. The record check drove git through subprocess, parsing --name-status -z by hand. Read the object database with pygit2 instead: revisions resolve through revparse, renames come from the diff's own similarity detection rather than status-letter parsing, and record contents are read from the commit trees, so the check sees committed state only and never the working tree. Run it with `uv run --script`, which resolves the dependency from the script's inline metadata. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/scripts/check_edition_records.py | 234 +++++++++++------- .github/workflows/ci.yml | 6 +- Cargo.lock | 2 +- .../src/declarations}/core/mod.rs | 0 .../src/declarations}/core/v2025_05.rs | 6 +- .../src/declarations}/core/v2025_06.rs | 6 +- .../src/declarations}/core/v2025_10.rs | 6 +- .../src/declarations}/core/v2026_07.rs | 6 +- .../src/declarations}/core/v2026_08.rs | 6 +- vortex-edition/src/declarations/mod.rs | 30 +++ .../src/declarations/unstable/mod.rs | 17 ++ .../src/declarations/unstable/v2025_05.rs | 20 ++ .../src/declarations/unstable/v2026_02.rs | 20 ++ .../src/declarations/unstable/v2026_04.rs | 27 ++ .../src/declarations/unstable/v2026_06.rs | 20 ++ vortex-edition/src/lib.rs | 2 + vortex/src/editions/core/v2026_08_2.rs | 21 -- vortex/src/editions/core/v2026_08_3.rs | 25 -- vortex/src/editions/mod.rs | 72 ++---- xtask/Cargo.toml | 2 +- xtask/src/generate_editions.rs | 8 +- 21 files changed, 326 insertions(+), 210 deletions(-) rename {vortex/src/editions => vortex-edition/src/declarations}/core/mod.rs (100%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2025_05.rs (92%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2025_06.rs (86%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2025_10.rs (87%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2026_07.rs (85%) rename {vortex/src/editions => vortex-edition/src/declarations}/core/v2026_08.rs (85%) create mode 100644 vortex-edition/src/declarations/mod.rs create mode 100644 vortex-edition/src/declarations/unstable/mod.rs create mode 100644 vortex-edition/src/declarations/unstable/v2025_05.rs create mode 100644 vortex-edition/src/declarations/unstable/v2026_02.rs create mode 100644 vortex-edition/src/declarations/unstable/v2026_04.rs create mode 100644 vortex-edition/src/declarations/unstable/v2026_06.rs delete mode 100644 vortex/src/editions/core/v2026_08_2.rs delete mode 100644 vortex/src/editions/core/v2026_08_3.rs diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py index a15240b9c95..ec4447c241e 100644 --- a/.github/scripts/check_edition_records.py +++ b/.github/scripts/check_edition_records.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.11" +# dependencies = ["pygit2>=1.14"] +# /// """Check that frozen edition records under `vortex/editions` never change. A record's mutability follows its edition. A draft is still being assembled, so its record may @@ -11,19 +15,24 @@ editions are only ever added going forward. Records are grouped by family, so `vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. +Both revisions are read straight out of the object database, so the check sees committed +state only and never the working tree. + Usage: - python3 check_edition_records.py --base origin/develop + uv run --script .github/scripts/check_edition_records.py --base origin/develop """ from __future__ import annotations import argparse import re -import subprocess import sys import tomllib -from pathlib import Path -from typing import Any +from pathlib import PurePosixPath +from typing import Any, Iterator + +import pygit2 +from pygit2.enums import DeltaStatus RECORD_DIR = "vortex/editions" @@ -36,42 +45,55 @@ # A record carries this exactly when the edition it records is frozen. FROZEN_MARKER = "min_vortex_version" +# Every way a record can change other than being added. Renames and copies carry an old path +# and a new one; the rest carry one. +CHANGE_VERBS = { + DeltaStatus.MODIFIED: "modifies", + DeltaStatus.DELETED: "deletes", + DeltaStatus.RENAMED: "renames", + DeltaStatus.COPIED: "copies", + DeltaStatus.TYPECHANGE: "retypes", +} + REMEDY = ( "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" - " vortex/src/editions// and regenerate the records with\n" + " vortex-edition/src/declarations// and regenerate the records with\n" " `cargo run -p xtask -- generate-editions`." ) -def git(*args: str) -> str: - result = subprocess.run(["git", *args], capture_output=True, text=True, check=False) - if result.returncode != 0: - sys.exit(f"git {' '.join(args)} failed:\n{result.stderr.strip()}") - return result.stdout +def parse_record(text: str, where: str) -> dict[str, Any]: + try: + return tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + sys.exit(f"{where} is not valid TOML: {error}") -def merge_base(base: str) -> str: - result = subprocess.run( - ["git", "merge-base", base, "HEAD"], capture_output=True, text=True, check=False - ) - if result.returncode != 0: - sys.exit( - f"cannot find a merge base between {base} and HEAD:\n" - f"{result.stderr.strip()}\n" - "The checkout is probably too shallow; this check needs `fetch-depth: 0`." - ) - return result.stdout.strip() +def read_record(commit: pygit2.Commit, path: str) -> dict[str, Any] | None: + """Parse a record out of a commit's tree, or None when it holds no such file.""" + try: + blob = commit.tree[path] + except KeyError: + return None + return parse_record(blob.data.decode(), f"{path} at {commit.short_id}") -def parse_record(text: str, path: str) -> dict[str, Any]: - try: - return tomllib.loads(text) - except tomllib.TOMLDecodeError as error: - sys.exit(f"{path} is not valid TOML: {error}") +def record_paths(commit: pygit2.Commit) -> Iterator[str]: + """Every record path in a commit, relative to the repository root.""" + def walk(tree: pygit2.Tree, prefix: str) -> Iterator[str]: + for entry in tree: + path = f"{prefix}/{entry.name}" + if isinstance(entry, pygit2.Tree): + yield from walk(entry, path) + elif entry.name.endswith(".toml"): + yield path -def record_at(base: str, path: str) -> dict[str, Any]: - return parse_record(git("show", f"{base}:{path}"), f"{path} at {base[:12]}") + try: + records = commit.tree[RECORD_DIR] + except KeyError: + return + yield from walk(records, RECORD_DIR) def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: @@ -86,36 +108,24 @@ def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: return match["family"], (int(match["year"]), int(match["month"]), int(match["version"])) -def changed_records(base: str) -> list[tuple[str, list[str]]]: - """The status and paths of every change to the record directory since `base`.""" - raw = git("diff", "--name-status", "-z", base, "HEAD", "--", RECORD_DIR) - fields = [field for field in raw.split("\0") if field] - changes: list[tuple[str, list[str]]] = [] - index = 0 - while index < len(fields): - status = fields[index] - # Renames and copies carry both the old and the new path. - count = 2 if status[0] in ("R", "C") else 1 - changes.append((status, fields[index + 1 : index + 1 + count])) - index += 1 + count - return changes +def changed_records(base: pygit2.Commit, head: pygit2.Commit) -> pygit2.Diff: + """The record directory's diff between two commits, with renames detected.""" + diff = base.tree.diff_to_tree(head.tree) + diff.find_similar() + return diff -def recorded_at(base: str) -> dict[str, tuple[int, int, int]]: - """The newest edition already recorded for each family at `base`.""" +def newest_recorded(commit: pygit2.Commit) -> dict[str, tuple[int, int, int]]: + """The newest edition already recorded for each family at `commit`.""" newest: dict[str, tuple[int, int, int]] = {} - listing = git("ls-tree", "-r", "--name-only", base, "--", RECORD_DIR) - for path in listing.splitlines(): - family, key = parse_name(Path(path).name) + for path in record_paths(commit): + family, key = parse_name(PurePosixPath(path).name) newest[family] = max(key, newest.get(family, (0, 0, 0))) return newest -def check_modification(before: dict[str, Any], path: str) -> list[str]: +def check_modification(before: dict[str, Any], after: dict[str, Any], name: str) -> list[str]: """A frozen record may not change at all; name the fields that did.""" - name = Path(path).name - after = parse_record(Path(path).read_text(), path) - changed = sorted( key for key in before.keys() | after.keys() if before.get(key) != after.get(key) ) @@ -129,60 +139,80 @@ def check_modification(before: dict[str, Any], path: str) -> list[str]: return [f"modifies the frozen record {name}: {', '.join(changed)}"] -def check(base: str) -> list[str]: +def check_addition( + path: str, record: dict[str, Any], newest: dict[str, tuple[int, int, int]] +) -> list[str]: + """A new record must extend its family's chronology, and be filed under it.""" + errors = [] + name = PurePosixPath(path).name + family, key = parse_name(name) + + previous = newest.get(family) + if previous is not None and key <= previous: + recorded = f"{previous[0]}.{previous[1]:02}.{previous[2]}" + errors.append( + f"adds {name}, which is not newer than the {family} edition already " + f"recorded ({family}{recorded}). Editions may only be added going forward." + ) + + # A record's family decides which chronology it extends, so the directory it sits in has + # to agree with the family its name declares. + directory = PurePosixPath(path).parent.name + if directory != family: + errors.append( + f"adds {name} under {directory}/, but it records a {family} edition; " + "records are grouped by family" + ) + + # The file name is the edition's identity, so it has to agree with the content. + edition = record.get("edition") + if edition is None: + errors.append(f"adds {name}, which has no `edition` field") + elif edition != name.removesuffix(".toml"): + errors.append( + f"adds {name}, which records edition {edition!r}; the file name must be the edition id" + ) + return errors + + +def under_record_dir(*paths: str | None) -> bool: + return any(path is not None and path.startswith(f"{RECORD_DIR}/") for path in paths) + + +def check(base: pygit2.Commit, head: pygit2.Commit) -> list[str]: errors: list[str] = [] added: list[str] = [] - for status, paths in changed_records(base): - if status == "A": - added.append(paths[0]) + for patch in changed_records(base, head): + delta = patch.delta + old_path, new_path = delta.old_file.path, delta.new_file.path + if not under_record_dir(old_path, new_path): + continue + + if delta.status == DeltaStatus.ADDED: + added.append(new_path) continue # Frozen-ness comes from the base revision, so a diff cannot unfreeze an edition and # then edit it. A draft's record is free to change, move, or go away with the draft. - before = record_at(base, paths[0]) - if FROZEN_MARKER not in before: + before = read_record(base, old_path) + if before is None or FROZEN_MARKER not in before: continue - if status == "M": - errors.extend(check_modification(before, paths[0])) + if delta.status == DeltaStatus.MODIFIED: + after = read_record(head, new_path) or {} + errors.extend(check_modification(before, after, PurePosixPath(new_path).name)) else: - verb = {"D": "deletes", "R": "renames", "C": "copies", "T": "retypes"} - errors.append( - f"{verb.get(status[0], 'changes')} the frozen record {' -> '.join(paths)}" - ) + verb = CHANGE_VERBS.get(delta.status, "changes") + moved = old_path if old_path == new_path else f"{old_path} -> {new_path}" + errors.append(f"{verb} the frozen record {moved}") - newest = recorded_at(base) + newest = newest_recorded(base) for path in sorted(added): - name = Path(path).name - family, key = parse_name(name) - - previous = newest.get(family) - if previous is not None and key <= previous: - recorded = f"{previous[0]}.{previous[1]:02}.{previous[2]}" - errors.append( - f"adds {name}, which is not newer than the {family} edition already " - f"recorded ({family}{recorded}). Editions may only be added going forward." - ) - - # A record's family decides which chronology it extends, so the directory it sits - # in has to agree with the family its name declares. - directory = Path(path).parent.name - if directory != family: - errors.append( - f"adds {name} under {directory}/, but it records a {family} edition; " - "records are grouped by family" - ) - - # The file name is the edition's identity, so it has to agree with the content. - edition = parse_record(Path(path).read_text(), path).get("edition") - if edition is None: - errors.append(f"adds {name}, which has no `edition` field") - elif edition != name.removesuffix(".toml"): - errors.append( - f"adds {name}, which records edition {edition!r}; the file name " - "must be the edition id" - ) + record = read_record(head, path) + if record is None: + continue + errors.extend(check_addition(path, record, newest)) return errors @@ -196,10 +226,24 @@ def main() -> int: ) args = parser.parse_args() - base = merge_base(args.base) - errors = check(base) + repo = pygit2.Repository(pygit2.discover_repository(".")) + try: + base_tip = repo.revparse_single(args.base).peel(pygit2.Commit) + except KeyError: + sys.exit(f"cannot resolve {args.base!r} in this repository") + + head = repo.head.peel(pygit2.Commit) + merge_base = repo.merge_base(base_tip.id, head.id) + if merge_base is None: + sys.exit( + f"{args.base} and HEAD have no common ancestor.\n" + "The checkout is probably too shallow; this check needs `fetch-depth: 0`." + ) + base = repo[merge_base] + + errors = check(base, head) if not errors: - print(f"{RECORD_DIR} preserves every frozen record against {args.base} ({base[:12]}).") + print(f"{RECORD_DIR} preserves every frozen record against {args.base} ({base.short_id}).") return 0 print(f"This change breaks the edition records in {RECORD_DIR}:\n", file=sys.stderr) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c0c56e738c..faa7f405908 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,10 +73,14 @@ jobs: with: # The check compares against the merge base, so it needs real history. fetch-depth: 0 + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false - name: Check edition records run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" - python3 .github/scripts/check_edition_records.py --base "$BASE" + uv run --script .github/scripts/check_edition_records.py --base "$BASE" python-lint: name: "Python (lint)" diff --git a/Cargo.lock b/Cargo.lock index 515379e5cde..f53a363cdcf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12200,7 +12200,7 @@ dependencies = [ "anyhow", "clap", "prost-build", - "vortex", + "vortex-edition", "xshell", ] diff --git a/vortex/src/editions/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs similarity index 100% rename from vortex/src/editions/core/mod.rs rename to vortex-edition/src/declarations/core/mod.rs diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex-edition/src/declarations/core/v2025_05.rs similarity index 92% rename from vortex/src/editions/core/v2025_05.rs rename to vortex-edition/src/declarations/core/v2025_05.rs index a25a6f971a4..115193619ba 100644 --- a/vortex/src/editions/core/v2025_05.rs +++ b/vortex-edition/src/declarations/core/v2025_05.rs @@ -3,9 +3,9 @@ //! The baseline `core` edition: stable encodings writable by Vortex 0.36.0. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The first edition of the `core` family, matching the first stable Vortex file release. pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex-edition/src/declarations/core/v2025_06.rs similarity index 86% rename from vortex/src/editions/core/v2025_06.rs rename to vortex-edition/src/declarations/core/v2025_06.rs index 015325b8429..7ebd3505799 100644 --- a/vortex/src/editions/core/v2025_06.rs +++ b/vortex-edition/src/declarations/core/v2025_06.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through June 2025. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The June 2025 edition of the `core` family. pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex-edition/src/declarations/core/v2025_10.rs similarity index 87% rename from vortex/src/editions/core/v2025_10.rs rename to vortex-edition/src/declarations/core/v2025_10.rs index ae21026b595..6124c9e94b3 100644 --- a/vortex/src/editions/core/v2025_10.rs +++ b/vortex-edition/src/declarations/core/v2025_10.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through October 2025. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The October 2025 edition of the `core` family. pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); diff --git a/vortex/src/editions/core/v2026_07.rs b/vortex-edition/src/declarations/core/v2026_07.rs similarity index 85% rename from vortex/src/editions/core/v2026_07.rs rename to vortex-edition/src/declarations/core/v2026_07.rs index 820c68cf7dd..78e0814d5a4 100644 --- a/vortex/src/editions/core/v2026_07.rs +++ b/vortex-edition/src/declarations/core/v2026_07.rs @@ -3,9 +3,9 @@ //! The `core` edition adding stable encodings released through July 2026. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The July 2026 edition of the `core` family. pub const CORE_2026_07_0: EditionId = EditionId::new("core", 2026, 7, 0); diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex-edition/src/declarations/core/v2026_08.rs similarity index 85% rename from vortex/src/editions/core/v2026_08.rs rename to vortex-edition/src/declarations/core/v2026_08.rs index 4d47cfbe027..904dfb4ef73 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex-edition/src/declarations/core/v2026_08.rs @@ -3,9 +3,9 @@ //! The August 2026 core edition adding the canonical Map encoding. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; /// The August 2026 core edition containing canonical Map arrays. pub const CORE_2026_08: EditionId = EditionId::new("core", 2026, 8, 0); diff --git a/vortex-edition/src/declarations/mod.rs b/vortex-edition/src/declarations/mod.rs new file mode 100644 index 00000000000..0df0f58974c --- /dev/null +++ b/vortex-edition/src/declarations/mod.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The first-party Vortex edition declarations, one module per edition. +//! +//! These are plain constants naming encodings by id, so they depend on nothing but the types +//! in this crate. That keeps them cheap to read: tooling that only needs to know what an +//! edition contains — `cargo run -p xtask -- generate-editions`, for one — can depend on +//! this crate alone rather than on the whole of `vortex`. +//! +//! The `vortex` facade re-exports everything here and owns the session wiring: registering +//! the declarations and selecting which of them the default writer may emit. + +pub mod core; +pub mod unstable; + +use crate::EditionDeclaration; + +/// The first-party Vortex edition declarations. +pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ + &core::v2025_05::DECLARATION, + &core::v2025_06::DECLARATION, + &core::v2025_10::DECLARATION, + &core::v2026_07::DECLARATION, + &core::v2026_08::DECLARATION, + &unstable::v2025_05::DECLARATION, + &unstable::v2026_02::DECLARATION, + &unstable::v2026_04::DECLARATION, + &unstable::v2026_06::DECLARATION, +]; diff --git a/vortex-edition/src/declarations/unstable/mod.rs b/vortex-edition/src/declarations/unstable/mod.rs new file mode 100644 index 00000000000..5544ba45c0f --- /dev/null +++ b/vortex-edition/src/declarations/unstable/mod.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `unstable` edition family: opt-in encodings without a frozen compatibility guarantee. +//! +//! One module per draft edition, each declaring the encodings that join the family at it. +//! Members of earlier editions are inherited and never restated. + +pub mod v2025_05; +pub mod v2026_02; +pub mod v2026_04; +pub mod v2026_06; + +pub use v2025_05::UNSTABLE_2025_05_0; +pub use v2026_02::UNSTABLE_2026_02_0; +pub use v2026_04::UNSTABLE_2026_04_0; +pub use v2026_06::UNSTABLE_2026_06_0; diff --git a/vortex-edition/src/declarations/unstable/v2025_05.rs b/vortex-edition/src/declarations/unstable/v2025_05.rs new file mode 100644 index 00000000000..1a992fd9937 --- /dev/null +++ b/vortex-edition/src/declarations/unstable/v2025_05.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The May 2025 `unstable` encoding cohort. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; + +/// The May 2025 draft edition of the `unstable` family. +pub const UNSTABLE_2025_05_0: EditionId = EditionId::new("unstable", 2025, 5, 0); + +/// The declaration of [`UNSTABLE_2025_05_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE_2025_05_0, + min_vortex_version: None, + }, + added: &[&"fastlanes.delta"], +}; diff --git a/vortex-edition/src/declarations/unstable/v2026_02.rs b/vortex-edition/src/declarations/unstable/v2026_02.rs new file mode 100644 index 00000000000..8e5faeeb28f --- /dev/null +++ b/vortex-edition/src/declarations/unstable/v2026_02.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The February 2026 `unstable` encoding cohort. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; + +/// The February 2026 draft edition of the `unstable` family. +pub const UNSTABLE_2026_02_0: EditionId = EditionId::new("unstable", 2026, 2, 0); + +/// The declaration of [`UNSTABLE_2026_02_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE_2026_02_0, + min_vortex_version: None, + }, + added: &[&"vortex.zstd_buffers"], +}; diff --git a/vortex-edition/src/declarations/unstable/v2026_04.rs b/vortex-edition/src/declarations/unstable/v2026_04.rs new file mode 100644 index 00000000000..d64bc07529e --- /dev/null +++ b/vortex-edition/src/declarations/unstable/v2026_04.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The April 2026 `unstable` encoding cohort. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; + +/// The April 2026 draft edition of the `unstable` family. +pub const UNSTABLE_2026_04_0: EditionId = EditionId::new("unstable", 2026, 4, 0); + +/// The declaration of [`UNSTABLE_2026_04_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE_2026_04_0, + min_vortex_version: None, + }, + added: &[ + &"vortex.parquet.variant", + &"vortex.patched", + &"vortex.tensor.cosine_similarity", + &"vortex.tensor.inner_product", + &"vortex.tensor.normalized", + &"vortex.tensor.l2_norm", + ], +}; diff --git a/vortex-edition/src/declarations/unstable/v2026_06.rs b/vortex-edition/src/declarations/unstable/v2026_06.rs new file mode 100644 index 00000000000..560b3b5bf02 --- /dev/null +++ b/vortex-edition/src/declarations/unstable/v2026_06.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The June 2026 `unstable` encoding cohort. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; + +/// The June 2026 draft edition of the `unstable` family. +pub const UNSTABLE_2026_06_0: EditionId = EditionId::new("unstable", 2026, 6, 0); + +/// The declaration of [`UNSTABLE_2026_06_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE_2026_06_0, + min_vortex_version: None, + }, + added: &[&"vortex.onpair"], +}; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index a8eaf8e8287..0f71e520721 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -22,6 +22,7 @@ //! and enables them on the default session. See the published spec at //! . +pub mod declarations; mod session; pub mod test_harness; #[cfg(test)] @@ -33,6 +34,7 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; +pub use declarations::EDITION_DECLARATIONS; pub use session::EditionSession; pub use session::EditionSessionExt; pub use session::EnabledEditions; diff --git a/vortex/src/editions/core/v2026_08_2.rs b/vortex/src/editions/core/v2026_08_2.rs deleted file mode 100644 index 6c423b2f838..00000000000 --- a/vortex/src/editions/core/v2026_08_2.rs +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The August 2026 draft core edition adding canonical Map arrays. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The third August 2026 edition of the `core` family. -pub const CORE_2026_08_2: EditionId = EditionId::new("core", 2026, 8, 2); - -/// The declaration of [`CORE_2026_08_2`] and the components that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: CORE_2026_08_2, - min_vortex_version: None, - }, - added: &[EditionMember::array(&"vortex.map")], -}; diff --git a/vortex/src/editions/core/v2026_08_3.rs b/vortex/src/editions/core/v2026_08_3.rs deleted file mode 100644 index 20c3e092b77..00000000000 --- a/vortex/src/editions/core/v2026_08_3.rs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The August 2026 draft core edition adding Variant arrays and UUID extension dtypes. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The fourth August 2026 edition of the `core` family. -pub const CORE_2026_08_3: EditionId = EditionId::new("core", 2026, 8, 3); - -/// The declaration of [`CORE_2026_08_3`] and the components that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: CORE_2026_08_3, - min_vortex_version: None, - }, - added: &[ - EditionMember::array(&"vortex.parquet.variant"), - EditionMember::array(&"vortex.variant"), - EditionMember::dtype(&"vortex.uuid"), - ], -}; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index a6dd8ee7fe9..39a811c7203 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -3,71 +3,49 @@ //! The Vortex edition declarations. //! -//! [`vortex_edition`] provides the types, session variables, and test harness. The actual -//! first-party declarations live here, one module per edition. The default session first -//! registers them with [`crate::editions::register_default_editions`] and then selects its write -//! policy with [`crate::editions::enable_default_editions`]. -//! -//! Members carry a [`crate::editions::ComponentKind`]: arrays a written array may use, extension -//! dtypes its schema may contain, and the aggregates zone maps record. Every kind is restricted to -//! its declared members, so an empty set permits no components of that kind. +//! [`vortex_edition`] provides the types, session variables, test harness, and the +//! first-party declarations themselves. This module re-exports them and owns the session +//! wiring: the default session first registers them with +//! [`crate::editions::register_default_editions`] and then selects its write policy with +//! [`crate::editions::enable_default_editions`]. //! //! The default file writer resolves the session's enabled editions at write time. The -//! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08_1`], and -//! additionally enables the latest preview edition when the `unstable_encodings` feature is +//! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08`], and +//! additionally enables the latest unstable edition when the `unstable_encodings` feature is //! selected. -pub mod core; -pub mod preview; #[cfg(test)] mod tests; -pub use vortex_edition::ComponentKind; +pub use vortex_edition::EDITION_DECLARATIONS; pub use vortex_edition::Edition; pub use vortex_edition::EditionDeclaration; pub use vortex_edition::EditionId; pub use vortex_edition::EditionInclusion; -pub use vortex_edition::EditionMember; pub use vortex_edition::EditionSession; pub use vortex_edition::EditionSessionExt; pub use vortex_edition::EnabledEditions; +pub use vortex_edition::declarations::core; +pub use vortex_edition::declarations::core::CORE_2025_05_0; +pub use vortex_edition::declarations::core::CORE_2025_06_0; +pub use vortex_edition::declarations::core::CORE_2025_10_0; +pub use vortex_edition::declarations::core::CORE_2026_07_0; +pub use vortex_edition::declarations::core::CORE_2026_08; +pub use vortex_edition::declarations::unstable; +pub use vortex_edition::declarations::unstable::UNSTABLE_2025_05_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_02_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_04_0; +pub use vortex_edition::declarations::unstable::UNSTABLE_2026_06_0; use vortex_error::VortexExpect; use vortex_error::vortex_err; use vortex_session::VortexSession; -pub use self::core::CORE_2025_05_0; -pub use self::core::CORE_2025_06_0; -pub use self::core::CORE_2025_10_0; -pub use self::core::CORE_2026_08_0; -pub use self::core::CORE_2026_08_1; -pub use self::core::CORE_2026_08_2; -pub use self::core::CORE_2026_08_3; -pub use self::preview::PREVIEW_2025_05_0; -pub use self::preview::PREVIEW_2026_02_0; -pub use self::preview::PREVIEW_2026_04_0; -pub use self::preview::PREVIEW_2026_06_0; - /// The `core` edition enabled for writing by the default Vortex session. -pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08_1; +pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08; -/// The `preview` edition enabled for writing by the default Vortex session when the +/// The `unstable` edition enabled for writing by the default Vortex session when the /// `unstable_encodings` feature is selected. -pub const DEFAULT_PREVIEW_EDITION: EditionId = PREVIEW_2026_06_0; - -/// The first-party Vortex edition declarations. -pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ - &core::v2025_05::DECLARATION, - &core::v2025_06::DECLARATION, - &core::v2025_10::DECLARATION, - &core::v2026_08::DECLARATION_0, - &core::v2026_08::DECLARATION_1, - &core::v2026_08_2::DECLARATION, - &core::v2026_08_3::DECLARATION, - &preview::v2025_05::DECLARATION, - &preview::v2026_02::DECLARATION, - &preview::v2026_04::DECLARATION, - &preview::v2026_06::DECLARATION, -]; +pub const DEFAULT_UNSTABLE_EDITION: EditionId = UNSTABLE_2026_06_0; /// Register the Vortex edition declarations with the session's [`EditionSession`]. pub fn register_default_editions(session: &VortexSession) { @@ -81,7 +59,7 @@ pub fn register_default_editions(session: &VortexSession) { /// Enable the default Vortex editions for writing. /// -/// This selects the newest frozen `core` edition and, when configured, the newest preview +/// This selects the newest frozen `core` edition and, when configured, the newest unstable /// edition. All declarations must have been registered first with /// [`register_default_editions`]. pub fn enable_default_editions(session: &VortexSession) { @@ -92,7 +70,7 @@ pub fn enable_default_editions(session: &VortexSession) { #[cfg(feature = "unstable_encodings")] session - .enable_edition(DEFAULT_PREVIEW_EDITION) + .enable_edition(DEFAULT_UNSTABLE_EDITION) .map_err(|e| vortex_err!("{e}")) - .vortex_expect("default preview edition is registered"); + .vortex_expect("default unstable edition is registered"); } diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 9878d2f6d1b..8bb8cd05125 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,7 +23,7 @@ test = false anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } prost-build = { workspace = true } -vortex = { workspace = true } +vortex-edition = { workspace = true } xshell = { workspace = true } [lints] diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 41e538f3619..519545c887e 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -6,7 +6,7 @@ //! Every declared edition gets one TOML file recording what it contains: the identifier, the //! minimum Vortex version whose reader supports it once frozen, and its full encoding set. //! Records are grouped by family — `vortex/editions/core/core2025.05.0.toml` — mirroring the -//! declarations in `vortex/src/editions`, since families version independently. +//! declarations in `vortex-edition/src/declarations`, since families version independently. //! //! A record's mutability follows its edition. A draft is still being assembled, so its record //! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a @@ -22,9 +22,9 @@ use std::path::PathBuf; use anyhow::Context; use anyhow::anyhow; -use vortex::editions::EDITION_DECLARATIONS; -use vortex::editions::Edition; -use vortex::editions::EditionSession; +use vortex_edition::EDITION_DECLARATIONS; +use vortex_edition::Edition; +use vortex_edition::EditionSession; const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; From 3c4b66816e65781731ab20dcebea04c023c417f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 10:56:58 +0000 Subject: [PATCH 07/18] Document the unstable edition family The spec described editions as frozen sets carrying a read-forever guarantee, then named unstable2026.06.0 once in passing without saying what the family is. It is the exception to the whole document: every unstable edition is a permanent draft, it never freezes, and it is only written when the unstable_encodings feature is selected. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- docs/specs/editions.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/specs/editions.md b/docs/specs/editions.md index b3c6d721107..f30f03feb70 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -178,6 +178,19 @@ selected editions, the write fails. function outside the selected editions fails the write. With `allow_unknown`, readers disable a zone map whose aggregate function they do not recognize; ignoring a zone map only reduces pruning and does not affect correctness. +## The `unstable` family + +Alongside `core` there is an `unstable` family, holding encodings that are still being +evaluated. It is the exception to everything above: every `unstable` edition is a permanent +draft, so the family never freezes and carries no read-compatibility guarantee at all. A file +written with these encodings is readable only by a build that knows them, and a future release +may stop supporting one. + +Because of that, the writer only emits them when you opt in — the default session enables the +newest `unstable` edition solely when the `unstable_encodings` cargo feature is selected. +Encodings graduate by being declared in a new `core` edition, which is where they pick up the +read-forever guarantee. + ## Edition registry Registry entries list the edition in which each component first appeared. Later editions in the same family inherit all From f72ecf018f4b089211d8db25656a7ef0a2a68709 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 11:03:52 +0000 Subject: [PATCH 08/18] Declare edition families, and record what each one is for A family had no definition anywhere in the code: it was a bare string repeated on each EditionId, a directory name, and prose in two module doc comments and the spec, none of it reachable from an edition. Nothing validated the name either, so a typo minted a family of one whose editions were unordered against every real edition. Declare families explicitly. EditionFamily carries the name and what the family is for, EditionSession registers them beside editions, and validate() now rejects an edition whose family was never declared, or a family that documents nothing. The exporter writes a family.toml beside each family's editions. That record is documentation rather than a contract, so unlike an edition record it stays editable and the append-only check exempts it. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/scripts/check_edition_records.py | 11 +++- vortex-edition/src/declarations/core/mod.rs | 11 ++++ vortex-edition/src/declarations/mod.rs | 4 ++ .../src/declarations/unstable/mod.rs | 12 ++++ vortex-edition/src/lib.rs | 36 ++++++++++++ vortex-edition/src/session.rs | 43 ++++++++++++++- vortex-edition/src/tests.rs | 46 ++++++++++++++++ vortex/editions/core/family.toml | 12 ++++ vortex/editions/unstable/family.toml | 13 +++++ vortex/src/editions/mod.rs | 12 +++- vortex/src/editions/tests.rs | 3 + xtask/src/generate_editions.rs | 55 +++++++++++++++++++ 12 files changed, 253 insertions(+), 5 deletions(-) create mode 100644 vortex/editions/core/family.toml create mode 100644 vortex/editions/unstable/family.toml diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py index ec4447c241e..863e6c4b762 100644 --- a/.github/scripts/check_edition_records.py +++ b/.github/scripts/check_edition_records.py @@ -13,7 +13,8 @@ A newly added record must also be newer than every edition already recorded for its family: editions are only ever added going forward. Records are grouped by family, so -`vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. +`vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. The +`family.toml` beside them documents the family rather than pinning a contract, so it is exempt. Both revisions are read straight out of the object database, so the check sees committed state only and never the working tree. @@ -45,6 +46,10 @@ # A record carries this exactly when the edition it records is frozen. FROZEN_MARKER = "min_vortex_version" +# Beside each family's editions sits a record of the family itself. That one is documentation +# rather than a contract, so it stays editable and these rules leave it alone. +FAMILY_FILE = "family.toml" + # Every way a record can change other than being added. Renames and copies carry an old path # and a new one; the rest carry one. CHANGE_VERBS = { @@ -86,7 +91,7 @@ def walk(tree: pygit2.Tree, prefix: str) -> Iterator[str]: path = f"{prefix}/{entry.name}" if isinstance(entry, pygit2.Tree): yield from walk(entry, path) - elif entry.name.endswith(".toml"): + elif entry.name.endswith(".toml") and entry.name != FAMILY_FILE: yield path try: @@ -188,6 +193,8 @@ def check(base: pygit2.Commit, head: pygit2.Commit) -> list[str]: old_path, new_path = delta.old_file.path, delta.new_file.path if not under_record_dir(old_path, new_path): continue + if PurePosixPath(new_path).name == FAMILY_FILE: + continue if delta.status == DeltaStatus.ADDED: added.append(new_path) diff --git a/vortex-edition/src/declarations/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs index 84a0dcb8a05..1bc8661c0c6 100644 --- a/vortex-edition/src/declarations/core/mod.rs +++ b/vortex-edition/src/declarations/core/mod.rs @@ -6,6 +6,17 @@ //! One module per edition, each declaring the edition and the components that join the //! family at it; members of earlier editions are inherited and never restated. +use crate::EditionFamily; + +/// The `core` family: what the default writer may emit. +pub static FAMILY: EditionFamily = EditionFamily { + name: "core", + doc: "The encodings the default file writer emits. Every core edition freezes, and a \ +frozen edition carries a read-forever guarantee: a file written with it stays readable by \ +every later Vortex release. New encodings join by being declared in a new edition; an \ +edition that has frozen never changes again.", +}; + pub mod v2025_05; pub mod v2025_06; pub mod v2025_10; diff --git a/vortex-edition/src/declarations/mod.rs b/vortex-edition/src/declarations/mod.rs index 0df0f58974c..07dd540d0ba 100644 --- a/vortex-edition/src/declarations/mod.rs +++ b/vortex-edition/src/declarations/mod.rs @@ -15,6 +15,10 @@ pub mod core; pub mod unstable; use crate::EditionDeclaration; +use crate::EditionFamily; + +/// The first-party edition families. Every family must be declared before its editions. +pub static EDITION_FAMILIES: &[&EditionFamily] = &[&core::FAMILY, &unstable::FAMILY]; /// The first-party Vortex edition declarations. pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ diff --git a/vortex-edition/src/declarations/unstable/mod.rs b/vortex-edition/src/declarations/unstable/mod.rs index 5544ba45c0f..96ea60e4cd9 100644 --- a/vortex-edition/src/declarations/unstable/mod.rs +++ b/vortex-edition/src/declarations/unstable/mod.rs @@ -6,6 +6,18 @@ //! One module per draft edition, each declaring the encodings that join the family at it. //! Members of earlier editions are inherited and never restated. +use crate::EditionFamily; + +/// The `unstable` family: opt-in encodings with no compatibility guarantee. +pub static FAMILY: EditionFamily = EditionFamily { + name: "unstable", + doc: "Opt-in encodings that are still being evaluated. Every unstable edition stays a \ +draft, so the family never freezes and carries no compatibility guarantee: a file written \ +with these encodings is readable only by a build that knows them, and a later release may \ +stop supporting one. The writer emits them only when the `unstable_encodings` feature is \ +selected. An encoding graduates by joining a core edition.", +}; + pub mod v2025_05; pub mod v2026_02; pub mod v2026_04; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index 0f71e520721..48e01ea7662 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -35,6 +35,7 @@ use std::fmt::Display; use std::fmt::Formatter; pub use declarations::EDITION_DECLARATIONS; +pub use declarations::EDITION_FAMILIES; pub use session::EditionSession; pub use session::EditionSessionExt; pub use session::EnabledEditions; @@ -111,6 +112,41 @@ impl Display for EditionId { } } +/// A family of editions: an independently versioned, additive group of encodings, registered +/// with [`EditionSession::declare_family`]. +/// +/// Every [`EditionId`] names one. Declaring the family is what makes the name real: +/// [`EditionSession::validate`] rejects an edition whose family was never declared, so a typo +/// cannot quietly mint a family of one. +#[derive(Clone, Copy, Debug)] +pub struct EditionFamily { + /// The family name, matching the [`EditionId::family`] of its editions, e.g. `core`. + pub name: &'static str, + /// What the family is for. Exported into the family's record, so a few sentences at + /// most: the long form belongs in the published spec. + pub doc: &'static str, +} + +impl EditionFamily { + /// Validate the family's form: a non-empty lowercase name and a non-empty doc. Checked + /// for every declared family by [`EditionSession::validate`]. + pub fn validate(&self) -> Result<(), EditionError> { + if self.name.is_empty() || !self.name.chars().all(|c| c.is_ascii_lowercase()) { + return Err(EditionError::new(format!( + "edition family {:?} must have a non-empty lowercase name, e.g. `core`", + self.name + ))); + } + if self.doc.trim().is_empty() { + return Err(EditionError::new(format!( + "edition family {} must document what it is for", + self.name + ))); + } + Ok(()) + } +} + /// An edition: a named set of encodings with a read-compatibility guarantee, registered with /// [`EditionSession::declare_edition`]. The set itself is computed from the registered /// [`EditionInclusion`]s by [`EditionSession::encodings_in`]. diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index b428062c17c..c1ccd45dabc 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -18,6 +18,7 @@ use crate::ComponentKind; use crate::Edition; use crate::EditionDeclaration; use crate::EditionError; +use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; use crate::parse_release; @@ -36,6 +37,8 @@ pub struct EditionSession { #[derive(Debug, Default)] struct Inner { + /// Keyed by family name. + families: BTreeMap, /// Keyed by the display form of the edition id. editions: BTreeMap, /// One map per component kind, each keyed by interned component id, because ids are @@ -97,6 +100,31 @@ impl EditionSession { Ok(()) } + /// Declare an edition family. Errors if a family with the same name is already + /// declared. Every family must be declared before [`EditionSession::validate`] will + /// accept editions belonging to it. + pub fn declare_family(&self, family: &EditionFamily) -> Result<(), EditionError> { + let mut inner = self.inner.write(); + if inner.families.contains_key(family.name) { + return Err(EditionError::new(format!( + "duplicate edition family {}", + family.name + ))); + } + inner.families.insert(family.name.to_string(), *family); + Ok(()) + } + + /// All declared families, sorted by name. + pub fn families(&self) -> Vec { + self.inner.read().families.values().copied().collect() + } + + /// Find a declared family by name. + pub fn find_family(&self, name: &str) -> Option { + self.inner.read().families.get(name).copied() + } + /// Declare an edition. Errors if an edition with the same id is already declared. pub fn declare_edition(&self, edition: Edition) -> Result<(), EditionError> { let mut inner = self.inner.write(); @@ -160,15 +188,26 @@ impl EditionSession { .collect() } - /// Validate all registered declarations. Errors on inclusions referencing undeclared - /// editions, editions out of chronological order within a family (unversioned drafts + /// Validate all registered declarations. Errors on editions in undeclared families, + /// inclusions referencing undeclared editions, editions out of chronological order within a family (unversioned drafts /// must be newest), malformed version strings, and members requiring a release newer /// than their edition declares. pub fn validate(&self) -> Result<(), EditionError> { let editions = self.editions(); + for family in self.families() { + family.validate()?; + } + for edition in &editions { edition.id.validate()?; + if self.find_family(edition.id.family).is_none() { + return Err(EditionError::new(format!( + "edition {} belongs to undeclared family {}; declare the family before \ + its editions", + edition.id, edition.id.family, + ))); + } if let Some(version) = edition.min_vortex_version && parse_release(version).is_none() { diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 09e1345615a..e023359760e 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -5,12 +5,23 @@ use vortex_session::VortexSession; use crate::Edition; use crate::EditionDeclaration; +use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; use crate::EditionSession; use crate::EditionSessionExt; use crate::EnabledEditions; +static TEST_FAMILY: EditionFamily = EditionFamily { + name: "test", + doc: "A family used by the unit tests.", +}; + +static OTHER_FAMILY: EditionFamily = EditionFamily { + name: "other", + doc: "A second family, for checking that families stay independent.", +}; + const FIRST: EditionId = EditionId::new("test", 2026, 1, 0); const SECOND: EditionId = EditionId::new("test", 2026, 7, 0); @@ -33,6 +44,9 @@ static DECLARATIONS: &[EditionDeclaration] = &[ fn session() -> EditionSession { let editions = EditionSession::empty(); + editions + .declare_family(&TEST_FAMILY) + .unwrap_or_else(|e| panic!("declaring the test family: {e}")); for declaration in DECLARATIONS { editions .declare(declaration) @@ -98,6 +112,7 @@ fn drafts_and_current() { // Freezing the first edition makes it current; the second stays a draft. let editions = EditionSession::empty(); + editions.declare_family(&TEST_FAMILY).unwrap(); editions .declare_edition(Edition { id: FIRST, @@ -179,8 +194,11 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi }; let session = VortexSession::empty().with::(); + session.editions().declare_family(&TEST_FAMILY)?; + session.editions().declare_family(&OTHER_FAMILY)?; session.register_edition(&DECLARATIONS[0])?; session.register_edition(&OTHER_DECLARATION)?; + session.editions().validate()?; session.enable_edition(FIRST)?; session.enable_edition(OTHER)?; @@ -274,3 +292,31 @@ fn edition_ids_order_within_family_only() { fn edition_id_display() { assert_eq!(FIRST.to_string(), "test2026.01.0"); } + +#[test] +fn families_must_be_declared_before_their_editions() -> Result<(), crate::EditionError> { + // An edition whose family was never declared: the name would otherwise be whatever the + // declaration happened to spell, and a typo would mint a family of one. + let editions = EditionSession::empty(); + editions.declare(&DECLARATIONS[0])?; + assert!(editions.validate().is_err()); + + editions.declare_family(&TEST_FAMILY)?; + editions.validate()?; + + // Declaring the same family twice is an error, as it is for editions. + assert!(editions.declare_family(&TEST_FAMILY).is_err()); + Ok(()) +} + +#[test] +fn families_must_document_themselves() { + let editions = EditionSession::empty(); + editions + .declare_family(&EditionFamily { + name: "undocumented", + doc: " ", + }) + .unwrap(); + assert!(editions.validate().is_err()); +} diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml new file mode 100644 index 00000000000..ebff1159dba --- /dev/null +++ b/vortex/editions/core/family.toml @@ -0,0 +1,12 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "core" + +doc = """ +The encodings the default file writer emits. Every core edition freezes, and a frozen +edition carries a read-forever guarantee: a file written with it stays readable by every +later Vortex release. New encodings join by being declared in a new edition; an edition that +has frozen never changes again. +""" diff --git a/vortex/editions/unstable/family.toml b/vortex/editions/unstable/family.toml new file mode 100644 index 00000000000..19c6b159176 --- /dev/null +++ b/vortex/editions/unstable/family.toml @@ -0,0 +1,13 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "unstable" + +doc = """ +Opt-in encodings that are still being evaluated. Every unstable edition stays a draft, so +the family never freezes and carries no compatibility guarantee: a file written with these +encodings is readable only by a build that knows them, and a later release may stop +supporting one. The writer emits them only when the `unstable_encodings` feature is +selected. An encoding graduates by joining a core edition. +""" diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 39a811c7203..2538a270901 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -18,8 +18,10 @@ mod tests; pub use vortex_edition::EDITION_DECLARATIONS; +pub use vortex_edition::EDITION_FAMILIES; pub use vortex_edition::Edition; pub use vortex_edition::EditionDeclaration; +pub use vortex_edition::EditionFamily; pub use vortex_edition::EditionId; pub use vortex_edition::EditionInclusion; pub use vortex_edition::EditionSession; @@ -47,8 +49,16 @@ pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08; /// `unstable_encodings` feature is selected. pub const DEFAULT_UNSTABLE_EDITION: EditionId = UNSTABLE_2026_06_0; -/// Register the Vortex edition declarations with the session's [`EditionSession`]. +/// Register the Vortex edition families and declarations with the session's +/// [`EditionSession`]. pub fn register_default_editions(session: &VortexSession) { + for family in EDITION_FAMILIES { + session + .editions() + .declare_family(family) + .map_err(|e| vortex_err!("{e}")) + .vortex_expect("edition families are valid"); + } for declaration in EDITION_DECLARATIONS { session .register_edition(declaration) diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index 810e93b0870..33edab68649 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -55,6 +55,9 @@ use super::PREVIEW_2026_06_0; fn session() -> Result { let session = EditionSession::empty(); + for family in super::EDITION_FAMILIES { + session.declare_family(family)?; + } for declaration in EDITION_DECLARATIONS { session.declare(declaration)?; } diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 519545c887e..5a23739f092 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -23,7 +23,9 @@ use std::path::PathBuf; use anyhow::Context; use anyhow::anyhow; use vortex_edition::EDITION_DECLARATIONS; +use vortex_edition::EDITION_FAMILIES; use vortex_edition::Edition; +use vortex_edition::EditionFamily; use vortex_edition::EditionSession; const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; @@ -38,6 +40,44 @@ const DRAFT_NOTE: &str = "\ # record changes with it. Recording a min_vortex_version freezes the edition, after which # this file may never change again."; +/// The file recording what a family is, beside that family's editions. +const FAMILY_FILE: &str = "family.toml"; + +/// Render a family's record: its name and what it is for. Unlike an edition record this is +/// documentation, not a contract, so it stays editable. +fn family_record(family: &EditionFamily) -> String { + let mut lines = vec![ + GENERATED_BY.to_string(), + "# This describes the family the editions beside it belong to.".to_string(), + String::new(), + format!("name = \"{}\"", family.name), + String::new(), + "doc = \"\"\"".to_string(), + ]; + lines.extend(wrap(family.doc, 92)); + lines.extend(["\"\"\"".to_string(), String::new()]); + lines.join("\n") +} + +/// Wrap prose to a column, so a long doc reads as a paragraph rather than one endless line. +fn wrap(text: &str, width: usize) -> Vec { + let mut lines = Vec::new(); + let mut line = String::new(); + for word in text.split_whitespace() { + if !line.is_empty() && line.len() + 1 + word.len() > width { + lines.push(std::mem::take(&mut line)); + } + if !line.is_empty() { + line.push(' '); + } + line.push_str(word); + } + if !line.is_empty() { + lines.push(line); + } + lines +} + /// Render one edition's record. Deterministic: every list is sorted by encoding id, so the /// generated bytes depend only on the declarations. fn record(session: &EditionSession, edition: &Edition) -> String { @@ -131,6 +171,11 @@ pub fn generate_editions() -> anyhow::Result<()> { fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; let session = EditionSession::empty(); + for family in EDITION_FAMILIES { + session + .declare_family(family) + .map_err(|error| anyhow!("declaring edition families: {error}"))?; + } for declaration in EDITION_DECLARATIONS { session .declare(declaration) @@ -141,6 +186,16 @@ pub fn generate_editions() -> anyhow::Result<()> { .map_err(|error| anyhow!("validating editions: {error}"))?; let mut expected = BTreeSet::new(); + for family in session.families() { + let relative = format!("{}/{FAMILY_FILE}", family.name); + let path = dir.join(&relative); + expected.insert(relative); + fs::create_dir_all(dir.join(family.name)) + .with_context(|| format!("creating the {} record directory", family.name))?; + fs::write(&path, family_record(&family)) + .with_context(|| format!("writing {}", path.display()))?; + } + for edition in session.editions() { let relative = format!("{}/{}.toml", edition.id.family, edition.id); let path = dir.join(&relative); From ffcde6e03208b118936857d5ef4048a8d6a4ac4d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 15:51:25 +0000 Subject: [PATCH 09/18] Port the frozen-record check to an xtask The check lived in a Python script run by a job of its own, which meant a second language, a second dependency manager, and a second CI job for a rule that belongs beside the exporter that writes the records. Move it into `cargo run -p xtask -- check-editions` and run it in the generated-files job, next to generate-fbs, generate-proto and generate-editions. The two halves now share their layout constants rather than restating the directory name and the family file in two languages. git2 reads the object database, mirroring what pygit2 did: revisions resolve through revparse, renames come from the diff's own similarity detection, and both revisions are read from commit trees, so the check still sees committed state only. The job's checkout gains fetch-depth: 0 for the merge base. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/scripts/check_edition_records.py | 264 ------------------- .github/workflows/ci.yml | 25 +- Cargo.lock | 45 ++++ Cargo.toml | 2 + xtask/Cargo.toml | 2 + xtask/src/check_editions.rs | 311 +++++++++++++++++++++++ xtask/src/generate_editions.rs | 9 +- xtask/src/main.rs | 10 + 8 files changed, 384 insertions(+), 284 deletions(-) delete mode 100644 .github/scripts/check_edition_records.py create mode 100644 xtask/src/check_editions.rs diff --git a/.github/scripts/check_edition_records.py b/.github/scripts/check_edition_records.py deleted file mode 100644 index 863e6c4b762..00000000000 --- a/.github/scripts/check_edition_records.py +++ /dev/null @@ -1,264 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.11" -# dependencies = ["pygit2>=1.14"] -# /// -"""Check that frozen edition records under `vortex/editions` never change. - -A record's mutability follows its edition. A draft is still being assembled, so its record may -change, be renamed, or be dropped. Freezing -- recording a `min_vortex_version` -- turns the -record into a read-forever contract, and from then on it may never change again. Whether a -record was frozen is read from the base revision, so a change cannot unfreeze an edition and -edit it in the same diff. - -A newly added record must also be newer than every edition already recorded for its family: -editions are only ever added going forward. Records are grouped by family, so -`vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. The -`family.toml` beside them documents the family rather than pinning a contract, so it is exempt. - -Both revisions are read straight out of the object database, so the check sees committed -state only and never the working tree. - -Usage: - uv run --script .github/scripts/check_edition_records.py --base origin/develop -""" - -from __future__ import annotations - -import argparse -import re -import sys -import tomllib -from pathlib import PurePosixPath -from typing import Any, Iterator - -import pygit2 -from pygit2.enums import DeltaStatus - -RECORD_DIR = "vortex/editions" - -# `core/core2026.08.0.toml`: the file name is the edition id and its directory is the family, -# so a record's identity is visible in the diff without reading the file. -RECORD_NAME = re.compile( - r"^(?P[a-z]+)(?P\d{4})\.(?P\d{2})\.(?P\d+)\.toml$" -) - -# A record carries this exactly when the edition it records is frozen. -FROZEN_MARKER = "min_vortex_version" - -# Beside each family's editions sits a record of the family itself. That one is documentation -# rather than a contract, so it stays editable and these rules leave it alone. -FAMILY_FILE = "family.toml" - -# Every way a record can change other than being added. Renames and copies carry an old path -# and a new one; the rest carry one. -CHANGE_VERBS = { - DeltaStatus.MODIFIED: "modifies", - DeltaStatus.DELETED: "deletes", - DeltaStatus.RENAMED: "renames", - DeltaStatus.COPIED: "copies", - DeltaStatus.TYPECHANGE: "retypes", -} - -REMEDY = ( - "A frozen edition is immutable. To add encodings, declare a NEW edition in\n" - " vortex-edition/src/declarations// and regenerate the records with\n" - " `cargo run -p xtask -- generate-editions`." -) - - -def parse_record(text: str, where: str) -> dict[str, Any]: - try: - return tomllib.loads(text) - except tomllib.TOMLDecodeError as error: - sys.exit(f"{where} is not valid TOML: {error}") - - -def read_record(commit: pygit2.Commit, path: str) -> dict[str, Any] | None: - """Parse a record out of a commit's tree, or None when it holds no such file.""" - try: - blob = commit.tree[path] - except KeyError: - return None - return parse_record(blob.data.decode(), f"{path} at {commit.short_id}") - - -def record_paths(commit: pygit2.Commit) -> Iterator[str]: - """Every record path in a commit, relative to the repository root.""" - - def walk(tree: pygit2.Tree, prefix: str) -> Iterator[str]: - for entry in tree: - path = f"{prefix}/{entry.name}" - if isinstance(entry, pygit2.Tree): - yield from walk(entry, path) - elif entry.name.endswith(".toml") and entry.name != FAMILY_FILE: - yield path - - try: - records = commit.tree[RECORD_DIR] - except KeyError: - return - yield from walk(records, RECORD_DIR) - - -def parse_name(name: str) -> tuple[str, tuple[int, int, int]]: - """Split a record file name into its family and its chronological sort key.""" - match = RECORD_NAME.match(name) - if match is None: - sys.exit( - f"{RECORD_DIR}/{name} is not a valid record name.\n" - "Records are named after the edition they record, e.g. " - "`core/core2026.08.0.toml`." - ) - return match["family"], (int(match["year"]), int(match["month"]), int(match["version"])) - - -def changed_records(base: pygit2.Commit, head: pygit2.Commit) -> pygit2.Diff: - """The record directory's diff between two commits, with renames detected.""" - diff = base.tree.diff_to_tree(head.tree) - diff.find_similar() - return diff - - -def newest_recorded(commit: pygit2.Commit) -> dict[str, tuple[int, int, int]]: - """The newest edition already recorded for each family at `commit`.""" - newest: dict[str, tuple[int, int, int]] = {} - for path in record_paths(commit): - family, key = parse_name(PurePosixPath(path).name) - newest[family] = max(key, newest.get(family, (0, 0, 0))) - return newest - - -def check_modification(before: dict[str, Any], after: dict[str, Any], name: str) -> list[str]: - """A frozen record may not change at all; name the fields that did.""" - changed = sorted( - key for key in before.keys() | after.keys() if before.get(key) != after.get(key) - ) - if not changed: - return [] - if FROZEN_MARKER in changed and FROZEN_MARKER not in after: - return [ - f"unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a " - "read-forever guarantee and may never return to draft" - ] - return [f"modifies the frozen record {name}: {', '.join(changed)}"] - - -def check_addition( - path: str, record: dict[str, Any], newest: dict[str, tuple[int, int, int]] -) -> list[str]: - """A new record must extend its family's chronology, and be filed under it.""" - errors = [] - name = PurePosixPath(path).name - family, key = parse_name(name) - - previous = newest.get(family) - if previous is not None and key <= previous: - recorded = f"{previous[0]}.{previous[1]:02}.{previous[2]}" - errors.append( - f"adds {name}, which is not newer than the {family} edition already " - f"recorded ({family}{recorded}). Editions may only be added going forward." - ) - - # A record's family decides which chronology it extends, so the directory it sits in has - # to agree with the family its name declares. - directory = PurePosixPath(path).parent.name - if directory != family: - errors.append( - f"adds {name} under {directory}/, but it records a {family} edition; " - "records are grouped by family" - ) - - # The file name is the edition's identity, so it has to agree with the content. - edition = record.get("edition") - if edition is None: - errors.append(f"adds {name}, which has no `edition` field") - elif edition != name.removesuffix(".toml"): - errors.append( - f"adds {name}, which records edition {edition!r}; the file name must be the edition id" - ) - return errors - - -def under_record_dir(*paths: str | None) -> bool: - return any(path is not None and path.startswith(f"{RECORD_DIR}/") for path in paths) - - -def check(base: pygit2.Commit, head: pygit2.Commit) -> list[str]: - errors: list[str] = [] - added: list[str] = [] - - for patch in changed_records(base, head): - delta = patch.delta - old_path, new_path = delta.old_file.path, delta.new_file.path - if not under_record_dir(old_path, new_path): - continue - if PurePosixPath(new_path).name == FAMILY_FILE: - continue - - if delta.status == DeltaStatus.ADDED: - added.append(new_path) - continue - - # Frozen-ness comes from the base revision, so a diff cannot unfreeze an edition and - # then edit it. A draft's record is free to change, move, or go away with the draft. - before = read_record(base, old_path) - if before is None or FROZEN_MARKER not in before: - continue - - if delta.status == DeltaStatus.MODIFIED: - after = read_record(head, new_path) or {} - errors.extend(check_modification(before, after, PurePosixPath(new_path).name)) - else: - verb = CHANGE_VERBS.get(delta.status, "changes") - moved = old_path if old_path == new_path else f"{old_path} -> {new_path}" - errors.append(f"{verb} the frozen record {moved}") - - newest = newest_recorded(base) - for path in sorted(added): - record = read_record(head, path) - if record is None: - continue - errors.extend(check_addition(path, record, newest)) - - return errors - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--base", - default="origin/develop", - help="the revision to compare against (default: origin/develop)", - ) - args = parser.parse_args() - - repo = pygit2.Repository(pygit2.discover_repository(".")) - try: - base_tip = repo.revparse_single(args.base).peel(pygit2.Commit) - except KeyError: - sys.exit(f"cannot resolve {args.base!r} in this repository") - - head = repo.head.peel(pygit2.Commit) - merge_base = repo.merge_base(base_tip.id, head.id) - if merge_base is None: - sys.exit( - f"{args.base} and HEAD have no common ancestor.\n" - "The checkout is probably too shallow; this check needs `fetch-depth: 0`." - ) - base = repo[merge_base] - - errors = check(base, head) - if not errors: - print(f"{RECORD_DIR} preserves every frozen record against {args.base} ({base.short_id}).") - return 0 - - print(f"This change breaks the edition records in {RECORD_DIR}:\n", file=sys.stderr) - for error in errors: - print(f" - it {error}", file=sys.stderr) - print(f"\n{REMEDY}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faa7f405908..9f2acf758fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,24 +64,6 @@ jobs: -c .yamllint.yaml \ .github/ - edition-records: - name: "Frozen edition records never change" - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - # The check compares against the merge base, so it needs real history. - fetch-depth: 0 - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false - - name: Check edition records - run: | - BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" - uv run --script .github/scripts/check_edition_records.py --base "$BASE" - python-lint: name: "Python (lint)" runs-on: >- @@ -747,6 +729,9 @@ jobs: with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # check-editions compares against the merge base, so it needs real history. + fetch-depth: 0 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -759,6 +744,10 @@ jobs: - name: "regenerate the edition records" run: | cargo run --profile ci -p xtask -- generate-editions + - name: "check frozen edition records never change" + run: | + BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" + cargo run --profile ci -p xtask -- check-editions --base "$BASE" - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi diff --git a/Cargo.lock b/Cargo.lock index f53a363cdcf..cf8cc6d7344 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4507,6 +4507,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "git2" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +dependencies = [ + "bitflags 2.13.1", + "libc", + "libgit2-sys", + "log", + "url", +] + [[package]] name = "glob" version = "0.3.4" @@ -6200,6 +6213,18 @@ dependencies = [ "cc", ] +[[package]] +name = "libgit2-sys" +version = "0.18.7+1.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.8.9" @@ -6285,6 +6310,18 @@ dependencies = [ "escape8259", ] +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "line-clipping" version = "0.3.8" @@ -10413,6 +10450,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -12199,7 +12242,9 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "git2", "prost-build", + "toml", "vortex-edition", "xshell", ] diff --git a/Cargo.toml b/Cargo.toml index ae164db13c2..ca3f47b6add 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -242,6 +242,7 @@ rstest = "0.26.1" rstest_reuse = "0.7.0" rustc-hash = "2.1.1" rustix = { version = "1.1", features = ["fs"] } +git2 = { version = "0.20", default-features = false } serde = "1.0.221" serde_json = "1.0.138" serde_test = "1.0.176" @@ -278,6 +279,7 @@ tokio-stream = "0.1.17" tokio-util = "0.7.17" tpchgen = "3.0.0" tpchgen-arrow = "3.0.0" +toml = "0.9" tracing = { version = "0.1.41", default-features = false } tracing-perfetto = "0.1.5" tracing-subscriber = "0.3" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 8bb8cd05125..fedd46b7dd8 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -22,7 +22,9 @@ test = false [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +git2 = { workspace = true } prost-build = { workspace = true } +toml = { workspace = true } vortex-edition = { workspace = true } xshell = { workspace = true } diff --git a/xtask/src/check_editions.rs b/xtask/src/check_editions.rs new file mode 100644 index 00000000000..56ed978bf51 --- /dev/null +++ b/xtask/src/check_editions.rs @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Check that frozen edition records under `vortex/editions` never change. +//! +//! A record's mutability follows its edition. A draft is still being assembled, so its record +//! may change, be renamed, or be dropped. Freezing — recording a `min_vortex_version` — turns +//! the record into a read-forever contract, and from then on it may never change again. +//! Whether a record was frozen is read from the base revision, so a change cannot unfreeze an +//! edition and edit it in the same diff. +//! +//! A newly added record must also be newer than every edition already recorded for its +//! family: editions are only ever added going forward. Records are grouped by family, so +//! `vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. The +//! `family.toml` beside them documents the family rather than pinning a contract, so it is +//! exempt. +//! +//! Both revisions are read out of the object database, so the check sees committed state only +//! and never the working tree. + +use std::collections::BTreeMap; +use std::path::Path; + +use anyhow::Context; +use anyhow::anyhow; +use anyhow::bail; +use git2::Commit; +use git2::Delta; +use git2::DiffFindOptions; +use git2::Repository; +use git2::TreeWalkMode; +use git2::TreeWalkResult; +use toml::Table; + +use crate::generate_editions::FAMILY_FILE; +use crate::generate_editions::RECORD_DIR; + +/// A record carries this key exactly when the edition it records is frozen. +const FROZEN_MARKER: &str = "min_vortex_version"; + +const REMEDY: &str = "\ +A frozen edition is immutable. To add encodings, declare a NEW edition in + vortex-edition/src/declarations// and regenerate the records with + `cargo run -p xtask -- generate-editions`."; + +/// An edition's position in its family's chronology, from `..`. +type Chronology = (u16, u8, u8); + +/// Split a record file name into its family and its place in that family's chronology. +fn parse_name(name: &str) -> anyhow::Result<(&str, Chronology)> { + let malformed = || { + anyhow!( + "{RECORD_DIR}/{name} is not a valid record name. Records are named after the \ + edition they record, e.g. `core/core2026.08.0.toml`." + ) + }; + + let stem = name.strip_suffix(".toml").ok_or_else(malformed)?; + let split = stem + .find(|c: char| c.is_ascii_digit()) + .ok_or_else(malformed)?; + let (family, version) = stem.split_at(split); + if family.is_empty() || !family.chars().all(|c| c.is_ascii_lowercase()) { + return Err(malformed()); + } + + let parts: Vec<&str> = version.split('.').collect(); + let [year, month, version] = parts.as_slice() else { + return Err(malformed()); + }; + if year.len() != 4 || month.len() != 2 { + return Err(malformed()); + } + let chronology = ( + year.parse().map_err(|_| malformed())?, + month.parse().map_err(|_| malformed())?, + version.parse().map_err(|_| malformed())?, + ); + Ok((family, chronology)) +} + +/// Parse a record out of a commit's tree, or `None` when it holds no such file. +fn read_record(repo: &Repository, commit: &Commit, path: &str) -> anyhow::Result> { + let Ok(entry) = commit.tree()?.get_path(Path::new(path)) else { + return Ok(None); + }; + let blob = entry.to_object(repo)?.peel_to_blob()?; + let text = std::str::from_utf8(blob.content()) + .with_context(|| format!("{path} at {} is not UTF-8", commit.id()))?; + Ok(Some(text.parse::().with_context(|| { + format!("{path} at {} is not valid TOML", commit.id()) + })?)) +} + +/// The newest edition already recorded for each family at `commit`. +fn newest_recorded( + repo: &Repository, + commit: &Commit, +) -> anyhow::Result> { + let mut newest = BTreeMap::new(); + let Ok(entry) = commit.tree()?.get_path(Path::new(RECORD_DIR)) else { + return Ok(newest); + }; + let records = entry.to_object(repo)?.peel_to_tree()?; + + let mut malformed = None; + records.walk(TreeWalkMode::PreOrder, |_, entry| { + let Some(name) = entry.name() else { + return TreeWalkResult::Ok; + }; + if !name.ends_with(".toml") || name == FAMILY_FILE { + return TreeWalkResult::Ok; + } + match parse_name(name) { + Ok((family, chronology)) => { + let slot = newest.entry(family.to_string()).or_insert(chronology); + *slot = (*slot).max(chronology); + TreeWalkResult::Ok + } + Err(error) => { + malformed = Some(error); + TreeWalkResult::Abort + } + } + })?; + match malformed { + Some(error) => Err(error), + None => Ok(newest), + } +} + +/// A frozen record may not change at all; name the fields that did. +fn check_modification(before: &Table, after: &Table, name: &str) -> Vec { + let mut changed: Vec<&str> = before + .keys() + .chain(after.keys()) + .map(String::as_str) + .filter(|key| before.get(*key) != after.get(*key)) + .collect(); + changed.sort_unstable(); + changed.dedup(); + + if changed.is_empty() { + return vec![]; + } + if changed.contains(&FROZEN_MARKER) && !after.contains_key(FROZEN_MARKER) { + return vec![format!( + "unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a \ + read-forever guarantee and may never return to draft" + )]; + } + vec![format!( + "modifies the frozen record {name}: {}", + changed.join(", ") + )] +} + +/// A new record must extend its family's chronology, and be filed under that family. +fn check_addition( + path: &str, + record: &Table, + newest: &BTreeMap, +) -> anyhow::Result> { + let mut errors = Vec::new(); + let name = Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("{path} has no file name"))?; + let (family, chronology) = parse_name(name)?; + + if let Some(previous) = newest.get(family) + && chronology <= *previous + { + errors.push(format!( + "adds {name}, which is not newer than the {family} edition already recorded \ + ({family}{}.{:02}.{}). Editions may only be added going forward.", + previous.0, previous.1, previous.2, + )); + } + + // A record's family decides which chronology it extends, so the directory it sits in has + // to agree with the family its name declares. + let directory = Path::new(path) + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + .unwrap_or_default(); + if directory != family { + errors.push(format!( + "adds {name} under {directory}/, but it records a {family} edition; records are \ + grouped by family" + )); + } + + // The file name is the edition's identity, so it has to agree with the content. + match record.get("edition").and_then(|edition| edition.as_str()) { + None => errors.push(format!("adds {name}, which has no `edition` field")), + Some(edition) if edition != name.trim_end_matches(".toml") => errors.push(format!( + "adds {name}, which records edition {edition:?}; the file name must be the \ + edition id" + )), + Some(_) => {} + } + Ok(errors) +} + +fn under_record_dir(path: Option<&Path>) -> bool { + path.is_some_and(|path| path.starts_with(RECORD_DIR)) +} + +fn is_family_record(path: Option<&Path>) -> bool { + path.is_some_and(|path| path.file_name().is_some_and(|name| name == FAMILY_FILE)) +} + +fn path_str(path: Option<&Path>) -> String { + path.map(|path| path.display().to_string()) + .unwrap_or_default() +} + +pub fn check_editions(base: &str) -> anyhow::Result<()> { + let repo = Repository::discover(".").context("opening the repository")?; + let base_tip = repo + .revparse_single(base) + .with_context(|| format!("cannot resolve {base:?} in this repository"))? + .peel_to_commit()?; + let head = repo.head()?.peel_to_commit()?; + + let merge_base = repo.merge_base(base_tip.id(), head.id()).with_context(|| { + format!( + "{base} and HEAD have no common ancestor. The checkout is probably too shallow; \ + this check needs `fetch-depth: 0`." + ) + })?; + let base_commit = repo.find_commit(merge_base)?; + + let mut diff = repo.diff_tree_to_tree(Some(&base_commit.tree()?), Some(&head.tree()?), None)?; + diff.find_similar(Some(DiffFindOptions::new().renames(true)))?; + + let newest = newest_recorded(&repo, &base_commit)?; + let mut errors = Vec::new(); + let mut added = Vec::new(); + + for delta in diff.deltas() { + let (old_path, new_path) = (delta.old_file().path(), delta.new_file().path()); + if !under_record_dir(old_path) && !under_record_dir(new_path) { + continue; + } + // The family record is documentation rather than a contract, so it stays editable. + if is_family_record(new_path) || is_family_record(old_path) { + continue; + } + + if delta.status() == Delta::Added { + added.push(path_str(new_path)); + continue; + } + + // Frozen-ness comes from the base revision, so a diff cannot unfreeze an edition and + // then edit it. A draft's record is free to change, move, or go away with the draft. + let old = path_str(old_path); + let Some(before) = read_record(&repo, &base_commit, &old)? else { + continue; + }; + if !before.contains_key(FROZEN_MARKER) { + continue; + } + + let new = path_str(new_path); + if delta.status() == Delta::Modified { + let after = read_record(&repo, &head, &new)?.unwrap_or_default(); + let name = Path::new(&new) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(&new); + errors.extend(check_modification(&before, &after, name)); + } else { + let verb = match delta.status() { + Delta::Deleted => "deletes", + Delta::Renamed => "renames", + Delta::Copied => "copies", + Delta::Typechange => "retypes", + _ => "changes", + }; + let moved = if old == new { + old.clone() + } else { + format!("{old} -> {new}") + }; + errors.push(format!("{verb} the frozen record {moved}")); + } + } + + added.sort(); + for path in &added { + if let Some(record) = read_record(&repo, &head, path)? { + errors.extend(check_addition(path, &record, &newest)?); + } + } + + if errors.is_empty() { + println!("{RECORD_DIR} preserves every frozen record against {base}."); + return Ok(()); + } + + let listed = errors + .iter() + .map(|error| format!(" - it {error}")) + .collect::>() + .join("\n"); + bail!("This change breaks the edition records in {RECORD_DIR}:\n\n{listed}\n\n{REMEDY}"); +} diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 5a23739f092..d4c4cd0cf8f 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -41,7 +41,10 @@ const DRAFT_NOTE: &str = "\ # this file may never change again."; /// The file recording what a family is, beside that family's editions. -const FAMILY_FILE: &str = "family.toml"; +pub const FAMILY_FILE: &str = "family.toml"; + +/// The edition records, relative to the repository root. +pub const RECORD_DIR: &str = "vortex/editions"; /// Render a family's record: its name and what it is for. Unlike an edition record this is /// documentation, not a contract, so it stays editable. @@ -167,7 +170,9 @@ fn records_a_frozen_edition(contents: &str) -> bool { } pub fn generate_editions() -> anyhow::Result<()> { - let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../vortex/editions"); + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(RECORD_DIR); fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; let session = EditionSession::empty(); diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 772dd953e0e..8cc582be233 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,12 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod check_editions; mod generate_editions; mod generate_fbs; mod generate_proto; use clap::Parser; +use crate::check_editions::check_editions; use crate::generate_editions::generate_editions; use crate::generate_fbs::generate_fbs; use crate::generate_proto::generate_proto; @@ -19,6 +21,13 @@ struct Xtask { #[derive(clap::Subcommand)] enum Commands { + /// Subcommand to check that frozen edition records never change. + #[command(name = "check-editions")] + CheckEditions { + /// The revision to compare against. + #[arg(long, default_value = "origin/develop")] + base: String, + }, /// Subcommand to regenerate the edition records under `vortex/editions`. #[command(name = "generate-editions")] Editions, @@ -33,6 +42,7 @@ enum Commands { fn main() -> anyhow::Result<()> { let cli = Xtask::parse(); match cli.command { + Commands::CheckEditions { base } => check_editions(&base)?, Commands::Editions => generate_editions()?, Commands::Flatbuffers => generate_fbs()?, Commands::Proto => generate_proto()?, From 0813fb89012023fc4b9bdf3ef05ea8759b92cadf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 16:04:41 +0000 Subject: [PATCH 10/18] Fix the CI failures from adding git2 Two, both mine. The workspace dependency list is sorted -- taplo.toml sets reorder_keys for it -- and I appended git2 and toml wherever the edit landed, which failed lint-toml. And git2 0.20 carries RUSTSEC-2026-0183 and RUSTSEC-2026-0184, two unsoundness advisories fixed in 0.21, which failed the advisories check. Neither reproduces on a `cargo deny check licenses` alone, which is all I had run. While here: run check-editions even when regenerating the records has already failed. The two look at different things -- one at the working tree, one at history -- and a job stops at its first failing step, so a stale record was hiding whether a frozen one had also been edited. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/workflows/ci.yml | 3 +++ Cargo.lock | 5 ++--- Cargo.toml | 4 ++-- xtask/src/check_editions.rs | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f2acf758fe..af5506efb05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -745,6 +745,9 @@ jobs: run: | cargo run --profile ci -p xtask -- generate-editions - name: "check frozen edition records never change" + # Independent of the regeneration above: a stale record must not mask a frozen one + # being edited, nor the other way round. + if: '!cancelled()' run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" cargo run --profile ci -p xtask -- check-editions --base "$BASE" diff --git a/Cargo.lock b/Cargo.lock index cf8cc6d7344..239cbafbd5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4509,15 +4509,14 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ "bitflags 2.13.1", "libc", "libgit2-sys", "log", - "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ca3f47b6add..37f66df1443 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,6 +171,7 @@ geo-types = "0.7.19" geoarrow = "0.8.0" geoarrow-cast = "0.8.0" get_dir = "0.5.0" +git2 = { version = "0.21", default-features = false } glob = "0.3.2" goldenfile = "1" half = { version = "2.7.1", features = ["std", "num-traits"] } @@ -242,7 +243,6 @@ rstest = "0.26.1" rstest_reuse = "0.7.0" rustc-hash = "2.1.1" rustix = { version = "1.1", features = ["fs"] } -git2 = { version = "0.20", default-features = false } serde = "1.0.221" serde_json = "1.0.138" serde_test = "1.0.176" @@ -277,9 +277,9 @@ thiserror = "2.0.3" tokio = { version = "1.52" } tokio-stream = "0.1.17" tokio-util = "0.7.17" +toml = "0.9" tpchgen = "3.0.0" tpchgen-arrow = "3.0.0" -toml = "0.9" tracing = { version = "0.1.41", default-features = false } tracing-perfetto = "0.1.5" tracing-subscriber = "0.3" diff --git a/xtask/src/check_editions.rs b/xtask/src/check_editions.rs index 56ed978bf51..a02179bc70d 100644 --- a/xtask/src/check_editions.rs +++ b/xtask/src/check_editions.rs @@ -105,7 +105,7 @@ fn newest_recorded( let mut malformed = None; records.walk(TreeWalkMode::PreOrder, |_, entry| { - let Some(name) = entry.name() else { + let Ok(name) = entry.name() else { return TreeWalkResult::Ok; }; if !name.ends_with(".toml") || name == FAMILY_FILE { From 2fcf7e89dbafedf56e787aedab29117a556777a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 16:04:57 +0000 Subject: [PATCH 11/18] Quote the step condition with double quotes yamllint's quoted-strings rule requires them, matching the `if` on duckdb-ready. Signed-off-by: "Joe Isaacs" Signed-off-by: Robert Kruszewski --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af5506efb05..e0a843f4429 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -747,7 +747,7 @@ jobs: - name: "check frozen edition records never change" # Independent of the regeneration above: a stale record must not mask a frozen one # being edited, nor the other way round. - if: '!cancelled()' + if: "!cancelled()" run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" cargo run --profile ci -p xtask -- check-editions --base "$BASE" From 7473a14a8f622c22ce6c9ba2acb753d9e8bf275f Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 20 Aug 2026 18:35:50 +0100 Subject: [PATCH 12/18] Adopt develop's edition set after the rebase Develop renamed the unstable family to preview, dropped the 2026.07 draft, and split core 2026.08 into four editions (#9340, #9478). Port those declarations into vortex-edition/src/declarations, regenerate the records per component kind, and teach the exporter the kind-aware API. Signed-off-by: Robert Kruszewski --- vortex-edition/src/declarations/core/mod.rs | 8 +- .../src/declarations/core/v2025_05.rs | 59 +++--- .../src/declarations/core/v2025_06.rs | 7 +- .../src/declarations/core/v2025_10.rs | 9 +- .../src/declarations/core/v2026_07.rs | 20 -- .../src/declarations/core/v2026_08.rs | 43 +++- .../src/declarations/core/v2026_08_2.rs | 21 ++ .../src/declarations/core/v2026_08_3.rs | 25 +++ vortex-edition/src/declarations/mod.rs | 20 +- .../src/declarations/preview/mod.rs | 29 +++ .../src/declarations/preview/v2025_05.rs | 21 ++ .../src/declarations/preview/v2026_02.rs | 21 ++ .../src/declarations/preview/v2026_04.rs | 29 +++ .../src/declarations/preview/v2026_06.rs | 21 ++ .../src/declarations/unstable/mod.rs | 29 --- .../src/declarations/unstable/v2025_05.rs | 20 -- .../src/declarations/unstable/v2026_02.rs | 20 -- .../src/declarations/unstable/v2026_04.rs | 27 --- .../src/declarations/unstable/v2026_06.rs | 20 -- vortex-edition/src/lib.rs | 192 +++++++++++++----- vortex-edition/src/tests.rs | 99 +++++++-- vortex/editions/core/core2025.05.0.toml | 36 +++- vortex/editions/core/core2025.06.0.toml | 26 ++- vortex/editions/core/core2025.10.0.toml | 26 ++- vortex/editions/core/core2026.08.0.toml | 45 +++- ...{core2026.07.0.toml => core2026.08.1.toml} | 42 +++- vortex/editions/core/core2026.08.2.toml | 76 +++++++ vortex/editions/core/core2026.08.3.toml | 82 ++++++++ vortex/editions/core/family.toml | 8 +- .../{unstable => preview}/family.toml | 8 +- .../preview2025.05.0.toml} | 20 +- .../preview2026.02.0.toml} | 20 +- .../preview2026.04.0.toml} | 28 ++- .../preview2026.06.0.toml} | 29 ++- vortex/src/editions/mod.rs | 38 ++-- xtask/src/generate_editions.rs | 77 ++++--- 36 files changed, 942 insertions(+), 359 deletions(-) delete mode 100644 vortex-edition/src/declarations/core/v2026_07.rs create mode 100644 vortex-edition/src/declarations/core/v2026_08_2.rs create mode 100644 vortex-edition/src/declarations/core/v2026_08_3.rs create mode 100644 vortex-edition/src/declarations/preview/mod.rs create mode 100644 vortex-edition/src/declarations/preview/v2025_05.rs create mode 100644 vortex-edition/src/declarations/preview/v2026_02.rs create mode 100644 vortex-edition/src/declarations/preview/v2026_04.rs create mode 100644 vortex-edition/src/declarations/preview/v2026_06.rs delete mode 100644 vortex-edition/src/declarations/unstable/mod.rs delete mode 100644 vortex-edition/src/declarations/unstable/v2025_05.rs delete mode 100644 vortex-edition/src/declarations/unstable/v2026_02.rs delete mode 100644 vortex-edition/src/declarations/unstable/v2026_04.rs delete mode 100644 vortex-edition/src/declarations/unstable/v2026_06.rs rename vortex/editions/core/{core2026.07.0.toml => core2026.08.1.toml} (59%) create mode 100644 vortex/editions/core/core2026.08.2.toml create mode 100644 vortex/editions/core/core2026.08.3.toml rename vortex/editions/{unstable => preview}/family.toml (60%) rename vortex/editions/{unstable/unstable2025.05.0.toml => preview/preview2025.05.0.toml} (53%) rename vortex/editions/{unstable/unstable2026.02.0.toml => preview/preview2026.02.0.toml} (55%) rename vortex/editions/{unstable/unstable2026.04.0.toml => preview/preview2026.04.0.toml} (59%) rename vortex/editions/{unstable/unstable2026.06.0.toml => preview/preview2026.06.0.toml} (55%) diff --git a/vortex-edition/src/declarations/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs index 1bc8661c0c6..0f7e983dcdc 100644 --- a/vortex-edition/src/declarations/core/mod.rs +++ b/vortex-edition/src/declarations/core/mod.rs @@ -11,10 +11,10 @@ use crate::EditionFamily; /// The `core` family: what the default writer may emit. pub static FAMILY: EditionFamily = EditionFamily { name: "core", - doc: "The encodings the default file writer emits. Every core edition freezes, and a \ -frozen edition carries a read-forever guarantee: a file written with it stays readable by \ -every later Vortex release. New encodings join by being declared in a new edition; an \ -edition that has frozen never changes again.", + doc: "The serialized components the default file writer emits. Every core edition \ +freezes, and a frozen edition carries a read-forever guarantee: a file written with it stays \ +readable by every later Vortex release. New components join by being declared in a new \ +edition; an edition that has frozen never changes again.", }; pub mod v2025_05; diff --git a/vortex-edition/src/declarations/core/v2025_05.rs b/vortex-edition/src/declarations/core/v2025_05.rs index 115193619ba..9ead57f1fc2 100644 --- a/vortex-edition/src/declarations/core/v2025_05.rs +++ b/vortex-edition/src/declarations/core/v2025_05.rs @@ -1,44 +1,53 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The baseline `core` edition: stable encodings writable by Vortex 0.36.0. +//! The baseline `core` edition: stable serialized components writable by Vortex 0.36.0. use crate::Edition; use crate::EditionDeclaration; use crate::EditionId; +use crate::EditionMember; /// The first edition of the `core` family, matching the first stable Vortex file release. pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); -/// The declaration of [`CORE_2025_05_0`] and the encodings that join the family at it. +/// The declaration of [`CORE_2025_05_0`] and the components that join the family at it. pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_05_0, min_vortex_version: Some("0.36.0"), }, added: &[ - &"fastlanes.bitpacked", - &"fastlanes.for", - &"vortex.alp", - &"vortex.alprd", - &"vortex.bool", - &"vortex.bytebool", - &"vortex.chunked", - &"vortex.constant", - &"vortex.datetimeparts", - &"vortex.decimal", - &"vortex.decimal_byte_parts", - &"vortex.dict", - &"vortex.ext", - &"vortex.fsst", - &"vortex.list", - &"vortex.null", - &"vortex.primitive", - &"vortex.runend", - &"vortex.sparse", - &"vortex.struct", - &"vortex.varbin", - &"vortex.varbinview", - &"vortex.zigzag", + EditionMember::array(&"fastlanes.bitpacked"), + EditionMember::array(&"fastlanes.for"), + EditionMember::array(&"vortex.alp"), + EditionMember::array(&"vortex.alprd"), + EditionMember::array(&"vortex.bool"), + EditionMember::array(&"vortex.bytebool"), + EditionMember::array(&"vortex.chunked"), + EditionMember::array(&"vortex.constant"), + EditionMember::array(&"vortex.datetimeparts"), + EditionMember::array(&"vortex.decimal"), + EditionMember::array(&"vortex.decimal_byte_parts"), + EditionMember::array(&"vortex.dict"), + EditionMember::array(&"vortex.ext"), + EditionMember::array(&"vortex.fsst"), + EditionMember::array(&"vortex.list"), + EditionMember::array(&"vortex.null"), + EditionMember::array(&"vortex.primitive"), + EditionMember::array(&"vortex.runend"), + EditionMember::array(&"vortex.sparse"), + EditionMember::array(&"vortex.struct"), + EditionMember::array(&"vortex.varbin"), + EditionMember::array(&"vortex.varbinview"), + EditionMember::array(&"vortex.zigzag"), + EditionMember::layout(&"vortex.chunked"), + EditionMember::layout(&"vortex.dict"), + EditionMember::layout(&"vortex.flat"), + EditionMember::layout(&"vortex.stats"), + EditionMember::layout(&"vortex.struct"), + EditionMember::dtype(&"vortex.date"), + EditionMember::dtype(&"vortex.time"), + EditionMember::dtype(&"vortex.timestamp"), ], }; diff --git a/vortex-edition/src/declarations/core/v2025_06.rs b/vortex-edition/src/declarations/core/v2025_06.rs index 7ebd3505799..a2acdb479fb 100644 --- a/vortex-edition/src/declarations/core/v2025_06.rs +++ b/vortex-edition/src/declarations/core/v2025_06.rs @@ -6,6 +6,7 @@ use crate::Edition; use crate::EditionDeclaration; use crate::EditionId; +use crate::EditionMember; /// The June 2025 edition of the `core` family. pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); @@ -16,5 +17,9 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { id: CORE_2025_06_0, min_vortex_version: Some("0.40.0"), }, - added: &[&"vortex.pco", &"vortex.sequence", &"vortex.zstd"], + added: &[ + EditionMember::array(&"vortex.pco"), + EditionMember::array(&"vortex.sequence"), + EditionMember::array(&"vortex.zstd"), + ], }; diff --git a/vortex-edition/src/declarations/core/v2025_10.rs b/vortex-edition/src/declarations/core/v2025_10.rs index 6124c9e94b3..1f5ad573e7b 100644 --- a/vortex-edition/src/declarations/core/v2025_10.rs +++ b/vortex-edition/src/declarations/core/v2025_10.rs @@ -6,6 +6,7 @@ use crate::Edition; use crate::EditionDeclaration; use crate::EditionId; +use crate::EditionMember; /// The October 2025 edition of the `core` family. pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); @@ -17,9 +18,9 @@ pub static DECLARATION: EditionDeclaration = EditionDeclaration { min_vortex_version: Some("0.54.0"), }, added: &[ - &"fastlanes.rle", - &"vortex.fixed_size_list", - &"vortex.listview", - &"vortex.masked", + EditionMember::array(&"fastlanes.rle"), + EditionMember::array(&"vortex.fixed_size_list"), + EditionMember::array(&"vortex.listview"), + EditionMember::array(&"vortex.masked"), ], }; diff --git a/vortex-edition/src/declarations/core/v2026_07.rs b/vortex-edition/src/declarations/core/v2026_07.rs deleted file mode 100644 index 78e0814d5a4..00000000000 --- a/vortex-edition/src/declarations/core/v2026_07.rs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The `core` edition adding stable encodings released through July 2026. - -use crate::Edition; -use crate::EditionDeclaration; -use crate::EditionId; - -/// The July 2026 edition of the `core` family. -pub const CORE_2026_07_0: EditionId = EditionId::new("core", 2026, 7, 0); - -/// The declaration of [`CORE_2026_07_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: CORE_2026_07_0, - min_vortex_version: Some("0.65.0"), - }, - added: &[&"vortex.variant"], -}; diff --git a/vortex-edition/src/declarations/core/v2026_08.rs b/vortex-edition/src/declarations/core/v2026_08.rs index 904dfb4ef73..176871ab38b 100644 --- a/vortex-edition/src/declarations/core/v2026_08.rs +++ b/vortex-edition/src/declarations/core/v2026_08.rs @@ -1,20 +1,49 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The August 2026 core edition adding the canonical Map encoding. +//! The frozen August 2026 core editions. use crate::Edition; use crate::EditionDeclaration; use crate::EditionId; +use crate::EditionMember; -/// The August 2026 core edition containing canonical Map arrays. -pub const CORE_2026_08: EditionId = EditionId::new("core", 2026, 8, 0); +/// The August 2026 core edition containing zoned layouts. +pub const CORE_2026_08_0: EditionId = EditionId::new("core", 2026, 8, 0); -/// The declaration of [`CORE_2026_08`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { +/// The declaration of [`CORE_2026_08_0`] and the components that join the family at it. +/// +/// The aggregates are the set the default writer records in zone maps. A strategy asking for an +/// aggregate outside this set fails the write instead of producing zone maps an older reader would +/// have to skip. +/// +/// `vortex.sum` is deliberately not a member: zone maps prune, a zone sum does not, and its +/// null-on-empty semantics were changed and reverted within a single week. The writer no longer +/// records it, so the two stay consistent. +pub static DECLARATION_0: EditionDeclaration = EditionDeclaration { edition: Edition { - id: CORE_2026_08, + id: CORE_2026_08_0, min_vortex_version: Some("0.84.0"), }, - added: &[&"vortex.map"], + added: &[ + EditionMember::layout(&"vortex.zoned"), + EditionMember::aggregate(&"vortex.bounded_max"), + EditionMember::aggregate(&"vortex.bounded_min"), + EditionMember::aggregate(&"vortex.max"), + EditionMember::aggregate(&"vortex.min"), + EditionMember::aggregate(&"vortex.nan_count"), + EditionMember::aggregate(&"vortex.null_count"), + ], +}; + +/// The second August 2026 edition of the `core` family, adding OnPair arrays. +pub const CORE_2026_08_1: EditionId = EditionId::new("core", 2026, 8, 1); + +/// The declaration of [`CORE_2026_08_1`] and the components that join the family at it. +pub static DECLARATION_1: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2026_08_1, + min_vortex_version: Some("0.84.0"), + }, + added: &[EditionMember::array(&"vortex.onpair")], }; diff --git a/vortex-edition/src/declarations/core/v2026_08_2.rs b/vortex-edition/src/declarations/core/v2026_08_2.rs new file mode 100644 index 00000000000..1869d2bc343 --- /dev/null +++ b/vortex-edition/src/declarations/core/v2026_08_2.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The August 2026 draft core edition adding canonical Map arrays. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; + +/// The third August 2026 edition of the `core` family. +pub const CORE_2026_08_2: EditionId = EditionId::new("core", 2026, 8, 2); + +/// The declaration of [`CORE_2026_08_2`] and the components that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2026_08_2, + min_vortex_version: None, + }, + added: &[EditionMember::array(&"vortex.map")], +}; diff --git a/vortex-edition/src/declarations/core/v2026_08_3.rs b/vortex-edition/src/declarations/core/v2026_08_3.rs new file mode 100644 index 00000000000..cf069c6f615 --- /dev/null +++ b/vortex-edition/src/declarations/core/v2026_08_3.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The August 2026 draft core edition adding Variant arrays and UUID extension dtypes. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; + +/// The fourth August 2026 edition of the `core` family. +pub const CORE_2026_08_3: EditionId = EditionId::new("core", 2026, 8, 3); + +/// The declaration of [`CORE_2026_08_3`] and the components that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2026_08_3, + min_vortex_version: None, + }, + added: &[ + EditionMember::array(&"vortex.parquet.variant"), + EditionMember::array(&"vortex.variant"), + EditionMember::dtype(&"vortex.uuid"), + ], +}; diff --git a/vortex-edition/src/declarations/mod.rs b/vortex-edition/src/declarations/mod.rs index 07dd540d0ba..58b8bc5b89d 100644 --- a/vortex-edition/src/declarations/mod.rs +++ b/vortex-edition/src/declarations/mod.rs @@ -3,7 +3,7 @@ //! The first-party Vortex edition declarations, one module per edition. //! -//! These are plain constants naming encodings by id, so they depend on nothing but the types +//! These are plain constants naming components by id, so they depend on nothing but the types //! in this crate. That keeps them cheap to read: tooling that only needs to know what an //! edition contains — `cargo run -p xtask -- generate-editions`, for one — can depend on //! this crate alone rather than on the whole of `vortex`. @@ -12,23 +12,25 @@ //! the declarations and selecting which of them the default writer may emit. pub mod core; -pub mod unstable; +pub mod preview; use crate::EditionDeclaration; use crate::EditionFamily; /// The first-party edition families. Every family must be declared before its editions. -pub static EDITION_FAMILIES: &[&EditionFamily] = &[&core::FAMILY, &unstable::FAMILY]; +pub static EDITION_FAMILIES: &[&EditionFamily] = &[&core::FAMILY, &preview::FAMILY]; /// The first-party Vortex edition declarations. pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ &core::v2025_05::DECLARATION, &core::v2025_06::DECLARATION, &core::v2025_10::DECLARATION, - &core::v2026_07::DECLARATION, - &core::v2026_08::DECLARATION, - &unstable::v2025_05::DECLARATION, - &unstable::v2026_02::DECLARATION, - &unstable::v2026_04::DECLARATION, - &unstable::v2026_06::DECLARATION, + &core::v2026_08::DECLARATION_0, + &core::v2026_08::DECLARATION_1, + &core::v2026_08_2::DECLARATION, + &core::v2026_08_3::DECLARATION, + &preview::v2025_05::DECLARATION, + &preview::v2026_02::DECLARATION, + &preview::v2026_04::DECLARATION, + &preview::v2026_06::DECLARATION, ]; diff --git a/vortex-edition/src/declarations/preview/mod.rs b/vortex-edition/src/declarations/preview/mod.rs new file mode 100644 index 00000000000..eba693f1719 --- /dev/null +++ b/vortex-edition/src/declarations/preview/mod.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `preview` edition family: opt-in components without a frozen compatibility guarantee. +//! +//! One module per draft edition, each declaring the components that join the family at it. +//! Members of earlier editions are inherited and never restated. + +use crate::EditionFamily; + +/// The `preview` family: opt-in components with no compatibility guarantee. +pub static FAMILY: EditionFamily = EditionFamily { + name: "preview", + doc: "Opt-in components that are still being evaluated. Every preview edition stays a \ +draft, so the family never freezes and carries no compatibility guarantee: a file written \ +with these components is readable only by a build that knows them, and a later release may \ +stop supporting one. The writer emits them only when the `unstable_encodings` feature is \ +selected. A component graduates by joining a core edition.", +}; + +pub mod v2025_05; +pub mod v2026_02; +pub mod v2026_04; +pub mod v2026_06; + +pub use v2025_05::PREVIEW_2025_05_0; +pub use v2026_02::PREVIEW_2026_02_0; +pub use v2026_04::PREVIEW_2026_04_0; +pub use v2026_06::PREVIEW_2026_06_0; diff --git a/vortex-edition/src/declarations/preview/v2025_05.rs b/vortex-edition/src/declarations/preview/v2025_05.rs new file mode 100644 index 00000000000..82028132e34 --- /dev/null +++ b/vortex-edition/src/declarations/preview/v2025_05.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The May 2025 `preview` encoding cohort. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; + +/// The May 2025 draft edition of the `preview` family. +pub const PREVIEW_2025_05_0: EditionId = EditionId::new("preview", 2025, 5, 0); + +/// The declaration of [`PREVIEW_2025_05_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: PREVIEW_2025_05_0, + min_vortex_version: None, + }, + added: &[EditionMember::array(&"fastlanes.delta")], +}; diff --git a/vortex-edition/src/declarations/preview/v2026_02.rs b/vortex-edition/src/declarations/preview/v2026_02.rs new file mode 100644 index 00000000000..0a0be9b2117 --- /dev/null +++ b/vortex-edition/src/declarations/preview/v2026_02.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The February 2026 `preview` encoding cohort. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; + +/// The February 2026 draft edition of the `preview` family. +pub const PREVIEW_2026_02_0: EditionId = EditionId::new("preview", 2026, 2, 0); + +/// The declaration of [`PREVIEW_2026_02_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: PREVIEW_2026_02_0, + min_vortex_version: None, + }, + added: &[EditionMember::array(&"vortex.zstd_buffers")], +}; diff --git a/vortex-edition/src/declarations/preview/v2026_04.rs b/vortex-edition/src/declarations/preview/v2026_04.rs new file mode 100644 index 00000000000..10f35a5db9e --- /dev/null +++ b/vortex-edition/src/declarations/preview/v2026_04.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The April 2026 `preview` component cohort. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; + +/// The April 2026 draft edition of the `preview` family. +pub const PREVIEW_2026_04_0: EditionId = EditionId::new("preview", 2026, 4, 0); + +/// The declaration of [`PREVIEW_2026_04_0`] and the components that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: PREVIEW_2026_04_0, + min_vortex_version: None, + }, + added: &[ + EditionMember::array(&"vortex.patched"), + EditionMember::array(&"vortex.tensor.cosine_similarity"), + EditionMember::array(&"vortex.tensor.inner_product"), + EditionMember::array(&"vortex.tensor.normalized"), + EditionMember::array(&"vortex.tensor.l2_norm"), + EditionMember::dtype(&"vortex.tensor.fixed_shape_tensor"), + EditionMember::dtype(&"vortex.tensor.vector"), + ], +}; diff --git a/vortex-edition/src/declarations/preview/v2026_06.rs b/vortex-edition/src/declarations/preview/v2026_06.rs new file mode 100644 index 00000000000..024ec45a810 --- /dev/null +++ b/vortex-edition/src/declarations/preview/v2026_06.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The June 2026 `preview` component cohort. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; + +/// The June 2026 draft edition of the `preview` family. +pub const PREVIEW_2026_06_0: EditionId = EditionId::new("preview", 2026, 6, 0); + +/// The declaration of [`PREVIEW_2026_06_0`] and the components that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: PREVIEW_2026_06_0, + min_vortex_version: None, + }, + added: &[EditionMember::layout(&"vortex.list")], +}; diff --git a/vortex-edition/src/declarations/unstable/mod.rs b/vortex-edition/src/declarations/unstable/mod.rs deleted file mode 100644 index 96ea60e4cd9..00000000000 --- a/vortex-edition/src/declarations/unstable/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The `unstable` edition family: opt-in encodings without a frozen compatibility guarantee. -//! -//! One module per draft edition, each declaring the encodings that join the family at it. -//! Members of earlier editions are inherited and never restated. - -use crate::EditionFamily; - -/// The `unstable` family: opt-in encodings with no compatibility guarantee. -pub static FAMILY: EditionFamily = EditionFamily { - name: "unstable", - doc: "Opt-in encodings that are still being evaluated. Every unstable edition stays a \ -draft, so the family never freezes and carries no compatibility guarantee: a file written \ -with these encodings is readable only by a build that knows them, and a later release may \ -stop supporting one. The writer emits them only when the `unstable_encodings` feature is \ -selected. An encoding graduates by joining a core edition.", -}; - -pub mod v2025_05; -pub mod v2026_02; -pub mod v2026_04; -pub mod v2026_06; - -pub use v2025_05::UNSTABLE_2025_05_0; -pub use v2026_02::UNSTABLE_2026_02_0; -pub use v2026_04::UNSTABLE_2026_04_0; -pub use v2026_06::UNSTABLE_2026_06_0; diff --git a/vortex-edition/src/declarations/unstable/v2025_05.rs b/vortex-edition/src/declarations/unstable/v2025_05.rs deleted file mode 100644 index 1a992fd9937..00000000000 --- a/vortex-edition/src/declarations/unstable/v2025_05.rs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The May 2025 `unstable` encoding cohort. - -use crate::Edition; -use crate::EditionDeclaration; -use crate::EditionId; - -/// The May 2025 draft edition of the `unstable` family. -pub const UNSTABLE_2025_05_0: EditionId = EditionId::new("unstable", 2025, 5, 0); - -/// The declaration of [`UNSTABLE_2025_05_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: UNSTABLE_2025_05_0, - min_vortex_version: None, - }, - added: &[&"fastlanes.delta"], -}; diff --git a/vortex-edition/src/declarations/unstable/v2026_02.rs b/vortex-edition/src/declarations/unstable/v2026_02.rs deleted file mode 100644 index 8e5faeeb28f..00000000000 --- a/vortex-edition/src/declarations/unstable/v2026_02.rs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The February 2026 `unstable` encoding cohort. - -use crate::Edition; -use crate::EditionDeclaration; -use crate::EditionId; - -/// The February 2026 draft edition of the `unstable` family. -pub const UNSTABLE_2026_02_0: EditionId = EditionId::new("unstable", 2026, 2, 0); - -/// The declaration of [`UNSTABLE_2026_02_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: UNSTABLE_2026_02_0, - min_vortex_version: None, - }, - added: &[&"vortex.zstd_buffers"], -}; diff --git a/vortex-edition/src/declarations/unstable/v2026_04.rs b/vortex-edition/src/declarations/unstable/v2026_04.rs deleted file mode 100644 index d64bc07529e..00000000000 --- a/vortex-edition/src/declarations/unstable/v2026_04.rs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The April 2026 `unstable` encoding cohort. - -use crate::Edition; -use crate::EditionDeclaration; -use crate::EditionId; - -/// The April 2026 draft edition of the `unstable` family. -pub const UNSTABLE_2026_04_0: EditionId = EditionId::new("unstable", 2026, 4, 0); - -/// The declaration of [`UNSTABLE_2026_04_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: UNSTABLE_2026_04_0, - min_vortex_version: None, - }, - added: &[ - &"vortex.parquet.variant", - &"vortex.patched", - &"vortex.tensor.cosine_similarity", - &"vortex.tensor.inner_product", - &"vortex.tensor.normalized", - &"vortex.tensor.l2_norm", - ], -}; diff --git a/vortex-edition/src/declarations/unstable/v2026_06.rs b/vortex-edition/src/declarations/unstable/v2026_06.rs deleted file mode 100644 index 560b3b5bf02..00000000000 --- a/vortex-edition/src/declarations/unstable/v2026_06.rs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The June 2026 `unstable` encoding cohort. - -use crate::Edition; -use crate::EditionDeclaration; -use crate::EditionId; - -/// The June 2026 draft edition of the `unstable` family. -pub const UNSTABLE_2026_06_0: EditionId = EditionId::new("unstable", 2026, 6, 0); - -/// The declaration of [`UNSTABLE_2026_06_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: UNSTABLE_2026_06_0, - min_vortex_version: None, - }, - added: &[&"vortex.onpair"], -}; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index 48e01ea7662..68be2a58e8f 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -1,19 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Definitions of Vortex *editions*: named, frozen sets of encodings that a writer may put in -//! a file, carrying a forever read-compatibility guarantee. +//! Definitions of Vortex *editions*: named, frozen sets of components that a writer may put +//! in a file, carrying a forever read-compatibility guarantee. //! //! Editions live on the session, like encodings do: [`EditionSession`] holds the registered //! editions and [`EnabledEditions`] selects which of them a writer may emit. Declarations //! are plain constants — an [`EditionId`] plus an [`Edition`] record, and one -//! [`EditionInclusion`] per encoding stating that it is a member of an edition *and every +//! [`EditionInclusion`] per member stating that it is a member of an edition *and every //! later edition of the same family*. Any crate can register declarations into a session, -//! so inclusions can live next to the encoding they describe. +//! so inclusions can live next to the component they describe. +//! +//! Every membership is typed by a [`ComponentKind`], and members are resolved one kind at a +//! time with [`EditionSessionExt::enabled_component_ids`]: the file writer restricts the +//! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets, never one +//! untyped set. //! //! An edition is a **draft** until its [`Edition::min_vortex_version`] is recorded — -//! recording it is the act of freezing. The per-edition encoding sets are computed from the -//! registered declarations by [`EditionSession::encodings_in`], and correctness is enforced +//! recording it is the act of freezing. The per-edition member sets are computed from the +//! registered declarations by [`EditionSession::components_in`], and correctness is enforced //! by unit tests: [`EditionSession::validate`] checks a whole registry, and //! [`test_harness::validate_edition`] validates one edition's constraints — call it once in //! the `#[cfg(test)]` module of each edition definition. @@ -43,7 +48,7 @@ use vortex_session::registry::Id; /// The identifier of an edition, e.g. `core2026.07.0`. /// -/// The `family` names an independently versioned, additive group of encodings (`core` is the +/// The `family` names an independently versioned, additive group of components (`core` is the /// set the default writer emits). The date components record when the edition was frozen and /// order editions chronologically *within* a family; there is no ordering across families. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -147,15 +152,45 @@ impl EditionFamily { } } -/// An edition: a named set of encodings with a read-compatibility guarantee, registered with +/// The kind of component an edition membership covers. +/// +/// Ids are unique per kind, not globally: a layout named `vortex.flat` and an array named +/// `vortex.flat` are different members. Every membership records its kind, and the writer +/// resolves one kind at a time, so the set restricting written arrays never restricts +/// written layouts. Further kinds (scalar functions, say) can be added the same way. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ComponentKind { + /// An array encoding, e.g. `vortex.alp`, registered in the session's array registry. + Array, + /// A layout encoding, e.g. `vortex.flat`, registered in the session's layout registry. + Layout, + /// An extension dtype, e.g. `vortex.timestamp`, registered in the session's dtype registry. + DType, + /// An aggregate function, e.g. `vortex.min`, written into zone maps and registered in + /// the session's aggregate function registry. + Aggregate, +} + +impl Display for ComponentKind { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Array => "array", + Self::Layout => "layout", + Self::DType => "dtype", + Self::Aggregate => "aggregate", + }) + } +} + +/// An edition: a named set of components with a read-compatibility guarantee, registered with /// [`EditionSession::declare_edition`]. The set itself is computed from the registered -/// [`EditionInclusion`]s by [`EditionSession::encodings_in`]. +/// [`EditionInclusion`]s by [`EditionSession::components_in`]. #[derive(Clone, Copy, Debug)] pub struct Edition { /// The edition identifier. Also carries the freeze date: `core2026.07.0` freezes in /// 2026-07. pub id: EditionId, - /// The minimum Vortex version whose reader supports every encoding in this edition. + /// The minimum Vortex version whose reader supports every member of this edition. /// /// Recording this is the act of freezing: an edition with `None` is a **draft** — being /// assembled, carrying no guarantee, free to change, never the default write target. @@ -171,83 +206,147 @@ impl Edition { } } -/// Declares that an encoding is a member of an edition — and of every later edition of the +/// Declares that a component is a member of an edition — and of every later edition of the /// same family. Registered with [`EditionSession::declare_inclusion`]. #[derive(Clone, Copy, Debug)] pub struct EditionInclusion { - /// The interned encoding id, e.g. `vortex.alp`. Globally unique across everything an - /// edition can cover: when layout encodings join editions, their ids must be distinct - /// from array encoding ids. - pub encoding_id: Id, - /// The first edition this encoding is a member of. + /// What the membership covers. Ids are unique per kind, so this is part of the + /// member's identity, not a label. + pub kind: ComponentKind, + /// The interned component id, e.g. `vortex.alp`. + pub component_id: Id, + /// The first edition this component is a member of. pub since: EditionId, - /// The earliest Vortex release able to read and execute this encoding, recorded from + /// The earliest Vortex release able to read and execute this component, recorded from /// evidence (e.g. compat-fixture history). `None` until recorded. pub required_vortex_release: Option<&'static str>, } -/// A source of an encoding id for edition declarations. +/// A source of a component id for edition declarations. /// /// Implemented for raw id strings (`"vortex.alp"`) and interned [`Id`]s here; encoding /// vtables implement it where they are defined, so a declaration can name the vtable -/// (`&Primitive`) instead of spelling its id. -pub trait AsEncodingId: Debug + Send + Sync { - /// The interned encoding id. - fn encoding_id(&self) -> Id; +/// (`&Primitive`) instead of spelling its id. The id alone does not say what kind of +/// component it names — [`EditionMember`] pairs it with a [`ComponentKind`]. +pub trait AsComponentId: Debug + Send + Sync { + /// The interned component id. + fn component_id(&self) -> Id; } -impl AsEncodingId for str { +impl AsComponentId for str { #[expect( clippy::disallowed_methods, - reason = "interning a dynamic encoding id at declaration time" + reason = "interning a dynamic component id at declaration time" )] - fn encoding_id(&self) -> Id { + fn component_id(&self) -> Id { Id::new(self) } } -impl AsEncodingId for Id { - fn encoding_id(&self) -> Id { +impl AsComponentId for Id { + fn component_id(&self) -> Id { *self } } -// `str` is unsized and cannot be a trait object, so declaration blocks (slices of -// `&dyn AsEncodingId`) name encodings as `&"vortex.alp"` through this impl. -impl AsEncodingId for &'static str { - fn encoding_id(&self) -> Id { - (**self).encoding_id() +// `str` is unsized and cannot be a trait object, so declaration blocks name components as +// `&"vortex.alp"` through this impl. +impl AsComponentId for &'static str { + fn component_id(&self) -> Id { + (**self).component_id() } } -/// Declares an edition together with the encodings that join the family at it, in one -/// block. Registered with [`EditionSession::declare`], which derives each encoding's +/// A component that joins an edition, named by id string or vtable and tagged with the kind +/// of registry it belongs to. Built with the per-kind constructors, so a declaration reads +/// as `EditionMember::array(&"vortex.alp")`. +#[derive(Clone, Copy, Debug)] +pub struct EditionMember { + /// What kind of component this is. + pub kind: ComponentKind, + /// The component, named by id string or by vtable. + pub component: &'static dyn AsComponentId, +} + +impl EditionMember { + /// An array encoding member, e.g. `vortex.alp`. + pub const fn array(component: &'static dyn AsComponentId) -> Self { + Self { + kind: ComponentKind::Array, + component, + } + } + + /// A layout member, e.g. `vortex.flat`. + pub const fn layout(component: &'static dyn AsComponentId) -> Self { + Self { + kind: ComponentKind::Layout, + component, + } + } + + /// An extension dtype member, e.g. `vortex.timestamp`. + pub const fn dtype(component: &'static dyn AsComponentId) -> Self { + Self { + kind: ComponentKind::DType, + component, + } + } + + /// An aggregate function member, e.g. `vortex.min`. + pub const fn aggregate(component: &'static dyn AsComponentId) -> Self { + Self { + kind: ComponentKind::Aggregate, + component, + } + } +} + +/// Declares an edition together with the components that join the family at it, in one +/// block. Registered with [`EditionSession::declare`], which derives each member's /// membership (`since` = the declared edition) from the block structure. #[derive(Clone, Copy, Debug)] pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, - /// The encodings that join the family at this edition, named by id string or by - /// vtable. Members of earlier editions are inherited and never restated. - pub added: &'static [&'static dyn AsEncodingId], + /// The components that join the family at this edition, each tagged with its + /// [`ComponentKind`]. Members of earlier editions are inherited and never restated. + pub added: &'static [EditionMember], } impl EditionInclusion { - /// Declare that an encoding is a member of `since` and every later edition of the same - /// family. The encoding can be named by id string or by vtable. - pub fn new(encoding: &E, since: EditionId) -> Self { + /// Declare that a component of `kind` is a member of `since` and every later edition of + /// the same family. The component can be named by id string or by vtable. + pub fn new( + kind: ComponentKind, + component: &C, + since: EditionId, + ) -> Self { Self { - encoding_id: encoding.encoding_id(), + kind, + component_id: component.component_id(), since, required_vortex_release: None, } } - /// Validate the declaration's form: a lowercase `namespace.name` encoding id and, if + /// Declare that an array encoding is a member of `since` and every later edition of the + /// same family. + pub fn array(encoding: &C, since: EditionId) -> Self { + Self::new(ComponentKind::Array, encoding, since) + } + + /// Declare that an extension dtype is a member of `since` and every later edition of the + /// same family. + pub fn dtype(dtype: &C, since: EditionId) -> Self { + Self::new(ComponentKind::DType, dtype, since) + } + + /// Validate the declaration's form: a lowercase `namespace.name` component id and, if /// recorded, a well-formed `major.minor.patch` release. Checked for every declared /// inclusion by [`EditionSession::validate`]. pub fn validate(&self) -> Result<(), EditionError> { - let id = self.encoding_id.as_str(); + let id = self.component_id.as_str(); let well_formed = !id.starts_with('.') && !id.ends_with('.') && id.contains('.') @@ -256,15 +355,16 @@ impl EditionInclusion { .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || "._-".contains(c)); if !well_formed { return Err(EditionError::new(format!( - "invalid encoding id {id:?}: expected lowercase `namespace.name`, e.g. \ - `vortex.alp`" + "invalid {} id {id:?}: expected lowercase `namespace.name`, e.g. `vortex.alp`", + self.kind ))); } if let Some(release) = self.required_vortex_release && parse_release(release).is_none() { return Err(EditionError::new(format!( - "encoding {id} declares malformed required_vortex_release {release:?}" + "{} {id} declares malformed required_vortex_release {release:?}", + self.kind ))); } Ok(()) diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index e023359760e..13f51588d32 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -3,11 +3,13 @@ use vortex_session::VortexSession; +use crate::ComponentKind; use crate::Edition; use crate::EditionDeclaration; use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; +use crate::EditionMember; use crate::EditionSession; use crate::EditionSessionExt; use crate::EnabledEditions; @@ -31,14 +33,17 @@ static DECLARATIONS: &[EditionDeclaration] = &[ id: FIRST, min_vortex_version: None, }, - added: &[&"test.alpha", &"test.beta"], + added: &[ + EditionMember::array(&"test.alpha"), + EditionMember::array(&"test.beta"), + ], }, EditionDeclaration { edition: Edition { id: SECOND, min_vortex_version: None, }, - added: &[&"test.gamma"], + added: &[EditionMember::array(&"test.gamma")], }, ]; @@ -70,19 +75,19 @@ fn editions_pass_the_test_harness() -> Result<(), crate::EditionError> { fn membership_is_transitive() { let editions = session(); - let first = editions.encodings_in(&FIRST); - let ids: Vec<&str> = first.iter().map(|i| i.encoding_id.as_str()).collect(); + let first = editions.components_in(&FIRST, ComponentKind::Array); + let ids: Vec<&str> = first.iter().map(|i| i.component_id.as_str()).collect(); assert_eq!(ids, ["test.alpha", "test.beta"]); // Members of the first edition are members of the second by inheritance, with their // `since` still recording the edition they actually joined in. - let second = editions.encodings_in(&SECOND); - let ids: Vec<&str> = second.iter().map(|i| i.encoding_id.as_str()).collect(); + let second = editions.components_in(&SECOND, ComponentKind::Array); + let ids: Vec<&str> = second.iter().map(|i| i.component_id.as_str()).collect(); assert_eq!(ids, ["test.alpha", "test.beta", "test.gamma"]); assert!( second .iter() - .filter(|i| i.encoding_id.as_str() != "test.gamma") + .filter(|i| i.component_id.as_str() != "test.gamma") .all(|i| i.since == FIRST) ); @@ -90,7 +95,7 @@ fn membership_is_transitive() { let added: Vec<&str> = second .iter() .filter(|i| i.since == SECOND) - .map(|i| i.encoding_id.as_str()) + .map(|i| i.component_id.as_str()) .collect(); assert_eq!(added, ["test.gamma"]); @@ -98,9 +103,16 @@ fn membership_is_transitive() { // never crosses families. assert!(first.iter().all(|i| i.since == FIRST)); let third = EditionId::new("test", 2026, 10, 0); - assert_eq!(editions.encodings_in(&third).len(), 3); + assert_eq!( + editions.components_in(&third, ComponentKind::Array).len(), + 3 + ); let other = EditionId::new("other", 2026, 10, 0); - assert!(editions.encodings_in(&other).is_empty()); + assert!( + editions + .components_in(&other, ComponentKind::Array) + .is_empty() + ); } #[test] @@ -149,12 +161,16 @@ fn registered_and_enabled_editions_are_separate() -> Result<(), crate::EditionEr session.register_edition(declaration)?; } - assert!(session.enabled_encoding_ids().is_empty()); + assert!( + session + .enabled_component_ids(ComponentKind::Array) + .is_empty() + ); session.enable_edition(FIRST)?; assert_eq!(session.enabled_editions().editions(), [FIRST]); assert_eq!( session - .enabled_encoding_ids() + .enabled_component_ids(ComponentKind::Array) .iter() .map(|id| id.as_str()) .collect::>(), @@ -163,13 +179,13 @@ fn registered_and_enabled_editions_are_separate() -> Result<(), crate::EditionEr session.enable_edition(SECOND)?; assert_eq!(session.enabled_editions().editions(), [SECOND]); - assert_eq!(session.enabled_encoding_ids().len(), 3); + assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 3); // Selecting an older edition in the same family replaces the newer one and removes // encodings that joined after it. session.enable_edition(FIRST)?; assert_eq!(session.enabled_editions().editions(), [FIRST]); - let enabled = session.enabled_encoding_ids(); + let enabled = session.enabled_component_ids(ComponentKind::Array); assert_eq!(enabled.len(), 2); assert!(enabled.iter().all(|id| id.as_str() != "test.gamma")); Ok(()) @@ -190,7 +206,7 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi id: OTHER, min_vortex_version: None, }, - added: &[&"other.delta"], + added: &[EditionMember::array(&"other.delta")], }; let session = VortexSession::empty().with::(); @@ -205,7 +221,7 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi let mut enabled = session.enabled_editions().editions(); enabled.sort_unstable(); assert_eq!(enabled, [OTHER, FIRST]); - assert_eq!(session.enabled_encoding_ids().len(), 3); + assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 3); Ok(()) } @@ -222,7 +238,7 @@ fn duplicate_declarations_error() { ); assert!( editions - .declare_inclusion(EditionInclusion::new("test.alpha", FIRST)) + .declare_inclusion(EditionInclusion::array("test.alpha", FIRST)) .is_err() ); } @@ -231,7 +247,7 @@ fn duplicate_declarations_error() { fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionError> { // An inclusion referencing an undeclared edition. let editions = EditionSession::empty(); - editions.declare_inclusion(EditionInclusion::new("test.alpha", FIRST))?; + editions.declare_inclusion(EditionInclusion::array("test.alpha", FIRST))?; assert!(editions.validate().is_err()); // A member requiring a release newer than its edition declares. @@ -242,7 +258,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro })?; editions.declare_inclusion(EditionInclusion { required_vortex_release: Some("0.80.0"), - ..EditionInclusion::new("test.alpha", FIRST) + ..EditionInclusion::array("test.alpha", FIRST) })?; assert!(editions.validate().is_err()); @@ -272,7 +288,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro id: FIRST, min_vortex_version: None, })?; - editions.declare_inclusion(EditionInclusion::new("Test.ALPHA", FIRST))?; + editions.declare_inclusion(EditionInclusion::array("Test.ALPHA", FIRST))?; assert!(editions.validate().is_err()); Ok(()) @@ -320,3 +336,46 @@ fn families_must_document_themselves() { .unwrap(); assert!(editions.validate().is_err()); } + +#[test] +fn kinds_are_resolved_independently() -> Result<(), crate::EditionError> { + // `test.alpha` is declared under both kinds: same id, two distinct members. + static MIXED: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: FIRST, + min_vortex_version: None, + }, + added: &[ + EditionMember::array(&"test.alpha"), + EditionMember::dtype(&"test.alpha"), + EditionMember::layout(&"test.alpha"), + EditionMember::layout(&"test.flat"), + ], + }; + + let session = VortexSession::empty().with::(); + session.register_edition(&MIXED)?; + session.enable_edition(FIRST)?; + + let ids = |kind| { + session + .enabled_component_ids(kind) + .iter() + .map(|id| id.to_string()) + .collect::>() + }; + // A layout never reaches the array registry, and what a writer may emit is the arrays. + assert_eq!(ids(ComponentKind::Array), ["test.alpha"]); + assert_eq!(ids(ComponentKind::DType), ["test.alpha"]); + assert_eq!(ids(ComponentKind::Layout), ["test.alpha", "test.flat"]); + assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 1); + + // A duplicate within one kind is still an error. + assert!( + session + .editions() + .declare_inclusion(EditionInclusion::array("test.alpha", FIRST)) + .is_err() + ); + Ok(()) +} diff --git a/vortex/editions/core/core2025.05.0.toml b/vortex/editions/core/core2025.05.0.toml index eac393d6ddf..66f9f70dd7d 100644 --- a/vortex/editions/core/core2025.05.0.toml +++ b/vortex/editions/core/core2025.05.0.toml @@ -8,8 +8,9 @@ edition = "core2025.05.0" family = "core" min_vortex_version = "0.36.0" -# The encodings that join the family at this edition. -added = [ +# The components that join the family at this edition. +[added] +arrays = [ "fastlanes.bitpacked", "fastlanes.for", "vortex.alp", @@ -34,10 +35,24 @@ added = [ "vortex.varbinview", "vortex.zigzag", ] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.bitpacked", "fastlanes.for", "vortex.alp", @@ -62,3 +77,16 @@ encodings = [ "vortex.varbinview", "vortex.zigzag", ] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [] diff --git a/vortex/editions/core/core2025.06.0.toml b/vortex/editions/core/core2025.06.0.toml index a4e63311f4c..21a306fb3c1 100644 --- a/vortex/editions/core/core2025.06.0.toml +++ b/vortex/editions/core/core2025.06.0.toml @@ -8,16 +8,21 @@ edition = "core2025.06.0" family = "core" min_vortex_version = "0.40.0" -# The encodings that join the family at this edition. -added = [ +# The components that join the family at this edition. +[added] +arrays = [ "vortex.pco", "vortex.sequence", "vortex.zstd", ] +layouts = [] +dtypes = [] +aggregates = [] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.bitpacked", "fastlanes.for", "vortex.alp", @@ -45,3 +50,16 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [] diff --git a/vortex/editions/core/core2025.10.0.toml b/vortex/editions/core/core2025.10.0.toml index 4e3bbba2d69..b83eb2c92fa 100644 --- a/vortex/editions/core/core2025.10.0.toml +++ b/vortex/editions/core/core2025.10.0.toml @@ -8,17 +8,22 @@ edition = "core2025.10.0" family = "core" min_vortex_version = "0.54.0" -# The encodings that join the family at this edition. -added = [ +# The components that join the family at this edition. +[added] +arrays = [ "fastlanes.rle", "vortex.fixed_size_list", "vortex.listview", "vortex.masked", ] +layouts = [] +dtypes = [] +aggregates = [] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.bitpacked", "fastlanes.for", "fastlanes.rle", @@ -50,3 +55,16 @@ encodings = [ "vortex.zigzag", "vortex.zstd", ] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [] diff --git a/vortex/editions/core/core2026.08.0.toml b/vortex/editions/core/core2026.08.0.toml index 82db60f548d..854e6b5fb2e 100644 --- a/vortex/editions/core/core2026.08.0.toml +++ b/vortex/editions/core/core2026.08.0.toml @@ -8,14 +8,26 @@ edition = "core2026.08.0" family = "core" min_vortex_version = "0.84.0" -# The encodings that join the family at this edition. -added = [ - "vortex.map", +# The components that join the family at this edition. +[added] +arrays = [] +layouts = [ + "vortex.zoned", +] +dtypes = [] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", ] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.bitpacked", "fastlanes.for", "fastlanes.rle", @@ -34,7 +46,6 @@ encodings = [ "vortex.fsst", "vortex.list", "vortex.listview", - "vortex.map", "vortex.masked", "vortex.null", "vortex.pco", @@ -45,7 +56,27 @@ encodings = [ "vortex.struct", "vortex.varbin", "vortex.varbinview", - "vortex.variant", "vortex.zigzag", "vortex.zstd", ] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", + "vortex.zoned", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] diff --git a/vortex/editions/core/core2026.07.0.toml b/vortex/editions/core/core2026.08.1.toml similarity index 59% rename from vortex/editions/core/core2026.07.0.toml rename to vortex/editions/core/core2026.08.1.toml index e49b6f0bf26..19ec877330c 100644 --- a/vortex/editions/core/core2026.07.0.toml +++ b/vortex/editions/core/core2026.08.1.toml @@ -4,18 +4,23 @@ # contains never changes again. Freezing a new edition adds a new file to this directory; # editing or deleting a frozen one is rejected by CI. -edition = "core2026.07.0" +edition = "core2026.08.1" family = "core" -min_vortex_version = "0.65.0" +min_vortex_version = "0.84.0" -# The encodings that join the family at this edition. -added = [ - "vortex.variant", +# The components that join the family at this edition. +[added] +arrays = [ + "vortex.onpair", ] +layouts = [] +dtypes = [] +aggregates = [] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.bitpacked", "fastlanes.for", "fastlanes.rle", @@ -36,6 +41,7 @@ encodings = [ "vortex.listview", "vortex.masked", "vortex.null", + "vortex.onpair", "vortex.pco", "vortex.primitive", "vortex.runend", @@ -44,7 +50,27 @@ encodings = [ "vortex.struct", "vortex.varbin", "vortex.varbinview", - "vortex.variant", "vortex.zigzag", "vortex.zstd", ] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", + "vortex.zoned", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] diff --git a/vortex/editions/core/core2026.08.2.toml b/vortex/editions/core/core2026.08.2.toml new file mode 100644 index 00000000000..a9c9543f20e --- /dev/null +++ b/vortex/editions/core/core2026.08.2.toml @@ -0,0 +1,76 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "core2026.08.2" +family = "core" + +# The components that join the family at this edition. +[added] +arrays = [ + "vortex.map", +] +layouts = [] +dtypes = [] +aggregates = [] + +# The edition's full membership: the components above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.onpair", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", + "vortex.zoned", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] diff --git a/vortex/editions/core/core2026.08.3.toml b/vortex/editions/core/core2026.08.3.toml new file mode 100644 index 00000000000..dc7dda51b71 --- /dev/null +++ b/vortex/editions/core/core2026.08.3.toml @@ -0,0 +1,82 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is a draft: it carries no guarantee and is still being assembled, so this +# record changes with it. Recording a min_vortex_version freezes the edition, after which +# this file may never change again. + +edition = "core2026.08.3" +family = "core" + +# The components that join the family at this edition. +[added] +arrays = [ + "vortex.parquet.variant", + "vortex.variant", +] +layouts = [] +dtypes = [ + "vortex.uuid", +] +aggregates = [] + +# The edition's full membership: the components above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.onpair", + "vortex.parquet.variant", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", + "vortex.zoned", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", + "vortex.uuid", +] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml index ebff1159dba..4ab2bc80dca 100644 --- a/vortex/editions/core/family.toml +++ b/vortex/editions/core/family.toml @@ -5,8 +5,8 @@ name = "core" doc = """ -The encodings the default file writer emits. Every core edition freezes, and a frozen -edition carries a read-forever guarantee: a file written with it stays readable by every -later Vortex release. New encodings join by being declared in a new edition; an edition that -has frozen never changes again. +The serialized components the default file writer emits. Every core edition freezes, and a +frozen edition carries a read-forever guarantee: a file written with it stays readable by +every later Vortex release. New components join by being declared in a new edition; an +edition that has frozen never changes again. """ diff --git a/vortex/editions/unstable/family.toml b/vortex/editions/preview/family.toml similarity index 60% rename from vortex/editions/unstable/family.toml rename to vortex/editions/preview/family.toml index 19c6b159176..de766d1db78 100644 --- a/vortex/editions/unstable/family.toml +++ b/vortex/editions/preview/family.toml @@ -2,12 +2,12 @@ # # This describes the family the editions beside it belong to. -name = "unstable" +name = "preview" doc = """ -Opt-in encodings that are still being evaluated. Every unstable edition stays a draft, so +Opt-in components that are still being evaluated. Every preview edition stays a draft, so the family never freezes and carries no compatibility guarantee: a file written with these -encodings is readable only by a build that knows them, and a later release may stop +components is readable only by a build that knows them, and a later release may stop supporting one. The writer emits them only when the `unstable_encodings` feature is -selected. An encoding graduates by joining a core edition. +selected. A component graduates by joining a core edition. """ diff --git a/vortex/editions/unstable/unstable2025.05.0.toml b/vortex/editions/preview/preview2025.05.0.toml similarity index 53% rename from vortex/editions/unstable/unstable2025.05.0.toml rename to vortex/editions/preview/preview2025.05.0.toml index b2b2ddb8325..653eb5df7a2 100644 --- a/vortex/editions/unstable/unstable2025.05.0.toml +++ b/vortex/editions/preview/preview2025.05.0.toml @@ -4,16 +4,24 @@ # record changes with it. Recording a min_vortex_version freezes the edition, after which # this file may never change again. -edition = "unstable2025.05.0" -family = "unstable" +edition = "preview2025.05.0" +family = "preview" -# The encodings that join the family at this edition. -added = [ +# The components that join the family at this edition. +[added] +arrays = [ "fastlanes.delta", ] +layouts = [] +dtypes = [] +aggregates = [] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.delta", ] +layouts = [] +dtypes = [] +aggregates = [] diff --git a/vortex/editions/unstable/unstable2026.02.0.toml b/vortex/editions/preview/preview2026.02.0.toml similarity index 55% rename from vortex/editions/unstable/unstable2026.02.0.toml rename to vortex/editions/preview/preview2026.02.0.toml index 00f1b0bef6f..ca1be1431ad 100644 --- a/vortex/editions/unstable/unstable2026.02.0.toml +++ b/vortex/editions/preview/preview2026.02.0.toml @@ -4,17 +4,25 @@ # record changes with it. Recording a min_vortex_version freezes the edition, after which # this file may never change again. -edition = "unstable2026.02.0" -family = "unstable" +edition = "preview2026.02.0" +family = "preview" -# The encodings that join the family at this edition. -added = [ +# The components that join the family at this edition. +[added] +arrays = [ "vortex.zstd_buffers", ] +layouts = [] +dtypes = [] +aggregates = [] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.delta", "vortex.zstd_buffers", ] +layouts = [] +dtypes = [] +aggregates = [] diff --git a/vortex/editions/unstable/unstable2026.04.0.toml b/vortex/editions/preview/preview2026.04.0.toml similarity index 59% rename from vortex/editions/unstable/unstable2026.04.0.toml rename to vortex/editions/preview/preview2026.04.0.toml index 7e721892046..dfecd6d3b21 100644 --- a/vortex/editions/unstable/unstable2026.04.0.toml +++ b/vortex/editions/preview/preview2026.04.0.toml @@ -4,24 +4,30 @@ # record changes with it. Recording a min_vortex_version freezes the edition, after which # this file may never change again. -edition = "unstable2026.04.0" -family = "unstable" +edition = "preview2026.04.0" +family = "preview" -# The encodings that join the family at this edition. -added = [ - "vortex.parquet.variant", +# The components that join the family at this edition. +[added] +arrays = [ "vortex.patched", "vortex.tensor.cosine_similarity", "vortex.tensor.inner_product", "vortex.tensor.l2_norm", "vortex.tensor.normalized", ] +layouts = [] +dtypes = [ + "vortex.tensor.fixed_shape_tensor", + "vortex.tensor.vector", +] +aggregates = [] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.delta", - "vortex.parquet.variant", "vortex.patched", "vortex.tensor.cosine_similarity", "vortex.tensor.inner_product", @@ -29,3 +35,9 @@ encodings = [ "vortex.tensor.normalized", "vortex.zstd_buffers", ] +layouts = [] +dtypes = [ + "vortex.tensor.fixed_shape_tensor", + "vortex.tensor.vector", +] +aggregates = [] diff --git a/vortex/editions/unstable/unstable2026.06.0.toml b/vortex/editions/preview/preview2026.06.0.toml similarity index 55% rename from vortex/editions/unstable/unstable2026.06.0.toml rename to vortex/editions/preview/preview2026.06.0.toml index c890c13627d..d4d41ab3b6b 100644 --- a/vortex/editions/unstable/unstable2026.06.0.toml +++ b/vortex/editions/preview/preview2026.06.0.toml @@ -4,20 +4,23 @@ # record changes with it. Recording a min_vortex_version freezes the edition, after which # this file may never change again. -edition = "unstable2026.06.0" -family = "unstable" +edition = "preview2026.06.0" +family = "preview" -# The encodings that join the family at this edition. -added = [ - "vortex.onpair", +# The components that join the family at this edition. +[added] +arrays = [] +layouts = [ + "vortex.list", ] +dtypes = [] +aggregates = [] -# The edition's full membership: the encodings above, plus every member of earlier +# The edition's full membership: the components above, plus every member of earlier # editions of the family. -encodings = [ +[components] +arrays = [ "fastlanes.delta", - "vortex.onpair", - "vortex.parquet.variant", "vortex.patched", "vortex.tensor.cosine_similarity", "vortex.tensor.inner_product", @@ -25,3 +28,11 @@ encodings = [ "vortex.tensor.normalized", "vortex.zstd_buffers", ] +layouts = [ + "vortex.list", +] +dtypes = [ + "vortex.tensor.fixed_shape_tensor", + "vortex.tensor.vector", +] +aggregates = [] diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 2538a270901..b5d7e22d200 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -9,14 +9,19 @@ //! [`crate::editions::register_default_editions`] and then selects its write policy with //! [`crate::editions::enable_default_editions`]. //! +//! Members carry a [`crate::editions::ComponentKind`]: arrays a written array may use, extension +//! dtypes its schema may contain, and the aggregates zone maps record. Every kind is restricted to +//! its declared members, so an empty set permits no components of that kind. +//! //! The default file writer resolves the session's enabled editions at write time. The -//! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08`], and -//! additionally enables the latest unstable edition when the `unstable_encodings` feature is +//! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08_1`], and +//! additionally enables the latest preview edition when the `unstable_encodings` feature is //! selected. #[cfg(test)] mod tests; +pub use vortex_edition::ComponentKind; pub use vortex_edition::EDITION_DECLARATIONS; pub use vortex_edition::EDITION_FAMILIES; pub use vortex_edition::Edition; @@ -24,6 +29,7 @@ pub use vortex_edition::EditionDeclaration; pub use vortex_edition::EditionFamily; pub use vortex_edition::EditionId; pub use vortex_edition::EditionInclusion; +pub use vortex_edition::EditionMember; pub use vortex_edition::EditionSession; pub use vortex_edition::EditionSessionExt; pub use vortex_edition::EnabledEditions; @@ -31,23 +37,25 @@ pub use vortex_edition::declarations::core; pub use vortex_edition::declarations::core::CORE_2025_05_0; pub use vortex_edition::declarations::core::CORE_2025_06_0; pub use vortex_edition::declarations::core::CORE_2025_10_0; -pub use vortex_edition::declarations::core::CORE_2026_07_0; -pub use vortex_edition::declarations::core::CORE_2026_08; -pub use vortex_edition::declarations::unstable; -pub use vortex_edition::declarations::unstable::UNSTABLE_2025_05_0; -pub use vortex_edition::declarations::unstable::UNSTABLE_2026_02_0; -pub use vortex_edition::declarations::unstable::UNSTABLE_2026_04_0; -pub use vortex_edition::declarations::unstable::UNSTABLE_2026_06_0; +pub use vortex_edition::declarations::core::CORE_2026_08_0; +pub use vortex_edition::declarations::core::CORE_2026_08_1; +pub use vortex_edition::declarations::core::CORE_2026_08_2; +pub use vortex_edition::declarations::core::CORE_2026_08_3; +pub use vortex_edition::declarations::preview; +pub use vortex_edition::declarations::preview::PREVIEW_2025_05_0; +pub use vortex_edition::declarations::preview::PREVIEW_2026_02_0; +pub use vortex_edition::declarations::preview::PREVIEW_2026_04_0; +pub use vortex_edition::declarations::preview::PREVIEW_2026_06_0; use vortex_error::VortexExpect; use vortex_error::vortex_err; use vortex_session::VortexSession; /// The `core` edition enabled for writing by the default Vortex session. -pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08; +pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08_1; -/// The `unstable` edition enabled for writing by the default Vortex session when the +/// The `preview` edition enabled for writing by the default Vortex session when the /// `unstable_encodings` feature is selected. -pub const DEFAULT_UNSTABLE_EDITION: EditionId = UNSTABLE_2026_06_0; +pub const DEFAULT_PREVIEW_EDITION: EditionId = PREVIEW_2026_06_0; /// Register the Vortex edition families and declarations with the session's /// [`EditionSession`]. @@ -69,7 +77,7 @@ pub fn register_default_editions(session: &VortexSession) { /// Enable the default Vortex editions for writing. /// -/// This selects the newest frozen `core` edition and, when configured, the newest unstable +/// This selects the newest frozen `core` edition and, when configured, the newest preview /// edition. All declarations must have been registered first with /// [`register_default_editions`]. pub fn enable_default_editions(session: &VortexSession) { @@ -80,7 +88,7 @@ pub fn enable_default_editions(session: &VortexSession) { #[cfg(feature = "unstable_encodings")] session - .enable_edition(DEFAULT_UNSTABLE_EDITION) + .enable_edition(DEFAULT_PREVIEW_EDITION) .map_err(|e| vortex_err!("{e}")) - .vortex_expect("default unstable edition is registered"); + .vortex_expect("default preview edition is registered"); } diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index d4c4cd0cf8f..e70a7718c88 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -4,14 +4,15 @@ //! Export the edition records under `vortex/editions`. //! //! Every declared edition gets one TOML file recording what it contains: the identifier, the -//! minimum Vortex version whose reader supports it once frozen, and its full encoding set. -//! Records are grouped by family — `vortex/editions/core/core2025.05.0.toml` — mirroring the -//! declarations in `vortex-edition/src/declarations`, since families version independently. +//! minimum Vortex version whose reader supports it once frozen, and its full component set, +//! one list per [`ComponentKind`]. Records are grouped by family — +//! `vortex/editions/core/core2025.05.0.toml` — mirroring the declarations in +//! `vortex-edition/src/declarations`, since families version independently. //! //! A record's mutability follows its edition. A draft is still being assembled, so its record //! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a //! contract carrying a read-forever guarantee, and from then on it may never change again. CI -//! enforces that against git history in `.github/scripts/check_edition_records.py`; this +//! enforces that against git history with `cargo run -p xtask -- check-editions`; this //! exporter enforces the two rules that history cannot see, refusing to delete a record or to //! unfreeze one. @@ -22,6 +23,7 @@ use std::path::PathBuf; use anyhow::Context; use anyhow::anyhow; +use vortex_edition::ComponentKind; use vortex_edition::EDITION_DECLARATIONS; use vortex_edition::EDITION_FAMILIES; use vortex_edition::Edition; @@ -81,24 +83,31 @@ fn wrap(text: &str, width: usize) -> Vec { lines } -/// Render one edition's record. Deterministic: every list is sorted by encoding id, so the -/// generated bytes depend only on the declarations. -fn record(session: &EditionSession, edition: &Edition) -> String { - let inclusions = session.encodings_in(&edition.id); - let members: BTreeSet<&str> = inclusions - .iter() - .map(|inclusion| inclusion.encoding_id.as_str()) - .collect(); - let added: BTreeSet<&str> = inclusions - .iter() - .filter(|inclusion| inclusion.since == edition.id) - .map(|inclusion| inclusion.encoding_id.as_str()) - .collect(); +/// Every component kind and the record key its members are listed under. +const KINDS: [(ComponentKind, &str); 4] = [ + (ComponentKind::Array, "arrays"), + (ComponentKind::Layout, "layouts"), + (ComponentKind::DType, "dtypes"), + (ComponentKind::Aggregate, "aggregates"), +]; - let list = |ids: &BTreeSet<&str>| -> Vec { - ids.iter().map(|id| format!(" \"{id}\",")).collect() - }; +/// Render one TOML list per component kind, each sorted by component id. +fn kind_lists(lines: &mut Vec, ids_of: impl Fn(ComponentKind) -> BTreeSet) { + for (kind, key) in KINDS { + let ids = ids_of(kind); + if ids.is_empty() { + lines.push(format!("{key} = []")); + continue; + } + lines.push(format!("{key} = [")); + lines.extend(ids.iter().map(|id| format!(" \"{id}\","))); + lines.push("]".to_string()); + } +} +/// Render one edition's record. Deterministic: every list is sorted by component id, so the +/// generated bytes depend only on the declarations. +fn record(session: &EditionSession, edition: &Edition) -> String { let note = if edition.is_draft() { DRAFT_NOTE } else { @@ -116,20 +125,32 @@ fn record(session: &EditionSession, edition: &Edition) -> String { } lines.extend([ String::new(), - "# The encodings that join the family at this edition.".to_string(), - "added = [".to_string(), + "# The components that join the family at this edition.".to_string(), + "[added]".to_string(), ]); - lines.extend(list(&added)); + kind_lists(&mut lines, |kind| { + session + .components_in(&edition.id, kind) + .iter() + .filter(|inclusion| inclusion.since == edition.id) + .map(|inclusion| inclusion.component_id.to_string()) + .collect() + }); lines.extend([ - "]".to_string(), String::new(), - "# The edition's full membership: the encodings above, plus every member of earlier" + "# The edition's full membership: the components above, plus every member of earlier" .to_string(), "# editions of the family.".to_string(), - "encodings = [".to_string(), + "[components]".to_string(), ]); - lines.extend(list(&members)); - lines.extend(["]".to_string(), String::new()]); + kind_lists(&mut lines, |kind| { + session + .components_in(&edition.id, kind) + .iter() + .map(|inclusion| inclusion.component_id.to_string()) + .collect() + }); + lines.push(String::new()); lines.join("\n") } From 5fcfde0c0d0143d24836573c29704e336cb2279b Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 20 Aug 2026 18:39:06 +0100 Subject: [PATCH 13/18] Require two approvals for edition changes, and document the freeze process Reviewers asked what the process for freezing an edition is and how these files stay protected. Document the declare/freeze/never-touch lifecycle in the editions spec, refresh the family section for the preview rename, and add a Policy Bot rule requiring two write-access approvals for changes under vortex/editions/ and vortex-edition/src/declarations/. Signed-off-by: Robert Kruszewski --- .policy.yml | 26 +++++++++++++++++++++++ docs/specs/editions.md | 39 +++++++++++++++++++++++++++++------ vortex-edition/src/session.rs | 6 +++--- 3 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 .policy.yml diff --git a/.policy.yml b/.policy.yml new file mode 100644 index 00000000000..cd451c75004 --- /dev/null +++ b/.policy.yml @@ -0,0 +1,26 @@ +# Policy Bot configuration: https://github.com/palantir/policy-bot +# +# Frozen editions carry a read-forever guarantee, so the declarations and their exported +# records are held to a higher bar than the rest of the repository. CI enforces that frozen +# records never change (`cargo run -p xtask -- check-editions`); this policy enforces that +# the changes the check does allow — new editions, draft edits, freezes — are seen by two +# reviewers instead of one. + +policy: + approval: + - edition declarations and records have two approvals + +approval_rules: + - name: edition declarations and records have two approvals + description: >- + Changes under vortex/editions/ or vortex-edition/src/declarations/ alter what the + edition compatibility contract says, so they need a second approval. + if: + changed_files: + paths: + - "^vortex/editions/.*$" + - "^vortex-edition/src/declarations/.*$" + requires: + count: 2 + permissions: + - "write" diff --git a/docs/specs/editions.md b/docs/specs/editions.md index f30f03feb70..e6273b13f45 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -178,19 +178,46 @@ selected editions, the write fails. function outside the selected editions fails the write. With `allow_unknown`, readers disable a zone map whose aggregate function they do not recognize; ignoring a zone map only reduces pruning and does not affect correctness. -## The `unstable` family +## The `preview` family -Alongside `core` there is an `unstable` family, holding encodings that are still being -evaluated. It is the exception to everything above: every `unstable` edition is a permanent +Alongside `core` there is a `preview` family, holding components that are still being +evaluated. It is the exception to everything above: every `preview` edition is a permanent draft, so the family never freezes and carries no read-compatibility guarantee at all. A file -written with these encodings is readable only by a build that knows them, and a future release +written with these components is readable only by a build that knows them, and a future release may stop supporting one. Because of that, the writer only emits them when you opt in — the default session enables the -newest `unstable` edition solely when the `unstable_encodings` cargo feature is selected. -Encodings graduate by being declared in a new `core` edition, which is where they pick up the +newest `preview` edition solely when the `unstable_encodings` cargo feature is selected. +Components graduate by being declared in a new `core` edition, which is where they pick up the read-forever guarantee. +## Declaring, freezing, and the edition records + +The first-party declarations live in `vortex-edition/src/declarations/`, one module per +edition. Each declared edition is exported as a TOML record under `vortex/editions/`, grouped +by family, by running: + +```sh +cargo run -p xtask -- generate-editions +``` + +Changing the declarations follows the edition's lifecycle: + +1. **Declare a draft.** Add a module declaring the new edition with `min_vortex_version: None` + and the members that join the family at it, then regenerate the records. A draft carries no + guarantee, so its declaration and record may change freely — or be dropped — while it is + assembled. +2. **Freeze it.** Once a release ships readers for every member, record that release as the + edition's `min_vortex_version` and regenerate the records. Freezing is the act of + publishing the read-forever guarantee. +3. **Never touch it again.** A frozen record is immutable: CI + (`cargo run -p xtask -- check-editions`) rejects any change that edits, renames, unfreezes, + or deletes a frozen record, and rejects new editions that do not extend their family's + chronology. To change what writers may emit, declare the next edition instead. + +The record files are the reviewable contract, so changes under `vortex/editions/` and +`vortex-edition/src/declarations/` additionally require two approvals to merge. + ## Edition registry Registry entries list the edition in which each component first appeared. Later editions in the same family inherit all diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index c1ccd45dabc..ad00fdeba61 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -189,9 +189,9 @@ impl EditionSession { } /// Validate all registered declarations. Errors on editions in undeclared families, - /// inclusions referencing undeclared editions, editions out of chronological order within a family (unversioned drafts - /// must be newest), malformed version strings, and members requiring a release newer - /// than their edition declares. + /// inclusions referencing undeclared editions, editions out of chronological order within + /// a family (unversioned drafts must be newest), malformed version strings, and members + /// requiring a release newer than their edition declares. pub fn validate(&self) -> Result<(), EditionError> { let editions = self.editions(); From 16b325c452b0824f79ecc6be83876dd83b6e82ca Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 21 Aug 2026 10:36:24 +0000 Subject: [PATCH 14/18] Declare the json and spatial edition families `EditionSession::validate` now rejects an edition whose family was never declared, but `vortex-json` and `vortex-spatial` mint their own families and only ever declared the edition. Their `*_edition_is_valid` tests failed with "belongs to undeclared family". Declare each family alongside its edition, inside the existing idempotency guard in `initialize`. Also drop `vortex/src/editions/preview/`, orphaned when the declarations moved to `vortex-edition/src/declarations/`: nothing declares the module, so the files were dead copies of the ones now in `vortex-edition`. Signed-off-by: Joe Isaacs --- vortex-json/src/editions.rs | 10 +++++++++ vortex-json/src/lib.rs | 5 +++++ vortex-spatial/src/editions.rs | 11 ++++++++++ vortex-spatial/src/lib.rs | 5 +++++ vortex/src/editions/preview/mod.rs | 17 --------------- vortex/src/editions/preview/v2025_05.rs | 21 ------------------ vortex/src/editions/preview/v2026_02.rs | 21 ------------------ vortex/src/editions/preview/v2026_04.rs | 29 ------------------------- vortex/src/editions/preview/v2026_06.rs | 27 ----------------------- 9 files changed, 31 insertions(+), 115 deletions(-) delete mode 100644 vortex/src/editions/preview/mod.rs delete mode 100644 vortex/src/editions/preview/v2025_05.rs delete mode 100644 vortex/src/editions/preview/v2026_02.rs delete mode 100644 vortex/src/editions/preview/v2026_04.rs delete mode 100644 vortex/src/editions/preview/v2026_06.rs diff --git a/vortex-json/src/editions.rs b/vortex-json/src/editions.rs index b3073c710fe..cbf69e29ecb 100644 --- a/vortex-json/src/editions.rs +++ b/vortex-json/src/editions.rs @@ -9,9 +9,19 @@ use vortex_edition::Edition; use vortex_edition::EditionDeclaration; +use vortex_edition::EditionFamily; use vortex_edition::EditionId; use vortex_edition::EditionMember; +/// The `json` family: the JSON extension dtype, declared here rather than in `core` because +/// a reader without this crate cannot resolve it. +pub static FAMILY: EditionFamily = EditionFamily { + name: "json", + doc: "The JSON extension dtype. JSON support is opt-in: a reader built without \ +`vortex-json` cannot resolve `vortex.json`, so the dtype is versioned independently of \ +`core` and a session enables this family only by initializing the crate.", +}; + /// The August 2026 draft edition of the `json` family. pub const JSON_2026_08: EditionId = EditionId::new("json", 2026, 8, 0); diff --git a/vortex-json/src/lib.rs b/vortex-json/src/lib.rs index 0a117eb35e7..0cf27f2db2f 100644 --- a/vortex-json/src/lib.rs +++ b/vortex-json/src/lib.rs @@ -39,6 +39,11 @@ pub fn initialize(session: &VortexSession) { // JSON is an opt-in durable dtype, so it belongs to an independently enabled edition family. // `initialize` is idempotent, hence the guard around declaration registration. if session.editions().find(&editions::JSON_2026_08).is_none() { + session + .editions() + .declare_family(&editions::FAMILY) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("JSON edition family is valid"); session .register_edition(&editions::DECLARATION) .map_err(|error| vortex_err!("{error}")) diff --git a/vortex-spatial/src/editions.rs b/vortex-spatial/src/editions.rs index e7e277d4f91..9bf437536c4 100644 --- a/vortex-spatial/src/editions.rs +++ b/vortex-spatial/src/editions.rs @@ -10,9 +10,20 @@ use vortex_edition::Edition; use vortex_edition::EditionDeclaration; +use vortex_edition::EditionFamily; use vortex_edition::EditionId; use vortex_edition::EditionMember; +/// The `spatial` family: the geometry dtypes and the AABB zone aggregate, declared here +/// rather than in `core` because a reader without this crate cannot resolve them. +pub static FAMILY: EditionFamily = EditionFamily { + name: "spatial", + doc: "The geometry extension dtypes and the axis-aligned bounding-box zone aggregate. \ +Spatial support is opt-in: a reader built without `vortex-spatial` cannot resolve \ +`vortex.st.*`, so these members are versioned independently of `core` and a session enables \ +this family only by initializing the crate.", +}; + /// The August 2026 draft edition of the `spatial` family. pub const SPATIAL_2026_08: EditionId = EditionId::new("spatial", 2026, 8, 0); diff --git a/vortex-spatial/src/lib.rs b/vortex-spatial/src/lib.rs index 919652e01c4..5ce1830cdd5 100644 --- a/vortex-spatial/src/lib.rs +++ b/vortex-spatial/src/lib.rs @@ -99,6 +99,11 @@ pub fn initialize(session: &VortexSession) { .find(&editions::SPATIAL_2026_08) .is_none() { + session + .editions() + .declare_family(&editions::FAMILY) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("spatial edition family is valid"); session .register_edition(&editions::DECLARATION) .map_err(|error| vortex_err!("{error}")) diff --git a/vortex/src/editions/preview/mod.rs b/vortex/src/editions/preview/mod.rs deleted file mode 100644 index cba3ba1dc6c..00000000000 --- a/vortex/src/editions/preview/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The `preview` edition family: opt-in components without a frozen compatibility guarantee. -//! -//! One module per draft edition, each declaring the components that join the family at it. -//! Members of earlier editions are inherited and never restated. - -pub mod v2025_05; -pub mod v2026_02; -pub mod v2026_04; -pub mod v2026_06; - -pub use v2025_05::PREVIEW_2025_05_0; -pub use v2026_02::PREVIEW_2026_02_0; -pub use v2026_04::PREVIEW_2026_04_0; -pub use v2026_06::PREVIEW_2026_06_0; diff --git a/vortex/src/editions/preview/v2025_05.rs b/vortex/src/editions/preview/v2025_05.rs deleted file mode 100644 index 818ec96b15a..00000000000 --- a/vortex/src/editions/preview/v2025_05.rs +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The May 2025 `preview` encoding cohort. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The May 2025 draft edition of the `preview` family. -pub const PREVIEW_2025_05_0: EditionId = EditionId::new("preview", 2025, 5, 0); - -/// The declaration of [`PREVIEW_2025_05_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: PREVIEW_2025_05_0, - min_vortex_version: None, - }, - added: &[EditionMember::array(&"fastlanes.delta")], -}; diff --git a/vortex/src/editions/preview/v2026_02.rs b/vortex/src/editions/preview/v2026_02.rs deleted file mode 100644 index 691e96c1850..00000000000 --- a/vortex/src/editions/preview/v2026_02.rs +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The February 2026 `preview` encoding cohort. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The February 2026 draft edition of the `preview` family. -pub const PREVIEW_2026_02_0: EditionId = EditionId::new("preview", 2026, 2, 0); - -/// The declaration of [`PREVIEW_2026_02_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: PREVIEW_2026_02_0, - min_vortex_version: None, - }, - added: &[EditionMember::array(&"vortex.zstd_buffers")], -}; diff --git a/vortex/src/editions/preview/v2026_04.rs b/vortex/src/editions/preview/v2026_04.rs deleted file mode 100644 index 4d1a5ede4b8..00000000000 --- a/vortex/src/editions/preview/v2026_04.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The April 2026 `preview` component cohort. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The April 2026 draft edition of the `preview` family. -pub const PREVIEW_2026_04_0: EditionId = EditionId::new("preview", 2026, 4, 0); - -/// The declaration of [`PREVIEW_2026_04_0`] and the components that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: PREVIEW_2026_04_0, - min_vortex_version: None, - }, - added: &[ - EditionMember::array(&"vortex.patched"), - EditionMember::array(&"vortex.tensor.cosine_similarity"), - EditionMember::array(&"vortex.tensor.inner_product"), - EditionMember::array(&"vortex.tensor.normalized"), - EditionMember::array(&"vortex.tensor.l2_norm"), - EditionMember::dtype(&"vortex.tensor.fixed_shape_tensor"), - EditionMember::dtype(&"vortex.tensor.vector"), - ], -}; diff --git a/vortex/src/editions/preview/v2026_06.rs b/vortex/src/editions/preview/v2026_06.rs deleted file mode 100644 index 3a888457c1f..00000000000 --- a/vortex/src/editions/preview/v2026_06.rs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The June 2026 `preview` component cohort. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The June 2026 draft edition of the `preview` family. -pub const PREVIEW_2026_06_0: EditionId = EditionId::new("preview", 2026, 6, 0); - -/// The declaration of [`PREVIEW_2026_06_0`] and the components that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: PREVIEW_2026_06_0, - min_vortex_version: None, - }, - added: &[ - EditionMember::layout(&"vortex.list"), - // Written only by CUDA-enabled sessions, which register the layout through - // `vortex_cuda::layout::register_cuda_layout`. A writer resolves layouts against the - // enabled editions, so the GPU flat layout has to be a member to be written at all. - EditionMember::layout(&"vortex.cuda_flat"), - ], -}; From 3a7e6f2c5e59b862c66a2a43847f0bca42bdde3a Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 21 Aug 2026 10:39:01 +0000 Subject: [PATCH 15/18] Add the SPDX header to .policy.yml `reuse lint` covers root YAML config, and `.policy.yml` was the one file in the tree without copyright and licensing information, so the reuse-check job failed. Header it the way `.yamllint.yaml` is. Signed-off-by: Joe Isaacs --- .policy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.policy.yml b/.policy.yml index cd451c75004..5958382f8da 100644 --- a/.policy.yml +++ b/.policy.yml @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + # Policy Bot configuration: https://github.com/palantir/policy-bot # # Frozen editions carry a read-forever guarantee, so the declarations and their exported From 7206477c8e60cc6905e2df8f7f26633c8e6a0fd9 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 25 Aug 2026 12:00:47 +0100 Subject: [PATCH 16/18] rework Signed-off-by: Robert Kruszewski --- .policy.yml | 29 -------- docs/specs/editions.md | 69 +++++++++++-------- policy.yml | 13 ++++ vortex-edition/src/declarations/core/mod.rs | 6 +- .../src/declarations/preview/mod.rs | 12 ++-- vortex-edition/src/lib.rs | 45 ++++++------ vortex/editions/core/core2026.08.2.toml | 6 +- vortex/editions/core/core2026.08.3.toml | 6 +- vortex/editions/core/family.toml | 5 +- vortex/editions/preview/family.toml | 12 ++-- vortex/editions/preview/preview2025.05.0.toml | 6 +- vortex/editions/preview/preview2026.02.0.toml | 6 +- vortex/editions/preview/preview2026.04.0.toml | 6 +- vortex/editions/preview/preview2026.06.0.toml | 6 +- xtask/src/check_editions.rs | 13 ++-- xtask/src/generate_editions.rs | 20 +++--- 16 files changed, 135 insertions(+), 125 deletions(-) delete mode 100644 .policy.yml diff --git a/.policy.yml b/.policy.yml deleted file mode 100644 index 5958382f8da..00000000000 --- a/.policy.yml +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors - -# Policy Bot configuration: https://github.com/palantir/policy-bot -# -# Frozen editions carry a read-forever guarantee, so the declarations and their exported -# records are held to a higher bar than the rest of the repository. CI enforces that frozen -# records never change (`cargo run -p xtask -- check-editions`); this policy enforces that -# the changes the check does allow — new editions, draft edits, freezes — are seen by two -# reviewers instead of one. - -policy: - approval: - - edition declarations and records have two approvals - -approval_rules: - - name: edition declarations and records have two approvals - description: >- - Changes under vortex/editions/ or vortex-edition/src/declarations/ alter what the - edition compatibility contract says, so they need a second approval. - if: - changed_files: - paths: - - "^vortex/editions/.*$" - - "^vortex-edition/src/declarations/.*$" - requires: - count: 2 - permissions: - - "write" diff --git a/docs/specs/editions.md b/docs/specs/editions.md index e6273b13f45..28e6f7eab1f 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -69,9 +69,11 @@ zone-map pruning rather than causing the file to be rejected. ## Writing with an edition -By default, the Vortex facade targets the newest frozen `core` edition. New components may first appear in a draft -edition before joining a later frozen `core` edition. If serialization would use a component outside the selected -editions, the write fails immediately. +By default, the Vortex facade targets the newest frozen `core` edition. Components maintained as part of Vortex first +belong to `preview` while their serialization is evolving. Components supplied by an optional plugin instead belong to +that plugin's standalone edition family, such as `spatial` or `json`. Once a non-plugin component's serialization is +stable, it moves into a new draft `core` edition. If serialization would use a component outside the selected editions, +the write fails immediately. Edition configuration belongs to the writer's Vortex session. Registering an edition makes its declaration available to the session; enabling it allows the writer to use its components. Enabling another edition in the same family replaces @@ -89,9 +91,14 @@ selected editions. ## How editions change -A frozen edition never changes: neither its component list nor the meaning of its component IDs may be altered. New -components are staged in a **draft** edition, whose contents may change. They become part of the compatibility guarantee -only when that draft is frozen as the next edition in its family. +A frozen edition never changes: neither its component list nor the meaning of its component IDs may be altered. A +component maintained as part of Vortex is staged in `preview` until its serialization is stable, then moves into a new +draft `core` edition. A component supplied by an optional plugin stays in that plugin's independently versioned family. + +A new stable `core` or plugin edition may freeze in the release in which it first ships. Until that release is cut, its +version is not known and the declaration keeps `min_vortex_version: None`. After the release is cut, the declaration is +updated with that newly released version, usually during development of the next release. This backfills the documented +minimum reader version; it does not delay the freeze or its read-forever compatibility guarantee. A component may later be deprecated, meaning that writers stop using it. Readers must continue to support it, so deprecation does not invalidate existing files. @@ -180,16 +187,17 @@ selected editions, the write fails. ## The `preview` family -Alongside `core` there is a `preview` family, holding components that are still being -evaluated. It is the exception to everything above: every `preview` edition is a permanent -draft, so the family never freezes and carries no read-compatibility guarantee at all. A file -written with these components is readable only by a build that knows them, and a future release -may stop supporting one. +Alongside `core` there is a `preview` family, holding non-plugin components whose serialization +is still being evaluated. It is the exception to everything above: every `preview` edition is a +permanent draft, so the family never freezes and carries no read-compatibility guarantee at all. +A file written with these components is readable only by a build that knows them, and a future +release may stop supporting one. Because of that, the writer only emits them when you opt in — the default session enables the newest `preview` edition solely when the `unstable_encodings` cargo feature is selected. -Components graduate by being declared in a new `core` edition, which is where they pick up the -read-forever guarantee. +Once a component's serialization is stable, it graduates by moving into a new `core` edition. +Components owned by optional plugins do not use `preview`; they live in standalone families such +as `spatial` and `json`, because a reader without the plugin cannot resolve them. ## Declaring, freezing, and the edition records @@ -203,20 +211,27 @@ cargo run -p xtask -- generate-editions Changing the declarations follows the edition's lifecycle: -1. **Declare a draft.** Add a module declaring the new edition with `min_vortex_version: None` - and the members that join the family at it, then regenerate the records. A draft carries no - guarantee, so its declaration and record may change freely — or be dropped — while it is - assembled. -2. **Freeze it.** Once a release ships readers for every member, record that release as the - edition's `min_vortex_version` and regenerate the records. Freezing is the act of - publishing the read-forever guarantee. -3. **Never touch it again.** A frozen record is immutable: CI +1. **Incubate the component.** Put a non-plugin component in `preview` while its serialization + can still change. Put a component supplied by an optional plugin in that plugin's standalone + family instead. These editions have `min_vortex_version: None` and carry no guarantee. +2. **Cut a stable edition.** Once a non-plugin component's serialization is stable, move it into + a new `core` edition. A stable plugin component remains in its plugin family. Declare the new + edition with `min_vortex_version: None`, regenerate the records, and ship it in a release. The + edition freezes as part of that release. For a core edition, its date is the freeze date. Its + minimum Vortex version cannot be populated yet because the release version is not known until + the release is cut. +3. **Backfill the released version.** After cutting the release, set `min_vortex_version` to that + newly released Vortex version — the version that first shipped readers for every member — and + regenerate the records. This update usually lands during development of the next release, but + it documents the freeze that already happened; it does not freeze the edition later. +4. **Never touch it again.** A frozen record is immutable: CI (`cargo run -p xtask -- check-editions`) rejects any change that edits, renames, unfreezes, or deletes a frozen record, and rejects new editions that do not extend their family's chronology. To change what writers may emit, declare the next edition instead. -The record files are the reviewable contract, so changes under `vortex/editions/` and -`vortex-edition/src/declarations/` additionally require two approvals to merge. +Changes under `vortex-edition/src/declarations/core/` require approval from `robert3005` or +`joseph-isaacs`. Generated records under `vortex/editions/core/` use the repository's normal +approval policy. ## Edition registry @@ -261,7 +276,7 @@ Minimum Vortex release: `0.84.0`. Minimum Vortex release: `0.84.0`. -- `array`: `vortex.map` +- `array`: `vortex.onpair` ### Draft editions @@ -269,12 +284,12 @@ Draft component lists may change and have no minimum reader or permanent compati #### `core2026.08.2` -- `array`: `vortex.parquet.variant`, `vortex.variant` -- `dtype`: `vortex.uuid` +- `array`: `vortex.map` #### `core2026.08.3` -- `array`: `vortex.onpair` +- `array`: `vortex.parquet.variant`, `vortex.variant` +- `dtype`: `vortex.uuid` #### `preview2025.05.0` diff --git a/policy.yml b/policy.yml index 35ad74194d5..8f1449fdbba 100644 --- a/policy.yml +++ b/policy.yml @@ -8,6 +8,7 @@ policy: - a vortex committer has approved - an untouched renovate pull request has an allowed approval - claude or codex authored pull requests have two committer approvals + - core edition changes have an edition owner approval disapproval: options: methods: @@ -86,3 +87,15 @@ approval_rules: count: 2 teams: - "vortex-data/committers" + + - name: core edition changes have an edition owner approval + description: "Changes to core edition declarations require an edition owner." + if: + changed_files: + paths: + - "^vortex-edition/src/declarations/core/.*$" + requires: + count: 1 + users: + - "robert3005" + - "joseph-isaacs" diff --git a/vortex-edition/src/declarations/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs index 0f7e983dcdc..16c9bd71ec9 100644 --- a/vortex-edition/src/declarations/core/mod.rs +++ b/vortex-edition/src/declarations/core/mod.rs @@ -13,8 +13,10 @@ pub static FAMILY: EditionFamily = EditionFamily { name: "core", doc: "The serialized components the default file writer emits. Every core edition \ freezes, and a frozen edition carries a read-forever guarantee: a file written with it stays \ -readable by every later Vortex release. New components join by being declared in a new \ -edition; an edition that has frozen never changes again.", +readable by every later Vortex release. A non-plugin component joins core once its \ +serialization is stable. Its edition may freeze in the release that cuts it; after that release \ +version is known, the declaration is backfilled with it as the minimum. A frozen edition never \ +changes.", }; pub mod v2025_05; diff --git a/vortex-edition/src/declarations/preview/mod.rs b/vortex-edition/src/declarations/preview/mod.rs index eba693f1719..05cad1a844d 100644 --- a/vortex-edition/src/declarations/preview/mod.rs +++ b/vortex-edition/src/declarations/preview/mod.rs @@ -11,11 +11,13 @@ use crate::EditionFamily; /// The `preview` family: opt-in components with no compatibility guarantee. pub static FAMILY: EditionFamily = EditionFamily { name: "preview", - doc: "Opt-in components that are still being evaluated. Every preview edition stays a \ -draft, so the family never freezes and carries no compatibility guarantee: a file written \ -with these components is readable only by a build that knows them, and a later release may \ -stop supporting one. The writer emits them only when the `unstable_encodings` feature is \ -selected. A component graduates by joining a core edition.", + doc: "Opt-in, non-plugin components whose serialization is still being evaluated. Every \ +preview edition stays a draft, so the family never freezes and carries no compatibility \ +guarantee: a file written with these components is readable only by a build that knows them, \ +and a later release may stop supporting one. The writer emits them only when the \ +`unstable_encodings` feature is selected. Once a component's serialization is stable, it \ +moves into a core edition. Components supplied by optional plugins instead live in standalone \ +families such as spatial and json.", }; pub mod v2025_05; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index 68be2a58e8f..d3d9cdd41a0 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Definitions of Vortex *editions*: named, frozen sets of components that a writer may put -//! in a file, carrying a forever read-compatibility guarantee. +//! Definitions of Vortex *editions*: named sets of components that a writer may put in a +//! file. Frozen editions carry a forever read-compatibility guarantee; draft editions do not. //! //! Editions live on the session, like encodings do: [`EditionSession`] holds the registered //! editions and [`EnabledEditions`] selects which of them a writer may emit. Declarations @@ -16,15 +16,16 @@ //! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets, never one //! untyped set. //! -//! An edition is a **draft** until its [`Edition::min_vortex_version`] is recorded — -//! recording it is the act of freezing. The per-edition member sets are computed from the -//! registered declarations by [`EditionSession::components_in`], and correctness is enforced -//! by unit tests: [`EditionSession::validate`] checks a whole registry, and -//! [`test_harness::validate_edition`] validates one edition's constraints — call it once in +//! An edition is represented as a **draft** until its [`Edition::min_vortex_version`] is +//! recorded. A stable edition may freeze in the release that cuts it; once that release version +//! is known, the field is backfilled to document the freeze. The per-edition member sets are +//! computed from the registered declarations by [`EditionSession::components_in`], and +//! correctness is enforced by unit tests: [`EditionSession::validate`] checks a whole registry, +//! and [`test_harness::validate_edition`] validates one edition's constraints — call it once in //! the `#[cfg(test)]` module of each edition definition. //! -//! The first-party edition declarations live in the public `vortex` crate, which registers -//! and enables them on the default session. See the published spec at +//! The first-party edition declarations live in this crate. The public `vortex` crate +//! re-exports them and registers and enables them on the default session. See the published spec at //! . pub mod declarations; @@ -49,17 +50,18 @@ use vortex_session::registry::Id; /// The identifier of an edition, e.g. `core2026.07.0`. /// /// The `family` names an independently versioned, additive group of components (`core` is the -/// set the default writer emits). The date components record when the edition was frozen and -/// order editions chronologically *within* a family; there is no ordering across families. +/// set the default writer emits). For `core`, the date components record when the edition freezes; +/// that date is prospective while the edition is still a draft. Dates order editions +/// chronologically *within* a family; there is no ordering across families. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct EditionId { /// The edition family, e.g. `core`. pub family: &'static str, - /// Year the edition was cut. + /// Year in the edition date. For `core`, this is the freeze year. pub year: u16, - /// Month the edition was cut. + /// Month in the edition date. For `core`, this is the freeze month. pub month: u8, - /// Distinguishes editions cut in the same month; normally `0`. + /// Distinguishes editions with the same family, year, and month; normally `0`. pub version: u8, } @@ -182,18 +184,19 @@ impl Display for ComponentKind { } } -/// An edition: a named set of components with a read-compatibility guarantee, registered with -/// [`EditionSession::declare_edition`]. The set itself is computed from the registered -/// [`EditionInclusion`]s by [`EditionSession::components_in`]. +/// An edition: a named set of components that can acquire a read-compatibility guarantee, +/// registered with [`EditionSession::declare_edition`]. The set itself is computed from the +/// registered [`EditionInclusion`]s by [`EditionSession::components_in`]. #[derive(Clone, Copy, Debug)] pub struct Edition { - /// The edition identifier. Also carries the freeze date: `core2026.07.0` freezes in - /// 2026-07. + /// The edition identifier. For a `core` edition, its date records when it freezes. pub id: EditionId, /// The minimum Vortex version whose reader supports every member of this edition. /// - /// Recording this is the act of freezing: an edition with `None` is a **draft** — being - /// assembled, carrying no guarantee, free to change, never the default write target. + /// A stable edition may freeze in the release that cuts it. Until that release is cut, its + /// version is not known and this remains `None`. The version is then backfilled to document + /// the already completed freeze and identify the first released reader supporting every + /// member. Preview editions remain drafts permanently. /// Validated against the members' [`EditionInclusion::required_vortex_release`] values: /// no member may require a version newer than the edition declares. pub min_vortex_version: Option<&'static str>, diff --git a/vortex/editions/core/core2026.08.2.toml b/vortex/editions/core/core2026.08.2.toml index a9c9543f20e..66b745c4c9e 100644 --- a/vortex/editions/core/core2026.08.2.toml +++ b/vortex/editions/core/core2026.08.2.toml @@ -1,8 +1,8 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again. +# This edition record has no documented compatibility guarantee. Its serialization may still be +# evolving, or it may be waiting for its release to be cut. After the release version is known, +# min_vortex_version is backfilled to document the freeze. A frozen record never changes. edition = "core2026.08.2" family = "core" diff --git a/vortex/editions/core/core2026.08.3.toml b/vortex/editions/core/core2026.08.3.toml index dc7dda51b71..10b377b299f 100644 --- a/vortex/editions/core/core2026.08.3.toml +++ b/vortex/editions/core/core2026.08.3.toml @@ -1,8 +1,8 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again. +# This edition record has no documented compatibility guarantee. Its serialization may still be +# evolving, or it may be waiting for its release to be cut. After the release version is known, +# min_vortex_version is backfilled to document the freeze. A frozen record never changes. edition = "core2026.08.3" family = "core" diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml index 4ab2bc80dca..0ac6aefb2df 100644 --- a/vortex/editions/core/family.toml +++ b/vortex/editions/core/family.toml @@ -7,6 +7,7 @@ name = "core" doc = """ The serialized components the default file writer emits. Every core edition freezes, and a frozen edition carries a read-forever guarantee: a file written with it stays readable by -every later Vortex release. New components join by being declared in a new edition; an -edition that has frozen never changes again. +every later Vortex release. A non-plugin component joins core once its serialization is +stable. Its edition may freeze in the release that cuts it; after that release version is +known, the declaration is backfilled with it as the minimum. A frozen edition never changes. """ diff --git a/vortex/editions/preview/family.toml b/vortex/editions/preview/family.toml index de766d1db78..824071d166a 100644 --- a/vortex/editions/preview/family.toml +++ b/vortex/editions/preview/family.toml @@ -5,9 +5,11 @@ name = "preview" doc = """ -Opt-in components that are still being evaluated. Every preview edition stays a draft, so -the family never freezes and carries no compatibility guarantee: a file written with these -components is readable only by a build that knows them, and a later release may stop -supporting one. The writer emits them only when the `unstable_encodings` feature is -selected. A component graduates by joining a core edition. +Opt-in, non-plugin components whose serialization is still being evaluated. Every preview +edition stays a draft, so the family never freezes and carries no compatibility guarantee: a +file written with these components is readable only by a build that knows them, and a later +release may stop supporting one. The writer emits them only when the `unstable_encodings` +feature is selected. Once a component's serialization is stable, it moves into a core +edition. Components supplied by optional plugins instead live in standalone families such as +spatial and json. """ diff --git a/vortex/editions/preview/preview2025.05.0.toml b/vortex/editions/preview/preview2025.05.0.toml index 653eb5df7a2..835bcd0329f 100644 --- a/vortex/editions/preview/preview2025.05.0.toml +++ b/vortex/editions/preview/preview2025.05.0.toml @@ -1,8 +1,8 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again. +# This edition record has no documented compatibility guarantee. Its serialization may still be +# evolving, or it may be waiting for its release to be cut. After the release version is known, +# min_vortex_version is backfilled to document the freeze. A frozen record never changes. edition = "preview2025.05.0" family = "preview" diff --git a/vortex/editions/preview/preview2026.02.0.toml b/vortex/editions/preview/preview2026.02.0.toml index ca1be1431ad..2d189b2d9a3 100644 --- a/vortex/editions/preview/preview2026.02.0.toml +++ b/vortex/editions/preview/preview2026.02.0.toml @@ -1,8 +1,8 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again. +# This edition record has no documented compatibility guarantee. Its serialization may still be +# evolving, or it may be waiting for its release to be cut. After the release version is known, +# min_vortex_version is backfilled to document the freeze. A frozen record never changes. edition = "preview2026.02.0" family = "preview" diff --git a/vortex/editions/preview/preview2026.04.0.toml b/vortex/editions/preview/preview2026.04.0.toml index dfecd6d3b21..31c210ab6f3 100644 --- a/vortex/editions/preview/preview2026.04.0.toml +++ b/vortex/editions/preview/preview2026.04.0.toml @@ -1,8 +1,8 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again. +# This edition record has no documented compatibility guarantee. Its serialization may still be +# evolving, or it may be waiting for its release to be cut. After the release version is known, +# min_vortex_version is backfilled to document the freeze. A frozen record never changes. edition = "preview2026.04.0" family = "preview" diff --git a/vortex/editions/preview/preview2026.06.0.toml b/vortex/editions/preview/preview2026.06.0.toml index d4d41ab3b6b..e61160a7ddf 100644 --- a/vortex/editions/preview/preview2026.06.0.toml +++ b/vortex/editions/preview/preview2026.06.0.toml @@ -1,8 +1,8 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again. +# This edition record has no documented compatibility guarantee. Its serialization may still be +# evolving, or it may be waiting for its release to be cut. After the release version is known, +# min_vortex_version is backfilled to document the freeze. A frozen record never changes. edition = "preview2026.06.0" family = "preview" diff --git a/xtask/src/check_editions.rs b/xtask/src/check_editions.rs index a02179bc70d..0c116a945d3 100644 --- a/xtask/src/check_editions.rs +++ b/xtask/src/check_editions.rs @@ -3,11 +3,12 @@ //! Check that frozen edition records under `vortex/editions` never change. //! -//! A record's mutability follows its edition. A draft is still being assembled, so its record -//! may change, be renamed, or be dropped. Freezing — recording a `min_vortex_version` — turns -//! the record into a read-forever contract, and from then on it may never change again. -//! Whether a record was frozen is read from the base revision, so a change cannot unfreeze an -//! edition and edit it in the same diff. +//! A draft record carries no compatibility guarantee and may change, be renamed, or be dropped. +//! Its serialization may still be evolving, or a stable edition may be waiting for its release to +//! be cut. A stable edition can freeze in that release; once the release version is known, +//! `min_vortex_version` is backfilled to document the freeze. The record then carries a +//! read-forever guarantee and may never change again. Whether a record was frozen is read from the +//! base revision, so a change cannot unfreeze an edition and edit it in the same diff. //! //! A newly added record must also be newer than every edition already recorded for its //! family: editions are only ever added going forward. Records are grouped by family, so @@ -35,7 +36,7 @@ use toml::Table; use crate::generate_editions::FAMILY_FILE; use crate::generate_editions::RECORD_DIR; -/// A record carries this key exactly when the edition it records is frozen. +/// A record carries this key once the edition's freeze has been documented. const FROZEN_MARKER: &str = "min_vortex_version"; const REMEDY: &str = "\ diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index e70a7718c88..c2e9df58fbc 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -9,12 +9,12 @@ //! `vortex/editions/core/core2025.05.0.toml` — mirroring the declarations in //! `vortex-edition/src/declarations`, since families version independently. //! -//! A record's mutability follows its edition. A draft is still being assembled, so its record -//! changes with it. Freezing — recording a `min_vortex_version` — turns the record into a -//! contract carrying a read-forever guarantee, and from then on it may never change again. CI -//! enforces that against git history with `cargo run -p xtask -- check-editions`; this -//! exporter enforces the two rules that history cannot see, refusing to delete a record or to -//! unfreeze one. +//! A draft record carries no compatibility guarantee. Its serialization may still be evolving, +//! or a stable edition may be waiting for its release to be cut. A stable edition can freeze in +//! that release; once the release version is known, `min_vortex_version` is backfilled to document +//! the freeze. The record then carries a read-forever guarantee and may never change again. CI +//! enforces that against git history with `cargo run -p xtask -- check-editions`; this exporter +//! enforces the two rules that history cannot see, refusing to delete a record or to unfreeze one. use std::collections::BTreeSet; use std::fs; @@ -38,9 +38,9 @@ const FROZEN_NOTE: &str = "\ # editing or deleting a frozen one is rejected by CI."; const DRAFT_NOTE: &str = "\ -# This edition is a draft: it carries no guarantee and is still being assembled, so this -# record changes with it. Recording a min_vortex_version freezes the edition, after which -# this file may never change again."; +# This edition record has no documented compatibility guarantee. Its serialization may still be +# evolving, or it may be waiting for its release to be cut. After the release version is known, +# min_vortex_version is backfilled to document the freeze. A frozen record never changes."; /// The file recording what a family is, beside that family's editions. pub const FAMILY_FILE: &str = "family.toml"; @@ -183,7 +183,7 @@ fn existing_records(dir: &Path) -> anyhow::Result> { Ok(records) } -/// A record carries a `min_vortex_version` exactly when the edition it records is frozen. +/// A record carries a `min_vortex_version` once its freeze has been documented. fn records_a_frozen_edition(contents: &str) -> bool { contents .lines() From d95b7699d03221724cbb672065871edbe2a3850c Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Wed, 26 Aug 2026 11:38:19 +0100 Subject: [PATCH 17/18] more Signed-off-by: Robert Kruszewski --- docs/specs/editions.md | 170 +++++++++++------- vortex-btrblocks/src/builder.rs | 21 ++- vortex-btrblocks/src/lib.rs | 3 + vortex-compressor/src/compressor/cascade.rs | 12 +- vortex-compressor/src/compressor/mod.rs | 41 +++++ vortex-compressor/src/compressor/tests.rs | 65 +++++++ vortex-compressor/src/lib.rs | 1 + vortex-compressor/src/scheme/mod.rs | 24 ++- vortex-edition/src/declarations/core/mod.rs | 23 +-- .../src/declarations/preview/mod.rs | 22 +-- vortex-edition/src/lib.rs | 126 ++++++++++--- vortex-edition/src/session.rs | 134 +++++++++++--- vortex-edition/src/tests.rs | 91 ++++++++-- vortex-file/src/strategy.rs | 20 +++ vortex-file/src/writer.rs | 18 +- vortex/editions/core/core2025.05.0.toml | 96 +++++----- vortex/editions/core/core2025.06.0.toml | 62 +++---- vortex/editions/core/core2025.10.0.toml | 72 ++++---- vortex/editions/core/core2026.08.0.toml | 64 +++---- vortex/editions/core/core2026.08.1.toml | 68 +++---- vortex/editions/core/core2026.08.2.toml | 77 ++++---- vortex/editions/core/core2026.08.3.toml | 83 ++++----- vortex/editions/core/family.toml | 13 +- vortex/editions/preview/family.toml | 14 +- vortex/editions/preview/preview2025.05.0.toml | 15 +- vortex/editions/preview/preview2026.02.0.toml | 17 +- vortex/editions/preview/preview2026.04.0.toml | 35 ++-- vortex/editions/preview/preview2026.06.0.toml | 25 +-- vortex/src/editions/mod.rs | 5 +- vortex/src/editions/tests.rs | 24 +++ vortex/src/lib.rs | 2 +- xtask/src/check_editions.rs | 16 +- xtask/src/generate_editions.rs | 59 +++--- 33 files changed, 1003 insertions(+), 515 deletions(-) diff --git a/docs/specs/editions.md b/docs/specs/editions.md index 28e6f7eab1f..01fd71fdbb3 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -1,17 +1,22 @@ # Editions Vortex files contain several kinds of serialized **component**: array encodings, layout encodings, extension dtypes, and -aggregate functions. An **edition** is a named set of these components. It controls what a writer may put in a file and, -once frozen, identifies the earliest Vortex release that supports every component in the set. +aggregate functions. An **edition** is a named set of these components. For each array encoding, it also pins an +**array writer version**: the compatible serialized features that compression schemes may produce under that array ID. +An edition controls what a writer may put in a file and, once frozen, identifies the earliest Vortex release that +supports every component and writer feature in the set. -Each component consists of a kind, an ID, and the wire contract for its metadata and payload. The compatibility -guarantee applies to that serialized contract, not to the in-memory implementation that reads or writes it. +An array writer version is only a write-time capability ceiling. It is not an in-memory array version, is not written as +a version tag, and never selects a reader. Each array ID has exactly one registered reader. A higher writer version may +authorize a compression scheme to populate compatible optional fields or properties that an earlier writer never +produced. An incompatible representation is a new array encoding with a new ID. Editions belong to independently versioned families and are cumulative within a family. Each edition includes all -components from the preceding edition in that family, plus any newly added components. A writer selects at most one -edition from each family and may use the union of their components. For example, selecting `core2026.08.1` and -`preview2026.06.0` allows stable components released through August 2026 and preview components released through -June 2026. +components and writer versions from the preceding edition in that family, plus any additions. A writer selects at most +one edition from each family and may use the union of their components. If enabled families mention the same array ID, +the writer resolves one effective version: the highest permitted writer version. It does not retain multiple array +versions. For example, selecting `core2026.08.1` and `preview2026.06.0` allows stable components released through August +2026 and preview components released through June 2026. The first frozen edition, `core2025.05.0`, contains the components that Vortex `0.36.0` could write. This marks the start of the Vortex file format's stability guarantee. Every Vortex release from `0.36.0` onward can read @@ -23,16 +28,16 @@ guarantee for any draft components written to the file. ## What an edition contains -An edition records every component by kind and ID. IDs are unique within a kind, but not across kinds: a layout named -`vortex.flat` and an array encoding with the same ID are distinct components. The writer therefore builds and enforces a -separate allowlist for each kind: +An edition records every component by kind and ID. Array entries additionally record one writer version. IDs are unique +within a kind, but not across kinds: a layout named `vortex.flat` and an array encoding with the same ID are distinct +components. The writer therefore builds and enforces a separate allowlist for each kind: -| Kind | Written | Enforced at | -|-------------|--------------------------------------------|------------------------------| -| `array` | every serialized array | array serialization context | -| `layout` | the footer's layout tree | layout serialization context | -| `dtype` | extension dtypes nested in the file schema | file writer | -| `aggregate` | zone maps in zoned layouts | the layout writer context | +| Kind | What it identifies | Used at | +|-------------|---------------------------------------------|-------------------------------| +| `array` | every serialized array and its writer limit | compression and serialization | +| `layout` | the footer's layout tree | layout serialization context | +| `dtype` | extension dtypes nested in the file schema | file writer | +| `aggregate` | zone maps in zoned layouts | layout writer context | Writing a component that is absent from the selected editions fails the write. This rule applies to every kind, including aggregates. Although a zone map is only an optimization and could be dropped, doing so would silently change @@ -42,7 +47,9 @@ Only aggregates that would actually be written are checked. If a column's dtype omits it and there is no edition violation. An empty allowlist permits no encodings. Collectively, the selected editions must declare every array encoding, layout -encoding, extension dtype, and aggregate function that the writer serializes. +encoding, extension dtype, and aggregate function that the writer serializes. For a given array ID, the resolved edition +set supplies exactly one writer ceiling. Compression schemes declare the minimum writer version required for the +specific representation they would produce and are filtered against that ceiling before doing expensive work. For example, `core2026.08.0` declares the aggregate functions that the default writer may store in zone maps: `min`, `max`, `bounded_min`, `bounded_max`, `nan_count`, and `null_count`. It does not declare `sum`, because the writer does @@ -69,11 +76,11 @@ zone-map pruning rather than causing the file to be rejected. ## Writing with an edition -By default, the Vortex facade targets the newest frozen `core` edition. Components maintained as part of Vortex first -belong to `preview` while their serialization is evolving. Components supplied by an optional plugin instead belong to -that plugin's standalone edition family, such as `spatial` or `json`. Once a non-plugin component's serialization is -stable, it moves into a new draft `core` edition. If serialization would use a component outside the selected editions, -the write fails immediately. +By default, the Vortex facade targets the newest frozen `core` edition. A new encoding or serialization feature that is +still evolving gets a new draft edition; later additions create later editions rather than changing an already +published feature set. Once a core-maintained feature is stable, it can join `preview` for explicit adoption without +changing the default writer. Components supplied by an optional plugin instead belong to that plugin's standalone +edition family, such as `spatial` or `json`. Edition configuration belongs to the writer's Vortex session. Registering an edition makes its declaration available to the session; enabling it allows the writer to use its components. Enabling another edition in the same family replaces @@ -89,11 +96,19 @@ Sessions created without the Vortex facade must register and enable their editio `with_allow_encodings` policy can further restrict array encodings, but cannot permit an encoding excluded by the selected editions. +The default file writer passes the resolved array writer versions into BtrBlocks. For each canonical input, BtrBlocks +removes a scheme if the representation it would produce needs an absent or newer writer version. This happens before +statistics generation, ratio estimation, sampling, and compression, so the writer does not compress an array and only +then discover that its selected edition forbids the result. The array allowlist remains a final check on the output. + ## How editions change -A frozen edition never changes: neither its component list nor the meaning of its component IDs may be altered. A -component maintained as part of Vortex is staged in `preview` until its serialization is stable, then moves into a new -draft `core` edition. A component supplied by an optional plugin stays in that plugin's independently versioned family. +A frozen edition never changes: neither its membership list nor the meaning of its component IDs or writer versions may +be altered. A new encoding or added encoding capability that has not stabilized gets its own new draft edition. Once a +component maintained as part of core is stable, it may join `preview`. Preview is an adoption boundary, not an +experimentation boundary: its serialized behavior should change only to fix a defect serious enough to block promotion +into core. Promotion into the default compatibility set happens through a later `core` edition. A component supplied by +an optional plugin stays in that plugin's independently versioned family. A new stable `core` or plugin edition may freeze in the release in which it first ships. Until that release is cut, its version is not known and the declaration keeps `min_vortex_version: None`. After the release is cut, the declaration is @@ -103,27 +118,38 @@ minimum reader version; it does not delay the freeze or its read-forever compati A component may later be deprecated, meaning that writers stop using it. Readers must continue to support it, so deprecation does not invalidate existing files. +Writer behavior evolves independently. Adding a compatible optional field to an array lets the one reader for that ID +understand the field, but does not authorize existing writers to populate it. A new edition raises that array's writer +version. Sessions targeting the earlier edition retain the earlier output behavior; users opt in by selecting the newer +edition. If the change is incompatible, it is a new array ID instead of a writer-version increase. + ## How serialized components evolve Editions govern serialized components, not in-memory representations. An in-memory representation may gain capabilities -or be replaced without changing an edition. On read, the plugin registered for a component ID constructs the current -in-memory representation. On write, the implementation selects a component that can represent the value and is allowed -by the selected editions. +or be replaced without changing an edition. On read, the single plugin registered for a component ID constructs the +current in-memory representation. On write, the implementation selects a component and writer behavior allowed by the +selected editions. An in-memory representation often has a single serialized component and uses the same ID in memory and on disk, but this is not required. Multiple component IDs may deserialize into the same in-memory representation. Editions constrain the ID stored in the file, because that is what the reader must understand. -### Compatible evolution keeps the ID +### Additive evolution keeps the ID + +A component may keep its ID when the new form is an additive, unambiguous extension of its serialized contract. The one +current reader must interpret every historical form correctly, using information already present in the array such as +its dtype or optional metadata fields. A new reader must use the old default when an optional field is absent. An older +reader may reject a form introduced by a later edition, but it must not silently misinterpret it; the later edition +identifies the newer minimum reader. -A component may keep its ID only if changes to its wire format are both **backward and forward compatible**: a new -reader must correctly interpret data from an old writer, and an old reader must correctly interpret data from a new -writer. For example, adding an optional field is compatible only if old readers can safely ignore it and new readers use -the correct default when it is absent. +Additive evolution may broaden what the wire format accepts, but it cannot change the meaning of data that existing +readers already accept. Removing or repurposing a field, redefining existing bytes, or making interpretation depend on a +separate reader-version choice are incompatible changes. -Compatible evolution may broaden what the wire format accepts, but it cannot change the meaning of data that existing -readers already accept. Removing or repurposing a field, redefining existing bytes, and requiring information that old -writers did not provide are all incompatible changes. +Reader and writer evolution are deliberately asymmetric. The one reader for an array ID may start accepting a compatible +optional field when the old default is well-defined. Existing edition selections must continue producing their old +form. A higher array writer version opts into populating the field, so upgrading Vortex alone does not silently change a +user's files. ### Incompatible evolution requires a new ID @@ -158,7 +184,12 @@ interior patches is read as a `Patched` array around a patch-free ALP array. Sim Readers do not negotiate versions. They resolve the component ID and deserialize it, or report an [unknown-component error](#resolving-an-unknown-component-error). -### Writing: select a permitted component +In particular, an array writer version does not create `v1` and `v2` reader registrations. A file contains its array ID, +dtype, metadata, children, and buffers. The reader resolves that ID once and the resulting plugin interprets the actual +serialized form. If a change would require selecting a different interpretation for the same bytes, it is incompatible +and needs a new array ID. + +### Writing: select a permitted component and writer behavior Writers choose a component that both represents the current value and belongs to the selected editions. This need not be the newest component: if an older component can represent the value exactly, the writer may continue to use it. If the @@ -167,16 +198,17 @@ preferred component is not permitted, the writer has two options: 1. **Translate.** If the value has a lossless translation to a permitted component, use that component. For example, a newer layout may write its zone statistics using an older statistics schema. 2. **Convert to canonical and recompress.** Otherwise, decompress the data to a canonical representation and recompress - it with the configured compressors, restricted to the selected editions. This is how arrays are handled today: the + it with the configured schemes, restricted to the selected editions. This is how arrays are handled today: the writer normalizes each chunk to a canonical representation, then lets the edition-filtered compressor choose the - final encoding. + final encoding and compatible serialized features. Both paths use the normal write pipeline and its configured compressors. If neither can express the data using the selected editions, the write fails. ### What this means for each kind -- **Arrays.** The array serialization context permits only encodings from the selected editions. +- **Arrays.** The array serialization context permits only encodings from the selected editions. BtrBlocks additionally + checks the required writer version for each candidate representation before evaluating its scheme. - **Layouts.** The layout strategy builds the layout tree at write time. When targeting an older edition, it must use structures available in that edition, such as plain chunked data in place of newer auxiliary layouts. - **Extension dtypes.** Before writing any bytes, the file writer recursively validates every extension dtype in the @@ -187,17 +219,19 @@ selected editions, the write fails. ## The `preview` family -Alongside `core` there is a `preview` family, holding non-plugin components whose serialization -is still being evaluated. It is the exception to everything above: every `preview` edition is a -permanent draft, so the family never freezes and carries no read-compatibility guarantee at all. -A file written with these components is readable only by a build that knows them, and a future -release may stop supporting one. +Alongside `core` there is a `preview` family for stabilized, core-maintained components and array writer-version +increases that are ready for explicit adoption but are not yet part of the default core writer. Preview behavior is +expected to remain compatible and should change only to fix a defect serious enough to block promotion into core. It +does not yet carry core's unconditional read-forever guarantee. + +The default writer does not adopt a higher writer version merely because its reader understands the corresponding +optional features. Users opt in by enabling the preview edition that raises the version. Today, builds using the +`unstable_encodings` Cargo feature also opt into registration and availability of the newest preview component set. -Because of that, the writer only emits them when you opt in — the default session enables the -newest `preview` edition solely when the `unstable_encodings` cargo feature is selected. -Once a component's serialization is stable, it graduates by moving into a new `core` edition. -Components owned by optional plugins do not use `preview`; they live in standalone families such -as `spatial` and `json`, because a reader without the plugin cannot resolve them. +Components that are still evolving belong to new draft editions rather than `preview`; each added +feature advances the edition so a file's capability set remains identifiable. Components owned by +optional plugins do not use `preview`; they live in standalone families such as `spatial` and +`json`, because a reader without the plugin cannot resolve them. ## Declaring, freezing, and the edition records @@ -211,20 +245,24 @@ cargo run -p xtask -- generate-editions Changing the declarations follows the edition's lifecycle: -1. **Incubate the component.** Put a non-plugin component in `preview` while its serialization - can still change. Put a component supplied by an optional plugin in that plugin's standalone - family instead. These editions have `min_vortex_version: None` and carry no guarantee. -2. **Cut a stable edition.** Once a non-plugin component's serialization is stable, move it into - a new `core` edition. A stable plugin component remains in its plugin family. Declare the new - edition with `min_vortex_version: None`, regenerate the records, and ship it in a release. The - edition freezes as part of that release. For a core edition, its date is the freeze date. Its - minimum Vortex version cannot be populated yet because the release version is not known until - the release is cut. -3. **Backfill the released version.** After cutting the release, set `min_vortex_version` to that +1. **Create a draft feature edition.** A new, not-yet-stable encoding or added capability gets a + new edition. Further capabilities advance to another edition instead of silently expanding an + existing record. A component supplied by an optional plugin uses that plugin's standalone + family. +2. **Publish stabilized core work in preview.** Once a core-maintained serialized feature is + stable, add it to a new `preview` edition. If compression schemes should begin populating a + compatible optional feature of an existing array, raise that array's writer version in the + preview edition. The core edition continues selecting the earlier version until the feature is + deliberately promoted. +3. **Cut a core edition.** Promote adopted preview members into a new `core` edition with + `min_vortex_version: None`, regenerate the records, and ship it in a release. The edition + freezes as part of that release. Its minimum Vortex version cannot be populated yet because the + release version is not known until the release is cut. +4. **Backfill the released version.** After cutting the release, set `min_vortex_version` to that newly released Vortex version — the version that first shipped readers for every member — and regenerate the records. This update usually lands during development of the next release, but it documents the freeze that already happened; it does not freeze the edition later. -4. **Never touch it again.** A frozen record is immutable: CI +5. **Never touch it again.** A frozen record is immutable: CI (`cargo run -p xtask -- check-editions`) rejects any change that edits, renames, unfreezes, or deletes a frozen record, and rejects new editions that do not extend their family's chronology. To change what writers may emit, declare the next edition instead. @@ -252,6 +290,8 @@ Minimum Vortex release: `0.36.0`. - `layout`: `vortex.chunked`, `vortex.dict`, `vortex.flat`, `vortex.stats`, `vortex.struct` - `dtype`: `vortex.date`, `vortex.time`, `vortex.timestamp` +All array entries currently use writer version 1 unless a later edition explicitly raises one. + #### `core2025.06.0` Minimum Vortex release: `0.40.0`. @@ -278,9 +318,11 @@ Minimum Vortex release: `0.84.0`. - `array`: `vortex.onpair` -### Draft editions +### Editions without a frozen core guarantee -Draft component lists may change and have no minimum reader or permanent compatibility guarantee. +These editions have no minimum reader version. Evolving features advance through new draft editions; stabilized preview +features are expected to remain compatible unless a defect is serious enough to block promotion into core. Optional +plugin families state their own policy. #### `core2026.08.2` diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index fe8072d5e66..b739d3c1b61 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -4,6 +4,7 @@ //! Builder for configuring `BtrBlocksCompressor` instances. use vortex_array::ArrayId; +use vortex_compressor::ArrayWriterVersions; use vortex_utils::aliases::hash_set::HashSet; use crate::BtrBlocksCompressor; @@ -91,12 +92,14 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[ #[derive(Debug, Clone)] pub struct BtrBlocksCompressorBuilder { schemes: Vec<&'static dyn Scheme>, + array_writer_versions: Option, } impl Default for BtrBlocksCompressorBuilder { fn default() -> Self { Self { schemes: ALL_SCHEMES.to_vec(), + array_writer_versions: None, } } } @@ -108,6 +111,7 @@ impl BtrBlocksCompressorBuilder { pub fn empty() -> Self { Self { schemes: Vec::new(), + array_writer_versions: None, } } @@ -214,9 +218,24 @@ impl BtrBlocksCompressorBuilder { self } + /// Constrains scheme selection to the enabled writer version of each array encoding. + /// + /// A scheme requiring an absent or newer version is rejected before it computes statistics, + /// estimates, samples, or compresses its input. This policy affects serialized output only; + /// it neither versions in-memory arrays nor changes reader registration. + pub fn with_array_writer_versions(mut self, versions: ArrayWriterVersions) -> Self { + self.array_writer_versions = Some(versions); + self + } + /// Builds the configured [`BtrBlocksCompressor`]. pub fn build(self) -> BtrBlocksCompressor { - BtrBlocksCompressor(CascadingCompressor::new(self.schemes)) + let compressor = CascadingCompressor::new(self.schemes); + let compressor = match self.array_writer_versions { + Some(versions) => compressor.with_array_writer_versions(versions), + None => compressor, + }; + BtrBlocksCompressor(compressor) } } diff --git a/vortex-btrblocks/src/lib.rs b/vortex-btrblocks/src/lib.rs index 1ca05c86b4e..19f0c4a6ce6 100644 --- a/vortex-btrblocks/src/lib.rs +++ b/vortex-btrblocks/src/lib.rs @@ -19,6 +19,8 @@ //! - **Cascaded Encoding**: Multiple compression layers can be applied for optimal results. //! - **Statistical Analysis**: Uses data sampling and statistics to predict compression ratios. //! - **Recursive Structure Handling**: Compresses nested structures like structs and lists. +//! - **Writer Compatibility**: Can reject schemes whose output needs a newer per-array writer +//! version before estimation or compression begins. //! //! # How It Works //! @@ -80,6 +82,7 @@ pub use builder::ALL_SCHEMES; pub use builder::BtrBlocksCompressorBuilder; pub use canonical_compressor::BtrBlocksCompressor; pub use schemes::patches::compress_patches; +pub use vortex_compressor::ArrayWriterVersions; pub use vortex_compressor::CascadingCompressor; pub use vortex_compressor::scheme::CompressorContext; pub use vortex_compressor::scheme::MAX_CASCADE; diff --git a/vortex-compressor/src/compressor/cascade.rs b/vortex-compressor/src/compressor/cascade.rs index 86d45d2c0d9..4e58a51e595 100644 --- a/vortex-compressor/src/compressor/cascade.rs +++ b/vortex-compressor/src/compressor/cascade.rs @@ -222,9 +222,9 @@ impl CascadingCompressor { /// The main scheme-selection entry point for a single leaf array. /// - /// Filters allowed schemes by [`matches`] and exclusion rules, merges their [`stats_options`] - /// into a single [`GenerateStatsOptions`], and picks the winner by estimated compression - /// ratio. + /// Filters allowed schemes by [`matches`], writer-version requirements, and exclusion rules, + /// merges their [`stats_options`] into a single [`GenerateStatsOptions`], and picks the winner + /// by estimated compression ratio. /// /// If a winner is found and its compressed output is actually smaller, that output is /// returned. Otherwise, the original array is returned unchanged. @@ -244,7 +244,11 @@ impl CascadingCompressor { .schemes .iter() .copied() - .filter(|s| s.matches(&canonical) && !self.is_excluded(*s, &compress_ctx)) + .filter(|s| { + s.matches(&canonical) + && self.writer_version_allows(*s, &canonical) + && !self.is_excluded(*s, &compress_ctx) + }) .collect(); let array: ArrayRef = canonical.into(); diff --git a/vortex-compressor/src/compressor/mod.rs b/vortex-compressor/src/compressor/mod.rs index a661970950c..96354ee19b8 100644 --- a/vortex-compressor/src/compressor/mod.rs +++ b/vortex-compressor/src/compressor/mod.rs @@ -9,6 +9,11 @@ mod sample; mod select; mod structural; +use std::collections::BTreeMap; +use std::sync::Arc; + +use vortex_array::ArrayId; + use crate::builtins::IntDictScheme; use crate::scheme::ChildSelection; use crate::scheme::DescendantExclusion; @@ -16,6 +21,12 @@ use crate::scheme::Scheme; use crate::scheme::SchemeExt; use crate::scheme::SchemeId; +/// The maximum compatible writer version enabled for each array encoding. +/// +/// This is a write-time policy. It is consulted before a scheme is estimated or run and is never +/// stored in an array or used to select a reader. +pub type ArrayWriterVersions = BTreeMap; + /// Synthetic scheme ID used for the compressor's own root-level cascading. pub(crate) const ROOT_SCHEME_ID: SchemeId = SchemeId { name: "vortex.compressor.root", @@ -46,6 +57,9 @@ pub struct CascadingCompressor { /// Descendant exclusion rules for the compressor's own cascading (e.g. excluding Dict from /// list offsets). root_exclusions: Vec, + + /// Per-array writer ceilings, when compression is constrained for serialization. + array_writer_versions: Option>, } impl CascadingCompressor { @@ -63,8 +77,35 @@ impl CascadingCompressor { Self { schemes, root_exclusions, + array_writer_versions: None, } } + + /// Constrains schemes to serialized features allowed by these per-array writer versions. + /// + /// Schemes whose required version is absent or newer than the configured ceiling are removed + /// before statistics, estimation, sampling, or compression. Without this policy, compression + /// is intended for in-memory use and all registered scheme versions remain eligible. + pub fn with_array_writer_versions(mut self, versions: ArrayWriterVersions) -> Self { + self.array_writer_versions = Some(Arc::new(versions)); + self + } + + /// Whether `scheme` can produce its representation of `canonical` under the writer policy. + fn writer_version_allows( + &self, + scheme: &dyn Scheme, + canonical: &vortex_array::Canonical, + ) -> bool { + let Some(enabled) = &self.array_writer_versions else { + return true; + }; + + scheme + .required_array_writer_versions(canonical) + .into_iter() + .all(|(id, required)| enabled.get(&id).is_some_and(|enabled| *enabled >= required)) + } } // NB: Cascading compression logic is located in `vortex-compressor/src/compressor/cascade.rs`. diff --git a/vortex-compressor/src/compressor/tests.rs b/vortex-compressor/src/compressor/tests.rs index ec14383ce36..5ac3f8a1052 100644 --- a/vortex-compressor/src/compressor/tests.rs +++ b/vortex-compressor/src/compressor/tests.rs @@ -2,6 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::sync::LazyLock; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use parking_lot::Mutex; use vortex_array::ArrayId; @@ -9,11 +11,13 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Constant; use vortex_array::arrays::Map; use vortex_array::arrays::NullArray; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builders::MapBuilder; @@ -60,6 +64,49 @@ fn matches_integer_primitive(canonical: &Canonical) -> bool { matches!(canonical, Canonical::Primitive(primitive) if primitive.ptype().is_int()) } +static WRITER_V2_WAS_ESTIMATED: AtomicBool = AtomicBool::new(false); + +#[derive(Debug)] +struct WriterV2Scheme; + +impl Scheme for WriterV2Scheme { + fn scheme_name(&self) -> &'static str { + "test.writer_v2" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn produced_encodings(&self) -> Vec { + vec![Primitive.id()] + } + + fn required_array_writer_versions(&self, _canonical: &Canonical) -> Vec<(ArrayId, u16)> { + vec![(Primitive.id(), 2)] + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + WRITER_V2_WAS_ESTIMATED.store(true, Ordering::Relaxed); + CompressionEstimate::Verdict(EstimateVerdict::Skip) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + unreachable!("the test scheme always skips") + } +} + #[derive(Debug)] struct DirectRatioScheme; @@ -697,6 +744,24 @@ fn ratio_tie_between_immediate_and_deferred_favors_immediate() -> VortexResult<( Ok(()) } +#[test] +fn writer_versions_filter_schemes_before_estimation() -> VortexResult<()> { + let array = PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + WRITER_V2_WAS_ESTIMATED.store(false, Ordering::Relaxed); + CascadingCompressor::new(vec![&WriterV2Scheme]) + .with_array_writer_versions([(Primitive.id(), 1)].into_iter().collect()) + .compress(&array, &mut exec_ctx)?; + assert!(!WRITER_V2_WAS_ESTIMATED.load(Ordering::Relaxed)); + + CascadingCompressor::new(vec![&WriterV2Scheme]) + .with_array_writer_versions([(Primitive.id(), 2)].into_iter().collect()) + .compress(&array, &mut exec_ctx)?; + assert!(WRITER_V2_WAS_ESTIMATED.load(Ordering::Relaxed)); + Ok(()) +} + #[test] fn all_null_array_compresses_to_constant() -> VortexResult<()> { let array = PrimitiveArray::new( diff --git a/vortex-compressor/src/lib.rs b/vortex-compressor/src/lib.rs index 55bb9b188f6..f99a385b0c3 100644 --- a/vortex-compressor/src/lib.rs +++ b/vortex-compressor/src/lib.rs @@ -67,6 +67,7 @@ pub mod scheme; pub mod stats; mod compressor; +pub use compressor::ArrayWriterVersions; pub use compressor::CascadingCompressor; mod trace; diff --git a/vortex-compressor/src/scheme/mod.rs b/vortex-compressor/src/scheme/mod.rs index de9e67690d4..f1b58c10512 100644 --- a/vortex-compressor/src/scheme/mod.rs +++ b/vortex-compressor/src/scheme/mod.rs @@ -56,9 +56,9 @@ impl fmt::Display for SchemeId { // TODO(connor): Remove all default implemented methods. /// A single compression encoding that the [`CascadingCompressor`] can select from. /// -/// The compressor evaluates every registered scheme whose [`matches`] returns `true` for a given -/// array, picks the one with the highest [`expected_compression_ratio`], and calls [`compress`] on -/// the winner. +/// The compressor evaluates every registered scheme whose [`matches`] returns `true` and whose +/// [`required_array_writer_versions`] fit the configured write policy, picks the one with the +/// highest [`expected_compression_ratio`], and calls [`compress`] on the winner. /// /// One of the key features of the compressor in this crate is that schemes may "cascade". A /// scheme's [`compress`] can call back into the compressor via @@ -113,6 +113,7 @@ impl fmt::Display for SchemeId { /// [`matches`]: Scheme::matches /// [`compress`]: Scheme::compress /// [`expected_compression_ratio`]: Scheme::expected_compression_ratio +/// [`required_array_writer_versions`]: Scheme::required_array_writer_versions /// [`stats_options`]: Scheme::stats_options /// [`num_children`]: Scheme::num_children /// [`descendant_exclusions`]: Scheme::descendant_exclusions @@ -131,6 +132,23 @@ pub trait Scheme: Debug + Send + Sync { /// Canonical arrays the scheme merely rearranges do not need to be declared. fn produced_encodings(&self) -> Vec; + /// The minimum writer version required for each array encoding this scheme would produce for + /// `canonical`. + /// + /// The default requires writer version 1 for every [`produced_encodings`](Self::produced_encodings) + /// entry. Override this when the same reader-compatible array encoding has optional serialized + /// fields or properties that only newer writers may populate. The compressor checks these + /// requirements before statistics, estimation, sampling, or compression. An incompatible + /// serialized representation must use a new array ID instead of a higher writer version. + /// This method must be cheap; if the exact output depends on later analysis, report the newest + /// version the scheme might produce. + fn required_array_writer_versions(&self, _canonical: &Canonical) -> Vec<(ArrayId, u16)> { + self.produced_encodings() + .into_iter() + .map(|id| (id, 1)) + .collect() + } + /// Returns the stats generation options this scheme requires. The compressor merges all /// eligible schemes' options before generating stats so that a single stats pass satisfies /// every scheme. diff --git a/vortex-edition/src/declarations/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs index 16c9bd71ec9..26b3f699810 100644 --- a/vortex-edition/src/declarations/core/mod.rs +++ b/vortex-edition/src/declarations/core/mod.rs @@ -1,22 +1,25 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The `core` edition family: the serialized components the default file writer emits. +//! The `core` edition family: serialized components available to the default file writer. //! -//! One module per edition, each declaring the edition and the components that join the -//! family at it; members of earlier editions are inherited and never restated. +//! One module per edition, each declaring the edition and the members that join the +//! family at it; members of earlier editions are inherited and never restated. Array members carry +//! the writer version that compression schemes may produce. use crate::EditionFamily; -/// The `core` family: what the default writer may emit. +/// The `core` family: serialized components available by default. pub static FAMILY: EditionFamily = EditionFamily { name: "core", - doc: "The serialized components the default file writer emits. Every core edition \ -freezes, and a frozen edition carries a read-forever guarantee: a file written with it stays \ -readable by every later Vortex release. A non-plugin component joins core once its \ -serialization is stable. Its edition may freeze in the release that cuts it; after that release \ -version is known, the declaration is backfilled with it as the minimum. A frozen edition never \ -changes.", + doc: "The serialized components available to the default file writer. Array memberships pin \ +the writer version compression schemes may produce. Every array ID still has one reader; an \ +incompatible serialized form must use a new ID. Every core edition freezes, and a \ +frozen edition carries a read-forever guarantee: a file written with it stays readable by every \ +later Vortex release. Stabilized non-plugin components and array writer-version upgrades \ +are adopted through preview before joining core. An edition may freeze in the release that cuts \ +it; after that release version is known, the declaration is backfilled with it as the minimum. A \ +frozen edition never changes.", }; pub mod v2025_05; diff --git a/vortex-edition/src/declarations/preview/mod.rs b/vortex-edition/src/declarations/preview/mod.rs index 05cad1a844d..7e15a309b55 100644 --- a/vortex-edition/src/declarations/preview/mod.rs +++ b/vortex-edition/src/declarations/preview/mod.rs @@ -1,23 +1,25 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The `preview` edition family: opt-in components without a frozen compatibility guarantee. +//! The `preview` edition family: stabilized core components and array writer-version upgrades +//! awaiting explicit adoption. //! -//! One module per draft edition, each declaring the components that join the family at it. +//! One module per draft edition, each declaring the members that join the family at it. //! Members of earlier editions are inherited and never restated. use crate::EditionFamily; -/// The `preview` family: opt-in components with no compatibility guarantee. +/// The `preview` family: stabilized, opt-in core functionality. pub static FAMILY: EditionFamily = EditionFamily { name: "preview", - doc: "Opt-in, non-plugin components whose serialization is still being evaluated. Every \ -preview edition stays a draft, so the family never freezes and carries no compatibility \ -guarantee: a file written with these components is readable only by a build that knows them, \ -and a later release may stop supporting one. The writer emits them only when the \ -`unstable_encodings` feature is selected. Once a component's serialization is stable, it \ -moves into a core edition. Components supplied by optional plugins instead live in standalone \ -families such as spatial and json.", + doc: "Stabilized, opt-in components and array writer-version upgrades maintained as \ +part of core but not yet adopted by the default core writer. Preview behavior is expected to \ +remain compatible and should change only to fix a defect serious enough to block promotion into \ +core. A writer-version upgrade lets compression schemes produce new optional fields or \ +properties; it never selects a reader. Users keep the earlier serialized form until they opt \ +into that edition. Experimental work \ +advances through new draft editions; optional plugins instead use standalone families such as \ +spatial and json.", }; pub mod v2025_05; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index d3d9cdd41a0..b5ab3ffb71a 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -1,8 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Definitions of Vortex *editions*: named sets of components that a writer may put in a -//! file. Frozen editions carry a forever read-compatibility guarantee; draft editions do not. +//! Definitions of Vortex *editions*: named sets of serialized components and the writer versions +//! compression schemes may produce for each array encoding. Frozen editions carry a forever +//! read-compatibility guarantee; draft editions do not. //! //! Editions live on the session, like encodings do: [`EditionSession`] holds the registered //! editions and [`EnabledEditions`] selects which of them a writer may emit. Declarations @@ -13,8 +14,12 @@ //! //! Every membership is typed by a [`ComponentKind`], and members are resolved one kind at a //! time with [`EditionSessionExt::enabled_component_ids`]: the file writer restricts the -//! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets, never one -//! untyped set. +//! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets. Array +//! memberships additionally carry a writer version. Compression schemes consult that version +//! before estimating or producing an array, so a compatible reader extension does not silently +//! change existing writer output. Writer versions are not stored in files and do not select a +//! reader: every array ID has exactly one registered reader. An incompatible serialized form must +//! therefore use a new array ID. //! //! An edition is represented as a **draft** until its [`Edition::min_vortex_version`] is //! recorded. A stable edition may freeze in the release that cuts it; once that release version @@ -49,9 +54,9 @@ use vortex_session::registry::Id; /// The identifier of an edition, e.g. `core2026.07.0`. /// -/// The `family` names an independently versioned, additive group of components (`core` is the -/// set the default writer emits). For `core`, the date components record when the edition freezes; -/// that date is prospective while the edition is still a draft. Dates order editions +/// The `family` names an independently versioned, additive group of members (`core` is the set +/// available to the default writer). For `core`, the date components record when the edition +/// freezes; that date is prospective while the edition is still a draft. Dates order editions /// chronologically *within* a family; there is no ordering across families. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct EditionId { @@ -119,7 +124,7 @@ impl Display for EditionId { } } -/// A family of editions: an independently versioned, additive group of encodings, registered +/// A family of editions: an independently versioned, additive group of members, registered /// with [`EditionSession::declare_family`]. /// /// Every [`EditionId`] names one. Declaring the family is what makes the name real: @@ -154,7 +159,7 @@ impl EditionFamily { } } -/// The kind of component an edition membership covers. +/// The kind of member an edition membership covers. /// /// Ids are unique per kind, not globally: a layout named `vortex.flat` and an array named /// `vortex.flat` are different members. Every membership records its kind, and the writer @@ -184,9 +189,13 @@ impl Display for ComponentKind { } } -/// An edition: a named set of components that can acquire a read-compatibility guarantee, -/// registered with [`EditionSession::declare_edition`]. The set itself is computed from the -/// registered [`EditionInclusion`]s by [`EditionSession::components_in`]. +/// The writer version assigned when an array encoding first joins an edition family. +pub const INITIAL_ARRAY_WRITER_VERSION: u16 = 1; + +/// An edition: a named set of serialized components and array writer versions that can +/// acquire a read-compatibility guarantee, registered with [`EditionSession::declare_edition`]. +/// The set itself is computed from the registered [`EditionInclusion`]s by +/// [`EditionSession::components_in`]. #[derive(Clone, Copy, Debug)] pub struct Edition { /// The edition identifier. For a `core` edition, its date records when it freezes. @@ -196,7 +205,9 @@ pub struct Edition { /// A stable edition may freeze in the release that cuts it. Until that release is cut, its /// version is not known and this remains `None`. The version is then backfilled to document /// the already completed freeze and identify the first released reader supporting every - /// member. Preview editions remain drafts permanently. + /// member. Preview editions remain drafts in this sense permanently: `draft` means that the + /// core read-forever guarantee has not been recorded, not that preview behavior is expected + /// to change. /// Validated against the members' [`EditionInclusion::required_vortex_release`] values: /// no member may require a version newer than the edition declares. pub min_vortex_version: Option<&'static str>, @@ -204,13 +215,17 @@ pub struct Edition { impl Edition { /// A draft is an edition whose `min_vortex_version` has not been recorded yet. + /// + /// This describes the absence of a frozen core compatibility guarantee, not necessarily the + /// implementation stability of its members. Stabilized preview editions are drafts too. pub fn is_draft(&self) -> bool { self.min_vortex_version.is_none() } } -/// Declares that a component is a member of an edition — and of every later edition of the -/// same family. Registered with [`EditionSession::declare_inclusion`]. +/// Declares that a serialized component is a member of an edition — and of every later edition of +/// the same family. Array components may be redeclared in a later edition with a higher writer +/// version. Registered with [`EditionSession::declare_inclusion`]. #[derive(Clone, Copy, Debug)] pub struct EditionInclusion { /// What the membership covers. Ids are unique per kind, so this is part of the @@ -218,10 +233,16 @@ pub struct EditionInclusion { pub kind: ComponentKind, /// The interned component id, e.g. `vortex.alp`. pub component_id: Id, + /// The compatible serialized features this edition permits writers to produce for an array. + /// + /// `None` for non-array members. This is a write-time capability ceiling, not a version of the + /// in-memory array or a read-time dispatch key. Each array ID has one reader; an incompatible + /// serialized form requires a new array ID. + pub array_writer_version: Option, /// The first edition this component is a member of. pub since: EditionId, - /// The earliest Vortex release able to read and execute this component, recorded from - /// evidence (e.g. compat-fixture history). `None` until recorded. + /// The earliest Vortex release supporting this member, recorded from evidence (e.g. + /// compat-fixture history for serialized components). `None` until recorded. pub required_vortex_release: Option<&'static str>, } @@ -260,23 +281,38 @@ impl AsComponentId for &'static str { } } -/// A component that joins an edition, named by id string or vtable and tagged with the kind -/// of registry it belongs to. Built with the per-kind constructors, so a declaration reads +/// A member that joins an edition, named by id string or vtable and tagged with its kind. +/// Built with the per-kind constructors, so a declaration reads /// as `EditionMember::array(&"vortex.alp")`. #[derive(Clone, Copy, Debug)] pub struct EditionMember { - /// What kind of component this is. + /// What kind of member this is. pub kind: ComponentKind, - /// The component, named by id string or by vtable. + /// The member, named by id string or by vtable. pub component: &'static dyn AsComponentId, + /// The writer version permitted for an array member, or `None` for every other kind. + pub array_writer_version: Option, } impl EditionMember { /// An array encoding member, e.g. `vortex.alp`. pub const fn array(component: &'static dyn AsComponentId) -> Self { + Self::array_writer_version(component, INITIAL_ARRAY_WRITER_VERSION) + } + + /// An array encoding member at a specific writer version. + /// + /// Use a later edition and a larger version when a writer may begin producing a new optional + /// field or another compatible serialized property that earlier writers never emitted. A + /// change requiring a different reader is a new array encoding, not a version increase. + pub const fn array_writer_version( + component: &'static dyn AsComponentId, + writer_version: u16, + ) -> Self { Self { kind: ComponentKind::Array, component, + array_writer_version: Some(writer_version), } } @@ -285,6 +321,7 @@ impl EditionMember { Self { kind: ComponentKind::Layout, component, + array_writer_version: None, } } @@ -293,6 +330,7 @@ impl EditionMember { Self { kind: ComponentKind::DType, component, + array_writer_version: None, } } @@ -301,19 +339,20 @@ impl EditionMember { Self { kind: ComponentKind::Aggregate, component, + array_writer_version: None, } } } -/// Declares an edition together with the components that join the family at it, in one -/// block. Registered with [`EditionSession::declare`], which derives each member's -/// membership (`since` = the declared edition) from the block structure. +/// Declares an edition together with its new members and array writer-version increases, in one +/// block. Registered with [`EditionSession::declare`], which derives each entry's membership +/// (`since` = the declared edition) from the block structure. #[derive(Clone, Copy, Debug)] pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, - /// The components that join the family at this edition, each tagged with its - /// [`ComponentKind`]. Members of earlier editions are inherited and never restated. + /// The members that join the family and array writer versions raised by this edition, each + /// tagged with its [`ComponentKind`]. Earlier entries are inherited and never restated. pub added: &'static [EditionMember], } @@ -328,6 +367,8 @@ impl EditionInclusion { Self { kind, component_id: component.component_id(), + array_writer_version: (kind == ComponentKind::Array) + .then_some(INITIAL_ARRAY_WRITER_VERSION), since, required_vortex_release: None, } @@ -339,6 +380,19 @@ impl EditionInclusion { Self::new(ComponentKind::Array, encoding, since) } + /// Declare that an array writer version is available in `since` and every later edition of + /// the same family, until superseded by a later writer version. + pub fn array_writer_version( + encoding: &C, + writer_version: u16, + since: EditionId, + ) -> Self { + Self { + array_writer_version: Some(writer_version), + ..Self::new(ComponentKind::Array, encoding, since) + } + } + /// Declare that an extension dtype is a member of `since` and every later edition of the /// same family. pub fn dtype(dtype: &C, since: EditionId) -> Self { @@ -362,6 +416,26 @@ impl EditionInclusion { self.kind ))); } + match (self.kind, self.array_writer_version) { + (ComponentKind::Array, Some(0)) => { + return Err(EditionError::new(format!( + "array {id} must have a non-zero writer version" + ))); + } + (ComponentKind::Array, Some(_)) => {} + (ComponentKind::Array, None) => { + return Err(EditionError::new(format!( + "array {id} must declare a writer version" + ))); + } + (_, Some(version)) => { + return Err(EditionError::new(format!( + "{} {id} cannot declare array writer version {version}", + self.kind + ))); + } + (_, None) => {} + } if let Some(release) = self.required_vortex_release && parse_release(release).is_none() { diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index ad00fdeba61..8f22de7cdd2 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -41,10 +41,10 @@ struct Inner { families: BTreeMap, /// Keyed by the display form of the edition id. editions: BTreeMap, - /// One map per component kind, each keyed by interned component id, because ids are - /// only unique within a kind. Resolving a kind scans that kind's map alone, never the - /// other kinds' entries. Ordered by kind, then by the id's string form. - inclusions: BTreeMap>, + /// One map per member kind, each keyed by interned member id, because ids are only unique + /// within a kind. An id has one inclusion history per family; array histories may contain + /// successive writer versions. Ordered by kind, then by the id's string form. + inclusions: BTreeMap>>, } /// Registry of enabled editions, keyed by interned edition family. @@ -85,17 +85,16 @@ impl EditionSession { } } - /// Declare an edition together with the components that join the family at it. Each - /// added member's membership (`since`) is the declared edition; members of earlier - /// editions are inherited and must not be restated. + /// Declare an edition together with the members and array writer-version increases added at + /// it. Each entry's membership (`since`) is the declared edition; earlier entries are + /// inherited and must not be restated. pub fn declare(&self, declaration: &EditionDeclaration) -> Result<(), EditionError> { self.declare_edition(declaration.edition)?; for member in declaration.added { - self.declare_inclusion(EditionInclusion::new( - member.kind, - member.component, - declaration.edition.id, - ))?; + self.declare_inclusion(EditionInclusion { + array_writer_version: member.array_writer_version, + ..EditionInclusion::new(member.kind, member.component, declaration.edition.id) + })?; } Ok(()) } @@ -136,19 +135,53 @@ impl EditionSession { Ok(()) } - /// Declare an edition inclusion. Errors if the component already has one: a component - /// belongs to exactly one family, with one membership interval. Kind is part of the - /// key, so an array encoding and a layout may share an id. + /// Declare an edition inclusion. A component may belong to multiple families. Within one + /// family, non-array members join once, while an array may be redeclared only with a larger + /// writer version in a later edition. Kind is part of the key, so an array encoding + /// and a layout may share an id. pub fn declare_inclusion(&self, inclusion: EditionInclusion) -> Result<(), EditionError> { let mut inner = self.inner.write(); let by_id = inner.inclusions.entry(inclusion.kind).or_default(); - if by_id.contains_key(&inclusion.component_id) { - return Err(EditionError::new(format!( - "duplicate edition inclusion for {} {}", - inclusion.kind, inclusion.component_id - ))); + let history = by_id.entry(inclusion.component_id).or_default(); + let previous = history + .iter() + .filter(|existing| existing.since.family == inclusion.since.family) + .max_by_key(|existing| { + ( + existing.since.year, + existing.since.month, + existing.since.version, + ) + }); + + if let Some(previous) = previous { + let is_later = previous.since != inclusion.since + && previous.since.is_at_or_before(&inclusion.since); + let is_array_upgrade = match ( + inclusion.kind, + previous.array_writer_version, + inclusion.array_writer_version, + ) { + (ComponentKind::Array, Some(previous), Some(next)) => next > previous, + _ => false, + }; + if !is_later || !is_array_upgrade { + return Err(EditionError::new(format!( + "{} {} already has a membership in family {}; only a later, higher array \ + writer version may supersede it", + inclusion.kind, inclusion.component_id, inclusion.since.family + ))); + } } - by_id.insert(inclusion.component_id, inclusion); + history.push(inclusion); + history.sort_by_key(|entry| { + ( + entry.since.family, + entry.since.year, + entry.since.month, + entry.since.version, + ) + }); Ok(()) } @@ -173,9 +206,10 @@ impl EditionSession { .rfind(|e| e.id.family == family && !e.is_draft()) } - /// Compute an edition's members of one kind: every declared inclusion of that kind in - /// the edition's family whose `since` is at or before it, sorted by component id. Only - /// that kind's declarations are scanned. + /// Compute an edition's members of one kind, sorted by component id. For each id, this returns + /// the newest inclusion in the edition's family whose `since` is at or before it. An array + /// writer-version upgrade therefore supersedes the older version without changing the array + /// ID or its single registered reader. Only that kind's declarations are scanned. pub fn components_in(&self, edition: &EditionId, kind: ComponentKind) -> Vec { let inner = self.inner.read(); let Some(by_id) = inner.inclusions.get(&kind) else { @@ -183,8 +217,19 @@ impl EditionSession { }; by_id .values() - .filter(|inclusion| inclusion.since.is_at_or_before(edition)) - .copied() + .filter_map(|history| { + history + .iter() + .filter(|inclusion| inclusion.since.is_at_or_before(edition)) + .max_by_key(|inclusion| { + ( + inclusion.since.year, + inclusion.since.month, + inclusion.since.version, + ) + }) + .copied() + }) .collect() } @@ -231,7 +276,12 @@ impl EditionSession { } let inner = self.inner.read(); - for inclusion in inner.inclusions.values().flat_map(|by_id| by_id.values()) { + for inclusion in inner + .inclusions + .values() + .flat_map(|by_id| by_id.values()) + .flatten() + { inclusion.validate()?; let Some(edition) = inner.editions.get(&inclusion.since.to_string()) else { @@ -315,8 +365,8 @@ pub trait EditionSessionExt: SessionExt { Ok(()) } - /// Resolve the ids of one [`ComponentKind`] across all enabled editions: what a writer - /// may emit for that kind. + /// Resolve the ids of one [`ComponentKind`] across all enabled editions: what a writer may + /// emit for that kind. /// /// Ids are only unique within a kind, so this never mixes kinds. An empty result means the /// enabled editions permit no components of this kind. @@ -335,6 +385,32 @@ pub trait EditionSessionExt: SessionExt { ids.dedup(); ids } + + /// Resolve one effective array writer version for every array ID across the enabled editions. + /// + /// Compression schemes use this map before sampling or compressing an array. An absent id is + /// not writable, and a writer-version upgrade in an opt-in family overrides an older core + /// version. The version is never used while reading; each array ID has one registered reader. + fn enabled_array_writer_versions(&self) -> BTreeMap { + let Some(enabled) = self.get_opt::() else { + return BTreeMap::new(); + }; + let editions = self.editions(); + let mut versions = BTreeMap::new(); + for inclusion in enabled + .editions() + .iter() + .flat_map(|edition| editions.components_in(edition, ComponentKind::Array)) + { + if let Some(version) = inclusion.array_writer_version { + versions + .entry(inclusion.component_id) + .and_modify(|enabled: &mut u16| *enabled = (*enabled).max(version)) + .or_insert(version); + } + } + versions + } } impl EditionSessionExt for S {} diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 13f51588d32..0d1863b9ec7 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -43,7 +43,10 @@ static DECLARATIONS: &[EditionDeclaration] = &[ id: SECOND, min_vortex_version: None, }, - added: &[EditionMember::array(&"test.gamma")], + added: &[ + EditionMember::array_writer_version(&"test.alpha", 2), + EditionMember::array(&"test.gamma"), + ], }, ]; @@ -72,24 +75,30 @@ fn editions_pass_the_test_harness() -> Result<(), crate::EditionError> { } #[test] -fn membership_is_transitive() { +fn membership_is_transitive() -> Result<(), crate::EditionError> { let editions = session(); let first = editions.components_in(&FIRST, ComponentKind::Array); let ids: Vec<&str> = first.iter().map(|i| i.component_id.as_str()).collect(); assert_eq!(ids, ["test.alpha", "test.beta"]); - // Members of the first edition are members of the second by inheritance, with their - // `since` still recording the edition they actually joined in. + // Members of the first edition are members of the second by inheritance. Alpha's writer v2 + // supersedes v1 without changing its array id or introducing read-time dispatch. let second = editions.components_in(&SECOND, ComponentKind::Array); let ids: Vec<&str> = second.iter().map(|i| i.component_id.as_str()).collect(); assert_eq!(ids, ["test.alpha", "test.beta", "test.gamma"]); - assert!( - second - .iter() - .filter(|i| i.component_id.as_str() != "test.gamma") - .all(|i| i.since == FIRST) - ); + let alpha = second + .iter() + .find(|i| i.component_id.as_str() == "test.alpha") + .ok_or_else(|| crate::EditionError::new("test.alpha is a member"))?; + assert_eq!(alpha.since, SECOND); + assert_eq!(alpha.array_writer_version, Some(2)); + let beta = second + .iter() + .find(|i| i.component_id.as_str() == "test.beta") + .ok_or_else(|| crate::EditionError::new("test.beta is a member"))?; + assert_eq!(beta.since, FIRST); + assert_eq!(beta.array_writer_version, Some(1)); // The second edition's delta is exactly the members declared at it. let added: Vec<&str> = second @@ -97,7 +106,7 @@ fn membership_is_transitive() { .filter(|i| i.since == SECOND) .map(|i| i.component_id.as_str()) .collect(); - assert_eq!(added, ["test.gamma"]); + assert_eq!(added, ["test.alpha", "test.gamma"]); // Inheritance never flows backwards, extends to later editions of the family, and // never crosses families. @@ -113,6 +122,7 @@ fn membership_is_transitive() { .components_in(&other, ComponentKind::Array) .is_empty() ); + Ok(()) } #[test] @@ -176,10 +186,22 @@ fn registered_and_enabled_editions_are_separate() -> Result<(), crate::EditionEr .collect::>(), ["test.alpha", "test.beta"] ); + assert_eq!( + session + .enabled_array_writer_versions() + .get(&"test.alpha".into()), + Some(&1) + ); session.enable_edition(SECOND)?; assert_eq!(session.enabled_editions().editions(), [SECOND]); assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 3); + assert_eq!( + session + .enabled_array_writer_versions() + .get(&"test.alpha".into()), + Some(&2) + ); // Selecting an older edition in the same family replaces the newer one and removes // encodings that joined after it. @@ -225,6 +247,40 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi Ok(()) } +#[test] +fn array_writer_versions_can_be_upgraded_by_an_opt_in_family() -> Result<(), crate::EditionError> { + const PREVIEW: EditionId = EditionId::new("other", 2026, 8, 0); + static PREVIEW_DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: PREVIEW, + min_vortex_version: None, + }, + added: &[EditionMember::array_writer_version(&"test.alpha", 2)], + }; + + let session = VortexSession::empty().with::(); + session.editions().declare_family(&TEST_FAMILY)?; + session.editions().declare_family(&OTHER_FAMILY)?; + session.register_edition(&DECLARATIONS[0])?; + session.register_edition(&PREVIEW_DECLARATION)?; + session.editions().validate()?; + session.enable_edition(FIRST)?; + session.enable_edition(PREVIEW)?; + + let versions = session.enabled_array_writer_versions(); + assert_eq!(versions.get(&"test.alpha".into()), Some(&2)); + assert_eq!(versions.get(&"test.beta".into()), Some(&1)); + assert_eq!( + session + .enabled_component_ids(ComponentKind::Array) + .iter() + .map(|id| id.as_str()) + .collect::>(), + ["test.alpha", "test.beta"] + ); + Ok(()) +} + #[test] fn duplicate_declarations_error() { let editions = session(); @@ -291,6 +347,19 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro editions.declare_inclusion(EditionInclusion::array("Test.ALPHA", FIRST))?; assert!(editions.validate().is_err()); + // Array writer versions start at one. + let editions = EditionSession::empty(); + editions.declare_edition(Edition { + id: FIRST, + min_vortex_version: None, + })?; + editions.declare_inclusion(EditionInclusion::array_writer_version( + "test.alpha", + 0, + FIRST, + ))?; + assert!(editions.validate().is_err()); + Ok(()) } diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index 9d4dbb90610..deee68bc3a4 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use vortex_array::ArrayId; use vortex_array::dtype::FieldPath; +use vortex_btrblocks::ArrayWriterVersions; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::integer::IntDictScheme; @@ -60,6 +61,7 @@ pub struct WriteStrategyBuilder { data_block_target_bytes: Option, field_writers: HashMap>, allow_encodings: Option>, + array_writer_versions: Option, flat_strategy: Option>, probe_compressor: Option>, /// Whether to write list fields using [`ListLayoutStrategy`]. @@ -78,6 +80,7 @@ impl Default for WriteStrategyBuilder { data_block_target_bytes: Some(ONE_MEG), field_writers: HashMap::new(), allow_encodings: None, + array_writer_versions: None, flat_strategy: None, probe_compressor: None, use_list_layout: use_experimental_list_layout(), @@ -139,6 +142,19 @@ impl WriteStrategyBuilder { self } + /// Configure the compatible serialized features the writer may produce for each array ID. + /// + /// The map's keys become the allowed array encodings. For the built-in BtrBlocks compressor, + /// schemes requiring an absent or newer writer version are excluded before estimation, + /// sampling, and compression. The flat writer validates every final array ID, including + /// output from an opaque custom compressor; an opaque compressor remains responsible for + /// honoring the writer versions because they are not read-time array tags. + pub fn with_array_writer_versions(mut self, versions: ArrayWriterVersions) -> Self { + self.allow_encodings = Some(versions.keys().copied().collect()); + self.array_writer_versions = Some(versions); + self + } + /// Override the flat layout strategy used for leaf chunks. /// /// By default, this uses [`FlatLayoutStrategy`]. This can be used to substitute a custom @@ -187,6 +203,10 @@ impl WriteStrategyBuilder { // regardless of the order in which the builder and the policy were configured. let compressor = match self.compressor { CompressorConfig::BtrBlocks(builder) => { + let builder = match self.array_writer_versions { + Some(versions) => builder.with_array_writer_versions(versions), + None => builder, + }; CompressorConfig::BtrBlocks(match &self.allow_encodings { Some(allow_encodings) => builder.retain_allowed_encodings(allow_encodings), None => builder, diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index e57bcb0ec86..194729ab793 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -68,9 +68,10 @@ use crate::segments::writer::BufferedSegmentSink; /// Configure a new writer, which can eventually be used to write an [`ArrayStream`] into a sink /// that implements [`VortexWrite`]. /// -/// All write strategies are restricted to the components in the session's enabled editions: an -/// array, layout, extension dtype, or zone-map aggregate outside them fails the write. An empty -/// component set therefore forbids writing any component of that kind. +/// The default write strategy is restricted to the components and array writer versions in the +/// session's enabled editions. An array, layout, extension dtype, or zone-map aggregate outside +/// them fails the write. An empty component set therefore forbids writing any component of that +/// kind. /// /// Construct with [`WriteOptionsSessionExt::write_options`] for normal use so the writer inherits /// the session's runtime, array registry, and memory configuration. @@ -97,12 +98,7 @@ impl VortexWriteOptions { /// Create a new [`VortexWriteOptions`] with the given session. pub fn new(session: VortexSession) -> Self { let strategy = WriteStrategyBuilder::default() - .with_allow_encodings( - session - .enabled_component_ids(ComponentKind::Array) - .into_iter() - .collect(), - ) + .with_array_writer_versions(session.enabled_array_writer_versions()) .build(); VortexWriteOptions { strategy, @@ -119,7 +115,9 @@ impl VortexWriteOptions { /// /// The strategy controls repartitioning, statistics layout, compression, and leaf segment /// emission. Use [`WriteStrategyBuilder`] when only a small part of the default strategy needs - /// customization. Replacing the strategy does not change the enabled-edition encoding policy. + /// customization. The final serializers still enforce the enabled-edition component IDs, but + /// a replacement compressor is responsible for applying array writer versions before it does + /// expensive work. pub fn with_strategy(mut self, strategy: Arc) -> Self { self.strategy = strategy; self diff --git a/vortex/editions/core/core2025.05.0.toml b/vortex/editions/core/core2025.05.0.toml index 66f9f70dd7d..c020a95a06d 100644 --- a/vortex/editions/core/core2025.05.0.toml +++ b/vortex/editions/core/core2025.05.0.toml @@ -8,32 +8,32 @@ edition = "core2025.05.0" family = "core" min_vortex_version = "0.36.0" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "fastlanes.bitpacked", - "fastlanes.for", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fsst", - "vortex.list", - "vortex.null", - "vortex.primitive", - "vortex.runend", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", + { id = "fastlanes.bitpacked", writer_version = 1 }, + { id = "fastlanes.for", writer_version = 1 }, + { id = "vortex.alp", writer_version = 1 }, + { id = "vortex.alprd", writer_version = 1 }, + { id = "vortex.bool", writer_version = 1 }, + { id = "vortex.bytebool", writer_version = 1 }, + { id = "vortex.chunked", writer_version = 1 }, + { id = "vortex.constant", writer_version = 1 }, + { id = "vortex.datetimeparts", writer_version = 1 }, + { id = "vortex.decimal", writer_version = 1 }, + { id = "vortex.decimal_byte_parts", writer_version = 1 }, + { id = "vortex.dict", writer_version = 1 }, + { id = "vortex.ext", writer_version = 1 }, + { id = "vortex.fsst", writer_version = 1 }, + { id = "vortex.list", writer_version = 1 }, + { id = "vortex.null", writer_version = 1 }, + { id = "vortex.primitive", writer_version = 1 }, + { id = "vortex.runend", writer_version = 1 }, + { id = "vortex.sparse", writer_version = 1 }, + { id = "vortex.struct", writer_version = 1 }, + { id = "vortex.varbin", writer_version = 1 }, + { id = "vortex.varbinview", writer_version = 1 }, + { id = "vortex.zigzag", writer_version = 1 }, ] layouts = [ "vortex.chunked", @@ -49,33 +49,33 @@ dtypes = [ ] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.bitpacked", - "fastlanes.for", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fsst", - "vortex.list", - "vortex.null", - "vortex.primitive", - "vortex.runend", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", + { id = "fastlanes.bitpacked", writer_version = 1 }, + { id = "fastlanes.for", writer_version = 1 }, + { id = "vortex.alp", writer_version = 1 }, + { id = "vortex.alprd", writer_version = 1 }, + { id = "vortex.bool", writer_version = 1 }, + { id = "vortex.bytebool", writer_version = 1 }, + { id = "vortex.chunked", writer_version = 1 }, + { id = "vortex.constant", writer_version = 1 }, + { id = "vortex.datetimeparts", writer_version = 1 }, + { id = "vortex.decimal", writer_version = 1 }, + { id = "vortex.decimal_byte_parts", writer_version = 1 }, + { id = "vortex.dict", writer_version = 1 }, + { id = "vortex.ext", writer_version = 1 }, + { id = "vortex.fsst", writer_version = 1 }, + { id = "vortex.list", writer_version = 1 }, + { id = "vortex.null", writer_version = 1 }, + { id = "vortex.primitive", writer_version = 1 }, + { id = "vortex.runend", writer_version = 1 }, + { id = "vortex.sparse", writer_version = 1 }, + { id = "vortex.struct", writer_version = 1 }, + { id = "vortex.varbin", writer_version = 1 }, + { id = "vortex.varbinview", writer_version = 1 }, + { id = "vortex.zigzag", writer_version = 1 }, ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2025.06.0.toml b/vortex/editions/core/core2025.06.0.toml index 21a306fb3c1..07552dac0ce 100644 --- a/vortex/editions/core/core2025.06.0.toml +++ b/vortex/editions/core/core2025.06.0.toml @@ -8,47 +8,47 @@ edition = "core2025.06.0" family = "core" min_vortex_version = "0.40.0" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "vortex.pco", - "vortex.sequence", - "vortex.zstd", + { id = "vortex.pco", writer_version = 1 }, + { id = "vortex.sequence", writer_version = 1 }, + { id = "vortex.zstd", writer_version = 1 }, ] layouts = [] dtypes = [] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.bitpacked", - "fastlanes.for", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fsst", - "vortex.list", - "vortex.null", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", - "vortex.zstd", + { id = "fastlanes.bitpacked", writer_version = 1 }, + { id = "fastlanes.for", writer_version = 1 }, + { id = "vortex.alp", writer_version = 1 }, + { id = "vortex.alprd", writer_version = 1 }, + { id = "vortex.bool", writer_version = 1 }, + { id = "vortex.bytebool", writer_version = 1 }, + { id = "vortex.chunked", writer_version = 1 }, + { id = "vortex.constant", writer_version = 1 }, + { id = "vortex.datetimeparts", writer_version = 1 }, + { id = "vortex.decimal", writer_version = 1 }, + { id = "vortex.decimal_byte_parts", writer_version = 1 }, + { id = "vortex.dict", writer_version = 1 }, + { id = "vortex.ext", writer_version = 1 }, + { id = "vortex.fsst", writer_version = 1 }, + { id = "vortex.list", writer_version = 1 }, + { id = "vortex.null", writer_version = 1 }, + { id = "vortex.pco", writer_version = 1 }, + { id = "vortex.primitive", writer_version = 1 }, + { id = "vortex.runend", writer_version = 1 }, + { id = "vortex.sequence", writer_version = 1 }, + { id = "vortex.sparse", writer_version = 1 }, + { id = "vortex.struct", writer_version = 1 }, + { id = "vortex.varbin", writer_version = 1 }, + { id = "vortex.varbinview", writer_version = 1 }, + { id = "vortex.zigzag", writer_version = 1 }, + { id = "vortex.zstd", writer_version = 1 }, ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2025.10.0.toml b/vortex/editions/core/core2025.10.0.toml index b83eb2c92fa..0aa9bf3e409 100644 --- a/vortex/editions/core/core2025.10.0.toml +++ b/vortex/editions/core/core2025.10.0.toml @@ -8,52 +8,52 @@ edition = "core2025.10.0" family = "core" min_vortex_version = "0.54.0" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "fastlanes.rle", - "vortex.fixed_size_list", - "vortex.listview", - "vortex.masked", + { id = "fastlanes.rle", writer_version = 1 }, + { id = "vortex.fixed_size_list", writer_version = 1 }, + { id = "vortex.listview", writer_version = 1 }, + { id = "vortex.masked", writer_version = 1 }, ] layouts = [] dtypes = [] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.masked", - "vortex.null", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", - "vortex.zstd", + { id = "fastlanes.bitpacked", writer_version = 1 }, + { id = "fastlanes.for", writer_version = 1 }, + { id = "fastlanes.rle", writer_version = 1 }, + { id = "vortex.alp", writer_version = 1 }, + { id = "vortex.alprd", writer_version = 1 }, + { id = "vortex.bool", writer_version = 1 }, + { id = "vortex.bytebool", writer_version = 1 }, + { id = "vortex.chunked", writer_version = 1 }, + { id = "vortex.constant", writer_version = 1 }, + { id = "vortex.datetimeparts", writer_version = 1 }, + { id = "vortex.decimal", writer_version = 1 }, + { id = "vortex.decimal_byte_parts", writer_version = 1 }, + { id = "vortex.dict", writer_version = 1 }, + { id = "vortex.ext", writer_version = 1 }, + { id = "vortex.fixed_size_list", writer_version = 1 }, + { id = "vortex.fsst", writer_version = 1 }, + { id = "vortex.list", writer_version = 1 }, + { id = "vortex.listview", writer_version = 1 }, + { id = "vortex.masked", writer_version = 1 }, + { id = "vortex.null", writer_version = 1 }, + { id = "vortex.pco", writer_version = 1 }, + { id = "vortex.primitive", writer_version = 1 }, + { id = "vortex.runend", writer_version = 1 }, + { id = "vortex.sequence", writer_version = 1 }, + { id = "vortex.sparse", writer_version = 1 }, + { id = "vortex.struct", writer_version = 1 }, + { id = "vortex.varbin", writer_version = 1 }, + { id = "vortex.varbinview", writer_version = 1 }, + { id = "vortex.zigzag", writer_version = 1 }, + { id = "vortex.zstd", writer_version = 1 }, ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2026.08.0.toml b/vortex/editions/core/core2026.08.0.toml index 854e6b5fb2e..947e2408345 100644 --- a/vortex/editions/core/core2026.08.0.toml +++ b/vortex/editions/core/core2026.08.0.toml @@ -8,7 +8,7 @@ edition = "core2026.08.0" family = "core" min_vortex_version = "0.84.0" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [] layouts = [ @@ -24,40 +24,40 @@ aggregates = [ "vortex.null_count", ] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.masked", - "vortex.null", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", - "vortex.zstd", + { id = "fastlanes.bitpacked", writer_version = 1 }, + { id = "fastlanes.for", writer_version = 1 }, + { id = "fastlanes.rle", writer_version = 1 }, + { id = "vortex.alp", writer_version = 1 }, + { id = "vortex.alprd", writer_version = 1 }, + { id = "vortex.bool", writer_version = 1 }, + { id = "vortex.bytebool", writer_version = 1 }, + { id = "vortex.chunked", writer_version = 1 }, + { id = "vortex.constant", writer_version = 1 }, + { id = "vortex.datetimeparts", writer_version = 1 }, + { id = "vortex.decimal", writer_version = 1 }, + { id = "vortex.decimal_byte_parts", writer_version = 1 }, + { id = "vortex.dict", writer_version = 1 }, + { id = "vortex.ext", writer_version = 1 }, + { id = "vortex.fixed_size_list", writer_version = 1 }, + { id = "vortex.fsst", writer_version = 1 }, + { id = "vortex.list", writer_version = 1 }, + { id = "vortex.listview", writer_version = 1 }, + { id = "vortex.masked", writer_version = 1 }, + { id = "vortex.null", writer_version = 1 }, + { id = "vortex.pco", writer_version = 1 }, + { id = "vortex.primitive", writer_version = 1 }, + { id = "vortex.runend", writer_version = 1 }, + { id = "vortex.sequence", writer_version = 1 }, + { id = "vortex.sparse", writer_version = 1 }, + { id = "vortex.struct", writer_version = 1 }, + { id = "vortex.varbin", writer_version = 1 }, + { id = "vortex.varbinview", writer_version = 1 }, + { id = "vortex.zigzag", writer_version = 1 }, + { id = "vortex.zstd", writer_version = 1 }, ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2026.08.1.toml b/vortex/editions/core/core2026.08.1.toml index 19ec877330c..976016d8abf 100644 --- a/vortex/editions/core/core2026.08.1.toml +++ b/vortex/editions/core/core2026.08.1.toml @@ -8,50 +8,50 @@ edition = "core2026.08.1" family = "core" min_vortex_version = "0.84.0" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "vortex.onpair", + { id = "vortex.onpair", writer_version = 1 }, ] layouts = [] dtypes = [] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.masked", - "vortex.null", - "vortex.onpair", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", - "vortex.zstd", + { id = "fastlanes.bitpacked", writer_version = 1 }, + { id = "fastlanes.for", writer_version = 1 }, + { id = "fastlanes.rle", writer_version = 1 }, + { id = "vortex.alp", writer_version = 1 }, + { id = "vortex.alprd", writer_version = 1 }, + { id = "vortex.bool", writer_version = 1 }, + { id = "vortex.bytebool", writer_version = 1 }, + { id = "vortex.chunked", writer_version = 1 }, + { id = "vortex.constant", writer_version = 1 }, + { id = "vortex.datetimeparts", writer_version = 1 }, + { id = "vortex.decimal", writer_version = 1 }, + { id = "vortex.decimal_byte_parts", writer_version = 1 }, + { id = "vortex.dict", writer_version = 1 }, + { id = "vortex.ext", writer_version = 1 }, + { id = "vortex.fixed_size_list", writer_version = 1 }, + { id = "vortex.fsst", writer_version = 1 }, + { id = "vortex.list", writer_version = 1 }, + { id = "vortex.listview", writer_version = 1 }, + { id = "vortex.masked", writer_version = 1 }, + { id = "vortex.null", writer_version = 1 }, + { id = "vortex.onpair", writer_version = 1 }, + { id = "vortex.pco", writer_version = 1 }, + { id = "vortex.primitive", writer_version = 1 }, + { id = "vortex.runend", writer_version = 1 }, + { id = "vortex.sequence", writer_version = 1 }, + { id = "vortex.sparse", writer_version = 1 }, + { id = "vortex.struct", writer_version = 1 }, + { id = "vortex.varbin", writer_version = 1 }, + { id = "vortex.varbinview", writer_version = 1 }, + { id = "vortex.zigzag", writer_version = 1 }, + { id = "vortex.zstd", writer_version = 1 }, ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2026.08.2.toml b/vortex/editions/core/core2026.08.2.toml index 66b745c4c9e..a5808b5d585 100644 --- a/vortex/editions/core/core2026.08.2.toml +++ b/vortex/editions/core/core2026.08.2.toml @@ -1,57 +1,58 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition record has no documented compatibility guarantee. Its serialization may still be -# evolving, or it may be waiting for its release to be cut. After the release version is known, -# min_vortex_version is backfilled to document the freeze. A frozen record never changes. +# This edition record has no core read-forever guarantee. It may describe an evolving feature, +# stabilized preview functionality awaiting adoption, or a release waiting to be cut. New +# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# backfilled to document its freeze. A frozen record never changes. edition = "core2026.08.2" family = "core" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "vortex.map", + { id = "vortex.map", writer_version = 1 }, ] layouts = [] dtypes = [] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.map", - "vortex.masked", - "vortex.null", - "vortex.onpair", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", - "vortex.zstd", + { id = "fastlanes.bitpacked", writer_version = 1 }, + { id = "fastlanes.for", writer_version = 1 }, + { id = "fastlanes.rle", writer_version = 1 }, + { id = "vortex.alp", writer_version = 1 }, + { id = "vortex.alprd", writer_version = 1 }, + { id = "vortex.bool", writer_version = 1 }, + { id = "vortex.bytebool", writer_version = 1 }, + { id = "vortex.chunked", writer_version = 1 }, + { id = "vortex.constant", writer_version = 1 }, + { id = "vortex.datetimeparts", writer_version = 1 }, + { id = "vortex.decimal", writer_version = 1 }, + { id = "vortex.decimal_byte_parts", writer_version = 1 }, + { id = "vortex.dict", writer_version = 1 }, + { id = "vortex.ext", writer_version = 1 }, + { id = "vortex.fixed_size_list", writer_version = 1 }, + { id = "vortex.fsst", writer_version = 1 }, + { id = "vortex.list", writer_version = 1 }, + { id = "vortex.listview", writer_version = 1 }, + { id = "vortex.map", writer_version = 1 }, + { id = "vortex.masked", writer_version = 1 }, + { id = "vortex.null", writer_version = 1 }, + { id = "vortex.onpair", writer_version = 1 }, + { id = "vortex.pco", writer_version = 1 }, + { id = "vortex.primitive", writer_version = 1 }, + { id = "vortex.runend", writer_version = 1 }, + { id = "vortex.sequence", writer_version = 1 }, + { id = "vortex.sparse", writer_version = 1 }, + { id = "vortex.struct", writer_version = 1 }, + { id = "vortex.varbin", writer_version = 1 }, + { id = "vortex.varbinview", writer_version = 1 }, + { id = "vortex.zigzag", writer_version = 1 }, + { id = "vortex.zstd", writer_version = 1 }, ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2026.08.3.toml b/vortex/editions/core/core2026.08.3.toml index 10b377b299f..092c912809a 100644 --- a/vortex/editions/core/core2026.08.3.toml +++ b/vortex/editions/core/core2026.08.3.toml @@ -1,17 +1,18 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition record has no documented compatibility guarantee. Its serialization may still be -# evolving, or it may be waiting for its release to be cut. After the release version is known, -# min_vortex_version is backfilled to document the freeze. A frozen record never changes. +# This edition record has no core read-forever guarantee. It may describe an evolving feature, +# stabilized preview functionality awaiting adoption, or a release waiting to be cut. New +# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# backfilled to document its freeze. A frozen record never changes. edition = "core2026.08.3" family = "core" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "vortex.parquet.variant", - "vortex.variant", + { id = "vortex.parquet.variant", writer_version = 1 }, + { id = "vortex.variant", writer_version = 1 }, ] layouts = [] dtypes = [ @@ -19,44 +20,44 @@ dtypes = [ ] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.map", - "vortex.masked", - "vortex.null", - "vortex.onpair", - "vortex.parquet.variant", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.variant", - "vortex.zigzag", - "vortex.zstd", + { id = "fastlanes.bitpacked", writer_version = 1 }, + { id = "fastlanes.for", writer_version = 1 }, + { id = "fastlanes.rle", writer_version = 1 }, + { id = "vortex.alp", writer_version = 1 }, + { id = "vortex.alprd", writer_version = 1 }, + { id = "vortex.bool", writer_version = 1 }, + { id = "vortex.bytebool", writer_version = 1 }, + { id = "vortex.chunked", writer_version = 1 }, + { id = "vortex.constant", writer_version = 1 }, + { id = "vortex.datetimeparts", writer_version = 1 }, + { id = "vortex.decimal", writer_version = 1 }, + { id = "vortex.decimal_byte_parts", writer_version = 1 }, + { id = "vortex.dict", writer_version = 1 }, + { id = "vortex.ext", writer_version = 1 }, + { id = "vortex.fixed_size_list", writer_version = 1 }, + { id = "vortex.fsst", writer_version = 1 }, + { id = "vortex.list", writer_version = 1 }, + { id = "vortex.listview", writer_version = 1 }, + { id = "vortex.map", writer_version = 1 }, + { id = "vortex.masked", writer_version = 1 }, + { id = "vortex.null", writer_version = 1 }, + { id = "vortex.onpair", writer_version = 1 }, + { id = "vortex.parquet.variant", writer_version = 1 }, + { id = "vortex.pco", writer_version = 1 }, + { id = "vortex.primitive", writer_version = 1 }, + { id = "vortex.runend", writer_version = 1 }, + { id = "vortex.sequence", writer_version = 1 }, + { id = "vortex.sparse", writer_version = 1 }, + { id = "vortex.struct", writer_version = 1 }, + { id = "vortex.varbin", writer_version = 1 }, + { id = "vortex.varbinview", writer_version = 1 }, + { id = "vortex.variant", writer_version = 1 }, + { id = "vortex.zigzag", writer_version = 1 }, + { id = "vortex.zstd", writer_version = 1 }, ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml index 0ac6aefb2df..4c8bfb388e9 100644 --- a/vortex/editions/core/family.toml +++ b/vortex/editions/core/family.toml @@ -5,9 +5,12 @@ name = "core" doc = """ -The serialized components the default file writer emits. Every core edition freezes, and a -frozen edition carries a read-forever guarantee: a file written with it stays readable by -every later Vortex release. A non-plugin component joins core once its serialization is -stable. Its edition may freeze in the release that cuts it; after that release version is -known, the declaration is backfilled with it as the minimum. A frozen edition never changes. +The serialized components available to the default file writer. Array memberships pin the +writer version compression schemes may produce. Every array ID still has one reader; an +incompatible serialized form must use a new ID. Every core edition freezes, and a frozen +edition carries a read-forever guarantee: a file written with it stays readable by every +later Vortex release. Stabilized non-plugin components and array writer-version upgrades are +adopted through preview before joining core. An edition may freeze in the release that cuts +it; after that release version is known, the declaration is backfilled with it as the +minimum. A frozen edition never changes. """ diff --git a/vortex/editions/preview/family.toml b/vortex/editions/preview/family.toml index 824071d166a..891d2e238f3 100644 --- a/vortex/editions/preview/family.toml +++ b/vortex/editions/preview/family.toml @@ -5,11 +5,11 @@ name = "preview" doc = """ -Opt-in, non-plugin components whose serialization is still being evaluated. Every preview -edition stays a draft, so the family never freezes and carries no compatibility guarantee: a -file written with these components is readable only by a build that knows them, and a later -release may stop supporting one. The writer emits them only when the `unstable_encodings` -feature is selected. Once a component's serialization is stable, it moves into a core -edition. Components supplied by optional plugins instead live in standalone families such as -spatial and json. +Stabilized, opt-in components and array writer-version upgrades maintained as part of core +but not yet adopted by the default core writer. Preview behavior is expected to remain +compatible and should change only to fix a defect serious enough to block promotion into +core. A writer-version upgrade lets compression schemes produce new optional fields or +properties; it never selects a reader. Users keep the earlier serialized form until they opt +into that edition. Experimental work advances through new draft editions; optional plugins +instead use standalone families such as spatial and json. """ diff --git a/vortex/editions/preview/preview2025.05.0.toml b/vortex/editions/preview/preview2025.05.0.toml index 835bcd0329f..409ac6524ac 100644 --- a/vortex/editions/preview/preview2025.05.0.toml +++ b/vortex/editions/preview/preview2025.05.0.toml @@ -1,26 +1,27 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition record has no documented compatibility guarantee. Its serialization may still be -# evolving, or it may be waiting for its release to be cut. After the release version is known, -# min_vortex_version is backfilled to document the freeze. A frozen record never changes. +# This edition record has no core read-forever guarantee. It may describe an evolving feature, +# stabilized preview functionality awaiting adoption, or a release waiting to be cut. New +# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# backfilled to document its freeze. A frozen record never changes. edition = "preview2025.05.0" family = "preview" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "fastlanes.delta", + { id = "fastlanes.delta", writer_version = 1 }, ] layouts = [] dtypes = [] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.delta", + { id = "fastlanes.delta", writer_version = 1 }, ] layouts = [] dtypes = [] diff --git a/vortex/editions/preview/preview2026.02.0.toml b/vortex/editions/preview/preview2026.02.0.toml index 2d189b2d9a3..adc290109cd 100644 --- a/vortex/editions/preview/preview2026.02.0.toml +++ b/vortex/editions/preview/preview2026.02.0.toml @@ -1,27 +1,28 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition record has no documented compatibility guarantee. Its serialization may still be -# evolving, or it may be waiting for its release to be cut. After the release version is known, -# min_vortex_version is backfilled to document the freeze. A frozen record never changes. +# This edition record has no core read-forever guarantee. It may describe an evolving feature, +# stabilized preview functionality awaiting adoption, or a release waiting to be cut. New +# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# backfilled to document its freeze. A frozen record never changes. edition = "preview2026.02.0" family = "preview" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "vortex.zstd_buffers", + { id = "vortex.zstd_buffers", writer_version = 1 }, ] layouts = [] dtypes = [] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.delta", - "vortex.zstd_buffers", + { id = "fastlanes.delta", writer_version = 1 }, + { id = "vortex.zstd_buffers", writer_version = 1 }, ] layouts = [] dtypes = [] diff --git a/vortex/editions/preview/preview2026.04.0.toml b/vortex/editions/preview/preview2026.04.0.toml index 31c210ab6f3..bb4fc6ce4e1 100644 --- a/vortex/editions/preview/preview2026.04.0.toml +++ b/vortex/editions/preview/preview2026.04.0.toml @@ -1,20 +1,21 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition record has no documented compatibility guarantee. Its serialization may still be -# evolving, or it may be waiting for its release to be cut. After the release version is known, -# min_vortex_version is backfilled to document the freeze. A frozen record never changes. +# This edition record has no core read-forever guarantee. It may describe an evolving feature, +# stabilized preview functionality awaiting adoption, or a release waiting to be cut. New +# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# backfilled to document its freeze. A frozen record never changes. edition = "preview2026.04.0" family = "preview" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [ - "vortex.patched", - "vortex.tensor.cosine_similarity", - "vortex.tensor.inner_product", - "vortex.tensor.l2_norm", - "vortex.tensor.normalized", + { id = "vortex.patched", writer_version = 1 }, + { id = "vortex.tensor.cosine_similarity", writer_version = 1 }, + { id = "vortex.tensor.inner_product", writer_version = 1 }, + { id = "vortex.tensor.l2_norm", writer_version = 1 }, + { id = "vortex.tensor.normalized", writer_version = 1 }, ] layouts = [] dtypes = [ @@ -23,17 +24,17 @@ dtypes = [ ] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.delta", - "vortex.patched", - "vortex.tensor.cosine_similarity", - "vortex.tensor.inner_product", - "vortex.tensor.l2_norm", - "vortex.tensor.normalized", - "vortex.zstd_buffers", + { id = "fastlanes.delta", writer_version = 1 }, + { id = "vortex.patched", writer_version = 1 }, + { id = "vortex.tensor.cosine_similarity", writer_version = 1 }, + { id = "vortex.tensor.inner_product", writer_version = 1 }, + { id = "vortex.tensor.l2_norm", writer_version = 1 }, + { id = "vortex.tensor.normalized", writer_version = 1 }, + { id = "vortex.zstd_buffers", writer_version = 1 }, ] layouts = [] dtypes = [ diff --git a/vortex/editions/preview/preview2026.06.0.toml b/vortex/editions/preview/preview2026.06.0.toml index e61160a7ddf..41866a8af72 100644 --- a/vortex/editions/preview/preview2026.06.0.toml +++ b/vortex/editions/preview/preview2026.06.0.toml @@ -1,13 +1,14 @@ # Generated by `cargo run -p xtask -- generate-editions`. # -# This edition record has no documented compatibility guarantee. Its serialization may still be -# evolving, or it may be waiting for its release to be cut. After the release version is known, -# min_vortex_version is backfilled to document the freeze. A frozen record never changes. +# This edition record has no core read-forever guarantee. It may describe an evolving feature, +# stabilized preview functionality awaiting adoption, or a release waiting to be cut. New +# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# backfilled to document its freeze. A frozen record never changes. edition = "preview2026.06.0" family = "preview" -# The components that join the family at this edition. +# Components and array writer-version increases added by this edition. [added] arrays = [] layouts = [ @@ -16,17 +17,17 @@ layouts = [ dtypes = [] aggregates = [] -# The edition's full membership: the components above, plus every member of earlier +# The edition's full membership: the members above, plus every member of earlier # editions of the family. [components] arrays = [ - "fastlanes.delta", - "vortex.patched", - "vortex.tensor.cosine_similarity", - "vortex.tensor.inner_product", - "vortex.tensor.l2_norm", - "vortex.tensor.normalized", - "vortex.zstd_buffers", + { id = "fastlanes.delta", writer_version = 1 }, + { id = "vortex.patched", writer_version = 1 }, + { id = "vortex.tensor.cosine_similarity", writer_version = 1 }, + { id = "vortex.tensor.inner_product", writer_version = 1 }, + { id = "vortex.tensor.l2_norm", writer_version = 1 }, + { id = "vortex.tensor.normalized", writer_version = 1 }, + { id = "vortex.zstd_buffers", writer_version = 1 }, ] layouts = [ "vortex.list", diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index b5d7e22d200..eb26a18ab3d 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -10,8 +10,9 @@ //! [`crate::editions::enable_default_editions`]. //! //! Members carry a [`crate::editions::ComponentKind`]: arrays a written array may use, extension -//! dtypes its schema may contain, and the aggregates zone maps record. Every kind is restricted to -//! its declared members, so an empty set permits no components of that kind. +//! dtypes its schema may contain, and aggregates zone maps record. Array memberships also pin the +//! writer version compression schemes may produce, preserving existing writer behavior +//! until a newer edition is explicitly selected. //! //! The default file writer resolves the session's enabled editions at write time. The //! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08_1`], and diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index 33edab68649..83b3edc931c 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -84,6 +84,24 @@ fn core_2026_08_1_dtype_set_is_pinned() { assert_eq!(ids, ["vortex.date", "vortex.time", "vortex.timestamp"]); } +#[test] +fn core_array_writer_versions_are_pinned() { + let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); + let arrays = session.components_in(&CORE_2026_08_1, ComponentKind::Array); + assert_eq!( + arrays + .iter() + .find(|inclusion| inclusion.component_id.as_str() == "vortex.pco") + .and_then(|inclusion| inclusion.array_writer_version), + Some(1) + ); + assert!( + arrays + .iter() + .all(|inclusion| inclusion.array_writer_version == Some(1)) + ); +} + #[test] fn core_2026_08_2_is_draft() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); @@ -228,6 +246,12 @@ fn default_session_enables_the_write_editions() { let session = VortexSession::default(); let enabled = session.enabled_editions().editions(); assert!(enabled.contains(&DEFAULT_CORE_EDITION)); + assert_eq!( + session + .enabled_array_writer_versions() + .get(&Id::from("vortex.pco")), + Some(&1) + ); #[cfg(feature = "unstable_encodings")] assert!(enabled.contains(&DEFAULT_PREVIEW_EDITION)); diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 1d1ab1252ac..98f5fe9e2b6 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -149,7 +149,7 @@ pub mod compressor { pub use vortex_btrblocks::SchemeId; } -/// Vortex editions: named, frozen sets of encodings with a read-compatibility guarantee. +/// Vortex editions: versioned sets of serialized components. pub mod editions; pub mod dtype { diff --git a/xtask/src/check_editions.rs b/xtask/src/check_editions.rs index 0c116a945d3..b3000d1eb0f 100644 --- a/xtask/src/check_editions.rs +++ b/xtask/src/check_editions.rs @@ -3,12 +3,14 @@ //! Check that frozen edition records under `vortex/editions` never change. //! -//! A draft record carries no compatibility guarantee and may change, be renamed, or be dropped. -//! Its serialization may still be evolving, or a stable edition may be waiting for its release to -//! be cut. A stable edition can freeze in that release; once the release version is known, -//! `min_vortex_version` is backfilled to document the freeze. The record then carries a -//! read-forever guarantee and may never change again. Whether a record was frozen is read from the -//! base revision, so a change cannot unfreeze an edition and edit it in the same diff. +//! A draft record carries no core read-forever guarantee and is not mechanically locked by this +//! check. It may describe evolving work, stabilized preview functionality awaiting adoption, or a +//! stable edition waiting for its release to be cut. Normal feature additions advance to a new +//! edition; exceptional corrections remain possible before a core freeze. A stable edition can +//! freeze in its release; once the release version is known, `min_vortex_version` is backfilled to +//! document the freeze. The record then carries a read-forever guarantee and may never change +//! again. Whether a record was frozen is read from the base revision, so a change cannot unfreeze +//! an edition and edit it in the same diff. //! //! A newly added record must also be newer than every edition already recorded for its //! family: editions are only ever added going forward. Records are grouped by family, so @@ -40,7 +42,7 @@ use crate::generate_editions::RECORD_DIR; const FROZEN_MARKER: &str = "min_vortex_version"; const REMEDY: &str = "\ -A frozen edition is immutable. To add encodings, declare a NEW edition in +A frozen edition is immutable. To add components or array writer versions, declare a NEW edition in vortex-edition/src/declarations// and regenerate the records with `cargo run -p xtask -- generate-editions`."; diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index c2e9df58fbc..745abdd5685 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -4,8 +4,9 @@ //! Export the edition records under `vortex/editions`. //! //! Every declared edition gets one TOML file recording what it contains: the identifier, the -//! minimum Vortex version whose reader supports it once frozen, and its full component set, -//! one list per [`ComponentKind`]. Records are grouped by family — +//! minimum Vortex version whose reader supports it once frozen, and its full member set. Array +//! members also record the writer version compression schemes may produce. Records are grouped by +//! family — //! `vortex/editions/core/core2025.05.0.toml` — mirroring the declarations in //! `vortex-edition/src/declarations`, since families version independently. //! @@ -28,6 +29,7 @@ use vortex_edition::EDITION_DECLARATIONS; use vortex_edition::EDITION_FAMILIES; use vortex_edition::Edition; use vortex_edition::EditionFamily; +use vortex_edition::EditionInclusion; use vortex_edition::EditionSession; const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; @@ -38,9 +40,10 @@ const FROZEN_NOTE: &str = "\ # editing or deleting a frozen one is rejected by CI."; const DRAFT_NOTE: &str = "\ -# This edition record has no documented compatibility guarantee. Its serialization may still be -# evolving, or it may be waiting for its release to be cut. After the release version is known, -# min_vortex_version is backfilled to document the freeze. A frozen record never changes."; +# This edition record has no core read-forever guarantee. It may describe an evolving feature, +# stabilized preview functionality awaiting adoption, or a release waiting to be cut. New +# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# backfilled to document its freeze. A frozen record never changes."; /// The file recording what a family is, beside that family's editions. pub const FAMILY_FILE: &str = "family.toml"; @@ -91,16 +94,37 @@ const KINDS: [(ComponentKind, &str); 4] = [ (ComponentKind::Aggregate, "aggregates"), ]; -/// Render one TOML list per component kind, each sorted by component id. -fn kind_lists(lines: &mut Vec, ids_of: impl Fn(ComponentKind) -> BTreeSet) { +/// Render one TOML list per component kind, each sorted by component id. Array entries also pin +/// the writer version that compression schemes may produce. +fn kind_lists( + lines: &mut Vec, + inclusions_of: impl Fn(ComponentKind) -> Vec, +) { for (kind, key) in KINDS { - let ids = ids_of(kind); - if ids.is_empty() { + let mut inclusions = inclusions_of(kind); + inclusions.sort_by_key(|inclusion| inclusion.component_id); + if inclusions.is_empty() { lines.push(format!("{key} = []")); continue; } lines.push(format!("{key} = [")); - lines.extend(ids.iter().map(|id| format!(" \"{id}\","))); + if kind == ComponentKind::Array { + lines.extend(inclusions.iter().map(|inclusion| { + let Some(writer_version) = inclusion.array_writer_version else { + unreachable!("validated array inclusion has a writer version") + }; + format!( + " {{ id = \"{}\", writer_version = {} }},", + inclusion.component_id, writer_version + ) + })); + } else { + lines.extend( + inclusions + .iter() + .map(|inclusion| format!(" \"{}\",", inclusion.component_id)), + ); + } lines.push("]".to_string()); } } @@ -125,31 +149,24 @@ fn record(session: &EditionSession, edition: &Edition) -> String { } lines.extend([ String::new(), - "# The components that join the family at this edition.".to_string(), + "# Components and array writer-version increases added by this edition.".to_string(), "[added]".to_string(), ]); kind_lists(&mut lines, |kind| { session .components_in(&edition.id, kind) - .iter() + .into_iter() .filter(|inclusion| inclusion.since == edition.id) - .map(|inclusion| inclusion.component_id.to_string()) .collect() }); lines.extend([ String::new(), - "# The edition's full membership: the components above, plus every member of earlier" + "# The edition's full membership: the members above, plus every member of earlier" .to_string(), "# editions of the family.".to_string(), "[components]".to_string(), ]); - kind_lists(&mut lines, |kind| { - session - .components_in(&edition.id, kind) - .iter() - .map(|inclusion| inclusion.component_id.to_string()) - .collect() - }); + kind_lists(&mut lines, |kind| session.components_in(&edition.id, kind)); lines.push(String::new()); lines.join("\n") } From a1f0616eb6ef920a394d6b8113a7886d4d5645da Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Wed, 26 Aug 2026 16:50:54 +0100 Subject: [PATCH 18/18] more Signed-off-by: Robert Kruszewski --- docs/specs/editions.md | 214 +++++----- encodings/alp/src/alp/plugin.rs | 85 ++-- encodings/fastlanes/src/bitpacking/plugin.rs | 84 ++-- encodings/fsst/src/array.rs | 61 +-- encodings/parquet-variant/src/vtable.rs | 2 +- encodings/zstd/src/zstd_buffers.rs | 48 +-- vortex-array/src/array/plugin.rs | 172 ++++++-- .../src/arrays/piecewise_sequence/tests.rs | 2 +- vortex-array/src/arrays/scalar_fn/plugin.rs | 42 +- vortex-array/src/serde.rs | 368 ++++++++++++++++-- vortex-array/src/session/mod.rs | 59 ++- vortex-btrblocks/src/builder.rs | 21 +- vortex-btrblocks/src/lib.rs | 3 - vortex-compressor/src/compressor/cascade.rs | 12 +- vortex-compressor/src/compressor/mod.rs | 41 -- vortex-compressor/src/compressor/tests.rs | 65 ---- vortex-compressor/src/lib.rs | 1 - vortex-compressor/src/scheme/mod.rs | 24 +- vortex-edition/src/declarations/core/mod.rs | 21 +- .../src/declarations/core/v2025_05.rs | 2 +- .../src/declarations/core/v2025_06.rs | 2 +- .../src/declarations/core/v2025_10.rs | 2 +- .../src/declarations/core/v2026_08.rs | 4 +- .../src/declarations/core/v2026_08_2.rs | 2 +- .../src/declarations/core/v2026_08_3.rs | 2 +- .../src/declarations/preview/mod.rs | 16 +- .../src/declarations/preview/v2025_05.rs | 2 +- .../src/declarations/preview/v2026_02.rs | 2 +- .../src/declarations/preview/v2026_04.rs | 2 +- .../src/declarations/preview/v2026_06.rs | 2 +- vortex-edition/src/lib.rs | 106 +---- vortex-edition/src/session.rs | 85 ++-- vortex-edition/src/tests.rs | 87 ++--- vortex-file/benches/split_collection.rs | 2 +- vortex-file/src/lib.rs | 2 +- vortex-file/src/strategy.rs | 20 - vortex-file/src/writer.rs | 26 +- vortex-file/tests/common/mod.rs | 2 +- vortex-json/src/editions.rs | 2 +- vortex-python-cuda/src/lib.rs | 14 +- vortex-python/src/arrays/mod.rs | 33 +- vortex-session/src/registry.rs | 14 +- vortex-spatial/src/editions.rs | 2 +- .../src/encodings/normalized/tests.rs | 53 ++- .../src/scalar_fns/cosine_similarity.rs | 19 +- vortex-tensor/src/scalar_fns/inner_product.rs | 19 +- vortex-tensor/src/scalar_fns/l2_norm.rs | 19 +- vortex-web/crate/src/wasm.rs | 5 +- vortex/editions/core/core2025.05.0.toml | 96 ++--- vortex/editions/core/core2025.06.0.toml | 62 +-- vortex/editions/core/core2025.10.0.toml | 72 ++-- vortex/editions/core/core2026.08.0.toml | 64 +-- vortex/editions/core/core2026.08.1.toml | 68 ++-- vortex/editions/core/core2026.08.2.toml | 70 ++-- vortex/editions/core/core2026.08.3.toml | 76 ++-- vortex/editions/core/family.toml | 16 +- vortex/editions/preview/family.toml | 11 +- vortex/editions/preview/preview2025.05.0.toml | 8 +- vortex/editions/preview/preview2026.02.0.toml | 10 +- vortex/editions/preview/preview2026.04.0.toml | 28 +- vortex/editions/preview/preview2026.06.0.toml | 18 +- vortex/src/editions/mod.rs | 8 +- vortex/src/editions/tests.rs | 320 ++------------- xtask/src/check_editions.rs | 6 +- xtask/src/generate_editions.rs | 46 +-- 65 files changed, 1422 insertions(+), 1430 deletions(-) diff --git a/docs/specs/editions.md b/docs/specs/editions.md index 01fd71fdbb3..3631fbbdce3 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -1,22 +1,19 @@ # Editions Vortex files contain several kinds of serialized **component**: array encodings, layout encodings, extension dtypes, and -aggregate functions. An **edition** is a named set of these components. For each array encoding, it also pins an -**array writer version**: the compatible serialized features that compression schemes may produce under that array ID. -An edition controls what a writer may put in a file and, once frozen, identifies the earliest Vortex release that -supports every component and writer feature in the set. +aggregate functions. An **edition** is a named set of their concrete wire IDs. It controls what a writer may put in a +file and, once frozen, identifies the earliest Vortex release that recognizes every ID in the set. -An array writer version is only a write-time capability ceiling. It is not an in-memory array version, is not written as -a version tag, and never selects a reader. Each array ID has exactly one registered reader. A higher writer version may -authorize a compression scheme to populate compatible optional fields or properties that an earlier writer never -produced. An incompatible representation is a new array encoding with a new ID. +Array versions are represented by different serialized array IDs, not by a numeric version attached to an in-memory +array. Several IDs may serialize and deserialize the same current in-memory representation. This makes compatibility +self-describing: an old reader resolves an ID it knows, and rejects a newer ID as unknown instead of silently +misinterpreting new metadata, children, or buffers. Editions belong to independently versioned families and are cumulative within a family. Each edition includes all -components and writer versions from the preceding edition in that family, plus any additions. A writer selects at most -one edition from each family and may use the union of their components. If enabled families mention the same array ID, -the writer resolves one effective version: the highest permitted writer version. It does not retain multiple array -versions. For example, selecting `core2026.08.1` and `preview2026.06.0` allows stable components released through August -2026 and preview components released through June 2026. +components from the preceding edition in that family, plus any additions. A writer selects at most one edition from +each family and may use the union of their component IDs. For example, selecting `core2026.08.1` and +`preview2026.06.0` allows stable components released through August 2026 and preview components released through June +2026. The first frozen edition, `core2025.05.0`, contains the components that Vortex `0.36.0` could write. This marks the start of the Vortex file format's stability guarantee. Every Vortex release from `0.36.0` onward can read @@ -28,13 +25,13 @@ guarantee for any draft components written to the file. ## What an edition contains -An edition records every component by kind and ID. Array entries additionally record one writer version. IDs are unique -within a kind, but not across kinds: a layout named `vortex.flat` and an array encoding with the same ID are distinct -components. The writer therefore builds and enforces a separate allowlist for each kind: +An edition records every component by kind and wire ID. IDs are unique within a kind, but not across kinds: a layout +named `vortex.flat` and an array encoding with the same ID are distinct components. The writer therefore builds and +enforces a separate allowlist for each kind: | Kind | What it identifies | Used at | |-------------|---------------------------------------------|-------------------------------| -| `array` | every serialized array and its writer limit | compression and serialization | +| `array` | a serialized array representation | array serialization | | `layout` | the footer's layout tree | layout serialization context | | `dtype` | extension dtypes nested in the file schema | file writer | | `aggregate` | zone maps in zoned layouts | layout writer context | @@ -46,10 +43,10 @@ the writer's configured pruning behavior. Only aggregates that would actually be written are checked. If a column's dtype cannot support an aggregate, the writer omits it and there is no edition violation. -An empty allowlist permits no encodings. Collectively, the selected editions must declare every array encoding, layout -encoding, extension dtype, and aggregate function that the writer serializes. For a given array ID, the resolved edition -set supplies exactly one writer ceiling. Compression schemes declare the minimum writer version required for the -specific representation they would produce and are filtered against that ceiling before doing expensive work. +An empty allowlist permits no components. Collectively, the selected editions must declare every serialized array ID, +layout encoding, extension dtype, and aggregate function that the writer writes. An array serializer may expose several +wire IDs for one in-memory encoding; it tries permitted IDs from oldest to newest and selects the first representation +that can encode the value losslessly. For example, `core2026.08.0` declares the aggregate functions that the default writer may store in zone maps: `min`, `max`, `bounded_min`, `bounded_max`, `nan_count`, and `null_count`. It does not declare `sum`, because the writer does @@ -93,69 +90,72 @@ You can change the default configuration to: `json` in addition to `core`. Sessions created without the Vortex facade must register and enable their editions before writing files. The lower-level -`with_allow_encodings` policy can further restrict array encodings, but cannot permit an encoding excluded by the -selected editions. +`with_allow_encodings` policy can separately restrict which in-memory encodings a compression strategy may produce. It +does not expand the serialized IDs permitted by the selected editions. -The default file writer passes the resolved array writer versions into BtrBlocks. For each canonical input, BtrBlocks -removes a scheme if the representation it would produce needs an absent or newer writer version. This happens before -statistics generation, ratio estimation, sampling, and compression, so the writer does not compress an array and only -then discover that its selected edition forbids the result. The array allowlist remains a final check on the output. +Compression and edition compatibility are separate. Compressors produce current in-memory arrays and do not select a +wire ID. At the flat-array boundary, the serializer receives the edition's allowed serialized IDs. For the in-memory +array it: + +1. considers its historical wire IDs from oldest to newest; +2. skips IDs absent from the selected editions; +3. tries to produce the metadata, buffers, and children for each remaining ID; and +4. uses the first lossless representation, or fails the write if none works. + +This permits an older compressor implementation to produce the current in-memory array while that array's serializer +safely downgrades it to the wire representation the compressor and target edition support. A custom layout or compressor +cannot bypass the check because the final array serializer owns wire-ID selection. ## How editions change -A frozen edition never changes: neither its membership list nor the meaning of its component IDs or writer versions may -be altered. A new encoding or added encoding capability that has not stabilized gets its own new draft edition. Once a -component maintained as part of core is stable, it may join `preview`. Preview is an adoption boundary, not an -experimentation boundary: its serialized behavior should change only to fix a defect serious enough to block promotion -into core. Promotion into the default compatibility set happens through a later `core` edition. A component supplied by -an optional plugin stays in that plugin's independently versioned family. +A frozen edition never changes: neither its membership list nor the meaning of its component IDs may be altered. A new +encoding or serialized array representation that has not stabilized gets its own new draft edition. Once a component +maintained as part of core is stable, it may join `preview`. Preview is an adoption boundary, not an experimentation +boundary: its serialized behavior should change only to fix a defect serious enough to block promotion into core. +Promotion into the default compatibility set happens through a later `core` edition. A component supplied by an +optional plugin stays in that plugin's independently versioned family. A new stable `core` or plugin edition may freeze in the release in which it first ships. Until that release is cut, its -version is not known and the declaration keeps `min_vortex_version: None`. After the release is cut, the declaration is +version is not known and the declaration keeps `min_library_version: None`. After the release is cut, the declaration is updated with that newly released version, usually during development of the next release. This backfills the documented minimum reader version; it does not delay the freeze or its read-forever compatibility guarantee. A component may later be deprecated, meaning that writers stop using it. Readers must continue to support it, so deprecation does not invalidate existing files. -Writer behavior evolves independently. Adding a compatible optional field to an array lets the one reader for that ID -understand the field, but does not authorize existing writers to populate it. A new edition raises that array's writer -version. Sessions targeting the earlier edition retain the earlier output behavior; users opt in by selecting the newer -edition. If the change is incompatible, it is a new array ID instead of a writer-version increase. +Writer behavior evolves independently from the in-memory representation. A change that an old reader must distinguish +uses a new serialized ID, even when the new deserializer produces the same in-memory array. Sessions targeting an older +edition continue selecting the older ID whenever the current value has a lossless downgrade. ## How serialized components evolve Editions govern serialized components, not in-memory representations. An in-memory representation may gain capabilities -or be replaced without changing an edition. On read, the single plugin registered for a component ID constructs the -current in-memory representation. On write, the implementation selects a component and writer behavior allowed by the -selected editions. - -An in-memory representation often has a single serialized component and uses the same ID in memory and on disk, but this -is not required. Multiple component IDs may deserialize into the same in-memory representation. Editions constrain the -ID stored in the file, because that is what the reader must understand. +or be replaced without changing an edition. Each in-memory array plugin owns the mapping between that representation and +its wire history: -### Additive evolution keeps the ID +- the serialized IDs its deserializer recognizes; +- one serializer that receives the permitted IDs and returns the earliest lossless variant as metadata, buffers, and + children; and +- a deserializer that receives the exact ID found in the file and constructs the current in-memory representation. -A component may keep its ID when the new form is an additive, unambiguous extension of its serialized contract. The one -current reader must interpret every historical form correctly, using information already present in the array such as -its dtype or optional metadata fields. A new reader must use the old default when an optional field is absent. An older -reader may reject a form introduced by a later edition, but it must not silently misinterpret it; the later edition -identifies the newer minimum reader. +An in-memory representation often has one serialized ID equal to its in-memory encoding ID, but this is only the simple +case. Editions constrain the ID stored in the file, because that is what an old reader can recognize. -Additive evolution may broaden what the wire format accepts, but it cannot change the meaning of data that existing -readers already accept. Removing or repurposing a field, redefining existing bytes, or making interpretation depend on a -separate reader-version choice are incompatible changes. +### Reader-visible evolution requires a new ID -Reader and writer evolution are deliberately asymmetric. The one reader for an array ID may start accepting a compatible -optional field when the old default is well-defined. Existing edition selections must continue producing their old -form. A higher array writer version opts into populating the field, so upgrading Vortex alone does not silently change a -user's files. +Any new form that an old reader does not already understand uses a new serialized ID. This includes additive metadata or +children when an old reader would accept the ID but reject or misinterpret the new combination. The ID is the capability +tag: readers do not consult the edition or negotiate a separate version while decoding an array. -### Incompatible evolution requires a new ID +Keeping an ID is safe only when the emitted representation remains within that ID's existing frozen contract. A writer +may choose a different but already-valid encoding of the same contract, and a reader may fix a bug or normalize the old +form into a newer in-memory structure. Neither action expands what the wire ID means. -An incompatible revision is a new component, with a new ID, registry entry, and edition membership. The old component -remains in the registry and must remain readable. The in-memory representation need not change: it can read and write -both components, choosing between them based on the value and the selected editions. +A new wire ID does not normally require a second in-memory array. The current plugin registers every historical ID, +serializes the current value under the oldest allowed lossless one, and deserializes all of them into the current type. +The old ID remains registered forever. If the compressor and serializer cannot preserve one common in-memory +representation and losslessly downgrade it, the change instead needs a new in-memory array, compressor, and +deserializer. Name successive incompatible revisions by appending a version to the same base name: `vortex.foo`, `vortex.foo_v2`, `vortex.foo_v3`. Do not give successor versions descriptive names. A linear naming scheme keeps the component's @@ -168,11 +168,32 @@ metadata includes `lower_part_count`, but readers of this component require that representation gains support for wide decimals, represented by a signed most-significant part and one or more unsigned 64-bit lower parts: -- A single-part array still serializes as `vortex.decimal_byte_parts` with - `lower_part_count = 0`, indistinguishable from files written before the change. -- An array with lower parts uses the new `vortex.decimal_byte_parts_v2` component, initially staged in a draft edition. +- The serializer first tries to construct the old single-signed-child form. If every value can be + represented that way, it emits `vortex.decimal_byte_parts` with `lower_part_count = 0`, even if + the current in-memory array has lower-part children. +- An array that cannot be collapsed into that old form losslessly uses the new + `vortex.decimal_byte_parts_v2` component, initially staged in a draft edition. - A new reader deserializes both IDs into the same in-memory representation. An older reader reports `vortex.decimal_byte_parts_v2` as unknown instead of trying to decode a wire format it does not support. +- When targeting an edition that permits only the old ID, serializing a value that can be collapsed succeeds; an + irreducibly multi-part value fails because no lossless downgrade exists. + +#### Example: Pco 8-bit integers + +The historical `vortex.pco` contract does not include `i8` or `u8`; readers implementing that contract must not be +sent an 8-bit Pco payload under the familiar ID. Adding 8-bit support keeps one current in-memory `Pco` array but adds +`vortex.pco_v2` as a serialized component: + +- The single Pco serializer emits `vortex.pco` for the primitive types covered by the old contract, even when both IDs + are permitted. +- For `i8` or `u8`, the earliest lossless form is `vortex.pco_v2`. A target edition without that ID rejects the write. +- The current deserializer registers both IDs. When given `vortex.pco`, it still rejects an 8-bit dtype; understanding + the v2 payload does not silently broaden the frozen v1 contract. +- The Pco compression scheme can sample and construct 8-bit Pco arrays without consulting editions. Wire selection + remains the serializer's responsibility. + +If writing an older edition must succeed for every input, its compression policy must choose an in-memory encoding +whose serializer has a permitted lossless form. It must not disguise the newer Pco form with the old ID. ### Reading: deserialize into the current representation @@ -181,34 +202,32 @@ in-memory representation rather than preserving a parallel legacy representation interior patches is read as a `Patched` array around a patch-free ALP array. Similarly, old zone maps, including `vortex.stats` layouts, are read by the machinery used for modern `vortex.zoned` layouts. -Readers do not negotiate versions. They resolve the component ID and deserialize it, or report an +Readers do not negotiate versions. They resolve the component ID, pass that exact ID to its deserializer, and either +construct the current in-memory array or report an [unknown-component error](#resolving-an-unknown-component-error). -In particular, an array writer version does not create `v1` and `v2` reader registrations. A file contains its array ID, -dtype, metadata, children, and buffers. The reader resolves that ID once and the resulting plugin interprets the actual -serialized form. If a change would require selecting a different interpretation for the same bytes, it is incompatible -and needs a new array ID. +A current deserializer must preserve each historical ID's contract. Recognizing a newer ID does not authorize it to +accept the newer metadata, child shape, dtype coverage, or buffer interpretation when the file carries an older ID. -### Writing: select a permitted component and writer behavior +A file contains its array ID, dtype, metadata, children, and buffers. A newer plugin may be registered under both +`vortex.foo` and `vortex.foo_v2`, but an older build is registered only under `vortex.foo`. This is what guarantees that +the older build rejects a v2 file before interpreting its contents. -Writers choose a component that both represents the current value and belongs to the selected editions. This need not be -the newest component: if an older component can represent the value exactly, the writer may continue to use it. If the -preferred component is not permitted, the writer has two options: +### Writing: select a permitted component and writer behavior -1. **Translate.** If the value has a lossless translation to a permitted component, use that component. For example, a - newer layout may write its zone statistics using an older statistics schema. -2. **Convert to canonical and recompress.** Otherwise, decompress the data to a canonical representation and recompress - it with the configured schemes, restricted to the selected editions. This is how arrays are handled today: the - writer normalizes each chunk to a canonical representation, then lets the edition-filtered compressor choose the - final encoding and compatible serialized features. +For each in-memory array, the writer calls its plugin's single serializer with the serialized IDs permitted by the +selected editions. The serializer owns the versioning logic and returns the earliest lossless variant. It may change +metadata, buffers, and children without constructing a legacy in-memory array. Returning `None` means no permitted +representation works, so the write fails. -Both paths use the normal write pipeline and its configured compressors. If neither can express the data using the -selected editions, the write fails. +This selection happens recursively after compression. Compressor output therefore remains an in-memory concern: a +compressor does not label its array with an edition or choose a wire version. Layouts, extension dtypes, and aggregates +perform their analogous compatibility checks at their own serialization boundaries. ### What this means for each kind -- **Arrays.** The array serialization context permits only encodings from the selected editions. BtrBlocks additionally - checks the required writer version for each candidate representation before evaluating its scheme. +- **Arrays.** The array serialization context permits only wire IDs from the selected editions. The in-memory array's + serializer chooses the oldest permitted lossless representation. - **Layouts.** The layout strategy builds the layout tree at write time. When targeting an older edition, it must use structures available in that edition, such as plain chunked data in place of newer auxiliary layouts. - **Extension dtypes.** Before writing any bytes, the file writer recursively validates every extension dtype in the @@ -219,14 +238,14 @@ selected editions, the write fails. ## The `preview` family -Alongside `core` there is a `preview` family for stabilized, core-maintained components and array writer-version -increases that are ready for explicit adoption but are not yet part of the default core writer. Preview behavior is -expected to remain compatible and should change only to fix a defect serious enough to block promotion into core. It -does not yet carry core's unconditional read-forever guarantee. +Alongside `core` there is a `preview` family for stabilized, core-maintained components and serialized array IDs that +are ready for explicit adoption but are not yet part of the default core writer. Preview behavior is expected to remain +compatible and should change only to fix a defect serious enough to block promotion into core. It does not yet carry +core's unconditional read-forever guarantee. -The default writer does not adopt a higher writer version merely because its reader understands the corresponding -optional features. Users opt in by enabling the preview edition that raises the version. Today, builds using the -`unstable_encodings` Cargo feature also opt into registration and availability of the newest preview component set. +The default writer does not emit a newer wire ID merely because its reader understands it. Users opt in by enabling the +preview edition containing that ID. Today, builds using the `unstable_encodings` Cargo feature also opt into registration +and availability of the newest preview component set. Components that are still evolving belong to new draft editions rather than `preview`; each added feature advances the edition so a file's capability set remains identifiable. Components owned by @@ -250,15 +269,14 @@ Changing the declarations follows the edition's lifecycle: existing record. A component supplied by an optional plugin uses that plugin's standalone family. 2. **Publish stabilized core work in preview.** Once a core-maintained serialized feature is - stable, add it to a new `preview` edition. If compression schemes should begin populating a - compatible optional feature of an existing array, raise that array's writer version in the - preview edition. The core edition continues selecting the earlier version until the feature is - deliberately promoted. + stable, give its wire representation a new array ID and add it to a new `preview` edition. The + core edition continues selecting an older compatible ID until the feature is deliberately + promoted. 3. **Cut a core edition.** Promote adopted preview members into a new `core` edition with - `min_vortex_version: None`, regenerate the records, and ship it in a release. The edition + `min_library_version: None`, regenerate the records, and ship it in a release. The edition freezes as part of that release. Its minimum Vortex version cannot be populated yet because the release version is not known until the release is cut. -4. **Backfill the released version.** After cutting the release, set `min_vortex_version` to that +4. **Backfill the released version.** After cutting the release, set `min_library_version` to that newly released Vortex version — the version that first shipped readers for every member — and regenerate the records. This update usually lands during development of the next release, but it documents the freeze that already happened; it does not freeze the edition later. @@ -290,8 +308,6 @@ Minimum Vortex release: `0.36.0`. - `layout`: `vortex.chunked`, `vortex.dict`, `vortex.flat`, `vortex.stats`, `vortex.struct` - `dtype`: `vortex.date`, `vortex.time`, `vortex.timestamp` -All array entries currently use writer version 1 unless a later edition explicitly raises one. - #### `core2025.06.0` Minimum Vortex release: `0.40.0`. diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index c14133109d1..0544952afe6 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -7,17 +7,18 @@ //! This enables zero-cost backward compatibility with previously written datasets. use vortex_array::Array; +use vortex_array::ArrayContext; +use vortex_array::ArrayDeserialization; use vortex_array::ArrayId; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; +use vortex_array::ArraySerialization; use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::serde::ArrayChildren; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_session::VortexSession; @@ -41,23 +42,31 @@ impl ArrayPlugin for ALPPatchedPlugin { fn serialize( &self, array: &ArrayRef, + ctx: &ArrayContext, session: &VortexSession, - ) -> VortexResult>> { + ) -> VortexResult> { // Delegate to ALP's metadata serde - ALP.serialize(array, session) + ArrayPlugin::serialize(&ALP, array, ctx, session) } fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult { + vortex_ensure!( + parts.serialized_id == self.id(), + "ALP plugin does not recognize serialized ID {}", + parts.serialized_id, + ); let alp_array = Array::::try_from_parts(ArrayVTable::deserialize( - &ALP, dtype, len, metadata, buffers, children, session, + &ALP, + parts.dtype, + parts.len, + parts.metadata, + parts.buffers, + parts.children, + session, )?) .map_err(|_| vortex_err!("ALP plugin should only deserialize vortex.alp"))?; @@ -91,6 +100,8 @@ mod tests { use std::f64::consts::PI; use std::sync::LazyLock; + use vortex_array::ArrayContext; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -133,7 +144,9 @@ mod tests { let array = alp_encoded.as_array(); - let metadata = SESSION.array_serialize(array)?.unwrap(); + let serialization = SESSION + .array_serialize(array, &ArrayContext::empty())? + .unwrap(); let children = array.children(); let buffers = array .buffers() @@ -142,11 +155,14 @@ mod tests { .collect::>(); let deserialized = ALPPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + ALPPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, )?; @@ -182,7 +198,9 @@ mod tests { let array = alp_encoded.as_array(); - let metadata = SESSION.array_serialize(array)?.unwrap(); + let serialization = SESSION + .array_serialize(array, &ArrayContext::empty())? + .unwrap(); let children = array.children(); let buffers = array .buffers() @@ -191,11 +209,14 @@ mod tests { .collect::>(); let deserialized = ALPPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + ALPPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, )?; @@ -213,7 +234,10 @@ mod tests { fn primitive_array_returns_error() { let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array(); - let metadata = SESSION.array_serialize(&array).unwrap().unwrap(); + let serialization = SESSION + .array_serialize(&array, &ArrayContext::empty()) + .unwrap() + .unwrap(); let children = array.children(); let buffers = array .buffers() @@ -223,11 +247,14 @@ mod tests { // This panics because PrimitiveArray has no children and ALP requires encoded child. let _result = ALPPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + ALPPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, ); } diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index a621d085514..43d8468d226 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -7,17 +7,18 @@ //! This enables zero-cost backward compatibility with previously written datasets. use vortex_array::Array; +use vortex_array::ArrayContext; +use vortex_array::ArrayDeserialization; use vortex_array::ArrayId; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; +use vortex_array::ArraySerialization; use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::serde::ArrayChildren; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_session::VortexSession; @@ -40,23 +41,31 @@ impl ArrayPlugin for BitPackedPatchedPlugin { fn serialize( &self, array: &ArrayRef, + ctx: &ArrayContext, session: &VortexSession, - ) -> VortexResult>> { + ) -> VortexResult> { // delegate to BitPacked VTable for serialization - BitPacked.serialize(array, session) + ArrayPlugin::serialize(&BitPacked, array, ctx, session) } fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult { + vortex_ensure!( + parts.serialized_id == self.id(), + "BitPacked plugin does not recognize serialized ID {}", + parts.serialized_id, + ); let bitpacked = Array::::try_from_parts(ArrayVTable::deserialize( - &BitPacked, dtype, len, metadata, buffers, children, session, + &BitPacked, + parts.dtype, + parts.len, + parts.metadata, + parts.buffers, + parts.children, + session, )?) .map_err(|_| vortex_err!("BitPacked plugin should only deserialize fastlanes.bitpacked"))?; @@ -93,6 +102,8 @@ impl ArrayPlugin for BitPackedPatchedPlugin { mod tests { use std::sync::LazyLock; + use vortex_array::ArrayContext; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -134,7 +145,9 @@ mod tests { let array = bitpacked.as_array(); - let metadata = SESSION.array_serialize(array)?.unwrap(); + let serialization = SESSION + .array_serialize(array, &ArrayContext::empty())? + .unwrap(); let children = array.children(); let buffers = array .buffers() @@ -143,11 +156,14 @@ mod tests { .collect::>(); let deserialized = BitPackedPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + BitPackedPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, )?; @@ -184,7 +200,9 @@ mod tests { let array = bitpacked.as_array(); - let metadata = SESSION.array_serialize(array)?.unwrap(); + let serialization = SESSION + .array_serialize(array, &ArrayContext::empty())? + .unwrap(); let children = array.children(); let buffers = array .buffers() @@ -193,11 +211,14 @@ mod tests { .collect::>(); let deserialized = BitPackedPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + BitPackedPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, )?; @@ -214,7 +235,9 @@ mod tests { fn primitive_array_returns_error() -> VortexResult<()> { let array = PrimitiveArray::from_iter([1i32, 2, 3]).into_array(); - let metadata = SESSION.array_serialize(&array)?.unwrap(); + let serialization = SESSION + .array_serialize(&array, &ArrayContext::empty())? + .unwrap(); let children = array.children(); let buffers = array .buffers() @@ -223,11 +246,14 @@ mod tests { .collect::>(); let result = BitPackedPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + BitPackedPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, ); diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index ef25c559b99..86b8fd0c34d 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -1033,6 +1033,7 @@ mod test { use fsst::Compressor; use fsst::Symbol; use prost::Message; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -1158,19 +1159,22 @@ mod test { let deserialized = ArrayPlugin::deserialize( &FSST, - &DType::Utf8(Nullability::NonNullable), - 2, - &FSSTMetadata { - uncompressed_lengths_ptype: fsst_array - .uncompressed_lengths() - .dtype() - .as_ptype() - .into(), - codes_offsets_ptype: fsst_array.codes_offsets().dtype().as_ptype().into(), - } - .encode_to_vec(), - &buffers, - &children.as_slice(), + ArrayDeserialization::new( + vortex_array::ArrayVTable::id(&FSST), + &DType::Utf8(Nullability::NonNullable), + 2, + &FSSTMetadata { + uncompressed_lengths_ptype: fsst_array + .uncompressed_lengths() + .dtype() + .as_ptype() + .into(), + codes_offsets_ptype: fsst_array.codes_offsets().dtype().as_ptype().into(), + } + .encode_to_vec(), + &buffers, + &children.as_slice(), + ), &array_session(), )?; @@ -1304,20 +1308,23 @@ mod test { let fsst = ArrayPlugin::deserialize( &FSST, - &DType::Utf8(Nullability::NonNullable), - 2, - &FSSTMetadata { - uncompressed_lengths_ptype: fsst_array - .uncompressed_lengths() - .dtype() - .as_ptype() - .into(), - // Legacy array did not store this field, use Protobuf default of 0. - codes_offsets_ptype: 0, - } - .encode_to_vec(), - &buffers, - &children.as_slice(), + ArrayDeserialization::new( + vortex_array::ArrayVTable::id(&FSST), + &DType::Utf8(Nullability::NonNullable), + 2, + &FSSTMetadata { + uncompressed_lengths_ptype: fsst_array + .uncompressed_lengths() + .dtype() + .as_ptype() + .into(), + // Legacy array did not store this field, use Protobuf default of 0. + codes_offsets_ptype: 0, + } + .encode_to_vec(), + &buffers, + &children.as_slice(), + ), &array_session(), )?; diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 302625e70ec..297198df438 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -410,7 +410,7 @@ mod tests { editions .declare_edition(Edition { id: TEST_EDITION, - min_vortex_version: None, + min_library_version: None, }) .map_err(|error| vortex_err!("{error}"))?; let component_ids = [ diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 5f7f785f71f..4f195eb7f27 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -10,6 +10,8 @@ use std::sync::Arc; use prost::Message as _; use vortex_array::Array; +use vortex_array::ArrayContext; +use vortex_array::ArrayDeserialization; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -60,43 +62,38 @@ impl ZstdBuffers { /// Compress every top-level buffer of `array` independently with zstd. /// - /// Children are preserved as slots and the wrapped array's serialized metadata is stored so the - /// original array can be rebuilt after decompression. + /// The wrapped array's earliest lossless serialized representation is captured so it can be + /// rebuilt after decompression, including any buffers or children selected by its serializer. pub fn compress( array: &ArrayRef, level: i32, session: &VortexSession, ) -> VortexResult { - let encoding_id = array.encoding_id(); - let metadata = session - .array_serialize(array)? + let serialization = session + .array_serialize(array, &ArrayContext::empty())? .ok_or_else(|| vortex_err!("[ZstdBuffers]: Array does not support serialization"))?; - let buffer_handles = array.buffer_handles(); - let children = array.children(); - let mut compressed_buffers = Vec::with_capacity(buffer_handles.len()); - let mut uncompressed_sizes = Vec::with_capacity(buffer_handles.len()); - let mut buffer_alignments = Vec::with_capacity(buffer_handles.len()); + let mut compressed_buffers = Vec::with_capacity(serialization.buffers.len()); + let mut uncompressed_sizes = Vec::with_capacity(serialization.buffers.len()); + let mut buffer_alignments = Vec::with_capacity(serialization.buffers.len()); let mut compressor = zstd::bulk::Compressor::new(level)?; - // Compression is currently CPU-only, so we gather all buffers on the host. - for handle in &buffer_handles { - buffer_alignments.push(u32::from(handle.alignment())); - let host_buf = handle.clone().try_to_host_sync()?; - uncompressed_sizes.push(host_buf.len() as u64); - let mut compressed = compressor.compress(&host_buf)?; + for buffer in &serialization.buffers { + buffer_alignments.push(u32::from(buffer.alignment())); + uncompressed_sizes.push(buffer.len() as u64); + let mut compressed = compressor.compress(buffer)?; compressed.shrink_to_fit(); compressed_buffers.push(BufferHandle::new_host(ByteBuffer::from(compressed))); } let data = ZstdBuffersData { - inner_encoding_id: encoding_id, - inner_metadata: metadata, + inner_encoding_id: serialization.serialized_id, + inner_metadata: serialization.metadata, compressed_buffers, uncompressed_sizes, buffer_alignments, }; - let slots: ArraySlots = children.into_iter().map(Some).collect(); + let slots: ArraySlots = serialization.children.into_iter().map(Some).collect(); let compressed = Array::try_from_parts( ArrayParts::new(ZstdBuffers, array.dtype().clone(), array.len(), data) .with_slots(slots), @@ -121,11 +118,14 @@ impl ZstdBuffers { let children: Vec = array.slots().iter().flatten().cloned().collect(); inner_vtable.deserialize( - array.dtype(), - array.len(), - &array.data().inner_metadata, - buffer_handles, - &children.as_slice(), + ArrayDeserialization::new( + array.data().inner_encoding_id, + array.dtype(), + array.len(), + &array.data().inner_metadata, + buffer_handles, + &children.as_slice(), + ), session, ) } diff --git a/vortex-array/src/array/plugin.rs b/vortex-array/src/array/plugin.rs index 66845eb9a0a..f30316f4d6f 100644 --- a/vortex-array/src/array/plugin.rs +++ b/vortex-array/src/array/plugin.rs @@ -6,9 +6,12 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_session::VortexSession; +use crate::ArrayContext; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; @@ -21,38 +24,129 @@ use crate::serde::ArrayChildren; /// Reference-counted array plugin. pub type ArrayPluginRef = Arc; -/// Registry trait for ID-based deserialization of arrays. +/// The wire representation produced by an in-memory array's serializer. /// -/// Plugins are registered in the session by their [`ArrayId`]. When a serialized array is -/// encountered, the session resolves the ID to the plugin and calls [`deserialize`] to reconstruct -/// the value as an [`ArrayRef`]. +/// A serializer may reuse the in-memory array's buffers and children with [`Self::from_array`], +/// or return different parts when an older wire representation requires a lossless structural +/// downgrade. +#[derive(Clone, Debug)] +pub struct ArraySerialization { + /// The concrete array ID to write on the wire. + pub serialized_id: ArrayId, + /// Encoding-specific metadata written into the array node. + pub metadata: Vec, + /// Top-level buffers written for this array node. + pub buffers: Vec, + /// Child arrays to serialize recursively. + pub children: Vec, +} + +impl ArraySerialization { + /// Create a wire representation from an ID, metadata, buffers, and children. + pub fn new( + serialized_id: ArrayId, + metadata: Vec, + buffers: Vec, + children: Vec, + ) -> Self { + Self { + serialized_id, + metadata, + buffers, + children, + } + } + + /// Reuse an in-memory array's buffers and children with the supplied serialized metadata. + pub fn from_array(serialized_id: ArrayId, array: &ArrayRef, metadata: Vec) -> Self { + Self::new(serialized_id, metadata, array.buffers(), array.children()) + } +} + +/// The borrowed wire components passed to an array deserializer. +pub struct ArrayDeserialization<'a> { + /// The exact array ID found on the wire. + pub serialized_id: ArrayId, + /// The logical dtype supplied by the containing format. + pub dtype: &'a DType, + /// The logical array length supplied by the containing format. + pub len: usize, + /// Encoding-specific metadata from the array node. + pub metadata: &'a [u8], + /// Top-level buffers referenced by the array node. + pub buffers: &'a [BufferHandle], + /// Lazily decoded child arrays referenced by the array node. + pub children: &'a dyn ArrayChildren, +} + +impl<'a> ArrayDeserialization<'a> { + /// Create borrowed deserialization input from a wire ID and its serialized components. + pub fn new( + serialized_id: ArrayId, + dtype: &'a DType, + len: usize, + metadata: &'a [u8], + buffers: &'a [BufferHandle], + children: &'a dyn ArrayChildren, + ) -> Self { + Self { + serialized_id, + dtype, + len, + metadata, + buffers, + children, + } + } +} + +/// Registry trait for serializing and deserializing an in-memory array representation. /// -/// [`deserialize`]: ArrayPlugin::deserialize +/// A plugin has one [`id`](Self::id) for the in-memory representation and one or more +/// [`serialized_ids`](Self::serialized_ids) for wire representations. Its single serializer +/// receives the permitted IDs and must produce the earliest lossless variant. This lets a current +/// compressor produce the current in-memory array while an older file edition safely downgrades +/// it during serialization. +/// +/// Every serialized ID is also registered for deserialization. A current plugin may therefore +/// deserialize several historical IDs into the same in-memory representation. A reader that +/// predates a newer ID has no registration for it and reports it as unknown instead of silently +/// interpreting an unsupported representation. pub trait ArrayPlugin: 'static + Send + Sync { - /// Returns the ID for this array encoding. - /// - /// During serde, this is the key the registry uses to find - /// this plugin instance and call the appropriate method on it. + /// Returns the ID of the in-memory array representation handled by this plugin. fn id(&self) -> ArrayId; - /// Serialize the array metadata. + /// Returns the serialized array IDs understood by this plugin, ordered oldest to newest. /// - /// This function will only be called for arrays where the encoding ID matches that of this - /// plugin. - fn serialize(&self, array: &ArrayRef, session: &VortexSession) - -> VortexResult>>; + /// The default uses the in-memory ID as the sole wire ID. Override this for an in-memory array + /// that has multiple serialized variants. IDs retained only for reading may also be included; + /// the single serializer need not select them. + fn serialized_ids(&self) -> Vec { + vec![self.id()] + } - /// Deserialize an array from serialized components. + /// Serialize `array` to the earliest lossless wire representation permitted by `ctx`. /// - /// The returned array doesn't necessary have to match this plugin's encoding ID. This is - /// useful for implementing back-compat logic and deserializing arrays into the new version. + /// This function is called only for arrays whose in-memory encoding matches [`id`](Self::id). + /// The returned ID must be declared by [`serialized_ids`](Self::serialized_ids) and permitted + /// by `ctx`. Return `Ok(None)` when no permitted variant represents the value losslessly. + fn serialize( + &self, + array: &ArrayRef, + ctx: &ArrayContext, + session: &VortexSession, + ) -> VortexResult>; + + /// Deserialize one recognized wire representation into the current in-memory array. + /// + /// `serialized_id` identifies the exact representation encountered on disk. The returned + /// array does not necessarily have to use this plugin's in-memory ID; this supports legacy + /// representations that are normalized into another current in-memory array. Implementations + /// must validate the contract of that exact ID rather than accepting every form understood by + /// the current in-memory representation under an older ID. fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult; @@ -79,27 +173,41 @@ impl ArrayPlugin for V { fn serialize( &self, array: &ArrayRef, + ctx: &ArrayContext, session: &VortexSession, - ) -> VortexResult>> { - assert_eq!( + ) -> VortexResult> { + vortex_ensure!( + self.id() == array.encoding_id(), + "array plugin {} cannot serialize in-memory array {}", self.id(), array.encoding_id(), - "Invoked for incorrect array ID" ); - V::serialize(array.as_::(), session) + if !ctx.is_allowed(&self.id()) { + return Ok(None); + } + Ok(V::serialize(array.as_::(), session)? + .map(|metadata| ArraySerialization::from_array(self.id(), array, metadata))) } fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult { + vortex_ensure!( + self.id() == parts.serialized_id, + "array plugin {} does not recognize serialized ID {}", + self.id(), + parts.serialized_id, + ); Ok(Array::::try_from_parts(V::deserialize( - self, dtype, len, metadata, buffers, children, session, + self, + parts.dtype, + parts.len, + parts.metadata, + parts.buffers, + parts.children, + session, )?)? .into_array()) } diff --git a/vortex-array/src/arrays/piecewise_sequence/tests.rs b/vortex-array/src/arrays/piecewise_sequence/tests.rs index a087404fee2..975935675f4 100644 --- a/vortex-array/src/arrays/piecewise_sequence/tests.rs +++ b/vortex-array/src/arrays/piecewise_sequence/tests.rs @@ -171,7 +171,7 @@ fn serialization_is_not_supported() -> VortexResult<()> { .unwrap_err(); assert!( err.to_string() - .contains("Array vortex.piecewise-sequence does not support serialization"), + .contains("Array vortex.piecewise-sequence cannot be represented"), "{err}" ); Ok(()) diff --git a/vortex-array/src/arrays/scalar_fn/plugin.rs b/vortex-array/src/arrays/scalar_fn/plugin.rs index 044f79362cd..b46f4c022bc 100644 --- a/vortex-array/src/arrays/scalar_fn/plugin.rs +++ b/vortex-array/src/arrays/scalar_fn/plugin.rs @@ -2,16 +2,19 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_session::VortexSession; +use crate::ArrayContext; +use crate::ArrayDeserialization; use crate::ArrayId; use crate::ArrayPlugin; use crate::ArrayRef; +use crate::ArraySerialization; use crate::IntoArray; use crate::arrays::ScalarFnArray; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayView; -use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::TypedScalarFnInstance; @@ -63,28 +66,43 @@ impl ArrayPlugin for ScalarFnArrayPlugi fn serialize( &self, array: &ArrayRef, + ctx: &ArrayContext, session: &VortexSession, - ) -> VortexResult>> { + ) -> VortexResult> { + if !ctx.is_allowed(&self.id()) { + return Ok(None); + } // We serialize the scalar function options, along with any scalar function array data. let scalar_fn = array.as_::>(); - ::serialize(&self.0, &scalar_fn, session) + Ok( + ::serialize(&self.0, &scalar_fn, session)? + .map(|metadata| ArraySerialization::from_array(self.id(), array, metadata)), + ) } fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult { - let parts = ::deserialize( - &self.0, dtype, len, metadata, children, session, + vortex_ensure!( + parts.serialized_id == self.id(), + "scalar function array plugin {} does not recognize serialized ID {}", + self.id(), + parts.serialized_id, + ); + let len = parts.len; + let scalar_parts = ::deserialize( + &self.0, + parts.dtype, + parts.len, + parts.metadata, + parts.children, + session, )?; Ok(ScalarFnArray::try_new_with_len( - TypedScalarFnInstance::new(self.0.clone(), parts.options).erased(), - parts.children, + TypedScalarFnInstance::new(self.0.clone(), scalar_parts.options).erased(), + scalar_parts.children, len, )? .into_array()) diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index f84ec182269..9ebacc27448 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -30,6 +30,7 @@ use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayContext; use crate::ArrayRef; use crate::ArraySlots; +use crate::array::ArrayDeserialization; use crate::array::ArrayId; use crate::array::new_foreign_array; use crate::buffer::BufferHandle; @@ -65,11 +66,10 @@ impl ArrayRef { session: &VortexSession, options: &SerializeOptions, ) -> VortexResult> { - // Collect all array buffers - let array_buffers = self - .depth_first_traversal() - .flat_map(|f| f.buffers()) - .collect::>(); + // Resolve the wire representation once. Serializers may choose historical IDs and may + // provide downgraded buffers or children that differ from the in-memory array tree. + let root = ArrayNodeFlatBuffer::try_new(ctx, session, self)?; + let array_buffers = root.array.buffers(); // Allocate result buffers, including a possible padding buffer for each. let mut buffers = vec![]; @@ -121,7 +121,6 @@ impl ArrayRef { // Set up the flatbuffer builder let mut fbb = FlatBufferBuilder::new(); - let root = ArrayNodeFlatBuffer::try_new(ctx, session, self)?; let fb_root = root.try_write_flatbuffer(&mut fbb)?; let fb_buffers = fbb.create_vector(&fb_buffers); @@ -158,20 +157,78 @@ impl ArrayRef { } } +#[derive(Clone, Debug)] +struct ArraySerializationTree { + source: ArrayRef, + serialized_id: ArrayId, + metadata: Vec, + buffers: Vec, + children: Vec, +} + +impl ArraySerializationTree { + fn try_new( + ctx: &ArrayContext, + session: &VortexSession, + source: &ArrayRef, + ) -> VortexResult { + let Some(serialization) = session.array_serialize(source, ctx)? else { + vortex_bail!( + "Array {} cannot be represented by any permitted serialized array ID", + source.encoding_id() + ); + }; + let children = serialization + .children + .iter() + .map(|child| Self::try_new(ctx, session, child)) + .collect::>>()?; + + Ok(Self { + source: source.clone(), + serialized_id: serialization.serialized_id, + metadata: serialization.metadata, + buffers: serialization.buffers, + children, + }) + } + + fn nbuffers_recursive(&self) -> usize { + self.buffers.len() + + self + .children + .iter() + .map(Self::nbuffers_recursive) + .sum::() + } + + fn buffers(&self) -> Vec { + let mut buffers = Vec::with_capacity(self.nbuffers_recursive()); + self.append_buffers(&mut buffers); + buffers + } + + fn append_buffers(&self, buffers: &mut Vec) { + buffers.extend(self.buffers.iter().cloned()); + for child in &self.children { + child.append_buffers(buffers); + } + } +} + /// A utility struct for creating an [`fba::ArrayNode`] flatbuffer. pub struct ArrayNodeFlatBuffer<'a> { ctx: &'a ArrayContext, - session: &'a VortexSession, - array: &'a ArrayRef, - buffer_idx: u16, + array: ArraySerializationTree, } impl<'a> ArrayNodeFlatBuffer<'a> { pub fn try_new( ctx: &'a ArrayContext, session: &'a VortexSession, - array: &'a ArrayRef, + array: &ArrayRef, ) -> VortexResult { + let array = ArraySerializationTree::try_new(ctx, session, array)?; let n_buffers_recursive = array.nbuffers_recursive(); if n_buffers_recursive > u16::MAX as usize { vortex_bail!( @@ -179,51 +236,42 @@ impl<'a> ArrayNodeFlatBuffer<'a> { n_buffers_recursive ); }; - Ok(Self { - ctx, - session, - array, - buffer_idx: 0, - }) + Ok(Self { ctx, array }) } pub fn try_write_flatbuffer<'fb>( &self, fbb: &mut FlatBufferBuilder<'fb>, ) -> VortexResult>> { - let encoding_idx = self.ctx.intern(&self.array.encoding_id()).ok_or_else(|| { - vortex_err!( - "Array encoding {} not permitted by ctx", - self.array.encoding_id() - ) - })?; + self.try_write_node(fbb, &self.array, 0) + } - let metadata_bytes = self.session.array_serialize(self.array)?.ok_or_else(|| { + fn try_write_node<'fb>( + &self, + fbb: &mut FlatBufferBuilder<'fb>, + array: &ArraySerializationTree, + buffer_idx: u16, + ) -> VortexResult>> { + let encoding_idx = self.ctx.intern(&array.serialized_id).ok_or_else(|| { vortex_err!( - "Array {} does not support serialization", - self.array.encoding_id() + "Serialized array ID {} not permitted by ctx", + array.serialized_id ) })?; - let metadata = Some(fbb.create_vector(metadata_bytes.as_slice())); + + let metadata = Some(fbb.create_vector(array.metadata.as_slice())); // Assign buffer indices for all child arrays. - let nbuffers = u16::try_from(self.array.nbuffers()) + let nbuffers = u16::try_from(array.buffers.len()) .map_err(|_| vortex_err!("Array can have at most u16::MAX buffers"))?; - let mut child_buffer_idx = self.buffer_idx + nbuffers; + let mut child_buffer_idx = buffer_idx + nbuffers; - let children = self - .array - .children() + let children = array + .children .iter() .map(|child| { // Update the number of buffers required. - let msg = ArrayNodeFlatBuffer { - ctx: self.ctx, - session: self.session, - array: child, - buffer_idx: child_buffer_idx, - } - .try_write_flatbuffer(fbb)?; + let msg = self.try_write_node(fbb, child, child_buffer_idx)?; child_buffer_idx = u16::try_from(child.nbuffers_recursive()) .ok() @@ -235,8 +283,8 @@ impl<'a> ArrayNodeFlatBuffer<'a> { .collect::>>()?; let children = Some(fbb.create_vector(&children)); - let buffers = Some(fbb.create_vector_from_iter((0..nbuffers).map(|i| i + self.buffer_idx))); - let stats = Some(self.array.statistics().write_flatbuffer(fbb)?); + let buffers = Some(fbb.create_vector_from_iter((0..nbuffers).map(|i| i + buffer_idx))); + let stats = Some(array.source.statistics().write_flatbuffer(fbb)?); Ok(fba::ArrayNode::create( fbb, @@ -336,8 +384,17 @@ impl SerializedArray { let buffers = self.collect_buffers()?; - let decoded = - plugin.deserialize(dtype, len, self.metadata(), &buffers, &children, session)?; + let decoded = plugin.deserialize( + ArrayDeserialization::new( + encoding_id, + dtype, + len, + self.metadata(), + &buffers, + &children, + ), + session, + )?; assert_eq!( decoded.len(), @@ -716,13 +773,240 @@ impl TryFrom for SerializedArray { #[cfg(test)] mod tests { + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + use vortex_buffer::ByteBufferMut; + use vortex_error::vortex_ensure; + use vortex_session::registry::CachedId; use super::*; + use crate::Array; + use crate::ArrayPlugin; + use crate::ArraySerialization; + use crate::ArrayVTable; use crate::IntoArray; use crate::array_session; + use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; + static SERIALIZER_CALLS: AtomicUsize = AtomicUsize::new(0); + + fn old_primitive_id() -> ArrayId { + ArrayVTable::id(&Primitive) + } + + fn new_primitive_id() -> ArrayId { + static ID: CachedId = CachedId::new("vortex.test.primitive_v2"); + *ID + } + + #[derive(Debug)] + struct VersionedPrimitivePlugin; + + impl ArrayPlugin for VersionedPrimitivePlugin { + fn id(&self) -> ArrayId { + old_primitive_id() + } + + fn serialized_ids(&self) -> Vec { + vec![old_primitive_id(), new_primitive_id()] + } + + fn serialize( + &self, + array: &ArrayRef, + ctx: &ArrayContext, + _session: &VortexSession, + ) -> VortexResult> { + vortex_ensure!( + array.encoding_id() == self.id(), + "versioned primitive serializer received {}", + array.encoding_id(), + ); + + let serialized_id = if array.len() <= 4 && ctx.is_allowed(&old_primitive_id()) { + old_primitive_id() + } else if ctx.is_allowed(&new_primitive_id()) { + new_primitive_id() + } else { + return Ok(None); + }; + + Ok(Some(ArraySerialization::from_array( + serialized_id, + array, + vec![], + ))) + } + + fn deserialize( + &self, + parts: ArrayDeserialization<'_>, + session: &VortexSession, + ) -> VortexResult { + vortex_ensure!( + parts.serialized_id == old_primitive_id() + || parts.serialized_id == new_primitive_id(), + "versioned primitive deserializer does not recognize {}", + parts.serialized_id, + ); + vortex_ensure!( + parts.serialized_id != old_primitive_id() || parts.len <= 4, + "old primitive wire ID cannot represent length {}", + parts.len, + ); + Ok(Array::::try_from_parts(ArrayVTable::deserialize( + &Primitive, + parts.dtype, + parts.len, + parts.metadata, + parts.buffers, + parts.children, + session, + )?)? + .into_array()) + } + } + + #[derive(Debug)] + struct CountingVersionedPrimitivePlugin; + + impl ArrayPlugin for CountingVersionedPrimitivePlugin { + fn id(&self) -> ArrayId { + VersionedPrimitivePlugin.id() + } + + fn serialized_ids(&self) -> Vec { + VersionedPrimitivePlugin.serialized_ids() + } + + fn serialize( + &self, + array: &ArrayRef, + ctx: &ArrayContext, + session: &VortexSession, + ) -> VortexResult> { + SERIALIZER_CALLS.fetch_add(1, Ordering::Relaxed); + VersionedPrimitivePlugin.serialize(array, ctx, session) + } + + fn deserialize( + &self, + parts: ArrayDeserialization<'_>, + session: &VortexSession, + ) -> VortexResult { + VersionedPrimitivePlugin.deserialize(parts, session) + } + } + + fn versioned_primitive_session() -> VortexSession { + let session = array_session(); + session.arrays().register(VersionedPrimitivePlugin); + session + } + + fn restricted_context(ids: &[ArrayId]) -> ArrayContext { + ArrayContext::new(ids.to_vec()).with_allowed_ids(ids.iter().copied().collect()) + } + + fn serialize_blob( + array: &ArrayRef, + ctx: &ArrayContext, + session: &VortexSession, + ) -> VortexResult { + let mut blob = ByteBufferMut::empty(); + for buffer in array.serialize(ctx, session, &SerializeOptions::default())? { + blob.extend_from_slice(buffer.as_ref()); + } + Ok(blob.freeze()) + } + + #[test] + fn one_serializer_selects_the_earliest_lossless_wire_id() -> VortexResult<()> { + let session = array_session(); + session.arrays().register(CountingVersionedPrimitivePlugin); + let ctx = restricted_context(&[old_primitive_id(), new_primitive_id()]); + let array = PrimitiveArray::from_iter([1i32, 2, 3, 4]).into_array(); + + SERIALIZER_CALLS.store(0, Ordering::Relaxed); + let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &session)?)?; + assert_eq!(SERIALIZER_CALLS.load(Ordering::Relaxed), 1); + assert_eq!( + ReadContext::new(ctx.to_ids()).resolve(serialized.encoding_id()), + Some(old_primitive_id()) + ); + Ok(()) + } + + #[test] + fn serializer_uses_a_newer_id_only_when_the_old_variant_cannot_represent_the_value() + -> VortexResult<()> { + let session = versioned_primitive_session(); + let ctx = restricted_context(&[old_primitive_id(), new_primitive_id()]); + let array = PrimitiveArray::from_iter(0..8i32).into_array(); + let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &session)?)?; + let read_ctx = ReadContext::new(ctx.to_ids()); + + assert_eq!( + read_ctx.resolve(serialized.encoding_id()), + Some(new_primitive_id()) + ); + let decoded = serialized.decode(array.dtype(), array.len(), &read_ctx, &session)?; + assert_eq!(decoded.encoding_id(), old_primitive_id()); + Ok(()) + } + + #[test] + fn serialization_fails_when_no_permitted_variant_is_lossless() -> VortexResult<()> { + let session = versioned_primitive_session(); + let ctx = restricted_context(&[old_primitive_id()]); + let array = PrimitiveArray::from_iter(0..8i32).into_array(); + + let error = array + .serialize(&ctx, &session, &SerializeOptions::default()) + .expect_err("the old wire variant cannot represent this value"); + assert!(error.to_string().contains("cannot be represented")); + Ok(()) + } + + #[test] + fn old_reader_rejects_a_new_serialized_id() -> VortexResult<()> { + let writer_session = versioned_primitive_session(); + let ctx = restricted_context(&[new_primitive_id()]); + let array = PrimitiveArray::from_iter(0..8i32).into_array(); + let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &writer_session)?)?; + let read_ctx = ReadContext::new(ctx.to_ids()); + + let old_session = array_session(); + let error = serialized + .decode(array.dtype(), array.len(), &read_ctx, &old_session) + .expect_err("an old reader must not recognize the new wire ID"); + assert!(error.to_string().contains("Unknown encoding")); + Ok(()) + } + + #[test] + fn deserializer_enforces_the_exact_wire_id_contract() -> VortexResult<()> { + let session = versioned_primitive_session(); + let write_ctx = restricted_context(&[new_primitive_id()]); + let array = PrimitiveArray::from_iter(0..8i32).into_array(); + let serialized = SerializedArray::try_from(serialize_blob(&array, &write_ctx, &session)?)?; + + // Interpret the encoded index as the old ID to simulate a file that uses the old tag for + // a representation outside that tag's frozen contract. + let error = serialized + .decode( + array.dtype(), + array.len(), + &ReadContext::new([old_primitive_id()]), + &session, + ) + .expect_err("the old wire contract must be enforced by the current deserializer"); + assert!(error.to_string().contains("old primitive wire ID")); + Ok(()) + } + /// A corrupt array tree can declare a buffer that extends past the backing segment. Slicing /// such a buffer must return a [`VortexError`] rather than panicking (see issue #8819). #[test] diff --git a/vortex-array/src/session/mod.rs b/vortex-array/src/session/mod.rs index 2f3fbb9e4e7..4a81efc0e6d 100644 --- a/vortex-array/src/session/mod.rs +++ b/vortex-array/src/session/mod.rs @@ -6,15 +6,19 @@ use std::sync::Arc; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; use vortex_session::ArcSwapMap; use vortex_session::SessionExt; use vortex_session::SessionGuard; use vortex_session::SessionVar; use vortex_session::registry::Id; +use crate::ArrayContext; use crate::ArrayRef; +use crate::array::ArrayId; use crate::array::ArrayPlugin; use crate::array::ArrayPluginRef; +use crate::array::ArraySerialization; use crate::arrays::Bool; use crate::arrays::Chunked; use crate::arrays::Constant; @@ -40,14 +44,17 @@ pub type ArrayRegistry = ArcSwapMap; #[derive(Clone, Debug)] pub struct ArraySession { - /// The set of registered array encodings. + /// Deserializers keyed by the array ID found on the wire. registry: ArrayRegistry, + /// Serializers keyed by the in-memory array encoding ID. + serializers: ArrayRegistry, } impl ArraySession { pub fn empty() -> ArraySession { Self { registry: ArrayRegistry::default(), + serializers: ArrayRegistry::default(), } } @@ -55,10 +62,20 @@ impl ArraySession { &self.registry } - /// Register a new array encoding, replacing any existing encoding with the same ID. + /// Register an in-memory array plugin and all of its recognized serialized IDs. + /// + /// This replaces any serializer with the same in-memory ID and any deserializer registered + /// under one of [`ArrayPlugin::serialized_ids`]. pub fn register(&self, plugin: P) { - self.registry - .insert(plugin.id(), Arc::new(plugin) as ArrayPluginRef); + let plugin = Arc::new(plugin) as ArrayPluginRef; + self.serializers.insert(plugin.id(), Arc::clone(&plugin)); + for serialized_id in plugin.serialized_ids() { + self.registry.insert(serialized_id, Arc::clone(&plugin)); + } + } + + fn serializer(&self, id: &ArrayId) -> Option { + self.serializers.get(id) } } @@ -66,6 +83,7 @@ impl Default for ArraySession { fn default() -> Self { let this = ArraySession { registry: ArrayRegistry::default(), + serializers: ArrayRegistry::default(), }; // Register the canonical encodings. @@ -112,16 +130,37 @@ pub trait ArraySessionExt: SessionExt { self.get::() } - /// Serialize an array using a plugin from the registry. - fn array_serialize(&self, array: &ArrayRef) -> VortexResult>> { - let Some(plugin) = self.arrays().registry.get(&array.encoding_id()) else { + /// Serialize an array using the oldest permitted wire ID that represents it losslessly. + fn array_serialize( + &self, + array: &ArrayRef, + ctx: &ArrayContext, + ) -> VortexResult> { + let Some(plugin) = self.arrays().serializer(&array.encoding_id()) else { vortex_bail!( - "Array {} is not registered for serializations", + "Array {} is not registered for serialization", array.encoding_id() ); }; - plugin.serialize(array, &self.session()) + let Some(serialization) = plugin.serialize(array, ctx, &self.session())? else { + return Ok(None); + }; + vortex_ensure!( + plugin + .serialized_ids() + .contains(&serialization.serialized_id), + "array serializer {} produced undeclared serialized ID {}", + array.encoding_id(), + serialization.serialized_id, + ); + vortex_ensure!( + ctx.is_allowed(&serialization.serialized_id), + "array serializer {} produced forbidden serialized ID {}", + array.encoding_id(), + serialization.serialized_id, + ); + Ok(Some(serialization)) } } @@ -141,6 +180,7 @@ mod tests { let session = VortexSession::empty().with::(); assert!(session.arrays().registry().contains_key(&Bool.id())); + assert!(session.arrays().serializer(&Bool.id()).is_some()); } #[test] @@ -148,5 +188,6 @@ mod tests { let session = VortexSession::empty().with_some(ArraySession::empty()); assert!(!session.arrays().registry().contains_key(&Bool.id())); + assert!(session.arrays().serializer(&Bool.id()).is_none()); } } diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index b739d3c1b61..fe8072d5e66 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -4,7 +4,6 @@ //! Builder for configuring `BtrBlocksCompressor` instances. use vortex_array::ArrayId; -use vortex_compressor::ArrayWriterVersions; use vortex_utils::aliases::hash_set::HashSet; use crate::BtrBlocksCompressor; @@ -92,14 +91,12 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[ #[derive(Debug, Clone)] pub struct BtrBlocksCompressorBuilder { schemes: Vec<&'static dyn Scheme>, - array_writer_versions: Option, } impl Default for BtrBlocksCompressorBuilder { fn default() -> Self { Self { schemes: ALL_SCHEMES.to_vec(), - array_writer_versions: None, } } } @@ -111,7 +108,6 @@ impl BtrBlocksCompressorBuilder { pub fn empty() -> Self { Self { schemes: Vec::new(), - array_writer_versions: None, } } @@ -218,24 +214,9 @@ impl BtrBlocksCompressorBuilder { self } - /// Constrains scheme selection to the enabled writer version of each array encoding. - /// - /// A scheme requiring an absent or newer version is rejected before it computes statistics, - /// estimates, samples, or compresses its input. This policy affects serialized output only; - /// it neither versions in-memory arrays nor changes reader registration. - pub fn with_array_writer_versions(mut self, versions: ArrayWriterVersions) -> Self { - self.array_writer_versions = Some(versions); - self - } - /// Builds the configured [`BtrBlocksCompressor`]. pub fn build(self) -> BtrBlocksCompressor { - let compressor = CascadingCompressor::new(self.schemes); - let compressor = match self.array_writer_versions { - Some(versions) => compressor.with_array_writer_versions(versions), - None => compressor, - }; - BtrBlocksCompressor(compressor) + BtrBlocksCompressor(CascadingCompressor::new(self.schemes)) } } diff --git a/vortex-btrblocks/src/lib.rs b/vortex-btrblocks/src/lib.rs index 19f0c4a6ce6..1ca05c86b4e 100644 --- a/vortex-btrblocks/src/lib.rs +++ b/vortex-btrblocks/src/lib.rs @@ -19,8 +19,6 @@ //! - **Cascaded Encoding**: Multiple compression layers can be applied for optimal results. //! - **Statistical Analysis**: Uses data sampling and statistics to predict compression ratios. //! - **Recursive Structure Handling**: Compresses nested structures like structs and lists. -//! - **Writer Compatibility**: Can reject schemes whose output needs a newer per-array writer -//! version before estimation or compression begins. //! //! # How It Works //! @@ -82,7 +80,6 @@ pub use builder::ALL_SCHEMES; pub use builder::BtrBlocksCompressorBuilder; pub use canonical_compressor::BtrBlocksCompressor; pub use schemes::patches::compress_patches; -pub use vortex_compressor::ArrayWriterVersions; pub use vortex_compressor::CascadingCompressor; pub use vortex_compressor::scheme::CompressorContext; pub use vortex_compressor::scheme::MAX_CASCADE; diff --git a/vortex-compressor/src/compressor/cascade.rs b/vortex-compressor/src/compressor/cascade.rs index 4e58a51e595..86d45d2c0d9 100644 --- a/vortex-compressor/src/compressor/cascade.rs +++ b/vortex-compressor/src/compressor/cascade.rs @@ -222,9 +222,9 @@ impl CascadingCompressor { /// The main scheme-selection entry point for a single leaf array. /// - /// Filters allowed schemes by [`matches`], writer-version requirements, and exclusion rules, - /// merges their [`stats_options`] into a single [`GenerateStatsOptions`], and picks the winner - /// by estimated compression ratio. + /// Filters allowed schemes by [`matches`] and exclusion rules, merges their [`stats_options`] + /// into a single [`GenerateStatsOptions`], and picks the winner by estimated compression + /// ratio. /// /// If a winner is found and its compressed output is actually smaller, that output is /// returned. Otherwise, the original array is returned unchanged. @@ -244,11 +244,7 @@ impl CascadingCompressor { .schemes .iter() .copied() - .filter(|s| { - s.matches(&canonical) - && self.writer_version_allows(*s, &canonical) - && !self.is_excluded(*s, &compress_ctx) - }) + .filter(|s| s.matches(&canonical) && !self.is_excluded(*s, &compress_ctx)) .collect(); let array: ArrayRef = canonical.into(); diff --git a/vortex-compressor/src/compressor/mod.rs b/vortex-compressor/src/compressor/mod.rs index 96354ee19b8..a661970950c 100644 --- a/vortex-compressor/src/compressor/mod.rs +++ b/vortex-compressor/src/compressor/mod.rs @@ -9,11 +9,6 @@ mod sample; mod select; mod structural; -use std::collections::BTreeMap; -use std::sync::Arc; - -use vortex_array::ArrayId; - use crate::builtins::IntDictScheme; use crate::scheme::ChildSelection; use crate::scheme::DescendantExclusion; @@ -21,12 +16,6 @@ use crate::scheme::Scheme; use crate::scheme::SchemeExt; use crate::scheme::SchemeId; -/// The maximum compatible writer version enabled for each array encoding. -/// -/// This is a write-time policy. It is consulted before a scheme is estimated or run and is never -/// stored in an array or used to select a reader. -pub type ArrayWriterVersions = BTreeMap; - /// Synthetic scheme ID used for the compressor's own root-level cascading. pub(crate) const ROOT_SCHEME_ID: SchemeId = SchemeId { name: "vortex.compressor.root", @@ -57,9 +46,6 @@ pub struct CascadingCompressor { /// Descendant exclusion rules for the compressor's own cascading (e.g. excluding Dict from /// list offsets). root_exclusions: Vec, - - /// Per-array writer ceilings, when compression is constrained for serialization. - array_writer_versions: Option>, } impl CascadingCompressor { @@ -77,35 +63,8 @@ impl CascadingCompressor { Self { schemes, root_exclusions, - array_writer_versions: None, } } - - /// Constrains schemes to serialized features allowed by these per-array writer versions. - /// - /// Schemes whose required version is absent or newer than the configured ceiling are removed - /// before statistics, estimation, sampling, or compression. Without this policy, compression - /// is intended for in-memory use and all registered scheme versions remain eligible. - pub fn with_array_writer_versions(mut self, versions: ArrayWriterVersions) -> Self { - self.array_writer_versions = Some(Arc::new(versions)); - self - } - - /// Whether `scheme` can produce its representation of `canonical` under the writer policy. - fn writer_version_allows( - &self, - scheme: &dyn Scheme, - canonical: &vortex_array::Canonical, - ) -> bool { - let Some(enabled) = &self.array_writer_versions else { - return true; - }; - - scheme - .required_array_writer_versions(canonical) - .into_iter() - .all(|(id, required)| enabled.get(&id).is_some_and(|enabled| *enabled >= required)) - } } // NB: Cascading compression logic is located in `vortex-compressor/src/compressor/cascade.rs`. diff --git a/vortex-compressor/src/compressor/tests.rs b/vortex-compressor/src/compressor/tests.rs index 5ac3f8a1052..ec14383ce36 100644 --- a/vortex-compressor/src/compressor/tests.rs +++ b/vortex-compressor/src/compressor/tests.rs @@ -2,8 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::sync::LazyLock; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; use parking_lot::Mutex; use vortex_array::ArrayId; @@ -11,13 +9,11 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::VTable; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Constant; use vortex_array::arrays::Map; use vortex_array::arrays::NullArray; -use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builders::MapBuilder; @@ -64,49 +60,6 @@ fn matches_integer_primitive(canonical: &Canonical) -> bool { matches!(canonical, Canonical::Primitive(primitive) if primitive.ptype().is_int()) } -static WRITER_V2_WAS_ESTIMATED: AtomicBool = AtomicBool::new(false); - -#[derive(Debug)] -struct WriterV2Scheme; - -impl Scheme for WriterV2Scheme { - fn scheme_name(&self) -> &'static str { - "test.writer_v2" - } - - fn matches(&self, canonical: &Canonical) -> bool { - matches_integer_primitive(canonical) - } - - fn produced_encodings(&self) -> Vec { - vec![Primitive.id()] - } - - fn required_array_writer_versions(&self, _canonical: &Canonical) -> Vec<(ArrayId, u16)> { - vec![(Primitive.id(), 2)] - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - WRITER_V2_WAS_ESTIMATED.store(true, Ordering::Relaxed); - CompressionEstimate::Verdict(EstimateVerdict::Skip) - } - - fn compress( - &self, - _compressor: &CascadingCompressor, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - unreachable!("the test scheme always skips") - } -} - #[derive(Debug)] struct DirectRatioScheme; @@ -744,24 +697,6 @@ fn ratio_tie_between_immediate_and_deferred_favors_immediate() -> VortexResult<( Ok(()) } -#[test] -fn writer_versions_filter_schemes_before_estimation() -> VortexResult<()> { - let array = PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array(); - let mut exec_ctx = SESSION.create_execution_ctx(); - - WRITER_V2_WAS_ESTIMATED.store(false, Ordering::Relaxed); - CascadingCompressor::new(vec![&WriterV2Scheme]) - .with_array_writer_versions([(Primitive.id(), 1)].into_iter().collect()) - .compress(&array, &mut exec_ctx)?; - assert!(!WRITER_V2_WAS_ESTIMATED.load(Ordering::Relaxed)); - - CascadingCompressor::new(vec![&WriterV2Scheme]) - .with_array_writer_versions([(Primitive.id(), 2)].into_iter().collect()) - .compress(&array, &mut exec_ctx)?; - assert!(WRITER_V2_WAS_ESTIMATED.load(Ordering::Relaxed)); - Ok(()) -} - #[test] fn all_null_array_compresses_to_constant() -> VortexResult<()> { let array = PrimitiveArray::new( diff --git a/vortex-compressor/src/lib.rs b/vortex-compressor/src/lib.rs index f99a385b0c3..55bb9b188f6 100644 --- a/vortex-compressor/src/lib.rs +++ b/vortex-compressor/src/lib.rs @@ -67,7 +67,6 @@ pub mod scheme; pub mod stats; mod compressor; -pub use compressor::ArrayWriterVersions; pub use compressor::CascadingCompressor; mod trace; diff --git a/vortex-compressor/src/scheme/mod.rs b/vortex-compressor/src/scheme/mod.rs index f1b58c10512..de9e67690d4 100644 --- a/vortex-compressor/src/scheme/mod.rs +++ b/vortex-compressor/src/scheme/mod.rs @@ -56,9 +56,9 @@ impl fmt::Display for SchemeId { // TODO(connor): Remove all default implemented methods. /// A single compression encoding that the [`CascadingCompressor`] can select from. /// -/// The compressor evaluates every registered scheme whose [`matches`] returns `true` and whose -/// [`required_array_writer_versions`] fit the configured write policy, picks the one with the -/// highest [`expected_compression_ratio`], and calls [`compress`] on the winner. +/// The compressor evaluates every registered scheme whose [`matches`] returns `true` for a given +/// array, picks the one with the highest [`expected_compression_ratio`], and calls [`compress`] on +/// the winner. /// /// One of the key features of the compressor in this crate is that schemes may "cascade". A /// scheme's [`compress`] can call back into the compressor via @@ -113,7 +113,6 @@ impl fmt::Display for SchemeId { /// [`matches`]: Scheme::matches /// [`compress`]: Scheme::compress /// [`expected_compression_ratio`]: Scheme::expected_compression_ratio -/// [`required_array_writer_versions`]: Scheme::required_array_writer_versions /// [`stats_options`]: Scheme::stats_options /// [`num_children`]: Scheme::num_children /// [`descendant_exclusions`]: Scheme::descendant_exclusions @@ -132,23 +131,6 @@ pub trait Scheme: Debug + Send + Sync { /// Canonical arrays the scheme merely rearranges do not need to be declared. fn produced_encodings(&self) -> Vec; - /// The minimum writer version required for each array encoding this scheme would produce for - /// `canonical`. - /// - /// The default requires writer version 1 for every [`produced_encodings`](Self::produced_encodings) - /// entry. Override this when the same reader-compatible array encoding has optional serialized - /// fields or properties that only newer writers may populate. The compressor checks these - /// requirements before statistics, estimation, sampling, or compression. An incompatible - /// serialized representation must use a new array ID instead of a higher writer version. - /// This method must be cheap; if the exact output depends on later analysis, report the newest - /// version the scheme might produce. - fn required_array_writer_versions(&self, _canonical: &Canonical) -> Vec<(ArrayId, u16)> { - self.produced_encodings() - .into_iter() - .map(|id| (id, 1)) - .collect() - } - /// Returns the stats generation options this scheme requires. The compressor merges all /// eligible schemes' options before generating stats so that a single stats pass satisfies /// every scheme. diff --git a/vortex-edition/src/declarations/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs index 26b3f699810..3dcf6ca6b0b 100644 --- a/vortex-edition/src/declarations/core/mod.rs +++ b/vortex-edition/src/declarations/core/mod.rs @@ -3,23 +3,22 @@ //! The `core` edition family: serialized components available to the default file writer. //! -//! One module per edition, each declaring the edition and the members that join the -//! family at it; members of earlier editions are inherited and never restated. Array members carry -//! the writer version that compression schemes may produce. +//! One module per edition, each declaring the edition and the serialized components that join the +//! family at it; members of earlier editions are inherited and never restated. use crate::EditionFamily; /// The `core` family: serialized components available by default. pub static FAMILY: EditionFamily = EditionFamily { name: "core", - doc: "The serialized components available to the default file writer. Array memberships pin \ -the writer version compression schemes may produce. Every array ID still has one reader; an \ -incompatible serialized form must use a new ID. Every core edition freezes, and a \ -frozen edition carries a read-forever guarantee: a file written with it stays readable by every \ -later Vortex release. Stabilized non-plugin components and array writer-version upgrades \ -are adopted through preview before joining core. An edition may freeze in the release that cuts \ -it; after that release version is known, the declaration is backfilled with it as the minimum. A \ -frozen edition never changes.", + doc: "The serialized components available to the default file writer. Each array ID names a \ +wire representation that old readers either recognize or reject; several IDs may deserialize \ +into one current in-memory array. Every core edition freezes, and a frozen edition carries a \ +read-forever guarantee: a file written with it stays readable by every later Vortex release. \ +Stabilized non-plugin components and newer serialized array IDs are adopted through preview \ +before joining core. An edition may freeze in the release that cuts it; after that release \ +version is known, the declaration is backfilled with it as the minimum. A frozen edition never \ +changes.", }; pub mod v2025_05; diff --git a/vortex-edition/src/declarations/core/v2025_05.rs b/vortex-edition/src/declarations/core/v2025_05.rs index 9ead57f1fc2..38ca6d75a10 100644 --- a/vortex-edition/src/declarations/core/v2025_05.rs +++ b/vortex-edition/src/declarations/core/v2025_05.rs @@ -15,7 +15,7 @@ pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_05_0, - min_vortex_version: Some("0.36.0"), + min_library_version: Some("0.36.0"), }, added: &[ EditionMember::array(&"fastlanes.bitpacked"), diff --git a/vortex-edition/src/declarations/core/v2025_06.rs b/vortex-edition/src/declarations/core/v2025_06.rs index a2acdb479fb..3d033d280ed 100644 --- a/vortex-edition/src/declarations/core/v2025_06.rs +++ b/vortex-edition/src/declarations/core/v2025_06.rs @@ -15,7 +15,7 @@ pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_06_0, - min_vortex_version: Some("0.40.0"), + min_library_version: Some("0.40.0"), }, added: &[ EditionMember::array(&"vortex.pco"), diff --git a/vortex-edition/src/declarations/core/v2025_10.rs b/vortex-edition/src/declarations/core/v2025_10.rs index 1f5ad573e7b..cfe2a79f0c3 100644 --- a/vortex-edition/src/declarations/core/v2025_10.rs +++ b/vortex-edition/src/declarations/core/v2025_10.rs @@ -15,7 +15,7 @@ pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_10_0, - min_vortex_version: Some("0.54.0"), + min_library_version: Some("0.54.0"), }, added: &[ EditionMember::array(&"fastlanes.rle"), diff --git a/vortex-edition/src/declarations/core/v2026_08.rs b/vortex-edition/src/declarations/core/v2026_08.rs index 176871ab38b..a9022d8d569 100644 --- a/vortex-edition/src/declarations/core/v2026_08.rs +++ b/vortex-edition/src/declarations/core/v2026_08.rs @@ -23,7 +23,7 @@ pub const CORE_2026_08_0: EditionId = EditionId::new("core", 2026, 8, 0); pub static DECLARATION_0: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_0, - min_vortex_version: Some("0.84.0"), + min_library_version: Some("0.84.0"), }, added: &[ EditionMember::layout(&"vortex.zoned"), @@ -43,7 +43,7 @@ pub const CORE_2026_08_1: EditionId = EditionId::new("core", 2026, 8, 1); pub static DECLARATION_1: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_1, - min_vortex_version: Some("0.84.0"), + min_library_version: Some("0.84.0"), }, added: &[EditionMember::array(&"vortex.onpair")], }; diff --git a/vortex-edition/src/declarations/core/v2026_08_2.rs b/vortex-edition/src/declarations/core/v2026_08_2.rs index 1869d2bc343..c4d1686ba1c 100644 --- a/vortex-edition/src/declarations/core/v2026_08_2.rs +++ b/vortex-edition/src/declarations/core/v2026_08_2.rs @@ -15,7 +15,7 @@ pub const CORE_2026_08_2: EditionId = EditionId::new("core", 2026, 8, 2); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_2, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"vortex.map")], }; diff --git a/vortex-edition/src/declarations/core/v2026_08_3.rs b/vortex-edition/src/declarations/core/v2026_08_3.rs index cf069c6f615..03dbe41500a 100644 --- a/vortex-edition/src/declarations/core/v2026_08_3.rs +++ b/vortex-edition/src/declarations/core/v2026_08_3.rs @@ -15,7 +15,7 @@ pub const CORE_2026_08_3: EditionId = EditionId::new("core", 2026, 8, 3); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_3, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::array(&"vortex.parquet.variant"), diff --git a/vortex-edition/src/declarations/preview/mod.rs b/vortex-edition/src/declarations/preview/mod.rs index 7e15a309b55..635e2095cb9 100644 --- a/vortex-edition/src/declarations/preview/mod.rs +++ b/vortex-edition/src/declarations/preview/mod.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The `preview` edition family: stabilized core components and array writer-version upgrades +//! The `preview` edition family: stabilized core components and serialized array representations //! awaiting explicit adoption. //! //! One module per draft edition, each declaring the members that join the family at it. @@ -12,14 +12,12 @@ use crate::EditionFamily; /// The `preview` family: stabilized, opt-in core functionality. pub static FAMILY: EditionFamily = EditionFamily { name: "preview", - doc: "Stabilized, opt-in components and array writer-version upgrades maintained as \ -part of core but not yet adopted by the default core writer. Preview behavior is expected to \ -remain compatible and should change only to fix a defect serious enough to block promotion into \ -core. A writer-version upgrade lets compression schemes produce new optional fields or \ -properties; it never selects a reader. Users keep the earlier serialized form until they opt \ -into that edition. Experimental work \ -advances through new draft editions; optional plugins instead use standalone families such as \ -spatial and json.", + doc: "Stabilized, opt-in components and serialized array representations maintained as part \ +of core but not yet adopted by the default core writer. Preview behavior is expected to remain \ +compatible and should change only to fix a defect serious enough to block promotion into core. \ +A new wire representation has a new array ID, even when it serializes and deserializes the same \ +in-memory array as an older ID. Experimental work advances through new draft editions; optional \ +plugins instead use standalone families such as spatial and json.", }; pub mod v2025_05; diff --git a/vortex-edition/src/declarations/preview/v2025_05.rs b/vortex-edition/src/declarations/preview/v2025_05.rs index 82028132e34..99937004bb1 100644 --- a/vortex-edition/src/declarations/preview/v2025_05.rs +++ b/vortex-edition/src/declarations/preview/v2025_05.rs @@ -15,7 +15,7 @@ pub const PREVIEW_2025_05_0: EditionId = EditionId::new("preview", 2025, 5, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: PREVIEW_2025_05_0, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"fastlanes.delta")], }; diff --git a/vortex-edition/src/declarations/preview/v2026_02.rs b/vortex-edition/src/declarations/preview/v2026_02.rs index 0a0be9b2117..e7852cad81f 100644 --- a/vortex-edition/src/declarations/preview/v2026_02.rs +++ b/vortex-edition/src/declarations/preview/v2026_02.rs @@ -15,7 +15,7 @@ pub const PREVIEW_2026_02_0: EditionId = EditionId::new("preview", 2026, 2, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: PREVIEW_2026_02_0, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"vortex.zstd_buffers")], }; diff --git a/vortex-edition/src/declarations/preview/v2026_04.rs b/vortex-edition/src/declarations/preview/v2026_04.rs index 10f35a5db9e..f8621da2506 100644 --- a/vortex-edition/src/declarations/preview/v2026_04.rs +++ b/vortex-edition/src/declarations/preview/v2026_04.rs @@ -15,7 +15,7 @@ pub const PREVIEW_2026_04_0: EditionId = EditionId::new("preview", 2026, 4, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: PREVIEW_2026_04_0, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::array(&"vortex.patched"), diff --git a/vortex-edition/src/declarations/preview/v2026_06.rs b/vortex-edition/src/declarations/preview/v2026_06.rs index 024ec45a810..15dc2197331 100644 --- a/vortex-edition/src/declarations/preview/v2026_06.rs +++ b/vortex-edition/src/declarations/preview/v2026_06.rs @@ -15,7 +15,7 @@ pub const PREVIEW_2026_06_0: EditionId = EditionId::new("preview", 2026, 6, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: PREVIEW_2026_06_0, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::layout(&"vortex.list")], }; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index b5ab3ffb71a..5a346b784d5 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -1,9 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Definitions of Vortex *editions*: named sets of serialized components and the writer versions -//! compression schemes may produce for each array encoding. Frozen editions carry a forever -//! read-compatibility guarantee; draft editions do not. +//! Definitions of Vortex *editions*: named sets of serialized component IDs. Frozen editions +//! carry a forever read-compatibility guarantee; draft editions do not. //! //! Editions live on the session, like encodings do: [`EditionSession`] holds the registered //! editions and [`EnabledEditions`] selects which of them a writer may emit. Declarations @@ -15,13 +14,12 @@ //! Every membership is typed by a [`ComponentKind`], and members are resolved one kind at a //! time with [`EditionSessionExt::enabled_component_ids`]: the file writer restricts the //! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets. Array -//! memberships additionally carry a writer version. Compression schemes consult that version -//! before estimating or producing an array, so a compatible reader extension does not silently -//! change existing writer output. Writer versions are not stored in files and do not select a -//! reader: every array ID has exactly one registered reader. An incompatible serialized form must -//! therefore use a new array ID. +//! memberships name wire IDs rather than in-memory array representations. An array plugin may +//! serialize one current in-memory representation under several historical IDs, choosing the +//! oldest permitted lossless representation. Readers resolve the ID stored in the file and either +//! deserialize it into the current representation or reject it as unknown. //! -//! An edition is represented as a **draft** until its [`Edition::min_vortex_version`] is +//! An edition is represented as a **draft** until its [`Edition::min_library_version`] is //! recorded. A stable edition may freeze in the release that cuts it; once that release version //! is known, the field is backfilled to document the freeze. The per-edition member sets are //! computed from the registered declarations by [`EditionSession::components_in`], and @@ -167,7 +165,8 @@ impl EditionFamily { /// written layouts. Further kinds (scalar functions, say) can be added the same way. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ComponentKind { - /// An array encoding, e.g. `vortex.alp`, registered in the session's array registry. + /// A serialized array representation, e.g. `vortex.alp`, registered in the session's array + /// registry. Array, /// A layout encoding, e.g. `vortex.flat`, registered in the session's layout registry. Layout, @@ -189,18 +188,15 @@ impl Display for ComponentKind { } } -/// The writer version assigned when an array encoding first joins an edition family. -pub const INITIAL_ARRAY_WRITER_VERSION: u16 = 1; - -/// An edition: a named set of serialized components and array writer versions that can -/// acquire a read-compatibility guarantee, registered with [`EditionSession::declare_edition`]. +/// An edition: a named set of serialized components that can acquire a read-compatibility +/// guarantee, registered with [`EditionSession::declare_edition`]. /// The set itself is computed from the registered [`EditionInclusion`]s by /// [`EditionSession::components_in`]. #[derive(Clone, Copy, Debug)] pub struct Edition { /// The edition identifier. For a `core` edition, its date records when it freezes. pub id: EditionId, - /// The minimum Vortex version whose reader supports every member of this edition. + /// The minimum library version whose reader supports every member of this edition. /// /// A stable edition may freeze in the release that cuts it. Until that release is cut, its /// version is not known and this remains `None`. The version is then backfilled to document @@ -210,22 +206,21 @@ pub struct Edition { /// to change. /// Validated against the members' [`EditionInclusion::required_vortex_release`] values: /// no member may require a version newer than the edition declares. - pub min_vortex_version: Option<&'static str>, + pub min_library_version: Option<&'static str>, } impl Edition { - /// A draft is an edition whose `min_vortex_version` has not been recorded yet. + /// A draft is an edition whose `min_library_version` has not been recorded yet. /// /// This describes the absence of a frozen core compatibility guarantee, not necessarily the /// implementation stability of its members. Stabilized preview editions are drafts too. pub fn is_draft(&self) -> bool { - self.min_vortex_version.is_none() + self.min_library_version.is_none() } } /// Declares that a serialized component is a member of an edition — and of every later edition of -/// the same family. Array components may be redeclared in a later edition with a higher writer -/// version. Registered with [`EditionSession::declare_inclusion`]. +/// the same family. Registered with [`EditionSession::declare_inclusion`]. #[derive(Clone, Copy, Debug)] pub struct EditionInclusion { /// What the membership covers. Ids are unique per kind, so this is part of the @@ -233,12 +228,6 @@ pub struct EditionInclusion { pub kind: ComponentKind, /// The interned component id, e.g. `vortex.alp`. pub component_id: Id, - /// The compatible serialized features this edition permits writers to produce for an array. - /// - /// `None` for non-array members. This is a write-time capability ceiling, not a version of the - /// in-memory array or a read-time dispatch key. Each array ID has one reader; an incompatible - /// serialized form requires a new array ID. - pub array_writer_version: Option, /// The first edition this component is a member of. pub since: EditionId, /// The earliest Vortex release supporting this member, recorded from evidence (e.g. @@ -290,29 +279,14 @@ pub struct EditionMember { pub kind: ComponentKind, /// The member, named by id string or by vtable. pub component: &'static dyn AsComponentId, - /// The writer version permitted for an array member, or `None` for every other kind. - pub array_writer_version: Option, } impl EditionMember { /// An array encoding member, e.g. `vortex.alp`. pub const fn array(component: &'static dyn AsComponentId) -> Self { - Self::array_writer_version(component, INITIAL_ARRAY_WRITER_VERSION) - } - - /// An array encoding member at a specific writer version. - /// - /// Use a later edition and a larger version when a writer may begin producing a new optional - /// field or another compatible serialized property that earlier writers never emitted. A - /// change requiring a different reader is a new array encoding, not a version increase. - pub const fn array_writer_version( - component: &'static dyn AsComponentId, - writer_version: u16, - ) -> Self { Self { kind: ComponentKind::Array, component, - array_writer_version: Some(writer_version), } } @@ -321,7 +295,6 @@ impl EditionMember { Self { kind: ComponentKind::Layout, component, - array_writer_version: None, } } @@ -330,7 +303,6 @@ impl EditionMember { Self { kind: ComponentKind::DType, component, - array_writer_version: None, } } @@ -339,20 +311,19 @@ impl EditionMember { Self { kind: ComponentKind::Aggregate, component, - array_writer_version: None, } } } -/// Declares an edition together with its new members and array writer-version increases, in one -/// block. Registered with [`EditionSession::declare`], which derives each entry's membership -/// (`since` = the declared edition) from the block structure. +/// Declares an edition together with its new members in one block. Registered with +/// [`EditionSession::declare`], which derives each entry's membership (`since` = the declared +/// edition) from the block structure. #[derive(Clone, Copy, Debug)] pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, - /// The members that join the family and array writer versions raised by this edition, each - /// tagged with its [`ComponentKind`]. Earlier entries are inherited and never restated. + /// The members that join the family at this edition, each tagged with its [`ComponentKind`]. + /// Earlier entries are inherited and never restated. pub added: &'static [EditionMember], } @@ -367,8 +338,6 @@ impl EditionInclusion { Self { kind, component_id: component.component_id(), - array_writer_version: (kind == ComponentKind::Array) - .then_some(INITIAL_ARRAY_WRITER_VERSION), since, required_vortex_release: None, } @@ -380,19 +349,6 @@ impl EditionInclusion { Self::new(ComponentKind::Array, encoding, since) } - /// Declare that an array writer version is available in `since` and every later edition of - /// the same family, until superseded by a later writer version. - pub fn array_writer_version( - encoding: &C, - writer_version: u16, - since: EditionId, - ) -> Self { - Self { - array_writer_version: Some(writer_version), - ..Self::new(ComponentKind::Array, encoding, since) - } - } - /// Declare that an extension dtype is a member of `since` and every later edition of the /// same family. pub fn dtype(dtype: &C, since: EditionId) -> Self { @@ -416,26 +372,6 @@ impl EditionInclusion { self.kind ))); } - match (self.kind, self.array_writer_version) { - (ComponentKind::Array, Some(0)) => { - return Err(EditionError::new(format!( - "array {id} must have a non-zero writer version" - ))); - } - (ComponentKind::Array, Some(_)) => {} - (ComponentKind::Array, None) => { - return Err(EditionError::new(format!( - "array {id} must declare a writer version" - ))); - } - (_, Some(version)) => { - return Err(EditionError::new(format!( - "{} {id} cannot declare array writer version {version}", - self.kind - ))); - } - (_, None) => {} - } if let Some(release) = self.required_vortex_release && parse_release(release).is_none() { diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index 8f22de7cdd2..a80327726df 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -42,8 +42,8 @@ struct Inner { /// Keyed by the display form of the edition id. editions: BTreeMap, /// One map per member kind, each keyed by interned member id, because ids are only unique - /// within a kind. An id has one inclusion history per family; array histories may contain - /// successive writer versions. Ordered by kind, then by the id's string form. + /// within a kind. An id may have one inclusion per family. Ordered by kind, then by the id's + /// string form. inclusions: BTreeMap>>, } @@ -85,16 +85,16 @@ impl EditionSession { } } - /// Declare an edition together with the members and array writer-version increases added at - /// it. Each entry's membership (`since`) is the declared edition; earlier entries are - /// inherited and must not be restated. + /// Declare an edition together with the members added at it. Each entry's membership + /// (`since`) is the declared edition; earlier entries are inherited and must not be restated. pub fn declare(&self, declaration: &EditionDeclaration) -> Result<(), EditionError> { self.declare_edition(declaration.edition)?; for member in declaration.added { - self.declare_inclusion(EditionInclusion { - array_writer_version: member.array_writer_version, - ..EditionInclusion::new(member.kind, member.component, declaration.edition.id) - })?; + self.declare_inclusion(EditionInclusion::new( + member.kind, + member.component, + declaration.edition.id, + ))?; } Ok(()) } @@ -135,10 +135,9 @@ impl EditionSession { Ok(()) } - /// Declare an edition inclusion. A component may belong to multiple families. Within one - /// family, non-array members join once, while an array may be redeclared only with a larger - /// writer version in a later edition. Kind is part of the key, so an array encoding - /// and a layout may share an id. + /// Declare an edition inclusion. A component may belong to multiple families but joins each + /// family only once. A newer wire representation uses a new component ID. Kind is part of the + /// key, so an array encoding and a layout may share an id. pub fn declare_inclusion(&self, inclusion: EditionInclusion) -> Result<(), EditionError> { let mut inner = self.inner.write(); let by_id = inner.inclusions.entry(inclusion.kind).or_default(); @@ -155,23 +154,10 @@ impl EditionSession { }); if let Some(previous) = previous { - let is_later = previous.since != inclusion.since - && previous.since.is_at_or_before(&inclusion.since); - let is_array_upgrade = match ( - inclusion.kind, - previous.array_writer_version, - inclusion.array_writer_version, - ) { - (ComponentKind::Array, Some(previous), Some(next)) => next > previous, - _ => false, - }; - if !is_later || !is_array_upgrade { - return Err(EditionError::new(format!( - "{} {} already has a membership in family {}; only a later, higher array \ - writer version may supersede it", - inclusion.kind, inclusion.component_id, inclusion.since.family - ))); - } + return Err(EditionError::new(format!( + "{} {} already joined family {} in edition {}", + inclusion.kind, inclusion.component_id, inclusion.since.family, previous.since, + ))); } history.push(inclusion); history.sort_by_key(|entry| { @@ -207,9 +193,8 @@ impl EditionSession { } /// Compute an edition's members of one kind, sorted by component id. For each id, this returns - /// the newest inclusion in the edition's family whose `since` is at or before it. An array - /// writer-version upgrade therefore supersedes the older version without changing the array - /// ID or its single registered reader. Only that kind's declarations are scanned. + /// its inclusion in the edition's family when it joined at or before the requested edition. + /// Only that kind's declarations are scanned. pub fn components_in(&self, edition: &EditionId, kind: ComponentKind) -> Vec { let inner = self.inner.read(); let Some(by_id) = inner.inclusions.get(&kind) else { @@ -253,11 +238,11 @@ impl EditionSession { edition.id, edition.id.family, ))); } - if let Some(version) = edition.min_vortex_version + if let Some(version) = edition.min_library_version && parse_release(version).is_none() { return Err(EditionError::new(format!( - "edition {} declares malformed min_vortex_version {version:?}", + "edition {} declares malformed min_library_version {version:?}", edition.id ))); } @@ -292,12 +277,12 @@ impl EditionSession { }; if let Some(required) = inclusion.required_vortex_release.and_then(parse_release) - && let Some(declared) = edition.min_vortex_version.and_then(parse_release) + && let Some(declared) = edition.min_library_version.and_then(parse_release) && required > declared { return Err(EditionError::new(format!( "{} {} requires release {}, newer than edition {}'s declared \ - min_vortex_version", + min_library_version", inclusion.kind, inclusion.component_id, inclusion.required_vortex_release.unwrap_or_default(), @@ -385,32 +370,6 @@ pub trait EditionSessionExt: SessionExt { ids.dedup(); ids } - - /// Resolve one effective array writer version for every array ID across the enabled editions. - /// - /// Compression schemes use this map before sampling or compressing an array. An absent id is - /// not writable, and a writer-version upgrade in an opt-in family overrides an older core - /// version. The version is never used while reading; each array ID has one registered reader. - fn enabled_array_writer_versions(&self) -> BTreeMap { - let Some(enabled) = self.get_opt::() else { - return BTreeMap::new(); - }; - let editions = self.editions(); - let mut versions = BTreeMap::new(); - for inclusion in enabled - .editions() - .iter() - .flat_map(|edition| editions.components_in(edition, ComponentKind::Array)) - { - if let Some(version) = inclusion.array_writer_version { - versions - .entry(inclusion.component_id) - .and_modify(|enabled: &mut u16| *enabled = (*enabled).max(version)) - .or_insert(version); - } - } - versions - } } impl EditionSessionExt for S {} diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 0d1863b9ec7..0da52a3506d 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -31,7 +31,7 @@ static DECLARATIONS: &[EditionDeclaration] = &[ EditionDeclaration { edition: Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::array(&"test.alpha"), @@ -41,10 +41,10 @@ static DECLARATIONS: &[EditionDeclaration] = &[ EditionDeclaration { edition: Edition { id: SECOND, - min_vortex_version: None, + min_library_version: None, }, added: &[ - EditionMember::array_writer_version(&"test.alpha", 2), + EditionMember::array(&"test.alpha_v2"), EditionMember::array(&"test.gamma"), ], }, @@ -82,23 +82,29 @@ fn membership_is_transitive() -> Result<(), crate::EditionError> { let ids: Vec<&str> = first.iter().map(|i| i.component_id.as_str()).collect(); assert_eq!(ids, ["test.alpha", "test.beta"]); - // Members of the first edition are members of the second by inheritance. Alpha's writer v2 - // supersedes v1 without changing its array id or introducing read-time dispatch. + // Members of the first edition are inherited. A newer wire representation has its own ID, so + // both the historical and current representations remain explicit members. let second = editions.components_in(&SECOND, ComponentKind::Array); let ids: Vec<&str> = second.iter().map(|i| i.component_id.as_str()).collect(); - assert_eq!(ids, ["test.alpha", "test.beta", "test.gamma"]); + assert_eq!( + ids, + ["test.alpha", "test.alpha_v2", "test.beta", "test.gamma"] + ); let alpha = second .iter() .find(|i| i.component_id.as_str() == "test.alpha") .ok_or_else(|| crate::EditionError::new("test.alpha is a member"))?; - assert_eq!(alpha.since, SECOND); - assert_eq!(alpha.array_writer_version, Some(2)); + assert_eq!(alpha.since, FIRST); + let alpha_v2 = second + .iter() + .find(|i| i.component_id.as_str() == "test.alpha_v2") + .ok_or_else(|| crate::EditionError::new("test.alpha_v2 is a member"))?; + assert_eq!(alpha_v2.since, SECOND); let beta = second .iter() .find(|i| i.component_id.as_str() == "test.beta") .ok_or_else(|| crate::EditionError::new("test.beta is a member"))?; assert_eq!(beta.since, FIRST); - assert_eq!(beta.array_writer_version, Some(1)); // The second edition's delta is exactly the members declared at it. let added: Vec<&str> = second @@ -106,7 +112,7 @@ fn membership_is_transitive() -> Result<(), crate::EditionError> { .filter(|i| i.since == SECOND) .map(|i| i.component_id.as_str()) .collect(); - assert_eq!(added, ["test.alpha", "test.gamma"]); + assert_eq!(added, ["test.alpha_v2", "test.gamma"]); // Inheritance never flows backwards, extends to later editions of the family, and // never crosses families. @@ -114,7 +120,7 @@ fn membership_is_transitive() -> Result<(), crate::EditionError> { let third = EditionId::new("test", 2026, 10, 0); assert_eq!( editions.components_in(&third, ComponentKind::Array).len(), - 3 + 4 ); let other = EditionId::new("other", 2026, 10, 0); assert!( @@ -138,13 +144,13 @@ fn drafts_and_current() { editions .declare_edition(Edition { id: FIRST, - min_vortex_version: Some("0.60.0"), + min_library_version: Some("0.60.0"), }) .unwrap(); editions .declare_edition(Edition { id: SECOND, - min_vortex_version: None, + min_library_version: None, }) .unwrap(); assert!(editions.validate().is_ok()); @@ -186,22 +192,9 @@ fn registered_and_enabled_editions_are_separate() -> Result<(), crate::EditionEr .collect::>(), ["test.alpha", "test.beta"] ); - assert_eq!( - session - .enabled_array_writer_versions() - .get(&"test.alpha".into()), - Some(&1) - ); - session.enable_edition(SECOND)?; assert_eq!(session.enabled_editions().editions(), [SECOND]); - assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 3); - assert_eq!( - session - .enabled_array_writer_versions() - .get(&"test.alpha".into()), - Some(&2) - ); + assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 4); // Selecting an older edition in the same family replaces the newer one and removes // encodings that joined after it. @@ -226,7 +219,7 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi static OTHER_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: OTHER, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"other.delta")], }; @@ -248,14 +241,14 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi } #[test] -fn array_writer_versions_can_be_upgraded_by_an_opt_in_family() -> Result<(), crate::EditionError> { +fn serialized_array_ids_can_be_added_by_an_opt_in_family() -> Result<(), crate::EditionError> { const PREVIEW: EditionId = EditionId::new("other", 2026, 8, 0); static PREVIEW_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: PREVIEW, - min_vortex_version: None, + min_library_version: None, }, - added: &[EditionMember::array_writer_version(&"test.alpha", 2)], + added: &[EditionMember::array(&"test.alpha_v2")], }; let session = VortexSession::empty().with::(); @@ -267,16 +260,13 @@ fn array_writer_versions_can_be_upgraded_by_an_opt_in_family() -> Result<(), cra session.enable_edition(FIRST)?; session.enable_edition(PREVIEW)?; - let versions = session.enabled_array_writer_versions(); - assert_eq!(versions.get(&"test.alpha".into()), Some(&2)); - assert_eq!(versions.get(&"test.beta".into()), Some(&1)); assert_eq!( session .enabled_component_ids(ComponentKind::Array) .iter() .map(|id| id.as_str()) .collect::>(), - ["test.alpha", "test.beta"] + ["test.alpha", "test.alpha_v2", "test.beta"] ); Ok(()) } @@ -288,7 +278,7 @@ fn duplicate_declarations_error() { editions .declare_edition(Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, }) .is_err() ); @@ -310,7 +300,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro let editions = EditionSession::empty(); editions.declare_edition(Edition { id: FIRST, - min_vortex_version: Some("0.70.0"), + min_library_version: Some("0.70.0"), })?; editions.declare_inclusion(EditionInclusion { required_vortex_release: Some("0.80.0"), @@ -322,11 +312,11 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro let editions = EditionSession::empty(); editions.declare_edition(Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, })?; editions.declare_edition(Edition { id: SECOND, - min_vortex_version: Some("0.70.0"), + min_library_version: Some("0.70.0"), })?; assert!(editions.validate().is_err()); @@ -334,7 +324,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro let editions = EditionSession::empty(); editions.declare_edition(Edition { id: EditionId::new("Test", 2026, 13, 0), - min_vortex_version: None, + min_library_version: None, })?; assert!(editions.validate().is_err()); @@ -342,24 +332,11 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro let editions = EditionSession::empty(); editions.declare_edition(Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, })?; editions.declare_inclusion(EditionInclusion::array("Test.ALPHA", FIRST))?; assert!(editions.validate().is_err()); - // Array writer versions start at one. - let editions = EditionSession::empty(); - editions.declare_edition(Edition { - id: FIRST, - min_vortex_version: None, - })?; - editions.declare_inclusion(EditionInclusion::array_writer_version( - "test.alpha", - 0, - FIRST, - ))?; - assert!(editions.validate().is_err()); - Ok(()) } @@ -412,7 +389,7 @@ fn kinds_are_resolved_independently() -> Result<(), crate::EditionError> { static MIXED: EditionDeclaration = EditionDeclaration { edition: Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::array(&"test.alpha"), diff --git a/vortex-file/benches/split_collection.rs b/vortex-file/benches/split_collection.rs index 4811e84c82d..7e043c57e4a 100644 --- a/vortex-file/benches/split_collection.rs +++ b/vortex-file/benches/split_collection.rs @@ -79,7 +79,7 @@ fn enable_all_registered_array_encodings(session: &VortexSession) { editions .declare_edition(Edition { id: BENCH_EDITION, - min_vortex_version: None, + min_library_version: None, }) .unwrap(); let component_ids = [ diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index d66aaa54a9b..f37e67764d1 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -215,7 +215,7 @@ pub(crate) fn enable_all_registered_array_encodings(session: &VortexSession) { editions .declare_edition(Edition { id: TEST_EDITION, - min_vortex_version: None, + min_library_version: None, }) .map_err(|error| vortex_err!("{error}")) .vortex_expect("test edition is valid"); diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index deee68bc3a4..9d4dbb90610 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -8,7 +8,6 @@ use std::sync::Arc; use vortex_array::ArrayId; use vortex_array::dtype::FieldPath; -use vortex_btrblocks::ArrayWriterVersions; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::integer::IntDictScheme; @@ -61,7 +60,6 @@ pub struct WriteStrategyBuilder { data_block_target_bytes: Option, field_writers: HashMap>, allow_encodings: Option>, - array_writer_versions: Option, flat_strategy: Option>, probe_compressor: Option>, /// Whether to write list fields using [`ListLayoutStrategy`]. @@ -80,7 +78,6 @@ impl Default for WriteStrategyBuilder { data_block_target_bytes: Some(ONE_MEG), field_writers: HashMap::new(), allow_encodings: None, - array_writer_versions: None, flat_strategy: None, probe_compressor: None, use_list_layout: use_experimental_list_layout(), @@ -142,19 +139,6 @@ impl WriteStrategyBuilder { self } - /// Configure the compatible serialized features the writer may produce for each array ID. - /// - /// The map's keys become the allowed array encodings. For the built-in BtrBlocks compressor, - /// schemes requiring an absent or newer writer version are excluded before estimation, - /// sampling, and compression. The flat writer validates every final array ID, including - /// output from an opaque custom compressor; an opaque compressor remains responsible for - /// honoring the writer versions because they are not read-time array tags. - pub fn with_array_writer_versions(mut self, versions: ArrayWriterVersions) -> Self { - self.allow_encodings = Some(versions.keys().copied().collect()); - self.array_writer_versions = Some(versions); - self - } - /// Override the flat layout strategy used for leaf chunks. /// /// By default, this uses [`FlatLayoutStrategy`]. This can be used to substitute a custom @@ -203,10 +187,6 @@ impl WriteStrategyBuilder { // regardless of the order in which the builder and the policy were configured. let compressor = match self.compressor { CompressorConfig::BtrBlocks(builder) => { - let builder = match self.array_writer_versions { - Some(versions) => builder.with_array_writer_versions(versions), - None => builder, - }; CompressorConfig::BtrBlocks(match &self.allow_encodings { Some(allow_encodings) => builder.retain_allowed_encodings(allow_encodings), None => builder, diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 194729ab793..3972e7e74e1 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -68,10 +68,9 @@ use crate::segments::writer::BufferedSegmentSink; /// Configure a new writer, which can eventually be used to write an [`ArrayStream`] into a sink /// that implements [`VortexWrite`]. /// -/// The default write strategy is restricted to the components and array writer versions in the -/// session's enabled editions. An array, layout, extension dtype, or zone-map aggregate outside -/// them fails the write. An empty component set therefore forbids writing any component of that -/// kind. +/// Serialized arrays, layouts, extension dtypes, and zone-map aggregates are restricted to the +/// component IDs in the session's enabled editions. An empty component set therefore forbids +/// writing any component of that kind. /// /// Construct with [`WriteOptionsSessionExt::write_options`] for normal use so the writer inherits /// the session's runtime, array registry, and memory configuration. @@ -97,9 +96,7 @@ impl WriteOptionsSessionExt for S {} impl VortexWriteOptions { /// Create a new [`VortexWriteOptions`] with the given session. pub fn new(session: VortexSession) -> Self { - let strategy = WriteStrategyBuilder::default() - .with_array_writer_versions(session.enabled_array_writer_versions()) - .build(); + let strategy = WriteStrategyBuilder::default().build(); VortexWriteOptions { strategy, buffered_bytes: BufferedBytesTracker::new(), @@ -115,9 +112,8 @@ impl VortexWriteOptions { /// /// The strategy controls repartitioning, statistics layout, compression, and leaf segment /// emission. Use [`WriteStrategyBuilder`] when only a small part of the default strategy needs - /// customization. The final serializers still enforce the enabled-edition component IDs, but - /// a replacement compressor is responsible for applying array writer versions before it does - /// expensive work. + /// customization. The final serializers still select only serialized array IDs permitted by + /// the enabled editions, independently of which in-memory encodings a compressor produces. pub fn with_strategy(mut self, strategy: Arc) -> Self { self.strategy = strategy; self @@ -354,14 +350,14 @@ impl VortexWriteOptions { } fn new_array_context(session: &VortexSession) -> ArrayContext { - // NOTE(os): Set up an array context with all enabled encodings pre-populated. + // NOTE(os): Set up an array context with all enabled serialized IDs pre-populated. // This is preferred for now over having an empty context here, because only the // serialised array order is deterministic. The serialisation of arrays are done // parallel and with an empty context they can register their encodings to the context // in different order, changing the written bytes from run to run. let enabled_encoding_ids = session.enabled_component_ids(ComponentKind::Array); ArrayContext::new(enabled_encoding_ids.iter().cloned().sorted().collect()) - // Only permit encodings in the enabled editions. + // Only permit serialized IDs in the enabled editions. .with_allowed_ids(enabled_encoding_ids.into_iter().collect()) } @@ -724,7 +720,7 @@ mod tests { static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"vortex.primitive")], }; @@ -748,7 +744,7 @@ mod tests { static ARRAYS_ONLY: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"vortex.primitive")], }; @@ -787,7 +783,7 @@ mod tests { static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::dtype(&"vortex.date")], }; diff --git a/vortex-file/tests/common/mod.rs b/vortex-file/tests/common/mod.rs index 4e1c85b030d..29b8ac0024f 100644 --- a/vortex-file/tests/common/mod.rs +++ b/vortex-file/tests/common/mod.rs @@ -21,7 +21,7 @@ pub fn enable_all_registered_array_encodings(session: &VortexSession) { editions .declare_edition(Edition { id: TEST_EDITION, - min_vortex_version: None, + min_library_version: None, }) .map_err(|error| vortex_err!("{error}")) .vortex_expect("test edition is valid"); diff --git a/vortex-json/src/editions.rs b/vortex-json/src/editions.rs index cbf69e29ecb..94e31871534 100644 --- a/vortex-json/src/editions.rs +++ b/vortex-json/src/editions.rs @@ -33,7 +33,7 @@ pub const JSON_2026_08: EditionId = EditionId::new("json", 2026, 8, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: JSON_2026_08, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::dtype(&"vortex.json")], }; diff --git a/vortex-python-cuda/src/lib.rs b/vortex-python-cuda/src/lib.rs index dff0a0fd03d..68f836be4e4 100644 --- a/vortex-python-cuda/src/lib.rs +++ b/vortex-python-cuda/src/lib.rs @@ -26,6 +26,7 @@ use pyo3::types::PyDict; use pyo3::types::PyList; use pyo3::types::PyTuple; use vortex::VortexSessionDefault; +use vortex::array::ArrayDeserialization; use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::buffer::BufferHandle; @@ -300,11 +301,14 @@ fn deserialize_metadata_tree( .get(&encoding_id) .ok_or_else(|| vortex_err!("Unknown array encoding: {}", metadata.encoding_id))?; let decoded = plugin.deserialize( - &dtype, - metadata.len, - &metadata.metadata, - &metadata.buffers, - &children, + ArrayDeserialization::new( + encoding_id, + &dtype, + metadata.len, + &metadata.metadata, + &metadata.buffers, + &children, + ), session, )?; vortex_ensure!( diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index 5daf57310d3..8adbbff949a 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -33,6 +33,7 @@ use pyo3::types::PyRange; use pyo3::types::PyRangeMethods; use pyo3::types::PyTuple; use pyo3_bytes::PyBytes as PyBufferBytes; +use vortex::array::ArrayContext; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; @@ -147,23 +148,27 @@ fn array_metadata_tuple<'py>( py: Python<'py>, array: &ArrayRef, ) -> PyVortexResult> { - let metadata = session().array_serialize(array)?.ok_or_else(|| { - PyValueError::new_err(format!( - "Array {} does not support metadata serialization", - array.encoding_id() - )) - })?; + let serialization = session() + .array_serialize(array, &ArrayContext::empty())? + .ok_or_else(|| { + PyValueError::new_err(format!( + "Array {} does not support serialization", + array.encoding_id() + )) + })?; let dtype = array.dtype().write_flatbuffer_bytes()?; - let buffers = array - .buffer_handles() + let buffers = serialization + .buffers .iter() - .map(|handle| export_buffer(py, handle).map(|cap| cap.into_any())) + .map(|buffer| { + export_buffer(py, &BufferHandle::new_host(buffer.clone())).map(|cap| cap.into_any()) + }) .collect::>>()?; let buffers = PyList::new(py, buffers)?; - let children = array - .children() + let children = serialization + .children .iter() .map(|child| array_metadata_tuple(py, child).map(|tuple| tuple.into_any())) .collect::>>()?; @@ -172,10 +177,12 @@ fn array_metadata_tuple<'py>( PyTuple::new( py, [ - array.encoding_id().to_string().into_py_any(py)?, + serialization.serialized_id.to_string().into_py_any(py)?, PyBytes::new(py, dtype.as_slice()).into_any().into(), array.len().into_py_any(py)?, - PyBytes::new(py, metadata.as_slice()).into_any().into(), + PyBytes::new(py, serialization.metadata.as_slice()) + .into_any() + .into(), buffers.into_any().into(), children.into_any().into(), ], diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index 330ae01f07a..f82fa0edf82 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -207,11 +207,19 @@ impl Interner { self } + /// Returns whether `id` may be interned by this context. + /// + /// An unrestricted context permits every ID. This check does not mutate the interner, so + /// serializers can probe compatible wire representations without recording failed choices. + pub fn is_allowed(&self, id: &Id) -> bool { + self.allowed + .as_ref() + .is_none_or(|allowed| allowed.contains(id)) + } + /// Intern an ID, returning its index. pub fn intern(&self, id: &Id) -> Option { - if let Some(allowed) = &self.allowed - && !allowed.contains(id) - { + if !self.is_allowed(id) { // ID not permitted, cannot intern. return None; } diff --git a/vortex-spatial/src/editions.rs b/vortex-spatial/src/editions.rs index 9bf437536c4..dda4efee0a7 100644 --- a/vortex-spatial/src/editions.rs +++ b/vortex-spatial/src/editions.rs @@ -34,7 +34,7 @@ pub const SPATIAL_2026_08: EditionId = EditionId::new("spatial", 2026, 8, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: SPATIAL_2026_08, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::dtype(&"vortex.st.box"), diff --git a/vortex-tensor/src/encodings/normalized/tests.rs b/vortex-tensor/src/encodings/normalized/tests.rs index 0687e1ef750..1a48a4e6caa 100644 --- a/vortex-tensor/src/encodings/normalized/tests.rs +++ b/vortex-tensor/src/encodings/normalized/tests.rs @@ -2,6 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use rstest::rstest; +use vortex_array::ArrayContext; +use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::ArrayVTable; @@ -652,16 +654,19 @@ fn serde_round_trip(#[case] input: ArrayRef) -> VortexResult<()> { let original = normalize(input, &mut ctx)?.into_array(); let children: Vec = original.children(); - let metadata = SESSION - .array_serialize(&original)? + let serialization = SESSION + .array_serialize(&original, &ArrayContext::empty())? .expect("Normalized must serialize"); let recovered = ArrayPlugin::deserialize( &Normalized, - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + ArrayVTable::id(&Normalized), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; @@ -678,10 +683,13 @@ fn serialization_carries_no_metadata() -> VortexResult<()> { let non_nullable = normalize(vector_array(2, &[3.0, 4.0, 1.0, 0.0])?, &mut ctx)?.into_array(); for array in [&nullable, &non_nullable] { - let bytes = SESSION - .array_serialize(array)? + let serialization = SESSION + .array_serialize(array, &ArrayContext::empty())? .expect("Normalized must serialize"); - assert!(bytes.is_empty(), "Normalized must not serialize metadata"); + assert!( + serialization.metadata.is_empty(), + "Normalized must not serialize metadata" + ); } assert_eq!(nullable.nchildren(), NormalizedSlots::COUNT); @@ -705,16 +713,19 @@ fn serde_round_trip_of_a_nullable_column_with_no_null_rows() -> VortexResult<()> assert!(original.dtype().is_nullable()); assert_eq!(children.len(), NormalizedSlots::COUNT - 1); - let metadata = SESSION - .array_serialize(&original)? + let serialization = SESSION + .array_serialize(&original, &ArrayContext::empty())? .expect("Normalized must serialize"); let recovered = ArrayPlugin::deserialize( &Normalized, - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + ArrayVTable::id(&Normalized), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; @@ -734,8 +745,12 @@ fn deserialize_rejects_validity_child_for_non_nullable_dtype() -> VortexResult<( BoolArray::from_iter([true, false]).into_array(), ]; - let error = ArrayPlugin::deserialize(&Normalized, &dtype, 2, &[], &[], &children, &SESSION) - .unwrap_err(); + let error = ArrayPlugin::deserialize( + &Normalized, + ArrayDeserialization::new(ArrayVTable::id(&Normalized), &dtype, 2, &[], &[], &children), + &SESSION, + ) + .unwrap_err(); assert!( error diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index 9f21653b4a2..ac5adb31202 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -310,6 +310,8 @@ impl CosineSimilarity { mod tests { use rstest::rstest; + use vortex_array::ArrayContext; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -755,17 +757,20 @@ mod tests { let original = CosineSimilarity::try_new(lhs.clone(), rhs.clone())?.into_array(); let plugin = ScalarFnArrayPlugin::new(CosineSimilarity); - let metadata = plugin - .serialize(&original, &SESSION)? + let serialization = plugin + .serialize(&original, &ArrayContext::empty(), &SESSION)? .expect("CosineSimilarity serialize must produce metadata"); let children = vec![lhs, rhs]; let recovered = plugin.deserialize( - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + plugin.id(), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index f10e697732d..83db4d322ee 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -280,6 +280,8 @@ fn inner_product_row(a: &[T], b: &[T]) -> T { mod tests { use rstest::rstest; + use vortex_array::ArrayContext; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -486,17 +488,20 @@ mod tests { let original = InnerProduct::try_new(lhs.clone(), rhs.clone())?.into_array(); let plugin = ScalarFnArrayPlugin::new(InnerProduct); - let metadata = plugin - .serialize(&original, &SESSION)? + let serialization = plugin + .serialize(&original, &ArrayContext::empty(), &SESSION)? .expect("InnerProduct serialize must produce metadata"); let children = vec![lhs, rhs]; let recovered = plugin.deserialize( - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + plugin.id(), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 0b00ec95aa4..240988ab43c 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -252,6 +252,8 @@ fn l2_norm_row(v: &[T]) -> T { mod tests { use rstest::rstest; + use vortex_array::ArrayContext; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::EmptyMetadata; @@ -430,17 +432,20 @@ mod tests { let original = L2Norm::try_new(child.clone())?.into_array(); let plugin = ScalarFnArrayPlugin::new(L2Norm); - let metadata = plugin - .serialize(&original, &SESSION)? + let serialization = plugin + .serialize(&original, &ArrayContext::empty(), &SESSION)? .expect("L2Norm serialize must produce metadata"); let children = vec![child]; let recovered = plugin.deserialize( - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + plugin.id(), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index b22440d88b0..912427c2467 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -18,6 +18,7 @@ use futures::FutureExt; use futures::TryStreamExt; use futures::future::BoxFuture; use serde::Serialize; +use vortex::array::ArrayContext; use vortex::array::ArrayRef; use vortex::array::VortexSessionExecute; use vortex::array::buffer::BufferHandle; @@ -647,10 +648,10 @@ fn build_array_encoding_tree_from_array( let buffer_handles = array.buffer_handles(); let buffer_lengths: Vec = buffer_handles.iter().map(|b| b.len()).collect(); let metadata_bytes = session - .array_serialize(array) + .array_serialize(array, &ArrayContext::empty()) .ok() .flatten() - .map(|m| m.len()) + .map(|serialization| serialization.metadata.len()) .unwrap_or(0); let named_children = array.named_children(); diff --git a/vortex/editions/core/core2025.05.0.toml b/vortex/editions/core/core2025.05.0.toml index c020a95a06d..5024be53c09 100644 --- a/vortex/editions/core/core2025.05.0.toml +++ b/vortex/editions/core/core2025.05.0.toml @@ -6,34 +6,34 @@ edition = "core2025.05.0" family = "core" -min_vortex_version = "0.36.0" +min_library_version = "0.36.0" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "fastlanes.bitpacked", writer_version = 1 }, - { id = "fastlanes.for", writer_version = 1 }, - { id = "vortex.alp", writer_version = 1 }, - { id = "vortex.alprd", writer_version = 1 }, - { id = "vortex.bool", writer_version = 1 }, - { id = "vortex.bytebool", writer_version = 1 }, - { id = "vortex.chunked", writer_version = 1 }, - { id = "vortex.constant", writer_version = 1 }, - { id = "vortex.datetimeparts", writer_version = 1 }, - { id = "vortex.decimal", writer_version = 1 }, - { id = "vortex.decimal_byte_parts", writer_version = 1 }, - { id = "vortex.dict", writer_version = 1 }, - { id = "vortex.ext", writer_version = 1 }, - { id = "vortex.fsst", writer_version = 1 }, - { id = "vortex.list", writer_version = 1 }, - { id = "vortex.null", writer_version = 1 }, - { id = "vortex.primitive", writer_version = 1 }, - { id = "vortex.runend", writer_version = 1 }, - { id = "vortex.sparse", writer_version = 1 }, - { id = "vortex.struct", writer_version = 1 }, - { id = "vortex.varbin", writer_version = 1 }, - { id = "vortex.varbinview", writer_version = 1 }, - { id = "vortex.zigzag", writer_version = 1 }, + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", ] layouts = [ "vortex.chunked", @@ -53,29 +53,29 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.bitpacked", writer_version = 1 }, - { id = "fastlanes.for", writer_version = 1 }, - { id = "vortex.alp", writer_version = 1 }, - { id = "vortex.alprd", writer_version = 1 }, - { id = "vortex.bool", writer_version = 1 }, - { id = "vortex.bytebool", writer_version = 1 }, - { id = "vortex.chunked", writer_version = 1 }, - { id = "vortex.constant", writer_version = 1 }, - { id = "vortex.datetimeparts", writer_version = 1 }, - { id = "vortex.decimal", writer_version = 1 }, - { id = "vortex.decimal_byte_parts", writer_version = 1 }, - { id = "vortex.dict", writer_version = 1 }, - { id = "vortex.ext", writer_version = 1 }, - { id = "vortex.fsst", writer_version = 1 }, - { id = "vortex.list", writer_version = 1 }, - { id = "vortex.null", writer_version = 1 }, - { id = "vortex.primitive", writer_version = 1 }, - { id = "vortex.runend", writer_version = 1 }, - { id = "vortex.sparse", writer_version = 1 }, - { id = "vortex.struct", writer_version = 1 }, - { id = "vortex.varbin", writer_version = 1 }, - { id = "vortex.varbinview", writer_version = 1 }, - { id = "vortex.zigzag", writer_version = 1 }, + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2025.06.0.toml b/vortex/editions/core/core2025.06.0.toml index 07552dac0ce..2838ea96b92 100644 --- a/vortex/editions/core/core2025.06.0.toml +++ b/vortex/editions/core/core2025.06.0.toml @@ -6,14 +6,14 @@ edition = "core2025.06.0" family = "core" -min_vortex_version = "0.40.0" +min_library_version = "0.40.0" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "vortex.pco", writer_version = 1 }, - { id = "vortex.sequence", writer_version = 1 }, - { id = "vortex.zstd", writer_version = 1 }, + "vortex.pco", + "vortex.sequence", + "vortex.zstd", ] layouts = [] dtypes = [] @@ -23,32 +23,32 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.bitpacked", writer_version = 1 }, - { id = "fastlanes.for", writer_version = 1 }, - { id = "vortex.alp", writer_version = 1 }, - { id = "vortex.alprd", writer_version = 1 }, - { id = "vortex.bool", writer_version = 1 }, - { id = "vortex.bytebool", writer_version = 1 }, - { id = "vortex.chunked", writer_version = 1 }, - { id = "vortex.constant", writer_version = 1 }, - { id = "vortex.datetimeparts", writer_version = 1 }, - { id = "vortex.decimal", writer_version = 1 }, - { id = "vortex.decimal_byte_parts", writer_version = 1 }, - { id = "vortex.dict", writer_version = 1 }, - { id = "vortex.ext", writer_version = 1 }, - { id = "vortex.fsst", writer_version = 1 }, - { id = "vortex.list", writer_version = 1 }, - { id = "vortex.null", writer_version = 1 }, - { id = "vortex.pco", writer_version = 1 }, - { id = "vortex.primitive", writer_version = 1 }, - { id = "vortex.runend", writer_version = 1 }, - { id = "vortex.sequence", writer_version = 1 }, - { id = "vortex.sparse", writer_version = 1 }, - { id = "vortex.struct", writer_version = 1 }, - { id = "vortex.varbin", writer_version = 1 }, - { id = "vortex.varbinview", writer_version = 1 }, - { id = "vortex.zigzag", writer_version = 1 }, - { id = "vortex.zstd", writer_version = 1 }, + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2025.10.0.toml b/vortex/editions/core/core2025.10.0.toml index 0aa9bf3e409..0962709ccf8 100644 --- a/vortex/editions/core/core2025.10.0.toml +++ b/vortex/editions/core/core2025.10.0.toml @@ -6,15 +6,15 @@ edition = "core2025.10.0" family = "core" -min_vortex_version = "0.54.0" +min_library_version = "0.54.0" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "fastlanes.rle", writer_version = 1 }, - { id = "vortex.fixed_size_list", writer_version = 1 }, - { id = "vortex.listview", writer_version = 1 }, - { id = "vortex.masked", writer_version = 1 }, + "fastlanes.rle", + "vortex.fixed_size_list", + "vortex.listview", + "vortex.masked", ] layouts = [] dtypes = [] @@ -24,36 +24,36 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.bitpacked", writer_version = 1 }, - { id = "fastlanes.for", writer_version = 1 }, - { id = "fastlanes.rle", writer_version = 1 }, - { id = "vortex.alp", writer_version = 1 }, - { id = "vortex.alprd", writer_version = 1 }, - { id = "vortex.bool", writer_version = 1 }, - { id = "vortex.bytebool", writer_version = 1 }, - { id = "vortex.chunked", writer_version = 1 }, - { id = "vortex.constant", writer_version = 1 }, - { id = "vortex.datetimeparts", writer_version = 1 }, - { id = "vortex.decimal", writer_version = 1 }, - { id = "vortex.decimal_byte_parts", writer_version = 1 }, - { id = "vortex.dict", writer_version = 1 }, - { id = "vortex.ext", writer_version = 1 }, - { id = "vortex.fixed_size_list", writer_version = 1 }, - { id = "vortex.fsst", writer_version = 1 }, - { id = "vortex.list", writer_version = 1 }, - { id = "vortex.listview", writer_version = 1 }, - { id = "vortex.masked", writer_version = 1 }, - { id = "vortex.null", writer_version = 1 }, - { id = "vortex.pco", writer_version = 1 }, - { id = "vortex.primitive", writer_version = 1 }, - { id = "vortex.runend", writer_version = 1 }, - { id = "vortex.sequence", writer_version = 1 }, - { id = "vortex.sparse", writer_version = 1 }, - { id = "vortex.struct", writer_version = 1 }, - { id = "vortex.varbin", writer_version = 1 }, - { id = "vortex.varbinview", writer_version = 1 }, - { id = "vortex.zigzag", writer_version = 1 }, - { id = "vortex.zstd", writer_version = 1 }, + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2026.08.0.toml b/vortex/editions/core/core2026.08.0.toml index 947e2408345..134d48afe54 100644 --- a/vortex/editions/core/core2026.08.0.toml +++ b/vortex/editions/core/core2026.08.0.toml @@ -6,9 +6,9 @@ edition = "core2026.08.0" family = "core" -min_vortex_version = "0.84.0" +min_library_version = "0.84.0" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [] layouts = [ @@ -28,36 +28,36 @@ aggregates = [ # editions of the family. [components] arrays = [ - { id = "fastlanes.bitpacked", writer_version = 1 }, - { id = "fastlanes.for", writer_version = 1 }, - { id = "fastlanes.rle", writer_version = 1 }, - { id = "vortex.alp", writer_version = 1 }, - { id = "vortex.alprd", writer_version = 1 }, - { id = "vortex.bool", writer_version = 1 }, - { id = "vortex.bytebool", writer_version = 1 }, - { id = "vortex.chunked", writer_version = 1 }, - { id = "vortex.constant", writer_version = 1 }, - { id = "vortex.datetimeparts", writer_version = 1 }, - { id = "vortex.decimal", writer_version = 1 }, - { id = "vortex.decimal_byte_parts", writer_version = 1 }, - { id = "vortex.dict", writer_version = 1 }, - { id = "vortex.ext", writer_version = 1 }, - { id = "vortex.fixed_size_list", writer_version = 1 }, - { id = "vortex.fsst", writer_version = 1 }, - { id = "vortex.list", writer_version = 1 }, - { id = "vortex.listview", writer_version = 1 }, - { id = "vortex.masked", writer_version = 1 }, - { id = "vortex.null", writer_version = 1 }, - { id = "vortex.pco", writer_version = 1 }, - { id = "vortex.primitive", writer_version = 1 }, - { id = "vortex.runend", writer_version = 1 }, - { id = "vortex.sequence", writer_version = 1 }, - { id = "vortex.sparse", writer_version = 1 }, - { id = "vortex.struct", writer_version = 1 }, - { id = "vortex.varbin", writer_version = 1 }, - { id = "vortex.varbinview", writer_version = 1 }, - { id = "vortex.zigzag", writer_version = 1 }, - { id = "vortex.zstd", writer_version = 1 }, + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2026.08.1.toml b/vortex/editions/core/core2026.08.1.toml index 976016d8abf..16c67acbc7e 100644 --- a/vortex/editions/core/core2026.08.1.toml +++ b/vortex/editions/core/core2026.08.1.toml @@ -6,12 +6,12 @@ edition = "core2026.08.1" family = "core" -min_vortex_version = "0.84.0" +min_library_version = "0.84.0" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "vortex.onpair", writer_version = 1 }, + "vortex.onpair", ] layouts = [] dtypes = [] @@ -21,37 +21,37 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.bitpacked", writer_version = 1 }, - { id = "fastlanes.for", writer_version = 1 }, - { id = "fastlanes.rle", writer_version = 1 }, - { id = "vortex.alp", writer_version = 1 }, - { id = "vortex.alprd", writer_version = 1 }, - { id = "vortex.bool", writer_version = 1 }, - { id = "vortex.bytebool", writer_version = 1 }, - { id = "vortex.chunked", writer_version = 1 }, - { id = "vortex.constant", writer_version = 1 }, - { id = "vortex.datetimeparts", writer_version = 1 }, - { id = "vortex.decimal", writer_version = 1 }, - { id = "vortex.decimal_byte_parts", writer_version = 1 }, - { id = "vortex.dict", writer_version = 1 }, - { id = "vortex.ext", writer_version = 1 }, - { id = "vortex.fixed_size_list", writer_version = 1 }, - { id = "vortex.fsst", writer_version = 1 }, - { id = "vortex.list", writer_version = 1 }, - { id = "vortex.listview", writer_version = 1 }, - { id = "vortex.masked", writer_version = 1 }, - { id = "vortex.null", writer_version = 1 }, - { id = "vortex.onpair", writer_version = 1 }, - { id = "vortex.pco", writer_version = 1 }, - { id = "vortex.primitive", writer_version = 1 }, - { id = "vortex.runend", writer_version = 1 }, - { id = "vortex.sequence", writer_version = 1 }, - { id = "vortex.sparse", writer_version = 1 }, - { id = "vortex.struct", writer_version = 1 }, - { id = "vortex.varbin", writer_version = 1 }, - { id = "vortex.varbinview", writer_version = 1 }, - { id = "vortex.zigzag", writer_version = 1 }, - { id = "vortex.zstd", writer_version = 1 }, + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.onpair", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2026.08.2.toml b/vortex/editions/core/core2026.08.2.toml index a5808b5d585..698014bdbce 100644 --- a/vortex/editions/core/core2026.08.2.toml +++ b/vortex/editions/core/core2026.08.2.toml @@ -2,16 +2,16 @@ # # This edition record has no core read-forever guarantee. It may describe an evolving feature, # stabilized preview functionality awaiting adoption, or a release waiting to be cut. New -# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# capabilities advance to a new edition; after a core release is known, min_library_version is # backfilled to document its freeze. A frozen record never changes. edition = "core2026.08.2" family = "core" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "vortex.map", writer_version = 1 }, + "vortex.map", ] layouts = [] dtypes = [] @@ -21,38 +21,38 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.bitpacked", writer_version = 1 }, - { id = "fastlanes.for", writer_version = 1 }, - { id = "fastlanes.rle", writer_version = 1 }, - { id = "vortex.alp", writer_version = 1 }, - { id = "vortex.alprd", writer_version = 1 }, - { id = "vortex.bool", writer_version = 1 }, - { id = "vortex.bytebool", writer_version = 1 }, - { id = "vortex.chunked", writer_version = 1 }, - { id = "vortex.constant", writer_version = 1 }, - { id = "vortex.datetimeparts", writer_version = 1 }, - { id = "vortex.decimal", writer_version = 1 }, - { id = "vortex.decimal_byte_parts", writer_version = 1 }, - { id = "vortex.dict", writer_version = 1 }, - { id = "vortex.ext", writer_version = 1 }, - { id = "vortex.fixed_size_list", writer_version = 1 }, - { id = "vortex.fsst", writer_version = 1 }, - { id = "vortex.list", writer_version = 1 }, - { id = "vortex.listview", writer_version = 1 }, - { id = "vortex.map", writer_version = 1 }, - { id = "vortex.masked", writer_version = 1 }, - { id = "vortex.null", writer_version = 1 }, - { id = "vortex.onpair", writer_version = 1 }, - { id = "vortex.pco", writer_version = 1 }, - { id = "vortex.primitive", writer_version = 1 }, - { id = "vortex.runend", writer_version = 1 }, - { id = "vortex.sequence", writer_version = 1 }, - { id = "vortex.sparse", writer_version = 1 }, - { id = "vortex.struct", writer_version = 1 }, - { id = "vortex.varbin", writer_version = 1 }, - { id = "vortex.varbinview", writer_version = 1 }, - { id = "vortex.zigzag", writer_version = 1 }, - { id = "vortex.zstd", writer_version = 1 }, + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.onpair", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/core2026.08.3.toml b/vortex/editions/core/core2026.08.3.toml index 092c912809a..9c0de576ece 100644 --- a/vortex/editions/core/core2026.08.3.toml +++ b/vortex/editions/core/core2026.08.3.toml @@ -2,17 +2,17 @@ # # This edition record has no core read-forever guarantee. It may describe an evolving feature, # stabilized preview functionality awaiting adoption, or a release waiting to be cut. New -# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# capabilities advance to a new edition; after a core release is known, min_library_version is # backfilled to document its freeze. A frozen record never changes. edition = "core2026.08.3" family = "core" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "vortex.parquet.variant", writer_version = 1 }, - { id = "vortex.variant", writer_version = 1 }, + "vortex.parquet.variant", + "vortex.variant", ] layouts = [] dtypes = [ @@ -24,40 +24,40 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.bitpacked", writer_version = 1 }, - { id = "fastlanes.for", writer_version = 1 }, - { id = "fastlanes.rle", writer_version = 1 }, - { id = "vortex.alp", writer_version = 1 }, - { id = "vortex.alprd", writer_version = 1 }, - { id = "vortex.bool", writer_version = 1 }, - { id = "vortex.bytebool", writer_version = 1 }, - { id = "vortex.chunked", writer_version = 1 }, - { id = "vortex.constant", writer_version = 1 }, - { id = "vortex.datetimeparts", writer_version = 1 }, - { id = "vortex.decimal", writer_version = 1 }, - { id = "vortex.decimal_byte_parts", writer_version = 1 }, - { id = "vortex.dict", writer_version = 1 }, - { id = "vortex.ext", writer_version = 1 }, - { id = "vortex.fixed_size_list", writer_version = 1 }, - { id = "vortex.fsst", writer_version = 1 }, - { id = "vortex.list", writer_version = 1 }, - { id = "vortex.listview", writer_version = 1 }, - { id = "vortex.map", writer_version = 1 }, - { id = "vortex.masked", writer_version = 1 }, - { id = "vortex.null", writer_version = 1 }, - { id = "vortex.onpair", writer_version = 1 }, - { id = "vortex.parquet.variant", writer_version = 1 }, - { id = "vortex.pco", writer_version = 1 }, - { id = "vortex.primitive", writer_version = 1 }, - { id = "vortex.runend", writer_version = 1 }, - { id = "vortex.sequence", writer_version = 1 }, - { id = "vortex.sparse", writer_version = 1 }, - { id = "vortex.struct", writer_version = 1 }, - { id = "vortex.varbin", writer_version = 1 }, - { id = "vortex.varbinview", writer_version = 1 }, - { id = "vortex.variant", writer_version = 1 }, - { id = "vortex.zigzag", writer_version = 1 }, - { id = "vortex.zstd", writer_version = 1 }, + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.onpair", + "vortex.parquet.variant", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", ] layouts = [ "vortex.chunked", diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml index 4c8bfb388e9..881cbfc9985 100644 --- a/vortex/editions/core/family.toml +++ b/vortex/editions/core/family.toml @@ -5,12 +5,12 @@ name = "core" doc = """ -The serialized components available to the default file writer. Array memberships pin the -writer version compression schemes may produce. Every array ID still has one reader; an -incompatible serialized form must use a new ID. Every core edition freezes, and a frozen -edition carries a read-forever guarantee: a file written with it stays readable by every -later Vortex release. Stabilized non-plugin components and array writer-version upgrades are -adopted through preview before joining core. An edition may freeze in the release that cuts -it; after that release version is known, the declaration is backfilled with it as the -minimum. A frozen edition never changes. +The serialized components available to the default file writer. Each array ID names a wire +representation that old readers either recognize or reject; several IDs may deserialize into +one current in-memory array. Every core edition freezes, and a frozen edition carries a +read-forever guarantee: a file written with it stays readable by every later Vortex release. +Stabilized non-plugin components and newer serialized array IDs are adopted through preview +before joining core. An edition may freeze in the release that cuts it; after that release +version is known, the declaration is backfilled with it as the minimum. A frozen edition +never changes. """ diff --git a/vortex/editions/preview/family.toml b/vortex/editions/preview/family.toml index 891d2e238f3..891e138bdaf 100644 --- a/vortex/editions/preview/family.toml +++ b/vortex/editions/preview/family.toml @@ -5,11 +5,10 @@ name = "preview" doc = """ -Stabilized, opt-in components and array writer-version upgrades maintained as part of core -but not yet adopted by the default core writer. Preview behavior is expected to remain +Stabilized, opt-in components and serialized array representations maintained as part of +core but not yet adopted by the default core writer. Preview behavior is expected to remain compatible and should change only to fix a defect serious enough to block promotion into -core. A writer-version upgrade lets compression schemes produce new optional fields or -properties; it never selects a reader. Users keep the earlier serialized form until they opt -into that edition. Experimental work advances through new draft editions; optional plugins -instead use standalone families such as spatial and json. +core. A new wire representation has a new array ID, even when it serializes and deserializes +the same in-memory array as an older ID. Experimental work advances through new draft +editions; optional plugins instead use standalone families such as spatial and json. """ diff --git a/vortex/editions/preview/preview2025.05.0.toml b/vortex/editions/preview/preview2025.05.0.toml index 409ac6524ac..3c427385876 100644 --- a/vortex/editions/preview/preview2025.05.0.toml +++ b/vortex/editions/preview/preview2025.05.0.toml @@ -2,16 +2,16 @@ # # This edition record has no core read-forever guarantee. It may describe an evolving feature, # stabilized preview functionality awaiting adoption, or a release waiting to be cut. New -# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# capabilities advance to a new edition; after a core release is known, min_library_version is # backfilled to document its freeze. A frozen record never changes. edition = "preview2025.05.0" family = "preview" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "fastlanes.delta", writer_version = 1 }, + "fastlanes.delta", ] layouts = [] dtypes = [] @@ -21,7 +21,7 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.delta", writer_version = 1 }, + "fastlanes.delta", ] layouts = [] dtypes = [] diff --git a/vortex/editions/preview/preview2026.02.0.toml b/vortex/editions/preview/preview2026.02.0.toml index adc290109cd..8640b5820d3 100644 --- a/vortex/editions/preview/preview2026.02.0.toml +++ b/vortex/editions/preview/preview2026.02.0.toml @@ -2,16 +2,16 @@ # # This edition record has no core read-forever guarantee. It may describe an evolving feature, # stabilized preview functionality awaiting adoption, or a release waiting to be cut. New -# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# capabilities advance to a new edition; after a core release is known, min_library_version is # backfilled to document its freeze. A frozen record never changes. edition = "preview2026.02.0" family = "preview" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "vortex.zstd_buffers", writer_version = 1 }, + "vortex.zstd_buffers", ] layouts = [] dtypes = [] @@ -21,8 +21,8 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.delta", writer_version = 1 }, - { id = "vortex.zstd_buffers", writer_version = 1 }, + "fastlanes.delta", + "vortex.zstd_buffers", ] layouts = [] dtypes = [] diff --git a/vortex/editions/preview/preview2026.04.0.toml b/vortex/editions/preview/preview2026.04.0.toml index bb4fc6ce4e1..3e15dd2c393 100644 --- a/vortex/editions/preview/preview2026.04.0.toml +++ b/vortex/editions/preview/preview2026.04.0.toml @@ -2,20 +2,20 @@ # # This edition record has no core read-forever guarantee. It may describe an evolving feature, # stabilized preview functionality awaiting adoption, or a release waiting to be cut. New -# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# capabilities advance to a new edition; after a core release is known, min_library_version is # backfilled to document its freeze. A frozen record never changes. edition = "preview2026.04.0" family = "preview" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [ - { id = "vortex.patched", writer_version = 1 }, - { id = "vortex.tensor.cosine_similarity", writer_version = 1 }, - { id = "vortex.tensor.inner_product", writer_version = 1 }, - { id = "vortex.tensor.l2_norm", writer_version = 1 }, - { id = "vortex.tensor.normalized", writer_version = 1 }, + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", ] layouts = [] dtypes = [ @@ -28,13 +28,13 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.delta", writer_version = 1 }, - { id = "vortex.patched", writer_version = 1 }, - { id = "vortex.tensor.cosine_similarity", writer_version = 1 }, - { id = "vortex.tensor.inner_product", writer_version = 1 }, - { id = "vortex.tensor.l2_norm", writer_version = 1 }, - { id = "vortex.tensor.normalized", writer_version = 1 }, - { id = "vortex.zstd_buffers", writer_version = 1 }, + "fastlanes.delta", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", + "vortex.zstd_buffers", ] layouts = [] dtypes = [ diff --git a/vortex/editions/preview/preview2026.06.0.toml b/vortex/editions/preview/preview2026.06.0.toml index 41866a8af72..f115faf1c9a 100644 --- a/vortex/editions/preview/preview2026.06.0.toml +++ b/vortex/editions/preview/preview2026.06.0.toml @@ -2,13 +2,13 @@ # # This edition record has no core read-forever guarantee. It may describe an evolving feature, # stabilized preview functionality awaiting adoption, or a release waiting to be cut. New -# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# capabilities advance to a new edition; after a core release is known, min_library_version is # backfilled to document its freeze. A frozen record never changes. edition = "preview2026.06.0" family = "preview" -# Components and array writer-version increases added by this edition. +# Components added by this edition. [added] arrays = [] layouts = [ @@ -21,13 +21,13 @@ aggregates = [] # editions of the family. [components] arrays = [ - { id = "fastlanes.delta", writer_version = 1 }, - { id = "vortex.patched", writer_version = 1 }, - { id = "vortex.tensor.cosine_similarity", writer_version = 1 }, - { id = "vortex.tensor.inner_product", writer_version = 1 }, - { id = "vortex.tensor.l2_norm", writer_version = 1 }, - { id = "vortex.tensor.normalized", writer_version = 1 }, - { id = "vortex.zstd_buffers", writer_version = 1 }, + "fastlanes.delta", + "vortex.patched", + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", + "vortex.zstd_buffers", ] layouts = [ "vortex.list", diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index eb26a18ab3d..850f5f5ee5d 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -9,10 +9,10 @@ //! [`crate::editions::register_default_editions`] and then selects its write policy with //! [`crate::editions::enable_default_editions`]. //! -//! Members carry a [`crate::editions::ComponentKind`]: arrays a written array may use, extension -//! dtypes its schema may contain, and aggregates zone maps record. Array memberships also pin the -//! writer version compression schemes may produce, preserving existing writer behavior -//! until a newer edition is explicitly selected. +//! Members carry a [`crate::editions::ComponentKind`]: serialized array IDs a writer may emit, +//! extension dtypes its schema may contain, and aggregates zone maps record. Array serializers +//! choose the oldest permitted lossless wire representation independently of the in-memory array +//! a compressor produced. //! //! The default file writer resolves the session's enabled editions at write time. The //! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08_1`], and diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index 83b3edc931c..3b6cf260757 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::sync::Arc; - use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -13,10 +11,7 @@ use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::field_path; use vortex_array::session::ArraySessionExt; -use vortex_array::stream::ArrayStreamExt; -use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_buffer::ByteBufferMut; use vortex_edition::ComponentKind; use vortex_edition::Edition; @@ -34,14 +29,10 @@ use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; use vortex_io::session::RuntimeSession; -use vortex_layout::LayoutStrategy; -use vortex_layout::layouts::compressed::CompressingStrategy; -use vortex_layout::layouts::flat::writer::FlatLayoutStrategy; use vortex_layout::session::LayoutSession; use vortex_sequence::Sequence; use vortex_session::VortexSession; use vortex_session::registry::Id; -use vortex_utils::aliases::hash_set::HashSet; use super::CORE_2025_05_0; use super::CORE_2026_08_0; @@ -84,24 +75,6 @@ fn core_2026_08_1_dtype_set_is_pinned() { assert_eq!(ids, ["vortex.date", "vortex.time", "vortex.timestamp"]); } -#[test] -fn core_array_writer_versions_are_pinned() { - let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); - let arrays = session.components_in(&CORE_2026_08_1, ComponentKind::Array); - assert_eq!( - arrays - .iter() - .find(|inclusion| inclusion.component_id.as_str() == "vortex.pco") - .and_then(|inclusion| inclusion.array_writer_version), - Some(1) - ); - assert!( - arrays - .iter() - .all(|inclusion| inclusion.array_writer_version == Some(1)) - ); -} - #[test] fn core_2026_08_2_is_draft() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); @@ -246,11 +219,10 @@ fn default_session_enables_the_write_editions() { let session = VortexSession::default(); let enabled = session.enabled_editions().editions(); assert!(enabled.contains(&DEFAULT_CORE_EDITION)); - assert_eq!( + assert!( session - .enabled_array_writer_versions() - .get(&Id::from("vortex.pco")), - Some(&1) + .enabled_component_ids(ComponentKind::Array) + .contains(&Id::from("vortex.pco")) ); #[cfg(feature = "unstable_encodings")] @@ -380,46 +352,6 @@ async fn default_session_writes_every_default_zone_aggregate() -> VortexResult<( Ok(()) } -/// Restrict arrays to the baseline core edition while allowing the modern zoned components that -/// the current default layout strategy writes. -fn baseline_core_array_session() -> VortexResult { - const SUPPORT_EDITION: EditionId = EditionId::new("writer-support", 2026, 8, 0); - static SUPPORT_DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: SUPPORT_EDITION, - min_vortex_version: None, - }, - added: &[ - EditionMember::layout(&"vortex.zoned"), - EditionMember::aggregate(&"vortex.bounded_max"), - EditionMember::aggregate(&"vortex.bounded_min"), - EditionMember::aggregate(&"vortex.max"), - EditionMember::aggregate(&"vortex.min"), - EditionMember::aggregate(&"vortex.nan_count"), - EditionMember::aggregate(&"vortex.null_count"), - ], - }; - - let session = array_session() - .with::() - .with::() - .with::(); - vortex_file::register_default_encodings(&session); - session - .register_edition(&super::core::v2025_05::DECLARATION) - .map_err(|error| vortex_err!("{error}"))?; - session - .register_edition(&SUPPORT_DECLARATION) - .map_err(|error| vortex_err!("{error}"))?; - session - .enable_edition(CORE_2025_05_0) - .map_err(|error| vortex_err!("{error}"))?; - session - .enable_edition(SUPPORT_EDITION) - .map_err(|error| vortex_err!("{error}"))?; - Ok(session) -} - fn sequential_integers() -> PrimitiveArray { PrimitiveArray::from_iter(0..65_536i32) } @@ -429,7 +361,7 @@ const WRITER_TEST_EDITION: EditionId = EditionId::new("writer-test", 2026, 7, 0) static WRITER_TEST_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: WRITER_TEST_EDITION, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::array(&"vortex.chunked"), @@ -513,7 +445,7 @@ fn session_declaring(members: &[(ComponentKind, Id)]) -> VortexResult Arc { - Arc::new(CompressingStrategy::new( - FlatLayoutStrategy::default(), - forbidden_sequence_compressor, - )) -} - -async fn assert_round_trip_encodings_are_enabled( - session: &VortexSession, - strategy: Option>, - array: ArrayRef, -) -> VortexResult<()> { - let mut buffer = ByteBufferMut::empty(); - let write_options = match strategy { - Some(strategy) => session.write_options().with_strategy(strategy), - None => session.write_options(), - }; - if let Err(error) = write_options - .write(&mut buffer, array.to_array_stream()) - .await - { - let message = error.to_string(); - if message.contains("not permitted by ctx") - || message.contains("normalize forbids encoding") - { - return Ok(()); - } - return Err(error); - } - - let round_tripped = session - .open_options() - .open_buffer(buffer)? - .scan()? - .into_array_stream()? - .read_all() - .await?; - let actual: HashSet<_> = round_tripped - .depth_first_traversal() - .map(|array| array.encoding_id()) - .collect(); - let allowed: HashSet<_> = session - .enabled_component_ids(ComponentKind::Array) - .into_iter() - .collect(); - let mut forbidden: Vec<_> = actual.difference(&allowed).map(|id| id.as_str()).collect(); - forbidden.sort_unstable(); - if !forbidden.is_empty() { - return Err(vortex_err!( - "round-tripped array contains encodings outside {WRITER_TEST_EDITION}: {forbidden:?}" - )); - } - - Ok(()) -} - -#[tokio::test] -async fn default_strategy_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - assert_round_trip_encodings_are_enabled(&session, None, sequential_integers().into_array()) - .await -} - +/// Compressors operate on the current in-memory array model and do not interpret edition wire +/// IDs. The serializer is the final compatibility boundary and rejects a compressor result when +/// none of its lossless wire variants is enabled. #[tokio::test] -async fn replacement_default_builder_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - assert_round_trip_encodings_are_enabled( - &session, - Some(WriteStrategyBuilder::default().build()), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn replacement_btrblocks_builder_round_trip_uses_only_enabled_encodings() -> VortexResult<()> -{ - let session = writer_test_session()?; - let strategy = WriteStrategyBuilder::default() - .with_btrblocks_builder(BtrBlocksCompressorBuilder::default()) - .build(); - assert_round_trip_encodings_are_enabled( - &session, - Some(strategy), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn opaque_compressor_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { +async fn serializer_rejects_unsupported_compressor_output() -> VortexResult<()> { let session = writer_test_session()?; let strategy = WriteStrategyBuilder::default() .with_compressor(forbidden_sequence_compressor) .build(); - assert_round_trip_encodings_are_enabled( - &session, - Some(strategy), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn custom_flat_strategy_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - let strategy = WriteStrategyBuilder::default() - .with_flat_strategy(Arc::new(FlatLayoutStrategy::default())) - .build(); - assert_round_trip_encodings_are_enabled( - &session, - Some(strategy), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn custom_field_writer_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - let strategy = WriteStrategyBuilder::default() - .with_field_writer(field_path!(values), custom_compressing_flat_strategy()) - .build(); - let array = - StructArray::from_fields(&[("values", sequential_integers().into_array())])?.into_array(); - assert_round_trip_encodings_are_enabled(&session, Some(strategy), array).await -} - -#[tokio::test] -async fn replacement_strategy_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - assert_round_trip_encodings_are_enabled( - &session, - Some(custom_compressing_flat_strategy()), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn replacement_flat_strategy_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - assert_round_trip_encodings_are_enabled( - &session, - Some(Arc::new(FlatLayoutStrategy::default())), - forbidden_sequence(65_536)?, - ) - .await -} - -#[tokio::test] -async fn probe_compressor_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - let strategy = WriteStrategyBuilder::default() - .with_probe_compressor(forbidden_sequence_compressor) - .build(); - assert_round_trip_encodings_are_enabled( - &session, - Some(strategy), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn default_writer_filters_compressor_to_enabled_editions() -> VortexResult<()> { - let session = baseline_core_array_session()?; let mut buffer = ByteBufferMut::empty(); - session + let error = session .write_options() + .with_strategy(strategy) .write( &mut buffer, sequential_integers().into_array().to_array_stream(), ) - .await?; + .await + .err() + .ok_or_else(|| vortex_err!("Sequence unexpectedly had a permitted wire variant"))?; + assert!( + error.to_string().contains( + "Array vortex.sequence cannot be represented by any permitted serialized array ID" + ), + "unexpected error: {error}" + ); Ok(()) } +/// The same compressor output is writable when its wire ID is enabled, without configuring the +/// compressor itself from the edition. #[tokio::test] -async fn configured_btrblocks_builder_uses_enabled_editions_in_either_order() -> VortexResult<()> { - let session = baseline_core_array_session()?; - let allowed: HashSet<_> = session - .enabled_component_ids(ComponentKind::Array) - .into_iter() - .collect(); - let strategies = [ - WriteStrategyBuilder::default() - .with_btrblocks_builder(BtrBlocksCompressorBuilder::default()) - .with_allow_encodings(allowed.clone()) - .build(), - WriteStrategyBuilder::default() - .with_allow_encodings(allowed) - .with_btrblocks_builder(BtrBlocksCompressorBuilder::default()) - .build(), - ]; - - for strategy in strategies { - let mut buffer = ByteBufferMut::empty(); - session - .write_options() - .with_strategy(strategy) - .write( - &mut buffer, - sequential_integers().into_array().to_array_stream(), - ) - .await?; - } - - Ok(()) -} +async fn serializer_accepts_supported_compressor_output() -> VortexResult<()> { + use crate::VortexSessionDefault; -#[tokio::test] -async fn opaque_compressor_cannot_write_outside_enabled_editions() -> VortexResult<()> { - let session = baseline_core_array_session()?; - let allowed = session - .enabled_component_ids(ComponentKind::Array) - .into_iter() - .collect(); + let session = VortexSession::default(); let strategy = WriteStrategyBuilder::default() - .with_compressor(BtrBlocksCompressorBuilder::default().build()) - .with_allow_encodings(allowed) + .with_compressor(forbidden_sequence_compressor) .build(); let mut buffer = ByteBufferMut::empty(); - let result = session + session .write_options() .with_strategy(strategy) .write( &mut buffer, sequential_integers().into_array().to_array_stream(), ) - .await; - let error = match result { - Ok(_) => { - return Err(vortex_err!( - "the unrestricted opaque compressor wrote an encoding outside core@2025.05" - )); - } - Err(error) => error, - }; - let message = error.to_string(); - assert!( - message.contains("normalize forbids encoding (vortex.sequence)"), - "unexpected error: {message}" - ); + .await?; Ok(()) } diff --git a/xtask/src/check_editions.rs b/xtask/src/check_editions.rs index b3000d1eb0f..1aac4c86743 100644 --- a/xtask/src/check_editions.rs +++ b/xtask/src/check_editions.rs @@ -7,7 +7,7 @@ //! check. It may describe evolving work, stabilized preview functionality awaiting adoption, or a //! stable edition waiting for its release to be cut. Normal feature additions advance to a new //! edition; exceptional corrections remain possible before a core freeze. A stable edition can -//! freeze in its release; once the release version is known, `min_vortex_version` is backfilled to +//! freeze in its release; once the release version is known, `min_library_version` is backfilled to //! document the freeze. The record then carries a read-forever guarantee and may never change //! again. Whether a record was frozen is read from the base revision, so a change cannot unfreeze //! an edition and edit it in the same diff. @@ -39,10 +39,10 @@ use crate::generate_editions::FAMILY_FILE; use crate::generate_editions::RECORD_DIR; /// A record carries this key once the edition's freeze has been documented. -const FROZEN_MARKER: &str = "min_vortex_version"; +const FROZEN_MARKER: &str = "min_library_version"; const REMEDY: &str = "\ -A frozen edition is immutable. To add components or array writer versions, declare a NEW edition in +A frozen edition is immutable. To add component IDs, declare a NEW edition in vortex-edition/src/declarations// and regenerate the records with `cargo run -p xtask -- generate-editions`."; diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs index 745abdd5685..62d4326e2f3 100644 --- a/xtask/src/generate_editions.rs +++ b/xtask/src/generate_editions.rs @@ -4,15 +4,14 @@ //! Export the edition records under `vortex/editions`. //! //! Every declared edition gets one TOML file recording what it contains: the identifier, the -//! minimum Vortex version whose reader supports it once frozen, and its full member set. Array -//! members also record the writer version compression schemes may produce. Records are grouped by -//! family — +//! minimum library version whose reader supports it once frozen, and its full member set. Records +//! are grouped by family — //! `vortex/editions/core/core2025.05.0.toml` — mirroring the declarations in //! `vortex-edition/src/declarations`, since families version independently. //! //! A draft record carries no compatibility guarantee. Its serialization may still be evolving, //! or a stable edition may be waiting for its release to be cut. A stable edition can freeze in -//! that release; once the release version is known, `min_vortex_version` is backfilled to document +//! that release; once the release version is known, `min_library_version` is backfilled to document //! the freeze. The record then carries a read-forever guarantee and may never change again. CI //! enforces that against git history with `cargo run -p xtask -- check-editions`; this exporter //! enforces the two rules that history cannot see, refusing to delete a record or to unfreeze one. @@ -42,7 +41,7 @@ const FROZEN_NOTE: &str = "\ const DRAFT_NOTE: &str = "\ # This edition record has no core read-forever guarantee. It may describe an evolving feature, # stabilized preview functionality awaiting adoption, or a release waiting to be cut. New -# capabilities advance to a new edition; after a core release is known, min_vortex_version is +# capabilities advance to a new edition; after a core release is known, min_library_version is # backfilled to document its freeze. A frozen record never changes."; /// The file recording what a family is, beside that family's editions. @@ -94,8 +93,7 @@ const KINDS: [(ComponentKind, &str); 4] = [ (ComponentKind::Aggregate, "aggregates"), ]; -/// Render one TOML list per component kind, each sorted by component id. Array entries also pin -/// the writer version that compression schemes may produce. +/// Render one TOML list per component kind, each sorted by component id. fn kind_lists( lines: &mut Vec, inclusions_of: impl Fn(ComponentKind) -> Vec, @@ -108,23 +106,11 @@ fn kind_lists( continue; } lines.push(format!("{key} = [")); - if kind == ComponentKind::Array { - lines.extend(inclusions.iter().map(|inclusion| { - let Some(writer_version) = inclusion.array_writer_version else { - unreachable!("validated array inclusion has a writer version") - }; - format!( - " {{ id = \"{}\", writer_version = {} }},", - inclusion.component_id, writer_version - ) - })); - } else { - lines.extend( - inclusions - .iter() - .map(|inclusion| format!(" \"{}\",", inclusion.component_id)), - ); - } + lines.extend( + inclusions + .iter() + .map(|inclusion| format!(" \"{}\",", inclusion.component_id)), + ); lines.push("]".to_string()); } } @@ -144,12 +130,12 @@ fn record(session: &EditionSession, edition: &Edition) -> String { format!("edition = \"{}\"", edition.id), format!("family = \"{}\"", edition.id.family), ]; - if let Some(min_vortex_version) = edition.min_vortex_version { - lines.push(format!("min_vortex_version = \"{min_vortex_version}\"")); + if let Some(min_library_version) = edition.min_library_version { + lines.push(format!("min_library_version = \"{min_library_version}\"")); } lines.extend([ String::new(), - "# Components and array writer-version increases added by this edition.".to_string(), + "# Components added by this edition.".to_string(), "[added]".to_string(), ]); kind_lists(&mut lines, |kind| { @@ -200,11 +186,11 @@ fn existing_records(dir: &Path) -> anyhow::Result> { Ok(records) } -/// A record carries a `min_vortex_version` once its freeze has been documented. +/// A record carries a `min_library_version` once its freeze has been documented. fn records_a_frozen_edition(contents: &str) -> bool { contents .lines() - .any(|line| line.starts_with("min_vortex_version = ")) + .any(|line| line.starts_with("min_library_version = ")) } pub fn generate_editions() -> anyhow::Result<()> { @@ -254,7 +240,7 @@ pub fn generate_editions() -> anyhow::Result<()> { { return Err(anyhow!( "{} is recorded as frozen but its declaration is now a draft.\n\ - An edition that recorded a min_vortex_version carries a read-forever \ + An edition that recorded a min_library_version carries a read-forever \ guarantee and may never return to draft.", edition.id, ));