From fe98adce223c75bd8465462fc90c6155fed28965 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 14:19:42 +0000 Subject: [PATCH 1/8] Add golden-corpus determinism tests for the default compressor Compress a fixed, seed-generated corpus covering every scheme's habitat (int monotone/sequence/low-cardinality/runs/sparse/null-heavy/negative/ random, ALP- and dict-friendly floats, FSST- and dict-friendly strings, binary, decimal, temporal, bool, struct, list) and snapshot each entry's full encoding tree with exact per-node byte counts via insta. Every entry is longer than 1024 values so the sampling-based estimation path is exercised, and each entry is compressed twice per run to assert determinism directly. These snapshots pin the default compressor's decisions: the upcoming cost-model refactor rungs cite zero snapshot churn as their proof of behavior preservation. Co-Authored-By: Claude Fable 5 Signed-off-by: Connor Tsui --- Cargo.lock | 1 + vortex-btrblocks/Cargo.toml | 1 + vortex-btrblocks/tests/golden.rs | 375 ++++++++++++++++++ ...lden__default__binary_low_cardinality.snap | 11 + .../golden__default__bool_random.snap | 7 + .../golden__default__decimal_prices.snap | 9 + .../golden__default__float_alp_prices.snap | 9 + ...golden__default__float_full_precision.snap | 15 + ...olden__default__float_low_cardinality.snap | 19 + .../golden__default__float_mostly_null.snap | 13 + ...den__default__int_arithmetic_sequence.snap | 7 + .../golden__default__int_low_cardinality.snap | 11 + .../golden__default__int_monotone_jitter.snap | 9 + .../golden__default__int_mostly_null.snap | 13 + .../golden__default__int_negatives.snap | 9 + .../snapshots/golden__default__int_runs.snap | 15 + .../golden__default__int_sparse_outliers.snap | 13 + .../golden__default__int_wide_random.snap | 7 + .../golden__default__list_of_int_runs.snap | 25 ++ ...lden__default__string_fsst_structured.snap | 15 + ...lden__default__string_low_cardinality.snap | 15 + .../golden__default__struct_mixed.snap | 23 ++ ...n__default__temporal_timestamp_micros.snap | 11 + 23 files changed, 633 insertions(+) create mode 100644 vortex-btrblocks/tests/golden.rs create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__binary_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__bool_random.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__decimal_prices.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__float_alp_prices.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__float_full_precision.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__float_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__float_mostly_null.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__int_arithmetic_sequence.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__int_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__int_mostly_null.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__int_negatives.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__int_runs.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__int_sparse_outliers.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__int_wide_random.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__string_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__struct_mixed.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap diff --git a/Cargo.lock b/Cargo.lock index 16df2a942c1..7a3b82f3dce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9928,6 +9928,7 @@ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ "codspeed-divan-compat", + "insta", "itertools 0.14.0", "num-traits", "pco", diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 5ee2bad5402..529c6d90bad 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -41,6 +41,7 @@ vortex-zstd = { workspace = true, optional = true } [dev-dependencies] divan = { workspace = true } +insta = { workspace = true } rstest = { workspace = true } test-with = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/vortex-btrblocks/tests/golden.rs b/vortex-btrblocks/tests/golden.rs new file mode 100644 index 00000000000..2cfebc22a9c --- /dev/null +++ b/vortex-btrblocks/tests/golden.rs @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Golden-corpus determinism tests for the default compressor. +//! +//! Compresses a fixed, seed-generated corpus and snapshots each entry's full encoding tree +//! and exact byte counts. These snapshots pin the default compressor's *decisions*: any +//! refactor of scheme selection (see the compressor cost-model track) must leave every +//! snapshot untouched, so snapshot churn in a later change is the reviewable signal of a +//! behavior change. +//! +//! The `default` variant pins the default feature set and [`BtrBlocksCompressor::default`]; +//! it is compiled out under `unstable_encodings`, whose scheme set differs (variants pinning +//! that feature set are gated separately). +//! +//! Every corpus entry is longer than 1024 values so the sampling-based estimation path is +//! exercised, and each entry is compressed twice per run to assert determinism directly. + +#![allow(clippy::cast_possible_truncation, clippy::tests_outside_test_module)] + +use std::fmt; +use std::sync::Arc; +use std::sync::LazyLock; + +use rand::RngExt; +use rand::SeedableRng; +use rand::rngs::StdRng; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::DecimalArray; +use vortex_array::arrays::ListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::StructArray; +use vortex_array::arrays::TemporalArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::display::EncodingSummaryExtractor; +use vortex_array::display::MetadataExtractor; +use vortex_array::display::TreeContext; +use vortex_array::display::TreeExtractor; +use vortex_array::dtype::DType; +use vortex_array::dtype::DecimalDType; +use vortex_array::dtype::Nullability; +use vortex_array::extension::datetime::TimeUnit; +use vortex_array::validity::Validity; +use vortex_btrblocks::BtrBlocksCompressor; +use vortex_buffer::Buffer; +use vortex_error::VortexResult; +use vortex_session::VortexSession; + +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); + +/// Number of values in each numeric corpus entry: comfortably above the 1024-value sampling +/// threshold so scheme selection runs on sampled estimates, as it does for real file chunks. +const N: usize = 16_384; + +/// Header extractor printing exact byte counts (the built-in [`NbytesExtractor`] rounds +/// through `humansize`, which could mask small size regressions). +/// +/// [`NbytesExtractor`]: vortex_array::display::NbytesExtractor +struct ExactNbytesExtractor; + +impl TreeExtractor for ExactNbytesExtractor { + fn write_header( + &self, + array: &ArrayRef, + _ctx: &TreeContext, + f: &mut fmt::Formatter<'_>, + ) -> fmt::Result { + write!(f, " nbytes={}", array.nbytes()) + } +} + +/// Renders the full snapshot content for one compressed corpus entry. +fn render(input: &ArrayRef, compressed: &ArrayRef) -> String { + format!( + "input: {}, len={}, nbytes={}\n{}", + input.dtype(), + input.len(), + input.nbytes(), + compressed + .tree_display_builder() + .with(EncodingSummaryExtractor) + .with(ExactNbytesExtractor) + .with(MetadataExtractor) + ) +} + +/// Compresses every corpus entry twice (direct determinism check) and snapshots the result +/// under `{variant}__{entry}`. +fn golden_corpus_snapshots(variant: &str, compressor: &BtrBlocksCompressor) -> VortexResult<()> { + for (name, array) in corpus()? { + let rendered = { + let mut exec_ctx = SESSION.create_execution_ctx(); + render(&array, &compressor.compress(&array, &mut exec_ctx)?) + }; + let rendered_again = { + let mut exec_ctx = SESSION.create_execution_ctx(); + render(&array, &compressor.compress(&array, &mut exec_ctx)?) + }; + assert_eq!( + rendered, rendered_again, + "compressing corpus entry {name} twice produced different results" + ); + + insta::assert_snapshot!(format!("{variant}__{name}"), rendered); + } + Ok(()) +} + +/// The fixed corpus: deterministic synthetic arrays covering each scheme's habitat. +fn corpus() -> VortexResult> { + Ok(vec![ + ("int_monotone_jitter", int_monotone_jitter()), + ("int_arithmetic_sequence", int_arithmetic_sequence()), + ("int_low_cardinality", int_low_cardinality()), + ("int_runs", int_runs()), + ("int_sparse_outliers", int_sparse_outliers()), + ("int_mostly_null", int_mostly_null()), + ("int_negatives", int_negatives()), + ("int_wide_random", int_wide_random()), + ("float_alp_prices", float_alp_prices()), + ("float_low_cardinality", float_low_cardinality()), + ("float_full_precision", float_full_precision()), + ("float_mostly_null", float_mostly_null()), + ("string_fsst_structured", string_fsst_structured()), + ("string_low_cardinality", string_low_cardinality()), + ("binary_low_cardinality", binary_low_cardinality()), + ("decimal_prices", decimal_prices()), + ("temporal_timestamp_micros", temporal_timestamp_micros()), + ("bool_random", bool_random()), + ("struct_mixed", struct_mixed()?), + ("list_of_int_runs", list_of_int_runs()?), + ]) +} + +/// Near-monotone u64 (timestamp-like): FoR/BitPacking habitat, Delta habitat when enabled. +fn int_monotone_jitter() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(101); + let mut value = 1_700_000_000_000u64; + let values: Buffer = (0..N) + .map(|_| { + value += 900 + rng.random_range(0..200); + value + }) + .collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Exact arithmetic sequence: Sequence habitat (distinct == len, no nulls). +fn int_arithmetic_sequence() -> ArrayRef { + let values: Buffer = (0..N as i64).map(|i| 10_000 + 7 * i).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// A handful of widely-spaced distinct values: IntDict habitat. +fn int_low_cardinality() -> ArrayRef { + const DISTINCT: [i64; 6] = [0, 123_400, 617_000, 1_234_000, 12_340_000, 37_020_000]; + let mut rng = StdRng::seed_from_u64(102); + let values: Buffer = (0..N).map(|_| DISTINCT[rng.random_range(0..6)]).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Long runs over a moderate value set: RunEnd/RLE habitat. +fn int_runs() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(103); + let mut values: Vec = Vec::with_capacity(N); + while values.len() < N { + let value = rng.random_range(-50_000..50_000i32); + let run = rng.random_range(8..25); + values.extend(std::iter::repeat_n(value, run)); + } + values.truncate(N); + PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable).into_array() +} + +/// One dominant value with rare large outliers: Sparse habitat. +fn int_sparse_outliers() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(104); + let values: Buffer = (0..N) + .map(|_| { + if rng.random_range(0..100) < 5 { + rng.random_range(1_000_000_000..2_000_000_000i64) + } else { + 1_000_000 + } + }) + .collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// 95% nulls over small values: null-dominated integer habitat. +fn int_mostly_null() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(105); + let mut validity: Vec = Vec::with_capacity(N); + let values: Buffer = (0..N) + .map(|_| { + let valid = rng.random_range(0..100) < 5; + validity.push(valid); + if valid { rng.random_range(0..1000) } else { 0 } + }) + .collect(); + PrimitiveArray::new( + values, + Validity::Array(BoolArray::from_iter(validity).into_array()), + ) + .into_array() +} + +/// Small values of mixed sign: ZigZag habitat. +fn int_negatives() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(106); + let values: Buffer = (0..N).map(|_| rng.random_range(-128..128i64)).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Full-width random u64: essentially incompressible; pins the "no scheme wins" path. +fn int_wide_random() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(107); + let values: Buffer = (0..N).map(|_| rng.random::()).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Two-decimal-digit "prices": ALP habitat. +fn float_alp_prices() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(108); + let values: Buffer = (0..N) + .map(|_| rng.random_range(0..10_000_000i64) as f64 / 100.0) + .collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// A handful of distinct floats: FloatDict habitat. +fn float_low_cardinality() -> ArrayRef { + const DISTINCT: [f64; 8] = [0.0, 0.5, 1.25, 2.75, 3.5, 10.125, 100.0625, 1000.03125]; + let mut rng = StdRng::seed_from_u64(109); + let values: Buffer = (0..N).map(|_| DISTINCT[rng.random_range(0..8)]).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Full-precision uniform floats: ALP-RD habitat. +fn float_full_precision() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(110); + let values: Buffer = (0..N).map(|_| rng.random::()).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// 95% nulls over floats: null-dominated sparse float habitat. +fn float_mostly_null() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(111); + let mut validity: Vec = Vec::with_capacity(N); + let values: Buffer = (0..N) + .map(|_| { + let valid = rng.random_range(0..100) < 5; + validity.push(valid); + if valid { + rng.random::() * 100.0 + } else { + 0.0 + } + }) + .collect(); + PrimitiveArray::new( + values, + Validity::Array(BoolArray::from_iter(validity).into_array()), + ) + .into_array() +} + +/// High-cardinality strings with shared substructure (emails): FSST habitat. +fn string_fsst_structured() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(112); + let strings: Vec = (0..N) + .map(|_| { + format!( + "user{:06}@example{}.com", + rng.random_range(0..1_000_000), + rng.random_range(0..100) + ) + }) + .collect(); + VarBinViewArray::from_iter_str(strings.iter().map(String::as_str)).into_array() +} + +/// A dozen distinct strings: StringDict habitat. +fn string_low_cardinality() -> ArrayRef { + const DISTINCT: [&str; 12] = [ + "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", + "juliett", "kilo", "lima", + ]; + let mut rng = StdRng::seed_from_u64(113); + let strings: Vec> = (0..N) + .map(|_| Some(DISTINCT[rng.random_range(0..12)])) + .collect(); + VarBinViewArray::from_iter(strings, DType::Utf8(Nullability::NonNullable)).into_array() +} + +/// Low-cardinality binary blobs: BinaryDict habitat (Zstd habitat under `compact`). +fn binary_low_cardinality() -> ArrayRef { + const DISTINCT: [&[u8]; 5] = [ + &[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08], + &[0xDE, 0xAD, 0xBE, 0xEF], + &[0x00; 16], + &[0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00], + &[0x42; 12], + ]; + let mut rng = StdRng::seed_from_u64(114); + let blobs: Vec> = (0..N) + .map(|_| Some(DISTINCT[rng.random_range(0..5)])) + .collect(); + VarBinViewArray::from_iter(blobs, DType::Binary(Nullability::NonNullable)).into_array() +} + +/// Two-decimal-place decimals: DecimalScheme (byte-parts) habitat. +fn decimal_prices() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(115); + let values: Buffer = (0..N).map(|_| rng.random_range(0..10_000_000i64)).collect(); + DecimalArray::new(values, DecimalDType::new(12, 2), Validity::NonNullable).into_array() +} + +/// Near-monotone microsecond timestamps: TemporalScheme (datetime-parts) habitat. +fn temporal_timestamp_micros() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(116); + let mut value = 1_700_000_000_000_000i64; + let values: Buffer = (0..N) + .map(|_| { + value += rng.random_range(1_000..1_000_000); + value + }) + .collect(); + TemporalArray::new_timestamp( + PrimitiveArray::new(values, Validity::NonNullable).into_array(), + TimeUnit::Microseconds, + Some(Arc::from("UTC")), + ) + .into_array() +} + +/// Random booleans: no bool scheme is registered, pinning the "stays canonical" path. +fn bool_random() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(117); + BoolArray::from_iter((0..N).map(|_| rng.random::())).into_array() +} + +/// Struct of int/string/float fields: pins the structural recursion path. +fn struct_mixed() -> VortexResult { + Ok(StructArray::from_fields(&[ + ("id", int_arithmetic_sequence()), + ("category", string_low_cardinality()), + ("value", float_alp_prices()), + ])? + .into_array()) +} + +/// Variable-length lists of run-heavy ints: pins the list offsets/elements path. +fn list_of_int_runs() -> VortexResult { + let mut rng = StdRng::seed_from_u64(118); + let elements = int_runs(); + let mut offsets: Vec = Vec::with_capacity(N / 4 + 1); + let mut offset = 0i32; + offsets.push(offset); + while (offset as usize) < N { + offset = (offset + rng.random_range(1..8)).min(N as i32); + offsets.push(offset); + } + let offsets = PrimitiveArray::new(Buffer::copy_from(&offsets), Validity::NonNullable); + Ok(ListArray::try_new(elements, offsets.into_array(), Validity::NonNullable)?.into_array()) +} + +#[cfg(not(feature = "unstable_encodings"))] +#[test] +fn golden_default() -> VortexResult<()> { + golden_corpus_snapshots("default", &BtrBlocksCompressor::default()) +} diff --git a/vortex-btrblocks/tests/snapshots/golden__default__binary_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__default__binary_low_cardinality.snap new file mode 100644 index 00000000000..56918c0d096 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__binary_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: binary, len=16384, nbytes=315856 +root: vortex.dict(binary, len=16384) nbytes=6240 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.varbinview(binary, len=5) nbytes=96 + metadata: diff --git a/vortex-btrblocks/tests/snapshots/golden__default__bool_random.snap b/vortex-btrblocks/tests/snapshots/golden__default__bool_random.snap new file mode 100644 index 00000000000..d80e2fa4464 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__bool_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: bool, len=16384, nbytes=2048 +root: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__decimal_prices.snap b/vortex-btrblocks/tests/snapshots/golden__default__decimal_prices.snap new file mode 100644 index 00000000000..f669755e4b1 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__decimal_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: decimal(12,2), len=16384, nbytes=131072 +root: vortex.decimal_byte_parts(decimal(12,2), len=16384) nbytes=49152 + metadata: + msp: fastlanes.bitpacked(i32, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__float_alp_prices.snap b/vortex-btrblocks/tests/snapshots/golden__default__float_alp_prices.snap new file mode 100644 index 00000000000..f70d90a301d --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__float_alp_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=49152 + metadata: exponents: e: 14, f: 12 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__float_full_precision.snap b/vortex-btrblocks/tests/snapshots/golden__default__float_full_precision.snap new file mode 100644 index 00000000000..487e7c7900b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__float_full_precision.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alprd(f64, len=16384) nbytes=112880 + metadata: right_bit_width: 52, patch_offset: 0 + left_parts: fastlanes.bitpacked(u16, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + right_parts: fastlanes.bitpacked(u64, len=16384) nbytes=106496 + metadata: bit_width: 52, offset: 0 + patch_indices: vortex.primitive(u16, len=60) nbytes=120 + metadata: ptype: u16 + patch_values: vortex.primitive(u16, len=60) nbytes=120 + metadata: ptype: u16 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__float_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__default__float_low_cardinality.snap new file mode 100644 index 00000000000..38d952262d6 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__float_low_cardinality.snap @@ -0,0 +1,19 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=10233 + metadata: exponents: e: 16, f: 12, patch_offset: 0 + encoded: vortex.dict(i64, len=16384) nbytes=6200 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.primitive(i64, len=7) nbytes=56 + metadata: ptype: i64 + patch_indices: vortex.primitive(u16, len=1996) nbytes=3992 + metadata: ptype: u16 + patch_values: vortex.constant(f64, len=1996) nbytes=9 + metadata: scalar: 1000.03125f64 + patch_chunk_offsets: vortex.primitive(u16, len=16) nbytes=32 + metadata: ptype: u16 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__float_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__default__float_mostly_null.snap new file mode 100644 index 00000000000..7b88d819472 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__float_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64?, len=16384, nbytes=133120 +root: vortex.sparse(f64?, len=16384) nbytes=8609 + metadata: fill_value: null + patch_indices: vortex.primitive(u16, len=850) nbytes=1700 + metadata: ptype: u16 + patch_values: vortex.primitive(f64?, len=850) nbytes=6907 + metadata: ptype: f64 + validity: vortex.bool(bool, len=850) nbytes=107 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_arithmetic_sequence.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_arithmetic_sequence.snap new file mode 100644 index 00000000000..873f3655a1f --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_arithmetic_sequence.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_low_cardinality.snap new file mode 100644 index 00000000000..8648200a138 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.dict(i64, len=16384) nbytes=6192 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.primitive(i64, len=6) nbytes=48 + metadata: ptype: i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap new file mode 100644 index 00000000000..4800aae289d --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: fastlanes.for(u64, len=16384) nbytes=49152 + metadata: reference: 1700000001036u64 + encoded: fastlanes.bitpacked(u64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_mostly_null.snap new file mode 100644 index 00000000000..73566369279 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32?, len=16384, nbytes=67584 +root: vortex.sparse(i32?, len=16384) nbytes=3031 + metadata: fill_value: null + patch_indices: vortex.primitive(u16, len=823) nbytes=1646 + metadata: ptype: u16 + patch_values: fastlanes.bitpacked(i32?, len=823) nbytes=1383 + metadata: bit_width: 10, offset: 0 + validity_child: vortex.bool(bool, len=823) nbytes=103 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_negatives.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_negatives.snap new file mode 100644 index 00000000000..24949893190 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_negatives.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: fastlanes.for(i64, len=16384) nbytes=16384 + metadata: reference: -128i64 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=16384 + metadata: bit_width: 8, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_runs.snap new file mode 100644 index 00000000000..271de8faf93 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_runs.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32, len=16384, nbytes=65536 +root: vortex.runend(i32, len=16384) nbytes=3968 + metadata: offset: 0 + ends: fastlanes.for(u16, len=1020) nbytes=1792 + metadata: reference: 13u16 + encoded: fastlanes.bitpacked(u16, len=1020) nbytes=1792 + metadata: bit_width: 14, offset: 0 + values: fastlanes.for(i32, len=1020) nbytes=2176 + metadata: reference: -49931i32 + encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 + metadata: bit_width: 17, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_sparse_outliers.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_sparse_outliers.snap new file mode 100644 index 00000000000..53a5d1e023c --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_sparse_outliers.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sparse(i64, len=16384) nbytes=5540 + metadata: fill_value: 1000000i64 + patch_indices: vortex.primitive(u16, len=848) nbytes=1696 + metadata: ptype: u16 + patch_values: fastlanes.for(i64, len=848) nbytes=3840 + metadata: reference: 1000830099i64 + encoded: fastlanes.bitpacked(i64, len=848) nbytes=3840 + metadata: bit_width: 30, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_wide_random.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_wide_random.snap new file mode 100644 index 00000000000..90e1894f910 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_wide_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: vortex.primitive(u64, len=16384) nbytes=131072 + metadata: ptype: u64 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap new file mode 100644 index 00000000000..c9554add05a --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap @@ -0,0 +1,25 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: list(i32), len=4066, nbytes=81804 +root: vortex.list(list(i32), len=4066) nbytes=11146 + metadata: + elements: vortex.runend(i32, len=16384) nbytes=3968 + metadata: offset: 0 + ends: fastlanes.for(u16, len=1020) nbytes=1792 + metadata: reference: 13u16 + encoded: fastlanes.bitpacked(u16, len=1020) nbytes=1792 + metadata: bit_width: 14, offset: 0 + values: fastlanes.for(i32, len=1020) nbytes=2176 + metadata: reference: -49931i32 + encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 + metadata: bit_width: 17, offset: 0 + offsets: fastlanes.bitpacked(u16, len=4067) nbytes=7178 + metadata: bit_width: 14, offset: 0 + patch_indices: vortex.primitive(u16, len=1) nbytes=2 + metadata: ptype: u16 + patch_values: vortex.constant(u16, len=1) nbytes=4 + metadata: scalar: 16384u16 + patch_chunk_offsets: vortex.primitive(u8, len=4) nbytes=4 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap new file mode 100644 index 00000000000..327f050b0d7 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=653785 +root: vortex.fsst(utf8, len=16384) nbytes=151382 + metadata: len: 16384, nsymbols: 223 + uncompressed_lengths: vortex.sparse(u8, len=16384) nbytes=3154 + metadata: fill_value: 24u8 + patch_indices: vortex.primitive(u16, len=1575) nbytes=3150 + metadata: ptype: u16 + patch_values: vortex.constant(u8, len=1575) nbytes=2 + metadata: scalar: 23u8 + codes_offsets: fastlanes.bitpacked(u32, len=16385) nbytes=36992 + metadata: bit_width: 17, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__string_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__default__string_low_cardinality.snap new file mode 100644 index 00000000000..ec79989d46a --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__string_low_cardinality.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=262144 +root: vortex.dict(utf8, len=16384) nbytes=8373 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.fsst(utf8, len=12) nbytes=181 + metadata: len: 12, nsymbols: 9 + uncompressed_lengths: vortex.primitive(u8, len=12) nbytes=12 + metadata: ptype: u8 + codes_offsets: vortex.primitive(u8, len=13) nbytes=13 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__struct_mixed.snap b/vortex-btrblocks/tests/snapshots/golden__default__struct_mixed.snap new file mode 100644 index 00000000000..be8f48b779b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__struct_mixed.snap @@ -0,0 +1,23 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: {id=i64, category=utf8, value=f64}, len=16384, nbytes=524288 +root: vortex.struct({id=i64, category=utf8, value=f64}, len=16384) nbytes=57525 + metadata: + id: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 + category: vortex.dict(utf8, len=16384) nbytes=8373 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.fsst(utf8, len=12) nbytes=181 + metadata: len: 12, nsymbols: 9 + uncompressed_lengths: vortex.primitive(u8, len=12) nbytes=12 + metadata: ptype: u8 + codes_offsets: vortex.primitive(u8, len=13) nbytes=13 + metadata: ptype: u8 + value: vortex.alp(f64, len=16384) nbytes=49152 + metadata: exponents: e: 14, f: 12 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap b/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap new file mode 100644 index 00000000000..6f38e8e6f5d --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: vortex.timestamp[µs, tz=UTC](i64), len=16384, nbytes=131072 +root: vortex.ext(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=67584 + metadata: + storage: fastlanes.for(i64, len=16384) nbytes=67584 + metadata: reference: 1700000000891673i64 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=67584 + metadata: bit_width: 33, offset: 0 From ba2c7e220d0244b8e77eab5bd1af319ceabb9ac4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 14:20:23 +0000 Subject: [PATCH 2/8] Pin unstable_encodings and with_compact compressors in the golden suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two feature-gated golden variants over the same corpus: 'unstable' (unstable_encodings, default builder) pins Delta and Sequence selection, and 'compact' (unstable_encodings + zstd + pco, with_compact) pins Zstd and Pco selection. The default-variant test stays compiled out under unstable_encodings since that feature changes ALL_SCHEMES. A dedicated CI job (btrblocks-golden) runs each of the three feature combinations explicitly. The variants are mutually exclusive at compile time, so the existing --all-features workspace test jobs compile the default variant out and it would otherwise only run incidentally in the default-features musl job. OnPairScheme is excluded from both variants: its dictionary training (upstream onpair crate) iterates randomly-seeded hashbrown maps, so its compressed output — and therefore its sampled estimate — differs from run to run. A nondeterministic scheme cannot serve as a golden baseline, and it violates the determinism contract in the Scheme docs; that is a pre-existing bug worth its own issue, not something this suite can pin. Co-Authored-By: Claude Fable 5 Signed-off-by: Connor Tsui --- .github/workflows/ci.yml | 32 +++++++++++++ vortex-btrblocks/tests/golden.rs | 46 +++++++++++++++++-- ...lden__compact__binary_low_cardinality.snap | 11 +++++ .../golden__compact__bool_random.snap | 7 +++ .../golden__compact__decimal_prices.snap | 9 ++++ .../golden__compact__float_alp_prices.snap | 9 ++++ ...golden__compact__float_full_precision.snap | 7 +++ ...olden__compact__float_low_cardinality.snap | 13 ++++++ .../golden__compact__float_mostly_null.snap | 13 ++++++ ...den__compact__int_arithmetic_sequence.snap | 7 +++ .../golden__compact__int_low_cardinality.snap | 11 +++++ .../golden__compact__int_monotone_jitter.snap | 7 +++ .../golden__compact__int_mostly_null.snap | 9 ++++ .../golden__compact__int_negatives.snap | 9 ++++ .../snapshots/golden__compact__int_runs.snap | 9 ++++ .../golden__compact__int_sparse_outliers.snap | 7 +++ .../golden__compact__int_wide_random.snap | 7 +++ .../golden__compact__list_of_int_runs.snap | 13 ++++++ ...lden__compact__string_fsst_structured.snap | 7 +++ ...lden__compact__string_low_cardinality.snap | 11 +++++ .../golden__compact__struct_mixed.snap | 19 ++++++++ ...n__compact__temporal_timestamp_micros.snap | 9 ++++ ...den__unstable__binary_low_cardinality.snap | 11 +++++ .../golden__unstable__bool_random.snap | 7 +++ .../golden__unstable__decimal_prices.snap | 9 ++++ .../golden__unstable__float_alp_prices.snap | 9 ++++ ...olden__unstable__float_full_precision.snap | 15 ++++++ ...lden__unstable__float_low_cardinality.snap | 19 ++++++++ .../golden__unstable__float_mostly_null.snap | 13 ++++++ ...en__unstable__int_arithmetic_sequence.snap | 7 +++ ...golden__unstable__int_low_cardinality.snap | 11 +++++ ...golden__unstable__int_monotone_jitter.snap | 15 ++++++ .../golden__unstable__int_mostly_null.snap | 13 ++++++ .../golden__unstable__int_negatives.snap | 9 ++++ .../snapshots/golden__unstable__int_runs.snap | 15 ++++++ ...golden__unstable__int_sparse_outliers.snap | 13 ++++++ .../golden__unstable__int_wide_random.snap | 7 +++ .../golden__unstable__list_of_int_runs.snap | 25 ++++++++++ ...den__unstable__string_fsst_structured.snap | 15 ++++++ ...den__unstable__string_low_cardinality.snap | 15 ++++++ .../golden__unstable__struct_mixed.snap | 23 ++++++++++ ...__unstable__temporal_timestamp_micros.snap | 13 ++++++ 42 files changed, 533 insertions(+), 3 deletions(-) create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__binary_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__bool_random.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__decimal_prices.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__float_alp_prices.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__float_full_precision.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__float_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__float_mostly_null.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_arithmetic_sequence.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_mostly_null.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_negatives.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_runs.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_sparse_outliers.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__int_wide_random.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__list_of_int_runs.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__string_fsst_structured.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__string_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__struct_mixed.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__compact__temporal_timestamp_micros.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__binary_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__bool_random.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__decimal_prices.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__float_alp_prices.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__float_full_precision.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__float_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__float_mostly_null.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__int_arithmetic_sequence.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__int_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__int_monotone_jitter.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__int_mostly_null.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__int_negatives.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__int_runs.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__int_sparse_outliers.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__int_wide_random.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__string_low_cardinality.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__struct_mixed.snap create mode 100644 vortex-btrblocks/tests/snapshots/golden__unstable__temporal_timestamp_micros.snap diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 372e2639bb0..11ff42a0187 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -452,6 +452,38 @@ jobs: alert-title: "Rust tests (linux-arm64) failed on develop" deduplication-key: ci-rust-test-linux-arm64-failure + btrblocks-golden: + name: "Rust (btrblocks golden corpus)" + timeout-minutes: 30 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=btrblocks-golden', github.run_id) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + # The golden suite pins the compressor's decisions per feature variant, and the + # variants are mutually exclusive at compile time: the `default` variant only exists + # without `unstable_encodings` (which changes ALL_SCHEMES), so the `--all-features` + # workspace test jobs cannot run it. Run each feature combination explicitly. + - name: Golden corpus (default features) + run: | + cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden + - name: Golden corpus (unstable_encodings) + run: | + cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden \ + --features unstable_encodings + - name: Golden corpus (compact, unstable_encodings + zstd + pco) + run: | + cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden \ + --features unstable_encodings,zstd,pco + build-java: name: "Java" runs-on: >- diff --git a/vortex-btrblocks/tests/golden.rs b/vortex-btrblocks/tests/golden.rs index 2cfebc22a9c..c60752b064a 100644 --- a/vortex-btrblocks/tests/golden.rs +++ b/vortex-btrblocks/tests/golden.rs @@ -9,9 +9,13 @@ //! snapshot untouched, so snapshot churn in a later change is the reviewable signal of a //! behavior change. //! -//! The `default` variant pins the default feature set and [`BtrBlocksCompressor::default`]; -//! it is compiled out under `unstable_encodings`, whose scheme set differs (variants pinning -//! that feature set are gated separately). +//! Three variants cover the feature matrix: +//! +//! - `default`: the default feature set and [`BtrBlocksCompressor::default`]. +//! - `unstable`: `unstable_encodings` enabled, default builder — pins Delta / OnPair +//! selection (compiled out of `ALL_SCHEMES` otherwise). +//! - `compact`: `unstable_encodings` + `zstd` + `pco`, with +//! [`BtrBlocksCompressorBuilder::with_compact`] — pins Zstd / Pco selection. //! //! Every corpus entry is longer than 1024 values so the sampling-based estimation path is //! exercised, and each entry is compressed twice per run to assert determinism directly. @@ -368,8 +372,44 @@ fn list_of_int_runs() -> VortexResult { Ok(ListArray::try_new(elements, offsets.into_array(), Validity::NonNullable)?.into_array()) } +/// Excludes OnPair from the golden compressors: its dictionary training (upstream `onpair` +/// crate) iterates randomly-seeded `hashbrown` maps, so its compressed output — and therefore +/// its sampled estimate — differs run-to-run. A nondeterministic scheme cannot serve as a +/// golden baseline; excluding it keeps the remaining unstable schemes pinned. +#[cfg(feature = "unstable_encodings")] +fn without_onpair( + builder: vortex_btrblocks::BtrBlocksCompressorBuilder, +) -> vortex_btrblocks::BtrBlocksCompressorBuilder { + use vortex_btrblocks::SchemeExt; + use vortex_btrblocks::schemes::string::OnPairScheme; + + builder.exclude_schemes([OnPairScheme.id()]) +} + #[cfg(not(feature = "unstable_encodings"))] #[test] fn golden_default() -> VortexResult<()> { golden_corpus_snapshots("default", &BtrBlocksCompressor::default()) } + +#[cfg(feature = "unstable_encodings")] +#[test] +fn golden_unstable() -> VortexResult<()> { + use vortex_btrblocks::BtrBlocksCompressorBuilder; + + golden_corpus_snapshots( + "unstable", + &without_onpair(BtrBlocksCompressorBuilder::default()).build(), + ) +} + +#[cfg(all(feature = "unstable_encodings", feature = "zstd", feature = "pco"))] +#[test] +fn golden_compact() -> VortexResult<()> { + use vortex_btrblocks::BtrBlocksCompressorBuilder; + + golden_corpus_snapshots( + "compact", + &without_onpair(BtrBlocksCompressorBuilder::default().with_compact()).build(), + ) +} diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__binary_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__compact__binary_low_cardinality.snap new file mode 100644 index 00000000000..d68474ce61c --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__binary_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: binary, len=16384, nbytes=315856 +root: vortex.dict(binary, len=16384) nbytes=6199 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.zstd(binary, len=5) nbytes=55 + metadata: nrows: 5, slice: 0..5 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__bool_random.snap b/vortex-btrblocks/tests/snapshots/golden__compact__bool_random.snap new file mode 100644 index 00000000000..d80e2fa4464 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__bool_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: bool, len=16384, nbytes=2048 +root: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__decimal_prices.snap b/vortex-btrblocks/tests/snapshots/golden__compact__decimal_prices.snap new file mode 100644 index 00000000000..780ef30a6b0 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__decimal_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: decimal(12,2), len=16384, nbytes=131072 +root: vortex.decimal_byte_parts(decimal(12,2), len=16384) nbytes=47666 + metadata: + msp: vortex.pco(i32, len=16384) nbytes=47666 + metadata: ptype: i32, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__float_alp_prices.snap b/vortex-btrblocks/tests/snapshots/golden__compact__float_alp_prices.snap new file mode 100644 index 00000000000..b1a3b41c4e7 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__float_alp_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=47699 + metadata: exponents: e: 14, f: 12 + encoded: vortex.pco(i64, len=16384) nbytes=47699 + metadata: ptype: i64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__float_full_precision.snap b/vortex-btrblocks/tests/snapshots/golden__compact__float_full_precision.snap new file mode 100644 index 00000000000..20fecf31e22 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__float_full_precision.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.pco(f64, len=16384) nbytes=110692 + metadata: ptype: f64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__float_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__compact__float_low_cardinality.snap new file mode 100644 index 00000000000..006f5a93b69 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__float_low_cardinality.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.dict(f64, len=16384) nbytes=6184 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.alp(f64, len=8) nbytes=40 + metadata: exponents: e: 16, f: 11 + encoded: vortex.pco(i64, len=8) nbytes=40 + metadata: ptype: i64, nrows: 8, slice: 0..8 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__float_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__compact__float_mostly_null.snap new file mode 100644 index 00000000000..134acccdc1c --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__float_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64?, len=16384, nbytes=133120 +root: vortex.sparse(f64?, len=16384) nbytes=7545 + metadata: fill_value: null + patch_indices: vortex.pco(u16, len=850) nbytes=636 + metadata: ptype: u16, nrows: 850, slice: 0..850 + patch_values: vortex.primitive(f64?, len=850) nbytes=6907 + metadata: ptype: f64 + validity: vortex.bool(bool, len=850) nbytes=107 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_arithmetic_sequence.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_arithmetic_sequence.snap new file mode 100644 index 00000000000..873f3655a1f --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_arithmetic_sequence.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_low_cardinality.snap new file mode 100644 index 00000000000..0cbe2e06814 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.dict(i64, len=16384) nbytes=6177 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.pco(i64, len=6) nbytes=33 + metadata: ptype: i64, nrows: 6, slice: 0..6 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter.snap new file mode 100644 index 00000000000..4a63aa0da71 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: vortex.pco(u64, len=16384) nbytes=15715 + metadata: ptype: u64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_mostly_null.snap new file mode 100644 index 00000000000..71e1c671a03 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_mostly_null.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32?, len=16384, nbytes=67584 +root: vortex.pco(i32?, len=16384) nbytes=3086 + metadata: ptype: i32, nrows: 16384, slice: 0..16384 + validity: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_negatives.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_negatives.snap new file mode 100644 index 00000000000..24949893190 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_negatives.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: fastlanes.for(i64, len=16384) nbytes=16384 + metadata: reference: -128i64 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=16384 + metadata: bit_width: 8, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_runs.snap new file mode 100644 index 00000000000..4ab2e46cead --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_runs.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32, len=16384, nbytes=65536 +root: vortex.zigzag(i32, len=16384) nbytes=2969 + metadata: + encoded: vortex.pco(u32, len=16384) nbytes=2969 + metadata: ptype: u32, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_sparse_outliers.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_sparse_outliers.snap new file mode 100644 index 00000000000..0303e0e4ef1 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_sparse_outliers.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.pco(i64, len=16384) nbytes=3817 + metadata: ptype: i64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_wide_random.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_wide_random.snap new file mode 100644 index 00000000000..90e1894f910 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_wide_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: vortex.primitive(u64, len=16384) nbytes=131072 + metadata: ptype: u64 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__list_of_int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__compact__list_of_int_runs.snap new file mode 100644 index 00000000000..88274daf4b1 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__list_of_int_runs.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: list(i32), len=4066, nbytes=81804 +root: vortex.list(list(i32), len=4066) nbytes=4434 + metadata: + elements: vortex.zigzag(i32, len=16384) nbytes=2969 + metadata: + encoded: vortex.pco(u32, len=16384) nbytes=2969 + metadata: ptype: u32, nrows: 16384, slice: 0..16384 + offsets: vortex.pco(u16, len=4067) nbytes=1465 + metadata: ptype: u16, nrows: 4067, slice: 0..4067 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__compact__string_fsst_structured.snap new file mode 100644 index 00000000000..49ad248087b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__string_fsst_structured.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=653785 +root: vortex.zstd(utf8, len=16384) nbytes=109119 + metadata: nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__string_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__compact__string_low_cardinality.snap new file mode 100644 index 00000000000..a65054ac178 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__string_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=262144 +root: vortex.dict(utf8, len=16384) nbytes=8287 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.zstd(utf8, len=12) nbytes=95 + metadata: nrows: 12, slice: 0..12 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__struct_mixed.snap b/vortex-btrblocks/tests/snapshots/golden__compact__struct_mixed.snap new file mode 100644 index 00000000000..83e34c583dc --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__struct_mixed.snap @@ -0,0 +1,19 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: {id=i64, category=utf8, value=f64}, len=16384, nbytes=524288 +root: vortex.struct({id=i64, category=utf8, value=f64}, len=16384) nbytes=55986 + metadata: + id: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 + category: vortex.dict(utf8, len=16384) nbytes=8287 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.zstd(utf8, len=12) nbytes=95 + metadata: nrows: 12, slice: 0..12 + value: vortex.alp(f64, len=16384) nbytes=47699 + metadata: exponents: e: 14, f: 12 + encoded: vortex.pco(i64, len=16384) nbytes=47699 + metadata: ptype: i64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__temporal_timestamp_micros.snap b/vortex-btrblocks/tests/snapshots/golden__compact__temporal_timestamp_micros.snap new file mode 100644 index 00000000000..0422808ce29 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__temporal_timestamp_micros.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: vortex.timestamp[µs, tz=UTC](i64), len=16384, nbytes=131072 +root: vortex.ext(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=40985 + metadata: + storage: vortex.pco(i64, len=16384) nbytes=40985 + metadata: ptype: i64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__binary_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__binary_low_cardinality.snap new file mode 100644 index 00000000000..56918c0d096 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__binary_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: binary, len=16384, nbytes=315856 +root: vortex.dict(binary, len=16384) nbytes=6240 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.varbinview(binary, len=5) nbytes=96 + metadata: diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__bool_random.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__bool_random.snap new file mode 100644 index 00000000000..d80e2fa4464 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__bool_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: bool, len=16384, nbytes=2048 +root: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__decimal_prices.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__decimal_prices.snap new file mode 100644 index 00000000000..f669755e4b1 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__decimal_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: decimal(12,2), len=16384, nbytes=131072 +root: vortex.decimal_byte_parts(decimal(12,2), len=16384) nbytes=49152 + metadata: + msp: fastlanes.bitpacked(i32, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__float_alp_prices.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__float_alp_prices.snap new file mode 100644 index 00000000000..f70d90a301d --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__float_alp_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=49152 + metadata: exponents: e: 14, f: 12 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__float_full_precision.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__float_full_precision.snap new file mode 100644 index 00000000000..487e7c7900b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__float_full_precision.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alprd(f64, len=16384) nbytes=112880 + metadata: right_bit_width: 52, patch_offset: 0 + left_parts: fastlanes.bitpacked(u16, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + right_parts: fastlanes.bitpacked(u64, len=16384) nbytes=106496 + metadata: bit_width: 52, offset: 0 + patch_indices: vortex.primitive(u16, len=60) nbytes=120 + metadata: ptype: u16 + patch_values: vortex.primitive(u16, len=60) nbytes=120 + metadata: ptype: u16 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__float_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__float_low_cardinality.snap new file mode 100644 index 00000000000..38d952262d6 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__float_low_cardinality.snap @@ -0,0 +1,19 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=10233 + metadata: exponents: e: 16, f: 12, patch_offset: 0 + encoded: vortex.dict(i64, len=16384) nbytes=6200 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.primitive(i64, len=7) nbytes=56 + metadata: ptype: i64 + patch_indices: vortex.primitive(u16, len=1996) nbytes=3992 + metadata: ptype: u16 + patch_values: vortex.constant(f64, len=1996) nbytes=9 + metadata: scalar: 1000.03125f64 + patch_chunk_offsets: vortex.primitive(u16, len=16) nbytes=32 + metadata: ptype: u16 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__float_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__float_mostly_null.snap new file mode 100644 index 00000000000..7b88d819472 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__float_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64?, len=16384, nbytes=133120 +root: vortex.sparse(f64?, len=16384) nbytes=8609 + metadata: fill_value: null + patch_indices: vortex.primitive(u16, len=850) nbytes=1700 + metadata: ptype: u16 + patch_values: vortex.primitive(f64?, len=850) nbytes=6907 + metadata: ptype: f64 + validity: vortex.bool(bool, len=850) nbytes=107 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_arithmetic_sequence.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_arithmetic_sequence.snap new file mode 100644 index 00000000000..873f3655a1f --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_arithmetic_sequence.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_low_cardinality.snap new file mode 100644 index 00000000000..8648200a138 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.dict(i64, len=16384) nbytes=6192 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.primitive(i64, len=6) nbytes=48 + metadata: ptype: i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_monotone_jitter.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_monotone_jitter.snap new file mode 100644 index 00000000000..f1ded4c3e59 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_monotone_jitter.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: fastlanes.delta(u64, len=16384) nbytes=19840 + metadata: offset: 0 + bases: vortex.primitive(u64, len=256) nbytes=2048 + metadata: ptype: u64 + deltas: vortex.dict(u64, len=16384) nbytes=17792 + metadata: all_values_referenced: true + codes: vortex.primitive(u8, len=16384) nbytes=16384 + metadata: ptype: u8 + values: fastlanes.bitpacked(u64, len=201) nbytes=1408 + metadata: bit_width: 11, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_mostly_null.snap new file mode 100644 index 00000000000..73566369279 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32?, len=16384, nbytes=67584 +root: vortex.sparse(i32?, len=16384) nbytes=3031 + metadata: fill_value: null + patch_indices: vortex.primitive(u16, len=823) nbytes=1646 + metadata: ptype: u16 + patch_values: fastlanes.bitpacked(i32?, len=823) nbytes=1383 + metadata: bit_width: 10, offset: 0 + validity_child: vortex.bool(bool, len=823) nbytes=103 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_negatives.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_negatives.snap new file mode 100644 index 00000000000..24949893190 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_negatives.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: fastlanes.for(i64, len=16384) nbytes=16384 + metadata: reference: -128i64 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=16384 + metadata: bit_width: 8, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_runs.snap new file mode 100644 index 00000000000..271de8faf93 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_runs.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32, len=16384, nbytes=65536 +root: vortex.runend(i32, len=16384) nbytes=3968 + metadata: offset: 0 + ends: fastlanes.for(u16, len=1020) nbytes=1792 + metadata: reference: 13u16 + encoded: fastlanes.bitpacked(u16, len=1020) nbytes=1792 + metadata: bit_width: 14, offset: 0 + values: fastlanes.for(i32, len=1020) nbytes=2176 + metadata: reference: -49931i32 + encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 + metadata: bit_width: 17, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_sparse_outliers.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_sparse_outliers.snap new file mode 100644 index 00000000000..53a5d1e023c --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_sparse_outliers.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sparse(i64, len=16384) nbytes=5540 + metadata: fill_value: 1000000i64 + patch_indices: vortex.primitive(u16, len=848) nbytes=1696 + metadata: ptype: u16 + patch_values: fastlanes.for(i64, len=848) nbytes=3840 + metadata: reference: 1000830099i64 + encoded: fastlanes.bitpacked(i64, len=848) nbytes=3840 + metadata: bit_width: 30, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_wide_random.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_wide_random.snap new file mode 100644 index 00000000000..90e1894f910 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_wide_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: vortex.primitive(u64, len=16384) nbytes=131072 + metadata: ptype: u64 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap new file mode 100644 index 00000000000..c9554add05a --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap @@ -0,0 +1,25 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: list(i32), len=4066, nbytes=81804 +root: vortex.list(list(i32), len=4066) nbytes=11146 + metadata: + elements: vortex.runend(i32, len=16384) nbytes=3968 + metadata: offset: 0 + ends: fastlanes.for(u16, len=1020) nbytes=1792 + metadata: reference: 13u16 + encoded: fastlanes.bitpacked(u16, len=1020) nbytes=1792 + metadata: bit_width: 14, offset: 0 + values: fastlanes.for(i32, len=1020) nbytes=2176 + metadata: reference: -49931i32 + encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 + metadata: bit_width: 17, offset: 0 + offsets: fastlanes.bitpacked(u16, len=4067) nbytes=7178 + metadata: bit_width: 14, offset: 0 + patch_indices: vortex.primitive(u16, len=1) nbytes=2 + metadata: ptype: u16 + patch_values: vortex.constant(u16, len=1) nbytes=4 + metadata: scalar: 16384u16 + patch_chunk_offsets: vortex.primitive(u8, len=4) nbytes=4 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap new file mode 100644 index 00000000000..327f050b0d7 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=653785 +root: vortex.fsst(utf8, len=16384) nbytes=151382 + metadata: len: 16384, nsymbols: 223 + uncompressed_lengths: vortex.sparse(u8, len=16384) nbytes=3154 + metadata: fill_value: 24u8 + patch_indices: vortex.primitive(u16, len=1575) nbytes=3150 + metadata: ptype: u16 + patch_values: vortex.constant(u8, len=1575) nbytes=2 + metadata: scalar: 23u8 + codes_offsets: fastlanes.bitpacked(u32, len=16385) nbytes=36992 + metadata: bit_width: 17, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__string_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__string_low_cardinality.snap new file mode 100644 index 00000000000..ec79989d46a --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__string_low_cardinality.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=262144 +root: vortex.dict(utf8, len=16384) nbytes=8373 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.fsst(utf8, len=12) nbytes=181 + metadata: len: 12, nsymbols: 9 + uncompressed_lengths: vortex.primitive(u8, len=12) nbytes=12 + metadata: ptype: u8 + codes_offsets: vortex.primitive(u8, len=13) nbytes=13 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__struct_mixed.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__struct_mixed.snap new file mode 100644 index 00000000000..be8f48b779b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__struct_mixed.snap @@ -0,0 +1,23 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: {id=i64, category=utf8, value=f64}, len=16384, nbytes=524288 +root: vortex.struct({id=i64, category=utf8, value=f64}, len=16384) nbytes=57525 + metadata: + id: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 + category: vortex.dict(utf8, len=16384) nbytes=8373 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.fsst(utf8, len=12) nbytes=181 + metadata: len: 12, nsymbols: 9 + uncompressed_lengths: vortex.primitive(u8, len=12) nbytes=12 + metadata: ptype: u8 + codes_offsets: vortex.primitive(u8, len=13) nbytes=13 + metadata: ptype: u8 + value: vortex.alp(f64, len=16384) nbytes=49152 + metadata: exponents: e: 14, f: 12 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__temporal_timestamp_micros.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__temporal_timestamp_micros.snap new file mode 100644 index 00000000000..a763fa28f4b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__temporal_timestamp_micros.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: vortex.timestamp[µs, tz=UTC](i64), len=16384, nbytes=131072 +root: vortex.ext(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=43008 + metadata: + storage: fastlanes.delta(i64, len=16384) nbytes=43008 + metadata: offset: 0 + bases: vortex.primitive(i64, len=256) nbytes=2048 + metadata: ptype: i64 + deltas: fastlanes.bitpacked(i64, len=16384) nbytes=40960 + metadata: bit_width: 20, offset: 0 From 29b6106fc254cec68649aed5608cf1a135b8a199 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 15:44:01 +0000 Subject: [PATCH 3/8] Cover btrblocks golden snapshots with a REUSE.toml annotation insta snapshot files do not allow leading comment lines, so license them via a nested REUSE.toml, mirroring vortex-bench's handling of its own snapshots. Co-Authored-By: Claude Fable 5 Signed-off-by: Connor Tsui --- .github/workflows/ci.yml | 2 +- vortex-btrblocks/REUSE.toml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 vortex-btrblocks/REUSE.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11ff42a0187..23af8e2c5ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -454,7 +454,7 @@ jobs: btrblocks-golden: name: "Rust (btrblocks golden corpus)" - timeout-minutes: 30 + timeout-minutes: 10 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=btrblocks-golden', github.run_id) diff --git a/vortex-btrblocks/REUSE.toml b/vortex-btrblocks/REUSE.toml new file mode 100644 index 00000000000..6bee2b6904f --- /dev/null +++ b/vortex-btrblocks/REUSE.toml @@ -0,0 +1,7 @@ +version = 1 + +# `insta` snapshot files do not allow leading comment lines. +[[annotations]] +path = "tests/snapshots/**.snap" +SPDX-FileCopyrightText = "Copyright the Vortex contributors" +SPDX-License-Identifier = "Apache-2.0" From ce7039ff44fde838522d984ed8467b5ffc3ff072 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 14:28:18 +0000 Subject: [PATCH 4/8] Introduce crate-private Candidate bookkeeping in scheme selection Pure refactor of choose_best_scheme and the sampling estimator: - estimate_compression_ratio_with_sampling now returns a SampledEstimate carrying the compressed sample array alongside its score, instead of measuring nbytes and dropping the array. - Selection bookkeeping switches from (scheme, EstimateScore) tuples to a crate-private Candidate carrying the mechanical facts about each option (scheme, score, input bytes, value count, sampled array, cascade ancestry). Candidates are still compared by ratio, ties still break by registration order, and the sampled array is dropped at the end of selection, so behavior is unchanged: same winners, same estimates, same output (golden suite has zero snapshot churn). The not-yet-read Candidate fields are the facts a pluggable cost model prices in the follow-up rungs (#7697) with no new Scheme API. Co-Authored-By: Claude Fable 5 Signed-off-by: Connor Tsui --- vortex-compressor/src/candidate.rs | 43 +++++++++++++++++++++++++++++ vortex-compressor/src/compressor.rs | 40 ++++++++++++++++++++------- vortex-compressor/src/estimate.rs | 31 +++++++++++++++------ vortex-compressor/src/lib.rs | 1 + 4 files changed, 97 insertions(+), 18 deletions(-) create mode 100644 vortex-compressor/src/candidate.rs diff --git a/vortex-compressor/src/candidate.rs b/vortex-compressor/src/candidate.rs new file mode 100644 index 00000000000..5b78e2120e1 --- /dev/null +++ b/vortex-compressor/src/candidate.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Selection-time candidate bookkeeping. + +use vortex_array::ArrayRef; + +use crate::estimate::EstimateScore; +use crate::scheme::Scheme; +use crate::scheme::SchemeId; + +/// The mechanical facts the compressor knows about one (scheme, estimate) option during +/// scheme selection. +/// +/// Selection currently reads only `scheme` and `score`; the remaining facts are carried so +/// that a pluggable cost model can price candidates without any new [`Scheme`] API +/// (see #7697). +pub(crate) struct Candidate { + /// The scheme that produced this estimate. + pub(crate) scheme: &'static dyn Scheme, + + /// The ranked estimate: an estimated or sample-measured compression ratio, or the + /// zero-byte special case. + pub(crate) score: EstimateScore, + + /// Uncompressed size in bytes of the array under selection. + #[expect(dead_code, reason = "consumed by the cost model in a follow-up")] + pub(crate) input_nbytes: u64, + + /// Number of values in the array under selection. + #[expect(dead_code, reason = "consumed by the cost model in a follow-up")] + pub(crate) n_values: u64, + + /// The compressed sample array, when this estimate came from sampling. Its encoding tree + /// is the best available prediction of the full-array encoding tree. Dropped at the end + /// of selection. + #[expect(dead_code, reason = "consumed by the cost model in a follow-up")] + pub(crate) sampled: Option, + + /// The cascade ancestry `(scheme_id, child_index)` of the selection site. + #[expect(dead_code, reason = "consumed by the cost model in a follow-up")] + pub(crate) cascade: Vec<(SchemeId, usize)>, +} diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 727aa4ed98b..0ea5b50d472 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -34,6 +34,7 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::builtins::IntDictScheme; +use crate::candidate::Candidate; use crate::constant; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; @@ -416,9 +417,20 @@ impl CascadingCompressor { compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult> { - let mut best: Option<(&'static dyn Scheme, EstimateScore)> = None; + let mut best: Option = None; let mut deferred: Vec<(&'static dyn Scheme, DeferredEstimate)> = Vec::new(); + let input_nbytes = data.array().nbytes(); + let n_values = data.array().len() as u64; + let new_candidate = |scheme, score, sampled| Candidate { + scheme, + score, + input_nbytes, + n_values, + sampled, + cascade: compress_ctx.cascade_history().to_vec(), + }; + // Pass 1: evaluate every immediate verdict. Stash deferred work for pass 2. { let _verdict_pass = trace::verdict_pass_span().entered(); @@ -432,7 +444,7 @@ impl CascadingCompressor { let score = EstimateScore::FiniteCompression(ratio); if is_better_score(score, best.as_ref()) { - best = Some((scheme, score)); + best = Some(new_candidate(scheme, score, None)); } } CompressionEstimate::Deferred(deferred_estimate) => { @@ -446,10 +458,10 @@ impl CascadingCompressor { // short-circuit with `Skip` when they cannot beat it. for (scheme, deferred_estimate) in deferred { let _span = trace::scheme_eval_span(scheme.id()).entered(); - let threshold: Option = best.map(|(_, score)| score); + let threshold: Option = best.as_ref().map(|candidate| candidate.score); match deferred_estimate { DeferredEstimate::Sample => { - let score = estimate_compression_ratio_with_sampling( + let estimate = estimate_compression_ratio_with_sampling( self, scheme, data.array(), @@ -457,8 +469,12 @@ impl CascadingCompressor { exec_ctx, )?; - if is_better_score(score, best.as_ref()) { - best = Some((scheme, score)); + if is_better_score(estimate.score, best.as_ref()) { + best = Some(new_candidate( + scheme, + estimate.score, + Some(estimate.sampled), + )); } } DeferredEstimate::Callback(callback) => { @@ -471,7 +487,7 @@ impl CascadingCompressor { let score = EstimateScore::FiniteCompression(ratio); if is_better_score(score, best.as_ref()) { - best = Some((scheme, score)); + best = Some(new_candidate(scheme, score, None)); } } } @@ -479,7 +495,9 @@ impl CascadingCompressor { } } - Ok(best.map(|(scheme, score)| (scheme, WinnerEstimate::Score(score)))) + // The sampled arrays carried by candidates are dropped here; only the winning scheme + // and its score survive selection. + Ok(best.map(|candidate| (candidate.scheme, WinnerEstimate::Score(candidate.score)))) } // TODO(connor): Lots of room for optimization here. @@ -1356,14 +1374,16 @@ mod tests { // Before the fix this panicked with: // "this must be present since `DictScheme` declared that we need distinct values" let mut exec_ctx = SESSION.create_execution_ctx(); - let score = estimate_compression_ratio_with_sampling( + let estimate = estimate_compression_ratio_with_sampling( &compressor, &FloatDictScheme, &array, ctx, &mut exec_ctx, )?; - assert!(matches!(score, EstimateScore::FiniteCompression(ratio) if ratio.is_finite())); + assert!( + matches!(estimate.score, EstimateScore::FiniteCompression(ratio) if ratio.is_finite()) + ); Ok(()) } } diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index 82d021353d2..f1ad74f635c 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -12,6 +12,7 @@ use vortex_array::IntoArray; use vortex_error::VortexResult; use crate::CascadingCompressor; +use crate::candidate::Candidate; use crate::ctx::CompressorContext; use crate::sample::SAMPLE_SIZE; use crate::sample::sample; @@ -179,12 +180,20 @@ impl WinnerEstimate { } } -/// Returns `true` if `score` beats the current best estimate. -pub(super) fn is_better_score( - score: EstimateScore, - best: Option<&(&'static dyn Scheme, EstimateScore)>, -) -> bool { - score.is_valid() && best.is_none_or(|(_, best_score)| score.beats(*best_score)) +/// Returns `true` if `score` beats the current best candidate. +pub(super) fn is_better_score(score: EstimateScore, best: Option<&Candidate>) -> bool { + score.is_valid() && best.is_none_or(|candidate| score.beats(candidate.score)) +} + +/// A sampling-based estimate: the ranked score together with the compressed sample array it +/// was measured on. +pub(crate) struct SampledEstimate { + /// The ranked estimate measured on the sample. + pub(crate) score: EstimateScore, + + /// The compressed sample array. Its encoding tree is the best available prediction of + /// the full-array encoding tree. + pub(crate) sampled: ArrayRef, } /// Estimates compression ratio by compressing a ~1% sample of the data. @@ -192,6 +201,9 @@ pub(super) fn is_better_score( /// Creates a new [`ArrayAndStats`] for the sample so that stats are generated from the sample, not /// the full array. /// +/// Returns the compressed sample alongside its score so the selection loop can keep the +/// sample's encoding tree for the winning candidate. +/// /// # Errors /// /// Returns an error if sample compression fails. @@ -201,7 +213,7 @@ pub(super) fn estimate_compression_ratio_with_sampling( array: &ArrayRef, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, -) -> VortexResult { +) -> VortexResult { let sample_array = if compress_ctx.is_sample() { array.clone() } else { @@ -232,7 +244,10 @@ pub(super) fn estimate_compression_ratio_with_sampling( trace::zero_byte_sample_result(scheme.id(), before); } - Ok(score) + Ok(SampledEstimate { + score, + sampled: compressed, + }) } impl fmt::Debug for DeferredEstimate { diff --git a/vortex-compressor/src/lib.rs b/vortex-compressor/src/lib.rs index 86df4fc4cba..b8db9f7db76 100644 --- a/vortex-compressor/src/lib.rs +++ b/vortex-compressor/src/lib.rs @@ -68,6 +68,7 @@ pub mod estimate; pub mod scheme; pub mod stats; +mod candidate; mod constant; mod sample; From 9af2884516ac79bc94f37b9a5073d5caaaf208de Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 14:33:23 +0000 Subject: [PATCH 5/8] Add cost module: Cost, CostModel trait, and the SizeCost default New vortex_compressor::cost module extracting selection policy into a pluggable model, per the compressor cost-model plan (#7697): - Cost: opaque finite f64 newtype, lower is better; units are defined per model since costs are only compared within one model. - CostModel: price a Candidate (None = reject) and price the canonical baseline every candidate must strictly beat. The module docs state the axioms that stay outside the model: AlwaysUse short-circuiting and the byte-acceptance gate (including its AnyScalarFn carve-out). - SizeCost: prices a candidate at Cost(-ratio) over the exact ratio signal and canonical at Cost(-1.0), reproducing ratio-argmax selection bit-exactly. Pricing off the ratio rather than recomputed bytes (input/ratio) avoids IEEE division collapsing strict ratio inequalities into ties and flipping winners via registration order. Candidate becomes public (it is the trait's argument), re-exported via the cost module. Unit tests pin the validity gate, pairwise ordering and tie behavior against the historical rules. Nothing consumes the module yet; the selection loop is wired to it in the next commit. Co-Authored-By: Claude Fable 5 Signed-off-by: Connor Tsui --- vortex-compressor/src/candidate.rs | 30 ++--- vortex-compressor/src/cost/mod.rs | 98 ++++++++++++++ vortex-compressor/src/cost/size.rs | 208 +++++++++++++++++++++++++++++ vortex-compressor/src/lib.rs | 1 + 4 files changed, 321 insertions(+), 16 deletions(-) create mode 100644 vortex-compressor/src/cost/mod.rs create mode 100644 vortex-compressor/src/cost/size.rs diff --git a/vortex-compressor/src/candidate.rs b/vortex-compressor/src/candidate.rs index 5b78e2120e1..8637862dc65 100644 --- a/vortex-compressor/src/candidate.rs +++ b/vortex-compressor/src/candidate.rs @@ -12,32 +12,30 @@ use crate::scheme::SchemeId; /// The mechanical facts the compressor knows about one (scheme, estimate) option during /// scheme selection. /// -/// Selection currently reads only `scheme` and `score`; the remaining facts are carried so -/// that a pluggable cost model can price candidates without any new [`Scheme`] API -/// (see #7697). -pub(crate) struct Candidate { +/// A `Candidate` is the argument to [`CostModel::cost`]: it carries everything a model can +/// price without any new [`Scheme`] API. Candidates are built by the compressor during +/// selection and dropped when selection ends. +/// +/// [`CostModel::cost`]: crate::cost::CostModel::cost +#[derive(Debug)] +pub struct Candidate { /// The scheme that produced this estimate. - pub(crate) scheme: &'static dyn Scheme, + pub scheme: &'static dyn Scheme, /// The ranked estimate: an estimated or sample-measured compression ratio, or the /// zero-byte special case. - pub(crate) score: EstimateScore, + pub score: EstimateScore, /// Uncompressed size in bytes of the array under selection. - #[expect(dead_code, reason = "consumed by the cost model in a follow-up")] - pub(crate) input_nbytes: u64, + pub input_nbytes: u64, /// Number of values in the array under selection. - #[expect(dead_code, reason = "consumed by the cost model in a follow-up")] - pub(crate) n_values: u64, + pub n_values: u64, /// The compressed sample array, when this estimate came from sampling. Its encoding tree - /// is the best available prediction of the full-array encoding tree. Dropped at the end - /// of selection. - #[expect(dead_code, reason = "consumed by the cost model in a follow-up")] - pub(crate) sampled: Option, + /// is the best available prediction of the full-array encoding tree. + pub sampled: Option, /// The cascade ancestry `(scheme_id, child_index)` of the selection site. - #[expect(dead_code, reason = "consumed by the cost model in a follow-up")] - pub(crate) cascade: Vec<(SchemeId, usize)>, + pub cascade: Vec<(SchemeId, usize)>, } diff --git a/vortex-compressor/src/cost/mod.rs b/vortex-compressor/src/cost/mod.rs new file mode 100644 index 00000000000..0f4e5c9ce75 --- /dev/null +++ b/vortex-compressor/src/cost/mod.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Pluggable cost models for scheme selection. +//! +//! A [`CostModel`] is the policy half of scheme selection: [`Scheme`]s produce mechanical +//! signals (estimated or sample-measured compression ratios, collected into a [`Candidate`]), +//! and the model prices each candidate. The compressor picks the candidate with the lowest +//! cost that (strictly) beats [`CostModel::canonical_cost`] — the price of leaving the array +//! in its canonical encoding. +//! +//! The default model is [`SizeCost`], which reproduces the compressor's historical +//! ratio-argmax selection bit-exactly. +//! +//! # What sits outside the model +//! +//! Two selection mechanisms are deliberately *not* routed through the cost model: +//! +//! - [`EstimateVerdict::AlwaysUse`] short-circuits selection entirely. It expresses semantic +//! normalization (e.g. decimal byte-parts, temporal decomposition), not a priced trade-off. +//! - The byte-acceptance gate: after the winning scheme compresses the full array, the result +//! is kept only if it is byte-wise smaller than its input. This is an axiom for **all** +//! models — compression never grows bytes. A model that prefers canonical (e.g. for speed) +//! expresses that by pricing every candidate at or above `canonical_cost`, so selection +//! returns no winner and the array stays canonical; the gate never forces a *bad* encoding, +//! only "no encoding". (The gate's `AnyScalarFn` carve-out is likewise semantic +//! denormalization, not cost.) +//! +//! # Determinism +//! +//! Cost models must be pure functions of the candidate and their own configuration: no +//! timing measurements, no I/O, no global state. The compressor's output must be a +//! deterministic function of its input and configuration. +//! +//! [`Scheme`]: crate::scheme::Scheme +//! [`EstimateVerdict::AlwaysUse`]: crate::estimate::EstimateVerdict::AlwaysUse + +mod size; + +use std::fmt::Debug; + +pub use size::SizeCost; + +pub use crate::candidate::Candidate; +use crate::stats::ArrayAndStats; + +/// An opaque, totally ordered cost. Lower is better. +/// +/// Units are defined by each [`CostModel`], not by the framework: costs are only ever +/// compared *within* one model on one compressor instance, so cross-model comparability is +/// intentionally not provided. +/// +/// Values must be finite. `NaN`/infinite prices are not representable orderings; a model +/// that cannot price a candidate rejects it by returning `None` from [`CostModel::cost`]. +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] +pub struct Cost(f64); + +impl Cost { + /// Creates a cost from a finite value. + /// + /// # Panics + /// + /// Panics in debug builds if `value` is not finite. + pub fn new(value: f64) -> Self { + debug_assert!(value.is_finite(), "Cost must be finite, got {value}"); + Self(value) + } + + /// Returns the raw cost value. + pub fn value(self) -> f64 { + self.0 + } +} + +/// Prices [`Candidate`]s during scheme selection. +/// +/// Implementations must be deterministic pure functions of their inputs and cheap: `cost` is +/// called once per scored candidate per selection site, i.e. `O(schemes × cascade levels)` +/// times per compressed chunk. +/// +/// Selection semantics: the winner is the candidate with the minimum cost, requiring +/// strictly lower cost both to displace the running best (ties break by scheme registration +/// order) and to beat [`canonical_cost`]. Returning `None` from [`cost`] rejects the +/// candidate outright. +/// +/// See the [module docs](self) for the acceptance axioms that apply to all models. +/// +/// [`canonical_cost`]: CostModel::canonical_cost +/// [`cost`]: CostModel::cost +pub trait CostModel: Debug + Send + Sync + 'static { + /// Estimated cost of choosing this candidate. Lower is better; `None` rejects the + /// candidate. + fn cost(&self, candidate: &Candidate) -> Option; + + /// Cost of leaving the array canonical — the baseline every candidate must strictly + /// beat to be selected. + fn canonical_cost(&self, data: &ArrayAndStats, n_values: u64) -> Cost; +} diff --git a/vortex-compressor/src/cost/size.rs b/vortex-compressor/src/cost/size.rs new file mode 100644 index 00000000000..ae3694b969c --- /dev/null +++ b/vortex-compressor/src/cost/size.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The default, size-only cost model. + +use crate::candidate::Candidate; +use crate::cost::Cost; +use crate::cost::CostModel; +use crate::estimate::EstimateScore; +use crate::stats::ArrayAndStats; + +/// The default cost model: maximize estimated compression ratio. +/// +/// `SizeCost` reproduces the compressor's historical ratio-argmax selection **bit-exactly**: +/// +/// - A candidate is priced at `Cost(-ratio)` over the exact ratio signal the candidate +/// carries (an analytic estimate or a sample-measured `before / after`), so argmin cost is +/// argmax ratio with identical tie behavior. Pricing off the ratio rather than recomputing +/// bytes (`input / ratio`) matters: IEEE division could collapse strict ratio inequalities +/// into cost ties and silently flip winners via the registration-order tie-break. +/// - The canonical baseline is `Cost(-1.0)`, so `cost < canonical_cost` reproduces the +/// historical `ratio > 1.0` validity gate. +/// - Zero-byte sample results and non-finite or subnormal ratios are rejected (`None`), +/// reproducing the historical estimate-validity checks. +#[derive(Debug, Default, Clone, Copy)] +pub struct SizeCost; + +impl CostModel for SizeCost { + fn cost(&self, candidate: &Candidate) -> Option { + match candidate.score { + EstimateScore::FiniteCompression(ratio) => { + (ratio.is_finite() && !ratio.is_subnormal()).then(|| Cost::new(-ratio)) + } + EstimateScore::ZeroBytes => None, + } + } + + fn canonical_cost(&self, _data: &ArrayAndStats, _n_values: u64) -> Cost { + Cost::new(-1.0) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use vortex_array::ArrayRef; + use vortex_array::Canonical; + use vortex_array::ExecutionCtx; + use vortex_array::IntoArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::validity::Validity; + use vortex_buffer::buffer; + use vortex_error::VortexResult; + + use super::*; + use crate::CascadingCompressor; + use crate::ctx::CompressorContext; + use crate::estimate::CompressionEstimate; + use crate::estimate::EstimateVerdict; + use crate::scheme::Scheme; + use crate::stats::GenerateStatsOptions; + + #[derive(Debug)] + struct TestScheme; + + impl Scheme for TestScheme { + fn scheme_name(&self) -> &'static str { + "test.size_cost" + } + + fn matches(&self, _canonical: &Canonical) -> bool { + true + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::Skip) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + fn candidate(score: EstimateScore) -> Candidate { + Candidate { + scheme: &TestScheme, + score, + input_nbytes: 1024, + n_values: 256, + sampled: None, + cascade: Vec::new(), + } + } + + fn canonical_cost() -> Cost { + let array = PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array(); + let data = ArrayAndStats::new(array, GenerateStatsOptions::default()); + SizeCost.canonical_cost(&data, 4) + } + + /// The historical estimate-validity rule that `SizeCost` must reproduce: finite, + /// non-subnormal, and strictly better than canonical (`ratio > 1.0`). + fn legacy_is_valid(ratio: f64) -> bool { + ratio.is_finite() && !ratio.is_subnormal() && ratio > 1.0 + } + + /// Every ratio regime: clear winners, near-1.0 boundaries, exact ties, sub-1.0, + /// non-positive, and non-finite/subnormal garbage. + const RATIOS: &[f64] = &[ + 100.0, + 3.0, + 2.0, + 1.0 + f64::EPSILON, + 1.0, + 1.0 - f64::EPSILON, + 0.5, + 0.0, + -1.0, + f64::MIN_POSITIVE / 2.0, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + ]; + + /// `cost < canonical_cost` must reproduce `legacy_is_valid` exactly, with non-priceable + /// ratios rejected as `None`. + #[test] + fn validity_matches_ratio_gate() { + let canonical = canonical_cost(); + for &ratio in RATIOS { + let cost = SizeCost.cost(&candidate(EstimateScore::FiniteCompression(ratio))); + let valid = cost.is_some_and(|cost| cost < canonical); + assert_eq!( + valid, + legacy_is_valid(ratio), + "validity mismatch for ratio {ratio}" + ); + } + } + + /// For every ordered pair of ratios, the cost ordering must equal the (reversed) ratio + /// ordering the selector historically used: a challenger displaces the best iff its + /// ratio is strictly greater. Equal ratios must produce equal costs (ties keep the + /// incumbent via strict `<`). + #[test] + fn cost_ordering_matches_ratio_ordering() { + let valid: Vec = RATIOS + .iter() + .copied() + .filter(|&r| legacy_is_valid(r)) + .collect(); + for &best in &valid { + for &challenger in &valid { + let best_cost = SizeCost + .cost(&candidate(EstimateScore::FiniteCompression(best))) + .expect("valid ratio must be priceable"); + let challenger_cost = SizeCost + .cost(&candidate(EstimateScore::FiniteCompression(challenger))) + .expect("valid ratio must be priceable"); + + let displaces = challenger_cost < best_cost; + assert_eq!( + displaces, + challenger > best, + "ordering mismatch: challenger {challenger} vs best {best}" + ); + assert_eq!( + challenger_cost == best_cost, + challenger == best, + "tie mismatch: challenger {challenger} vs best {best}" + ); + } + } + } + + #[rstest] + #[case::nan(f64::NAN)] + #[case::infinite(f64::INFINITY)] + #[case::neg_infinite(f64::NEG_INFINITY)] + #[case::subnormal(f64::MIN_POSITIVE / 2.0)] + fn unpriceable_ratios_are_rejected(#[case] ratio: f64) { + assert!( + SizeCost + .cost(&candidate(EstimateScore::FiniteCompression(ratio))) + .is_none() + ); + } + + #[test] + fn zero_bytes_is_rejected() { + assert!( + SizeCost + .cost(&candidate(EstimateScore::ZeroBytes)) + .is_none() + ); + } +} diff --git a/vortex-compressor/src/lib.rs b/vortex-compressor/src/lib.rs index b8db9f7db76..b821c865202 100644 --- a/vortex-compressor/src/lib.rs +++ b/vortex-compressor/src/lib.rs @@ -63,6 +63,7 @@ //! with a short `jq` query. pub mod builtins; +pub mod cost; pub mod ctx; pub mod estimate; pub mod scheme; From 83988a0a47de077c28120b6b866370071b8cde54 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 14:37:05 +0000 Subject: [PATCH 6/8] Wire scheme selection to the cost model; selection becomes argmin cost choose_best_scheme now prices every scored candidate through the compressor's CostModel: the winner is the argmin-cost candidate, and eligibility is cost strictly below CostModel::canonical_cost. Strict < comparisons preserve registration-order tie-breaking exactly as before. The legacy comparators whose logic moved into SizeCost and the selection loop (is_better_score, EstimateScore::{is_valid, beats}) are deleted. CascadingCompressor gains with_cost_model(Arc), holding SizeCost by default, so default behavior is unchanged (bit-exact under SizeCost by decision 1 of the cost-model plan; golden suite shows zero snapshot churn). AlwaysUse short-circuiting, the deferred-callback ratio threshold, and the byte-acceptance gate are untouched. The winner_compress trace span gains estimated_cost alongside estimated_ratio, keeping estimator observability at parity for non-ratio models. WinnerEstimate::Score becomes a struct variant carrying the winning cost; the choose_best_scheme unit tests' match patterns are updated mechanically for the new variant shape. Co-Authored-By: Claude Fable 5 Signed-off-by: Connor Tsui --- vortex-compressor/src/compressor.rs | 96 ++++++++++++++++++++++------- vortex-compressor/src/estimate.rs | 45 +++++--------- vortex-compressor/src/trace.rs | 7 ++- 3 files changed, 95 insertions(+), 53 deletions(-) diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 0ea5b50d472..d11d045b89c 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -3,6 +3,8 @@ //! Cascading array compression implementation. +use std::sync::Arc; + use vortex_array::ArrayRef; use vortex_array::ArraySlots; use vortex_array::Canonical; @@ -36,6 +38,9 @@ use vortex_error::VortexResult; use crate::builtins::IntDictScheme; use crate::candidate::Candidate; use crate::constant; +use crate::cost::Cost; +use crate::cost::CostModel; +use crate::cost::SizeCost; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; use crate::estimate::DeferredEstimate; @@ -43,7 +48,6 @@ use crate::estimate::EstimateScore; use crate::estimate::EstimateVerdict; use crate::estimate::WinnerEstimate; use crate::estimate::estimate_compression_ratio_with_sampling; -use crate::estimate::is_better_score; use crate::scheme::ChildSelection; use crate::scheme::DescendantExclusion; use crate::scheme::Scheme; @@ -91,10 +95,13 @@ pub struct CascadingCompressor { /// Descendant exclusion rules for the compressor's own cascading (e.g. excluding Dict from /// list offsets). root_exclusions: Vec, + + /// The cost model pricing candidates during scheme selection. + cost_model: Arc, } impl CascadingCompressor { - /// Creates a new compressor with the given schemes. + /// Creates a new compressor with the given schemes and the default [`SizeCost`] model. /// /// Root-level exclusion rules (e.g. excluding Dict from list offsets) are built /// automatically. @@ -108,9 +115,18 @@ impl CascadingCompressor { Self { schemes, root_exclusions, + cost_model: Arc::new(SizeCost), } } + /// Replaces the cost model used to price candidates during scheme selection. + /// + /// The default is [`SizeCost`], which maximizes estimated compression ratio. + pub fn with_cost_model(mut self, cost_model: Arc) -> Self { + self.cost_model = cost_model; + self + } + /// Compresses an array using cascading adaptive compression. /// /// First canonicalizes and compacts the array, then applies optimal compression schemes. @@ -343,7 +359,7 @@ impl CascadingCompressor { let actual_ratio = (after_nbytes != 0).then(|| before_nbytes as f64 / after_nbytes as f64); let accepted = after_nbytes < before_nbytes; - trace::record_winner_compress_result(after_nbytes, None, actual_ratio, accepted); + trace::record_winner_compress_result(after_nbytes, None, None, actual_ratio, accepted); return if accepted { Ok(compressed) @@ -384,6 +400,7 @@ impl CascadingCompressor { trace::record_winner_compress_result( after_nbytes, winner_estimate.trace_ratio(), + winner_estimate.trace_cost(), actual_ratio, accepted, ); @@ -398,6 +415,10 @@ impl CascadingCompressor { /// Calls [`expected_compression_ratio`] on each candidate and returns the winning scheme along /// with its resolved winner estimate, or `None` if no scheme beats the canonical encoding. /// + /// Each scored candidate is priced by the compressor's [`CostModel`]; the winner is the + /// candidate with the minimum cost, and only candidates pricing strictly below + /// [`CostModel::canonical_cost`] are eligible. + /// /// Selection runs in two passes. Pass 1 evaluates every immediate /// [`CompressionEstimate::Verdict`] and tracks the running best. [`Scheme`]s returning /// [`CompressionEstimate::Deferred`] are stashed for pass 2 so that we do not make any @@ -407,7 +428,8 @@ impl CascadingCompressor { /// current best [`EstimateScore`] as an early-exit hint so the callback can return /// [`EstimateVerdict::Skip`] without doing expensive work when it cannot beat the threshold. /// - /// Ties are broken by registration order within each pass. + /// Ties are broken by registration order within each pass (displacing the running best + /// requires a strictly lower cost). /// /// [`expected_compression_ratio`]: Scheme::expected_compression_ratio fn choose_best_scheme( @@ -417,9 +439,13 @@ impl CascadingCompressor { compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult> { - let mut best: Option = None; + let mut best: Option<(Cost, Candidate)> = None; let mut deferred: Vec<(&'static dyn Scheme, DeferredEstimate)> = Vec::new(); + let canonical_cost = self + .cost_model + .canonical_cost(data, data.array().len() as u64); + let input_nbytes = data.array().nbytes(); let n_values = data.array().len() as u64; let new_candidate = |scheme, score, sampled| Candidate { @@ -442,9 +468,10 @@ impl CascadingCompressor { } CompressionEstimate::Verdict(EstimateVerdict::Ratio(ratio)) => { let score = EstimateScore::FiniteCompression(ratio); + let candidate = new_candidate(scheme, score, None); - if is_better_score(score, best.as_ref()) { - best = Some(new_candidate(scheme, score, None)); + if let Some(cost) = self.price(&candidate, canonical_cost, best.as_ref()) { + best = Some((cost, candidate)); } } CompressionEstimate::Deferred(deferred_estimate) => { @@ -458,7 +485,8 @@ impl CascadingCompressor { // short-circuit with `Skip` when they cannot beat it. for (scheme, deferred_estimate) in deferred { let _span = trace::scheme_eval_span(scheme.id()).entered(); - let threshold: Option = best.as_ref().map(|candidate| candidate.score); + let threshold: Option = + best.as_ref().map(|(_, candidate)| candidate.score); match deferred_estimate { DeferredEstimate::Sample => { let estimate = estimate_compression_ratio_with_sampling( @@ -468,13 +496,10 @@ impl CascadingCompressor { compress_ctx.clone(), exec_ctx, )?; + let candidate = new_candidate(scheme, estimate.score, Some(estimate.sampled)); - if is_better_score(estimate.score, best.as_ref()) { - best = Some(new_candidate( - scheme, - estimate.score, - Some(estimate.sampled), - )); + if let Some(cost) = self.price(&candidate, canonical_cost, best.as_ref()) { + best = Some((cost, candidate)); } } DeferredEstimate::Callback(callback) => { @@ -485,9 +510,12 @@ impl CascadingCompressor { } EstimateVerdict::Ratio(ratio) => { let score = EstimateScore::FiniteCompression(ratio); + let candidate = new_candidate(scheme, score, None); - if is_better_score(score, best.as_ref()) { - best = Some(new_candidate(scheme, score, None)); + if let Some(cost) = + self.price(&candidate, canonical_cost, best.as_ref()) + { + best = Some((cost, candidate)); } } } @@ -496,8 +524,30 @@ impl CascadingCompressor { } // The sampled arrays carried by candidates are dropped here; only the winning scheme - // and its score survive selection. - Ok(best.map(|candidate| (candidate.scheme, WinnerEstimate::Score(candidate.score)))) + // and its estimate survive selection. + Ok(best.map(|(cost, candidate)| { + ( + candidate.scheme, + WinnerEstimate::Score { + score: candidate.score, + cost, + }, + ) + })) + } + + /// Prices a candidate and returns its cost iff it becomes the new best: the model must + /// price it (`Some`), and the cost must be strictly below both the canonical baseline and + /// the best so far. Strict `<` preserves registration-order tie-breaking. + fn price( + &self, + candidate: &Candidate, + canonical_cost: Cost, + best: Option<&(Cost, Candidate)>, + ) -> Option { + let cost = self.cost_model.cost(candidate)?; + (cost < canonical_cost && best.is_none_or(|&(best_cost, _)| cost < best_cost)) + .then_some(cost) } // TODO(connor): Lots of room for optimization here. @@ -1032,7 +1082,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(2.0)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(2.0), .. })) if scheme.id() == DirectRatioScheme.id() )); Ok(()) @@ -1054,7 +1104,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(3.0)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(3.0), .. })) if scheme.id() == CallbackRatioScheme.id() )); Ok(()) @@ -1076,7 +1126,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(100.0)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(100.0), .. })) if scheme.id() == HugeRatioScheme.id() )); Ok(()) @@ -1098,7 +1148,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(100.0)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(100.0), .. })) if scheme.id() == HugeRatioScheme.id() )); Ok(()) @@ -1326,7 +1376,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(r)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(r), .. })) if scheme.id() == DirectRatioScheme.id() && r == 2.0 )); Ok(()) diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index f1ad74f635c..e195fb6e7ab 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -12,7 +12,7 @@ use vortex_array::IntoArray; use vortex_error::VortexResult; use crate::CascadingCompressor; -use crate::candidate::Candidate; +use crate::cost::Cost; use crate::ctx::CompressorContext; use crate::sample::SAMPLE_SIZE; use crate::sample::sample; @@ -138,27 +138,6 @@ impl EstimateScore { Self::ZeroBytes => None, } } - - /// Returns whether this estimate is eligible to compete. - fn is_valid(self) -> bool { - match self { - Self::FiniteCompression(ratio) => { - ratio.is_finite() && !ratio.is_subnormal() && ratio > 1.0 - } - Self::ZeroBytes => false, - } - } - - /// Returns whether this estimate beats another valid estimate. - fn beats(self, other: Self) -> bool { - match (self, other) { - (Self::ZeroBytes, _) => false, - (Self::FiniteCompression(_), Self::ZeroBytes) => true, - (Self::FiniteCompression(ratio), Self::FiniteCompression(best_ratio)) => { - ratio > best_ratio - } - } - } } /// Winner estimate carried from scheme selection into result tracing. @@ -166,8 +145,13 @@ impl EstimateScore { pub(super) enum WinnerEstimate { /// The scheme must be used immediately. AlwaysUse, - /// The scheme won by a ranked estimate. - Score(EstimateScore), + /// The scheme won by a ranked estimate, priced by the compressor's cost model. + Score { + /// The winning candidate's ranked estimate. + score: EstimateScore, + /// The winning candidate's cost under the compressor's cost model. + cost: Cost, + }, } impl WinnerEstimate { @@ -175,14 +159,17 @@ impl WinnerEstimate { pub(super) fn trace_ratio(self) -> Option { match self { Self::AlwaysUse => None, - Self::Score(score) => score.finite_ratio(), + Self::Score { score, .. } => score.finite_ratio(), } } -} -/// Returns `true` if `score` beats the current best candidate. -pub(super) fn is_better_score(score: EstimateScore, best: Option<&Candidate>) -> bool { - score.is_valid() && best.is_none_or(|candidate| score.beats(candidate.score)) + /// Returns the traceable cost for the winning estimate. + pub(super) fn trace_cost(self) -> Option { + match self { + Self::AlwaysUse => None, + Self::Score { cost, .. } => Some(cost.value()), + } + } } /// A sampling-based estimate: the ranked score together with the compressed sample array it diff --git a/vortex-compressor/src/trace.rs b/vortex-compressor/src/trace.rs index 84027272b31..e6cdffed997 100644 --- a/vortex-compressor/src/trace.rs +++ b/vortex-compressor/src/trace.rs @@ -87,7 +87,7 @@ pub(super) fn zero_byte_sample_result(scheme: SchemeId, sampled_before: u64) { /// Builds a span covering the winning scheme's full-array compression. /// /// `scheme_chosen` and `input_nbytes` are known up front. `compressed_nbytes`, -/// `estimated_ratio`, `achieved_ratio`, and `accepted` are filled in by +/// `estimated_ratio`, `estimated_cost`, `achieved_ratio`, and `accepted` are filled in by /// [`record_winner_compress_result`] once the encode completes. #[inline] pub(super) fn winner_compress_span(scheme: SchemeId, before_nbytes: u64) -> tracing::Span { @@ -98,6 +98,7 @@ pub(super) fn winner_compress_span(scheme: SchemeId, before_nbytes: u64) -> trac input_nbytes = before_nbytes, compressed_nbytes = tracing::field::Empty, estimated_ratio = tracing::field::Empty, + estimated_cost = tracing::field::Empty, achieved_ratio = tracing::field::Empty, accepted = tracing::field::Empty, ) @@ -108,6 +109,7 @@ pub(super) fn winner_compress_span(scheme: SchemeId, before_nbytes: u64) -> trac pub(super) fn record_winner_compress_result( compressed_nbytes: u64, estimated_ratio: Option, + estimated_cost: Option, achieved_ratio: Option, accepted: bool, ) { @@ -116,6 +118,9 @@ pub(super) fn record_winner_compress_result( if let Some(r) = estimated_ratio { span.record("estimated_ratio", r); } + if let Some(c) = estimated_cost { + span.record("estimated_cost", c); + } if let Some(r) = achieved_ratio { span.record("achieved_ratio", r); } From f19c5e7fecd5eb085f25aedac4a6b0fc552ebb0e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 14:42:51 +0000 Subject: [PATCH 7/8] Add CostModel::lower_bound; compressor prunes hopeless deferred work CostModel gains lower_bound(scheme, data) -> Option: a bound on the best cost any candidate from the scheme could achieve. Pass 2 of choose_best_scheme now skips a scheme's deferred sampling or callback outright when the bound is not strictly below the best cost so far. This is the hook that later lets decode-heavy models skip expensive FSST sampling when it cannot win. The default implementation returns None (never prune), which SizeCost keeps, so behavior is unchanged (zero golden snapshot churn). The plan sketched the signature as returning Cost with a never-prunes default, but Cost must be finite and its units are model-defined, so no sentinel 'never prunes' value exists; Option expresses the default honestly. Unit tests cover pruning behind a best candidate and the no-best case where pruning must not fire. Co-Authored-By: Claude Fable 5 Signed-off-by: Connor Tsui --- vortex-compressor/src/compressor.rs | 80 +++++++++++++++++++++++++++++ vortex-compressor/src/cost/mod.rs | 15 ++++++ 2 files changed, 95 insertions(+) diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index d11d045b89c..9c53e1b0359 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -485,6 +485,16 @@ impl CascadingCompressor { // short-circuit with `Skip` when they cannot beat it. for (scheme, deferred_estimate) in deferred { let _span = trace::scheme_eval_span(scheme.id()).entered(); + + // Model-side pruning: skip the deferred work outright when even this scheme's + // best case cannot strictly beat the best candidate so far. + if let Some((best_cost, _)) = best.as_ref() + && let Some(lower_bound) = self.cost_model.lower_bound(scheme.id(), data) + && lower_bound >= *best_cost + { + continue; + } + let threshold: Option = best.as_ref().map(|(_, candidate)| candidate.score); match deferred_estimate { @@ -1382,6 +1392,76 @@ mod tests { Ok(()) } + /// Delegates pricing to [`SizeCost`] but bounds every scheme at a cost that can never + /// win, so all deferred work is pruned as soon as any best candidate exists. + #[derive(Debug)] + struct PruneAllDeferredModel; + + impl CostModel for PruneAllDeferredModel { + fn cost(&self, candidate: &Candidate) -> Option { + SizeCost.cost(candidate) + } + + fn canonical_cost(&self, data: &ArrayAndStats, n_values: u64) -> Cost { + SizeCost.canonical_cost(data, n_values) + } + + fn lower_bound(&self, _scheme: SchemeId, _data: &ArrayAndStats) -> Option { + Some(Cost::new(f64::MAX)) + } + } + + #[test] + fn lower_bound_prunes_deferred_work_behind_a_best() -> VortexResult<()> { + // `CallbackRatioScheme`'s deferred `Ratio(3.0)` would beat `DirectRatioScheme`'s + // immediate `Ratio(2.0)` (see `callback_ratio_competes_numerically`), but the model + // bounds it above the running best, so its deferred work is pruned. + let compressor = CascadingCompressor::new(vec![&DirectRatioScheme, &CallbackRatioScheme]) + .with_cost_model(Arc::new(PruneAllDeferredModel)); + let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CallbackRatioScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + let winner = compressor.choose_best_scheme( + &schemes, + &data, + CompressorContext::new(), + &mut exec_ctx, + )?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(2.0), .. })) + if scheme.id() == DirectRatioScheme.id() + )); + Ok(()) + } + + #[test] + fn lower_bound_never_prunes_without_a_best() -> VortexResult<()> { + // With no best candidate there is nothing to prune against, so the deferred + // callback still runs even under a prune-everything bound. + let compressor = CascadingCompressor::new(vec![&CallbackRatioScheme]) + .with_cost_model(Arc::new(PruneAllDeferredModel)); + let schemes: [&'static dyn Scheme; 1] = [&CallbackRatioScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + let winner = compressor.choose_best_scheme( + &schemes, + &data, + CompressorContext::new(), + &mut exec_ctx, + )?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(3.0), .. })) + if scheme.id() == CallbackRatioScheme.id() + )); + Ok(()) + } + #[test] fn all_null_array_compresses_to_constant() -> VortexResult<()> { let array = PrimitiveArray::new( diff --git a/vortex-compressor/src/cost/mod.rs b/vortex-compressor/src/cost/mod.rs index 0f4e5c9ce75..049659b045f 100644 --- a/vortex-compressor/src/cost/mod.rs +++ b/vortex-compressor/src/cost/mod.rs @@ -42,6 +42,7 @@ use std::fmt::Debug; pub use size::SizeCost; pub use crate::candidate::Candidate; +use crate::scheme::SchemeId; use crate::stats::ArrayAndStats; /// An opaque, totally ordered cost. Lower is better. @@ -95,4 +96,18 @@ pub trait CostModel: Debug + Send + Sync + 'static { /// Cost of leaving the array canonical — the baseline every candidate must strictly /// beat to be selected. fn canonical_cost(&self, data: &ArrayAndStats, n_values: u64) -> Cost; + + /// A lower bound on the cost any candidate from `scheme` could possibly achieve on + /// `data`, or `None` (the default) if the model offers no bound. + /// + /// The compressor skips a scheme's deferred estimation work (sampling, callbacks) + /// outright when the bound is not strictly below the best cost observed so far, since + /// even the scheme's best case could not win selection. Models that can bound a + /// scheme's payoff from cheap facts (e.g. minimum plausible output bytes at zero decode + /// cost) can prune expensive sampling here; a sound bound never exceeds the cost of any + /// candidate the scheme could actually produce. + fn lower_bound(&self, scheme: SchemeId, data: &ArrayAndStats) -> Option { + let _ = (scheme, data); + None + } } From d7a3b3ac84dab26c5945f64466658f280049dda4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 14:51:08 +0000 Subject: [PATCH 8/8] Replace EstimateFn's ratio threshold with the SkipThreshold handle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING (changelog/break): the EstimateFn callback signature changes — the Option best-so-far parameter becomes a SkipThreshold handle owned by the compressor. Migration is mechanical: replace best_so_far.and_then(EstimateScore::finite_ratio) + 'max_ratio <= t' with threshold.best_case_ratio_cannot_win(max_ratio); the handle also exposes the old ratio view via best_ratio(). The handle wraps the best cost so far, the compressor's cost model, and the selection-site facts. best_case_ratio_cannot_win prices the calling scheme's hypothetical best-case candidate under the model and compares it against the best cost, so scheme-side early exits now speak cost rather than hardcoding ratio comparisons. Under the default SizeCost it reduces to exactly the historical 'max_ratio <= best_ratio' skip (pinned by unit tests across tie and one-ulp boundaries, and through the compressor-built handle), so behavior is unchanged: zero golden snapshot churn across all three variants. Delta and Sequence — the two callback users — migrate to the handle with identical max-ratio expressions (full_width * DELTA_PENALTY and len / 2), and gain end-to-end selection tests on both sides of their thresholds: winning over a low incumbent and losing at the exact best-case tie, plus Delta's min_ratio skip. Co-Authored-By: Claude Fable 5 Signed-off-by: Connor Tsui --- vortex-btrblocks/src/schemes/integer/delta.rs | 133 +++++++++++++++++- .../src/schemes/integer/sequence.rs | 109 +++++++++++++- vortex-compressor/src/compressor.rs | 118 ++++++++++++---- vortex-compressor/src/cost/size.rs | 52 +++++++ vortex-compressor/src/estimate.rs | 117 +++++++++++++-- 5 files changed, 483 insertions(+), 46 deletions(-) diff --git a/vortex-btrblocks/src/schemes/integer/delta.rs b/vortex-btrblocks/src/schemes/integer/delta.rs index f25bc6ce466..bcb3076a7b6 100644 --- a/vortex-btrblocks/src/schemes/integer/delta.rs +++ b/vortex-btrblocks/src/schemes/integer/delta.rs @@ -14,7 +14,6 @@ use vortex_compressor::builtins::IntDictScheme; use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::estimate::CompressionEstimate; use vortex_compressor::estimate::DeferredEstimate; -use vortex_compressor::estimate::EstimateScore; use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; @@ -133,14 +132,13 @@ impl Scheme for DeltaScheme { // delta-encodes the array and measures the residual range. let min_ratio = self.min_ratio; CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - move |_compressor, data, best_so_far, _ctx, exec_ctx| { + move |_compressor, data, threshold, _ctx, exec_ctx| { let primitive = data.array().clone().execute::(exec_ctx)?; let full_width = primitive.ptype().bit_width() as f64; // Delta's best case is residuals collapsing to a single bit. If even that, after // the penalty, can't beat the incumbent, skip before doing the encode work. - let threshold = best_so_far.and_then(EstimateScore::finite_ratio); - if threshold.is_some_and(|t| full_width * DELTA_PENALTY <= t) { + if threshold.best_case_ratio_cannot_win(full_width * DELTA_PENALTY) { return Ok(EstimateVerdict::Skip); } @@ -197,3 +195,130 @@ impl Scheme for DeltaScheme { Delta::try_new(compressed_bases, compressed_deltas, 0, len).map(IntoArray::into_array) } } + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use rand::RngExt; + use rand::SeedableRng; + use rand::rngs::StdRng; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Constant; + use vortex_array::arrays::ConstantArray; + use vortex_array::dtype::Nullability; + use vortex_array::scalar::Scalar; + use vortex_array::validity::Validity; + use vortex_buffer::Buffer; + use vortex_session::VortexSession; + + use super::*; + + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); + + /// Immediate fixed-ratio competitor; its `compress` emits a tiny constant array so the + /// winner is observable from the output encoding. + #[derive(Debug)] + struct FixedRatioScheme { + ratio: f64, + } + + impl Scheme for FixedRatioScheme { + fn scheme_name(&self) -> &'static str { + "test.fixed_ratio" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::Ratio(self.ratio)) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(ConstantArray::new( + Scalar::primitive(0u64, Nullability::NonNullable), + data.array_len(), + ) + .into_array()) + } + } + + /// Near-monotone u64 data: Delta's residual span is a few bits, so its real penalized + /// ratio is comfortably above 2.0 but nowhere near its 64-bit best case. + fn monotone_jitter_u64() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(42); + let mut value = 1_700_000_000_000u64; + let values: Buffer = (0..4096) + .map(|_| { + value += 900 + rng.random_range(0..200); + value + }) + .collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() + } + + /// With the incumbent below Delta's achievable ratio, the callback must proceed past + /// the best-case threshold check and win with its measured estimate. + #[test] + fn delta_wins_over_low_threshold() -> VortexResult<()> { + static COMPETITOR: FixedRatioScheme = FixedRatioScheme { ratio: 2.0 }; + static DELTA: DeltaScheme = DeltaScheme::new(1.25); + let compressor = CascadingCompressor::new(vec![&COMPETITOR, &DELTA]); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&monotone_jitter_u64(), &mut exec_ctx)?; + + assert!(compressed.is::()); + Ok(()) + } + + /// With the incumbent at exactly Delta's best case (`full_width * DELTA_PENALTY`, the + /// same expression the callback computes), Delta must not be chosen — the same decision + /// the pre-threshold-handle `max_ratio <= best` skip produced on this input. + #[test] + fn delta_loses_at_best_case_tie() -> VortexResult<()> { + static COMPETITOR: FixedRatioScheme = FixedRatioScheme { + ratio: 64.0 * DELTA_PENALTY, + }; + static DELTA: DeltaScheme = DeltaScheme::new(1.25); + let compressor = CascadingCompressor::new(vec![&COMPETITOR, &DELTA]); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&monotone_jitter_u64(), &mut exec_ctx)?; + + assert!(compressed.is::()); + Ok(()) + } + + /// Wide random residuals put Delta's penalized ratio below its `min_ratio`, so with no + /// competitor the array must stay canonical. + #[test] + fn delta_skips_below_min_ratio() -> VortexResult<()> { + static DELTA: DeltaScheme = DeltaScheme::new(1.25); + let compressor = CascadingCompressor::new(vec![&DELTA]); + + let mut rng = StdRng::seed_from_u64(43); + let values: Buffer = (0..4096).map(|_| rng.random::()).collect(); + let array = PrimitiveArray::new(values, Validity::NonNullable).into_array(); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&array, &mut exec_ctx)?; + + assert!(!compressed.is::()); + Ok(()) + } +} diff --git a/vortex-btrblocks/src/schemes/integer/sequence.rs b/vortex-btrblocks/src/schemes/integer/sequence.rs index 604d7567d00..cbebc242cce 100644 --- a/vortex-btrblocks/src/schemes/integer/sequence.rs +++ b/vortex-btrblocks/src/schemes/integer/sequence.rs @@ -12,7 +12,6 @@ use vortex_compressor::builtins::IntDictScheme; use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::estimate::CompressionEstimate; use vortex_compressor::estimate::DeferredEstimate; -use vortex_compressor::estimate::EstimateScore; use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; @@ -94,7 +93,7 @@ impl Scheme for SequenceScheme { // TODO(connor): `sequence_encode` allocates the encoded array just to confirm feasibility. // A cheaper `is_sequence` probe would let us skip the allocation entirely. CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, best_so_far, _ctx, exec_ctx| { + |_compressor, data, threshold, _ctx, exec_ctx| { // `SequenceArray` stores exactly two scalars (base and multiplier), so the best // achievable compression ratio is `array_len / 2`. let compressed_size = 2usize; @@ -102,8 +101,7 @@ impl Scheme for SequenceScheme { // If we cannot beat the best so far, then we do not want to even try sequence // encoding the data. - let threshold = best_so_far.and_then(EstimateScore::finite_ratio); - if threshold.is_some_and(|t| max_ratio <= t) { + if threshold.best_case_ratio_cannot_win(max_ratio) { return Ok(EstimateVerdict::Skip); } @@ -134,3 +132,106 @@ impl Scheme for SequenceScheme { .ok_or_else(|| vortex_err!("cannot sequence encode array")) } } + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use vortex_array::ExecutionCtx; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Constant; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::dtype::Nullability; + use vortex_array::scalar::Scalar; + use vortex_array::validity::Validity; + use vortex_buffer::Buffer; + use vortex_compressor::estimate::EstimateVerdict; + use vortex_sequence::Sequence; + use vortex_session::VortexSession; + + use super::*; + + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); + + /// Number of values in the test arrays; Sequence's best case is `LEN / 2`. + const LEN: usize = 2048; + + /// Immediate fixed-ratio competitor; its `compress` emits a tiny constant array so the + /// winner is observable from the output encoding. + #[derive(Debug)] + struct FixedRatioScheme { + ratio: f64, + } + + impl Scheme for FixedRatioScheme { + fn scheme_name(&self) -> &'static str { + "test.fixed_ratio" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::Ratio(self.ratio)) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(ConstantArray::new( + Scalar::primitive(0i64, Nullability::NonNullable), + data.array_len(), + ) + .into_array()) + } + } + + /// An exact arithmetic sequence, encodable by [`SequenceScheme`]. + fn arithmetic_sequence() -> ArrayRef { + let values: Buffer = (0..LEN as i64).map(|i| 10_000 + 7 * i).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() + } + + /// With the incumbent below Sequence's best case, the callback must proceed past the + /// threshold check, trial-encode, and win. + #[test] + fn sequence_wins_over_low_threshold() -> VortexResult<()> { + static COMPETITOR: FixedRatioScheme = FixedRatioScheme { ratio: 2.0 }; + let compressor = CascadingCompressor::new(vec![&COMPETITOR, &SequenceScheme]); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&arithmetic_sequence(), &mut exec_ctx)?; + + assert!(compressed.is::()); + Ok(()) + } + + /// With the incumbent at exactly Sequence's best case (`LEN / 2`), Sequence must not be + /// chosen — the same decision the pre-threshold-handle `max_ratio <= best` skip produced + /// on this input. + #[test] + fn sequence_loses_at_best_case_tie() -> VortexResult<()> { + static COMPETITOR: FixedRatioScheme = FixedRatioScheme { + ratio: LEN as f64 / 2.0, + }; + let compressor = CascadingCompressor::new(vec![&COMPETITOR, &SequenceScheme]); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&arithmetic_sequence(), &mut exec_ctx)?; + + assert!(compressed.is::()); + Ok(()) + } +} diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 9c53e1b0359..9e71fbbb615 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -46,6 +46,7 @@ use crate::estimate::CompressionEstimate; use crate::estimate::DeferredEstimate; use crate::estimate::EstimateScore; use crate::estimate::EstimateVerdict; +use crate::estimate::SkipThreshold; use crate::estimate::WinnerEstimate; use crate::estimate::estimate_compression_ratio_with_sampling; use crate::scheme::ChildSelection; @@ -424,9 +425,9 @@ impl CascadingCompressor { /// [`CompressionEstimate::Deferred`] are stashed for pass 2 so that we do not make any /// expensive computations if we don't have to. /// - /// Pass 2 evaluates the deferred work and, for each [`DeferredEstimate::Callback`], passes the - /// current best [`EstimateScore`] as an early-exit hint so the callback can return - /// [`EstimateVerdict::Skip`] without doing expensive work when it cannot beat the threshold. + /// Pass 2 evaluates the deferred work and, for each [`DeferredEstimate::Callback`], passes a + /// [`SkipThreshold`] wrapping the current best as an early-exit hint so the callback can + /// return [`EstimateVerdict::Skip`] without doing expensive work when it cannot win. /// /// Ties are broken by registration order within each pass (displacing the running best /// requires a strictly lower cost). @@ -495,8 +496,6 @@ impl CascadingCompressor { continue; } - let threshold: Option = - best.as_ref().map(|(_, candidate)| candidate.score); match deferred_estimate { DeferredEstimate::Sample => { let estimate = estimate_compression_ratio_with_sampling( @@ -513,6 +512,15 @@ impl CascadingCompressor { } } DeferredEstimate::Callback(callback) => { + let threshold = SkipThreshold::new( + best.as_ref() + .map(|(cost, candidate)| (*cost, candidate.score)), + Arc::clone(&self.cost_model), + scheme, + input_nbytes, + n_values, + compress_ctx.cascade_history().to_vec(), + ); match callback(self, data, threshold, compress_ctx.clone(), exec_ctx)? { EstimateVerdict::Skip => {} EstimateVerdict::AlwaysUse => { @@ -834,7 +842,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::AlwaysUse), + |_compressor, _data, _threshold, _ctx, _exec_ctx| Ok(EstimateVerdict::AlwaysUse), ))) } @@ -868,7 +876,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Skip), + |_compressor, _data, _threshold, _ctx, _exec_ctx| Ok(EstimateVerdict::Skip), ))) } @@ -902,7 +910,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Ratio(3.0)), + |_compressor, _data, _threshold, _ctx, _exec_ctx| Ok(EstimateVerdict::Ratio(3.0)), ))) } @@ -1182,11 +1190,11 @@ mod tests { Ok(()) } - // Observer helper used by threshold-related tests. Captures the `best_so_far` value the - // compressor passes to its deferred callback. `OBSERVER_LOCK` serializes tests that share - // `OBSERVED_THRESHOLD` so they do not race. + // Observer helper used by threshold-related tests. Captures the threshold handle's + // ratio view as the compressor passes it to a deferred callback. `OBSERVER_LOCK` + // serializes tests that share `OBSERVED_THRESHOLD` so they do not race. static OBSERVER_LOCK: Mutex<()> = Mutex::new(()); - static OBSERVED_THRESHOLD: Mutex>> = Mutex::new(None); + static OBSERVED_THRESHOLD: Mutex>> = Mutex::new(None); #[derive(Debug)] struct ThresholdObservingScheme; @@ -1207,8 +1215,53 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, best_so_far, _ctx, _exec_ctx| { - *OBSERVED_THRESHOLD.lock() = Some(best_so_far); + |_compressor, _data, threshold, _ctx, _exec_ctx| { + *OBSERVED_THRESHOLD.lock() = Some(threshold.best_ratio()); + Ok(EstimateVerdict::Skip) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + /// Records the answers the compressor-built [`SkipThreshold`] gives at, just above, and + /// below the incumbent ratio. Shares `OBSERVER_LOCK` with the other observing tests. + static OBSERVED_CANNOT_WIN: Mutex> = Mutex::new(None); + + #[derive(Debug)] + struct CannotWinObservingScheme; + + impl Scheme for CannotWinObservingScheme { + fn scheme_name(&self) -> &'static str { + "test.cannot_win_observing" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, _data, threshold, _ctx, _exec_ctx| { + *OBSERVED_CANNOT_WIN.lock() = Some(( + threshold.best_case_ratio_cannot_win(2.0), + threshold.best_case_ratio_cannot_win(2.0f64.next_up()), + threshold.best_case_ratio_cannot_win(1.9), + )); Ok(EstimateVerdict::Skip) }, ))) @@ -1225,6 +1278,27 @@ mod tests { } } + /// The compressor-built threshold must reproduce the historical `max_ratio <= best` + /// skip rule exactly: a best case that ties or loses to the incumbent cannot win, one + /// that is strictly greater can. + #[test] + fn threshold_cannot_win_matches_ratio_rule() -> VortexResult<()> { + let _guard = OBSERVER_LOCK.lock(); + *OBSERVED_CANNOT_WIN.lock() = None; + + let compressor = + CascadingCompressor::new(vec![&DirectRatioScheme, &CannotWinObservingScheme]); + let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CannotWinObservingScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; + + // Incumbent ratio is 2.0: a tie cannot win, the next float up can, a loss cannot. + assert_eq!(*OBSERVED_CANNOT_WIN.lock(), Some((true, false, true))); + Ok(()) + } + #[derive(Debug)] struct CallbackMatchingRatioScheme; @@ -1244,7 +1318,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Ratio(2.0)), + |_compressor, _data, _threshold, _ctx, _exec_ctx| Ok(EstimateVerdict::Ratio(2.0)), ))) } @@ -1298,10 +1372,7 @@ mod tests { compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; let observed = *OBSERVED_THRESHOLD.lock(); - assert!(matches!( - observed, - Some(Some(EstimateScore::FiniteCompression(r))) if r == 2.0 - )); + assert_eq!(observed, Some(Some(2.0))); Ok(()) } @@ -1319,8 +1390,8 @@ mod tests { compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; - // The observing callback was invoked (outer `Some`) and `best_so_far` was `None` (inner - // `None`) because the zero-byte sample is never stored as the best. + // The observing callback was invoked (outer `Some`) and the threshold's ratio view + // was `None` (inner `None`) because the zero-byte sample is never stored as the best. let observed = *OBSERVED_THRESHOLD.lock(); assert_eq!(observed, Some(None)); Ok(()) @@ -1359,10 +1430,7 @@ mod tests { compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; let observed = *OBSERVED_THRESHOLD.lock(); - assert!(matches!( - observed, - Some(Some(EstimateScore::FiniteCompression(r))) if r == 3.0 - )); + assert_eq!(observed, Some(Some(3.0))); Ok(()) } diff --git a/vortex-compressor/src/cost/size.rs b/vortex-compressor/src/cost/size.rs index ae3694b969c..00a71d6ad6f 100644 --- a/vortex-compressor/src/cost/size.rs +++ b/vortex-compressor/src/cost/size.rs @@ -205,4 +205,56 @@ mod tests { .is_none() ); } + + /// Under `SizeCost`, [`SkipThreshold::best_case_ratio_cannot_win`] must reduce to + /// exactly the historical `max_ratio <= best_ratio` skip rule, including exact ties and + /// one-ulp boundaries; unpriceable best cases must never skip. + #[test] + fn skip_threshold_reduces_to_ratio_comparison() { + use std::sync::Arc; + + use crate::estimate::SkipThreshold; + + for &best_ratio in &[1.5, 2.0, 3.0, 60.8, 1024.0] { + let threshold = SkipThreshold::new( + Some(( + Cost::new(-best_ratio), + EstimateScore::FiniteCompression(best_ratio), + )), + Arc::new(SizeCost), + &TestScheme, + 1024, + 256, + Vec::new(), + ); + assert_eq!(threshold.best_ratio(), Some(best_ratio)); + + let max_ratios = [ + 0.5, + 1.0, + best_ratio.next_down(), + best_ratio, + best_ratio.next_up(), + best_ratio * 2.0, + ]; + for max_ratio in max_ratios { + assert_eq!( + threshold.best_case_ratio_cannot_win(max_ratio), + max_ratio <= best_ratio, + "mismatch for max_ratio {max_ratio} vs best {best_ratio}" + ); + } + + // Best cases SizeCost cannot price never skip: the callback proceeds and its + // real estimate is rejected later, exactly as before. + assert!(!threshold.best_case_ratio_cannot_win(f64::NAN)); + assert!(!threshold.best_case_ratio_cannot_win(f64::INFINITY)); + } + + // Without a best candidate there is no threshold to lose to. + let no_best = + SkipThreshold::new(None, Arc::new(SizeCost), &TestScheme, 1024, 256, Vec::new()); + assert!(!no_best.best_case_ratio_cannot_win(1e9)); + assert_eq!(no_best.best_ratio(), None); + } } diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index e195fb6e7ab..df832f5d743 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -4,6 +4,7 @@ //! Compression ratio estimation types and sampling-based estimation. use std::fmt; +use std::sync::Arc; use vortex_array::ArrayRef; use vortex_array::Canonical; @@ -12,39 +13,129 @@ use vortex_array::IntoArray; use vortex_error::VortexResult; use crate::CascadingCompressor; +use crate::candidate::Candidate; use crate::cost::Cost; +use crate::cost::CostModel; use crate::ctx::CompressorContext; use crate::sample::SAMPLE_SIZE; use crate::sample::sample; use crate::sample::sample_count_approx_one_percent; use crate::scheme::Scheme; use crate::scheme::SchemeExt; +use crate::scheme::SchemeId; use crate::stats::ArrayAndStats; use crate::trace; /// Closure type for [`DeferredEstimate::Callback`]. /// -/// The compressor calls this with the same arguments it would pass to sampling, plus the best -/// [`EstimateScore`] observed so far (if any). The closure must resolve directly to a terminal -/// [`EstimateVerdict`]. +/// The compressor calls this with the same arguments it would pass to sampling, plus a +/// [`SkipThreshold`] handle wrapping the best candidate observed so far. The closure must +/// resolve directly to a terminal [`EstimateVerdict`]. /// -/// The `best_so_far` threshold is an early-exit hint. If your scheme's maximum achievable -/// compression ratio is not strictly greater than -/// `best_so_far.and_then(EstimateScore::finite_ratio)`, you should return -/// [`EstimateVerdict::Skip`]. Returning a ratio equal to the threshold is permitted but will -/// lose to the prior best due to strict `>` tie-breaking in the selector. Use the threshold -/// only as an early-exit hint, never to perform additional work. +/// The threshold is an early-exit hint. If your scheme knows its maximum achievable +/// compression ratio, ask [`SkipThreshold::best_case_ratio_cannot_win`] before doing +/// expensive work and return [`EstimateVerdict::Skip`] when it answers `true`. Returning an +/// estimate that merely ties the threshold is permitted but will lose to the prior best, +/// since displacing the running best requires a strictly better (lower-cost) candidate. Use +/// the threshold only as an early-exit hint, never to perform additional work. #[rustfmt::skip] pub type EstimateFn = dyn FnOnce( &CascadingCompressor, &ArrayAndStats, - Option, + SkipThreshold, CompressorContext, &mut ExecutionCtx, ) -> VortexResult + Send + Sync; +/// Early-exit threshold handle passed to [`DeferredEstimate::Callback`] closures. +/// +/// Owned and constructed by the compressor during selection; wraps the best candidate +/// observed so far together with the compressor's [`CostModel`] and the selection-site +/// facts needed to price a hypothetical best-case candidate. Schemes keep their side of the +/// bargain — knowing their own best case — and ask the handle whether that best case could +/// still win. +pub struct SkipThreshold { + /// The best candidate so far: its cost and its ranked estimate. `None` when no + /// candidate has been stored yet. + best: Option<(Cost, EstimateScore)>, + + /// The compressor's cost model. + model: Arc, + + /// The scheme whose deferred estimate is being resolved. + scheme: &'static dyn Scheme, + + /// Uncompressed size in bytes of the array under selection. + input_nbytes: u64, + + /// Number of values in the array under selection. + n_values: u64, + + /// The cascade ancestry `(scheme_id, child_index)` of the selection site. + cascade: Vec<(SchemeId, usize)>, +} + +impl SkipThreshold { + /// Creates a threshold handle for `scheme` at a selection site. + /// + /// Normally the compressor builds these during selection. The constructor is public so + /// scheme implementors can unit-test their callbacks' skip decisions. + pub fn new( + best: Option<(Cost, EstimateScore)>, + model: Arc, + scheme: &'static dyn Scheme, + input_nbytes: u64, + n_values: u64, + cascade: Vec<(SchemeId, usize)>, + ) -> Self { + Self { + best, + model, + scheme, + input_nbytes, + n_values, + cascade, + } + } + + /// Returns `true` if a candidate achieving `max_ratio` — the best case the calling + /// scheme could possibly report — still could not displace the best candidate observed + /// so far, so the scheme should return [`EstimateVerdict::Skip`] without doing expensive + /// work. + /// + /// The handle prices a hypothetical candidate carrying `max_ratio` under the + /// compressor's cost model and compares it against the best cost so far; displacement + /// requires a strictly lower cost. Under the default [`SizeCost`] model this reduces to + /// exactly the historical `max_ratio <= best_ratio` skip. Returns `false` when there is + /// no best candidate yet or the best case cannot be priced. + /// + /// [`SizeCost`]: crate::cost::SizeCost + pub fn best_case_ratio_cannot_win(&self, max_ratio: f64) -> bool { + let Some((best_cost, _)) = self.best else { + return false; + }; + let best_case = Candidate { + scheme: self.scheme, + score: EstimateScore::FiniteCompression(max_ratio), + input_nbytes: self.input_nbytes, + n_values: self.n_values, + sampled: None, + cascade: self.cascade.clone(), + }; + self.model + .cost(&best_case) + .is_some_and(|cost| cost >= best_cost) + } + + /// The best finite compression ratio observed so far, if any — the pre-cost-model view + /// of the threshold, kept for callbacks that reason in ratio space. + pub fn best_ratio(&self) -> Option { + self.best.and_then(|(_, score)| score.finite_ratio()) + } +} + /// The result of a [`Scheme`]'s compression ratio estimation. /// /// This type is returned by [`Scheme::expected_compression_ratio`] to tell the compressor how @@ -97,9 +188,9 @@ pub enum DeferredEstimate { /// it cannot request more sampling or another deferred callback. /// /// The compressor evaluates all immediate [`CompressionEstimate::Verdict`] results before - /// invoking any deferred callback, and passes the best [`EstimateScore`] observed so far to - /// the callback. This lets the callback return [`EstimateVerdict::Skip`] without performing - /// expensive work when its maximum achievable ratio cannot beat the current best. See + /// invoking any deferred callback, and passes a [`SkipThreshold`] wrapping the best + /// candidate observed so far. This lets the callback return [`EstimateVerdict::Skip`] + /// without performing expensive work when its maximum achievable estimate cannot win. See /// [`EstimateFn`] for the full contract. Callback(Box), }