diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 372e2639bb0..23af8e2c5ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -452,6 +452,38 @@ jobs: alert-title: "Rust tests (linux-arm64) failed on develop" deduplication-key: ci-rust-test-linux-arm64-failure + btrblocks-golden: + name: "Rust (btrblocks golden corpus)" + timeout-minutes: 10 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=btrblocks-golden', github.run_id) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + # The golden suite pins the compressor's decisions per feature variant, and the + # variants are mutually exclusive at compile time: the `default` variant only exists + # without `unstable_encodings` (which changes ALL_SCHEMES), so the `--all-features` + # workspace test jobs cannot run it. Run each feature combination explicitly. + - name: Golden corpus (default features) + run: | + cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden + - name: Golden corpus (unstable_encodings) + run: | + cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden \ + --features unstable_encodings + - name: Golden corpus (compact, unstable_encodings + zstd + pco) + run: | + cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden \ + --features unstable_encodings,zstd,pco + build-java: name: "Java" runs-on: >- diff --git a/Cargo.lock b/Cargo.lock index 16df2a942c1..7a3b82f3dce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9928,6 +9928,7 @@ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ "codspeed-divan-compat", + "insta", "itertools 0.14.0", "num-traits", "pco", diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 5ee2bad5402..529c6d90bad 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -41,6 +41,7 @@ vortex-zstd = { workspace = true, optional = true } [dev-dependencies] divan = { workspace = true } +insta = { workspace = true } rstest = { workspace = true } test-with = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/vortex-btrblocks/REUSE.toml b/vortex-btrblocks/REUSE.toml new file mode 100644 index 00000000000..6bee2b6904f --- /dev/null +++ b/vortex-btrblocks/REUSE.toml @@ -0,0 +1,7 @@ +version = 1 + +# `insta` snapshot files do not allow leading comment lines. +[[annotations]] +path = "tests/snapshots/**.snap" +SPDX-FileCopyrightText = "Copyright the Vortex contributors" +SPDX-License-Identifier = "Apache-2.0" diff --git a/vortex-btrblocks/src/schemes/integer/delta.rs b/vortex-btrblocks/src/schemes/integer/delta.rs index f25bc6ce466..bcb3076a7b6 100644 --- a/vortex-btrblocks/src/schemes/integer/delta.rs +++ b/vortex-btrblocks/src/schemes/integer/delta.rs @@ -14,7 +14,6 @@ use vortex_compressor::builtins::IntDictScheme; use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::estimate::CompressionEstimate; use vortex_compressor::estimate::DeferredEstimate; -use vortex_compressor::estimate::EstimateScore; use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; @@ -133,14 +132,13 @@ impl Scheme for DeltaScheme { // delta-encodes the array and measures the residual range. let min_ratio = self.min_ratio; CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - move |_compressor, data, best_so_far, _ctx, exec_ctx| { + move |_compressor, data, threshold, _ctx, exec_ctx| { let primitive = data.array().clone().execute::(exec_ctx)?; let full_width = primitive.ptype().bit_width() as f64; // Delta's best case is residuals collapsing to a single bit. If even that, after // the penalty, can't beat the incumbent, skip before doing the encode work. - let threshold = best_so_far.and_then(EstimateScore::finite_ratio); - if threshold.is_some_and(|t| full_width * DELTA_PENALTY <= t) { + if threshold.best_case_ratio_cannot_win(full_width * DELTA_PENALTY) { return Ok(EstimateVerdict::Skip); } @@ -197,3 +195,130 @@ impl Scheme for DeltaScheme { Delta::try_new(compressed_bases, compressed_deltas, 0, len).map(IntoArray::into_array) } } + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use rand::RngExt; + use rand::SeedableRng; + use rand::rngs::StdRng; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Constant; + use vortex_array::arrays::ConstantArray; + use vortex_array::dtype::Nullability; + use vortex_array::scalar::Scalar; + use vortex_array::validity::Validity; + use vortex_buffer::Buffer; + use vortex_session::VortexSession; + + use super::*; + + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); + + /// Immediate fixed-ratio competitor; its `compress` emits a tiny constant array so the + /// winner is observable from the output encoding. + #[derive(Debug)] + struct FixedRatioScheme { + ratio: f64, + } + + impl Scheme for FixedRatioScheme { + fn scheme_name(&self) -> &'static str { + "test.fixed_ratio" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::Ratio(self.ratio)) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(ConstantArray::new( + Scalar::primitive(0u64, Nullability::NonNullable), + data.array_len(), + ) + .into_array()) + } + } + + /// Near-monotone u64 data: Delta's residual span is a few bits, so its real penalized + /// ratio is comfortably above 2.0 but nowhere near its 64-bit best case. + fn monotone_jitter_u64() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(42); + let mut value = 1_700_000_000_000u64; + let values: Buffer = (0..4096) + .map(|_| { + value += 900 + rng.random_range(0..200); + value + }) + .collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() + } + + /// With the incumbent below Delta's achievable ratio, the callback must proceed past + /// the best-case threshold check and win with its measured estimate. + #[test] + fn delta_wins_over_low_threshold() -> VortexResult<()> { + static COMPETITOR: FixedRatioScheme = FixedRatioScheme { ratio: 2.0 }; + static DELTA: DeltaScheme = DeltaScheme::new(1.25); + let compressor = CascadingCompressor::new(vec![&COMPETITOR, &DELTA]); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&monotone_jitter_u64(), &mut exec_ctx)?; + + assert!(compressed.is::()); + Ok(()) + } + + /// With the incumbent at exactly Delta's best case (`full_width * DELTA_PENALTY`, the + /// same expression the callback computes), Delta must not be chosen — the same decision + /// the pre-threshold-handle `max_ratio <= best` skip produced on this input. + #[test] + fn delta_loses_at_best_case_tie() -> VortexResult<()> { + static COMPETITOR: FixedRatioScheme = FixedRatioScheme { + ratio: 64.0 * DELTA_PENALTY, + }; + static DELTA: DeltaScheme = DeltaScheme::new(1.25); + let compressor = CascadingCompressor::new(vec![&COMPETITOR, &DELTA]); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&monotone_jitter_u64(), &mut exec_ctx)?; + + assert!(compressed.is::()); + Ok(()) + } + + /// Wide random residuals put Delta's penalized ratio below its `min_ratio`, so with no + /// competitor the array must stay canonical. + #[test] + fn delta_skips_below_min_ratio() -> VortexResult<()> { + static DELTA: DeltaScheme = DeltaScheme::new(1.25); + let compressor = CascadingCompressor::new(vec![&DELTA]); + + let mut rng = StdRng::seed_from_u64(43); + let values: Buffer = (0..4096).map(|_| rng.random::()).collect(); + let array = PrimitiveArray::new(values, Validity::NonNullable).into_array(); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&array, &mut exec_ctx)?; + + assert!(!compressed.is::()); + Ok(()) + } +} diff --git a/vortex-btrblocks/src/schemes/integer/sequence.rs b/vortex-btrblocks/src/schemes/integer/sequence.rs index 604d7567d00..cbebc242cce 100644 --- a/vortex-btrblocks/src/schemes/integer/sequence.rs +++ b/vortex-btrblocks/src/schemes/integer/sequence.rs @@ -12,7 +12,6 @@ use vortex_compressor::builtins::IntDictScheme; use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::estimate::CompressionEstimate; use vortex_compressor::estimate::DeferredEstimate; -use vortex_compressor::estimate::EstimateScore; use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; @@ -94,7 +93,7 @@ impl Scheme for SequenceScheme { // TODO(connor): `sequence_encode` allocates the encoded array just to confirm feasibility. // A cheaper `is_sequence` probe would let us skip the allocation entirely. CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, best_so_far, _ctx, exec_ctx| { + |_compressor, data, threshold, _ctx, exec_ctx| { // `SequenceArray` stores exactly two scalars (base and multiplier), so the best // achievable compression ratio is `array_len / 2`. let compressed_size = 2usize; @@ -102,8 +101,7 @@ impl Scheme for SequenceScheme { // If we cannot beat the best so far, then we do not want to even try sequence // encoding the data. - let threshold = best_so_far.and_then(EstimateScore::finite_ratio); - if threshold.is_some_and(|t| max_ratio <= t) { + if threshold.best_case_ratio_cannot_win(max_ratio) { return Ok(EstimateVerdict::Skip); } @@ -134,3 +132,106 @@ impl Scheme for SequenceScheme { .ok_or_else(|| vortex_err!("cannot sequence encode array")) } } + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use vortex_array::ExecutionCtx; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Constant; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::dtype::Nullability; + use vortex_array::scalar::Scalar; + use vortex_array::validity::Validity; + use vortex_buffer::Buffer; + use vortex_compressor::estimate::EstimateVerdict; + use vortex_sequence::Sequence; + use vortex_session::VortexSession; + + use super::*; + + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); + + /// Number of values in the test arrays; Sequence's best case is `LEN / 2`. + const LEN: usize = 2048; + + /// Immediate fixed-ratio competitor; its `compress` emits a tiny constant array so the + /// winner is observable from the output encoding. + #[derive(Debug)] + struct FixedRatioScheme { + ratio: f64, + } + + impl Scheme for FixedRatioScheme { + fn scheme_name(&self) -> &'static str { + "test.fixed_ratio" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::Ratio(self.ratio)) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(ConstantArray::new( + Scalar::primitive(0i64, Nullability::NonNullable), + data.array_len(), + ) + .into_array()) + } + } + + /// An exact arithmetic sequence, encodable by [`SequenceScheme`]. + fn arithmetic_sequence() -> ArrayRef { + let values: Buffer = (0..LEN as i64).map(|i| 10_000 + 7 * i).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() + } + + /// With the incumbent below Sequence's best case, the callback must proceed past the + /// threshold check, trial-encode, and win. + #[test] + fn sequence_wins_over_low_threshold() -> VortexResult<()> { + static COMPETITOR: FixedRatioScheme = FixedRatioScheme { ratio: 2.0 }; + let compressor = CascadingCompressor::new(vec![&COMPETITOR, &SequenceScheme]); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&arithmetic_sequence(), &mut exec_ctx)?; + + assert!(compressed.is::()); + Ok(()) + } + + /// With the incumbent at exactly Sequence's best case (`LEN / 2`), Sequence must not be + /// chosen — the same decision the pre-threshold-handle `max_ratio <= best` skip produced + /// on this input. + #[test] + fn sequence_loses_at_best_case_tie() -> VortexResult<()> { + static COMPETITOR: FixedRatioScheme = FixedRatioScheme { + ratio: LEN as f64 / 2.0, + }; + let compressor = CascadingCompressor::new(vec![&COMPETITOR, &SequenceScheme]); + + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&arithmetic_sequence(), &mut exec_ctx)?; + + assert!(compressed.is::()); + Ok(()) + } +} diff --git a/vortex-btrblocks/tests/golden.rs b/vortex-btrblocks/tests/golden.rs new file mode 100644 index 00000000000..c60752b064a --- /dev/null +++ b/vortex-btrblocks/tests/golden.rs @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Golden-corpus determinism tests for the default compressor. +//! +//! Compresses a fixed, seed-generated corpus and snapshots each entry's full encoding tree +//! and exact byte counts. These snapshots pin the default compressor's *decisions*: any +//! refactor of scheme selection (see the compressor cost-model track) must leave every +//! snapshot untouched, so snapshot churn in a later change is the reviewable signal of a +//! behavior change. +//! +//! Three variants cover the feature matrix: +//! +//! - `default`: the default feature set and [`BtrBlocksCompressor::default`]. +//! - `unstable`: `unstable_encodings` enabled, default builder — pins Delta / OnPair +//! selection (compiled out of `ALL_SCHEMES` otherwise). +//! - `compact`: `unstable_encodings` + `zstd` + `pco`, with +//! [`BtrBlocksCompressorBuilder::with_compact`] — pins Zstd / Pco selection. +//! +//! Every corpus entry is longer than 1024 values so the sampling-based estimation path is +//! exercised, and each entry is compressed twice per run to assert determinism directly. + +#![allow(clippy::cast_possible_truncation, clippy::tests_outside_test_module)] + +use std::fmt; +use std::sync::Arc; +use std::sync::LazyLock; + +use rand::RngExt; +use rand::SeedableRng; +use rand::rngs::StdRng; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::DecimalArray; +use vortex_array::arrays::ListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::StructArray; +use vortex_array::arrays::TemporalArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::display::EncodingSummaryExtractor; +use vortex_array::display::MetadataExtractor; +use vortex_array::display::TreeContext; +use vortex_array::display::TreeExtractor; +use vortex_array::dtype::DType; +use vortex_array::dtype::DecimalDType; +use vortex_array::dtype::Nullability; +use vortex_array::extension::datetime::TimeUnit; +use vortex_array::validity::Validity; +use vortex_btrblocks::BtrBlocksCompressor; +use vortex_buffer::Buffer; +use vortex_error::VortexResult; +use vortex_session::VortexSession; + +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); + +/// Number of values in each numeric corpus entry: comfortably above the 1024-value sampling +/// threshold so scheme selection runs on sampled estimates, as it does for real file chunks. +const N: usize = 16_384; + +/// Header extractor printing exact byte counts (the built-in [`NbytesExtractor`] rounds +/// through `humansize`, which could mask small size regressions). +/// +/// [`NbytesExtractor`]: vortex_array::display::NbytesExtractor +struct ExactNbytesExtractor; + +impl TreeExtractor for ExactNbytesExtractor { + fn write_header( + &self, + array: &ArrayRef, + _ctx: &TreeContext, + f: &mut fmt::Formatter<'_>, + ) -> fmt::Result { + write!(f, " nbytes={}", array.nbytes()) + } +} + +/// Renders the full snapshot content for one compressed corpus entry. +fn render(input: &ArrayRef, compressed: &ArrayRef) -> String { + format!( + "input: {}, len={}, nbytes={}\n{}", + input.dtype(), + input.len(), + input.nbytes(), + compressed + .tree_display_builder() + .with(EncodingSummaryExtractor) + .with(ExactNbytesExtractor) + .with(MetadataExtractor) + ) +} + +/// Compresses every corpus entry twice (direct determinism check) and snapshots the result +/// under `{variant}__{entry}`. +fn golden_corpus_snapshots(variant: &str, compressor: &BtrBlocksCompressor) -> VortexResult<()> { + for (name, array) in corpus()? { + let rendered = { + let mut exec_ctx = SESSION.create_execution_ctx(); + render(&array, &compressor.compress(&array, &mut exec_ctx)?) + }; + let rendered_again = { + let mut exec_ctx = SESSION.create_execution_ctx(); + render(&array, &compressor.compress(&array, &mut exec_ctx)?) + }; + assert_eq!( + rendered, rendered_again, + "compressing corpus entry {name} twice produced different results" + ); + + insta::assert_snapshot!(format!("{variant}__{name}"), rendered); + } + Ok(()) +} + +/// The fixed corpus: deterministic synthetic arrays covering each scheme's habitat. +fn corpus() -> VortexResult> { + Ok(vec![ + ("int_monotone_jitter", int_monotone_jitter()), + ("int_arithmetic_sequence", int_arithmetic_sequence()), + ("int_low_cardinality", int_low_cardinality()), + ("int_runs", int_runs()), + ("int_sparse_outliers", int_sparse_outliers()), + ("int_mostly_null", int_mostly_null()), + ("int_negatives", int_negatives()), + ("int_wide_random", int_wide_random()), + ("float_alp_prices", float_alp_prices()), + ("float_low_cardinality", float_low_cardinality()), + ("float_full_precision", float_full_precision()), + ("float_mostly_null", float_mostly_null()), + ("string_fsst_structured", string_fsst_structured()), + ("string_low_cardinality", string_low_cardinality()), + ("binary_low_cardinality", binary_low_cardinality()), + ("decimal_prices", decimal_prices()), + ("temporal_timestamp_micros", temporal_timestamp_micros()), + ("bool_random", bool_random()), + ("struct_mixed", struct_mixed()?), + ("list_of_int_runs", list_of_int_runs()?), + ]) +} + +/// Near-monotone u64 (timestamp-like): FoR/BitPacking habitat, Delta habitat when enabled. +fn int_monotone_jitter() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(101); + let mut value = 1_700_000_000_000u64; + let values: Buffer = (0..N) + .map(|_| { + value += 900 + rng.random_range(0..200); + value + }) + .collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Exact arithmetic sequence: Sequence habitat (distinct == len, no nulls). +fn int_arithmetic_sequence() -> ArrayRef { + let values: Buffer = (0..N as i64).map(|i| 10_000 + 7 * i).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// A handful of widely-spaced distinct values: IntDict habitat. +fn int_low_cardinality() -> ArrayRef { + const DISTINCT: [i64; 6] = [0, 123_400, 617_000, 1_234_000, 12_340_000, 37_020_000]; + let mut rng = StdRng::seed_from_u64(102); + let values: Buffer = (0..N).map(|_| DISTINCT[rng.random_range(0..6)]).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Long runs over a moderate value set: RunEnd/RLE habitat. +fn int_runs() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(103); + let mut values: Vec = Vec::with_capacity(N); + while values.len() < N { + let value = rng.random_range(-50_000..50_000i32); + let run = rng.random_range(8..25); + values.extend(std::iter::repeat_n(value, run)); + } + values.truncate(N); + PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable).into_array() +} + +/// One dominant value with rare large outliers: Sparse habitat. +fn int_sparse_outliers() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(104); + let values: Buffer = (0..N) + .map(|_| { + if rng.random_range(0..100) < 5 { + rng.random_range(1_000_000_000..2_000_000_000i64) + } else { + 1_000_000 + } + }) + .collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// 95% nulls over small values: null-dominated integer habitat. +fn int_mostly_null() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(105); + let mut validity: Vec = Vec::with_capacity(N); + let values: Buffer = (0..N) + .map(|_| { + let valid = rng.random_range(0..100) < 5; + validity.push(valid); + if valid { rng.random_range(0..1000) } else { 0 } + }) + .collect(); + PrimitiveArray::new( + values, + Validity::Array(BoolArray::from_iter(validity).into_array()), + ) + .into_array() +} + +/// Small values of mixed sign: ZigZag habitat. +fn int_negatives() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(106); + let values: Buffer = (0..N).map(|_| rng.random_range(-128..128i64)).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Full-width random u64: essentially incompressible; pins the "no scheme wins" path. +fn int_wide_random() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(107); + let values: Buffer = (0..N).map(|_| rng.random::()).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Two-decimal-digit "prices": ALP habitat. +fn float_alp_prices() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(108); + let values: Buffer = (0..N) + .map(|_| rng.random_range(0..10_000_000i64) as f64 / 100.0) + .collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// A handful of distinct floats: FloatDict habitat. +fn float_low_cardinality() -> ArrayRef { + const DISTINCT: [f64; 8] = [0.0, 0.5, 1.25, 2.75, 3.5, 10.125, 100.0625, 1000.03125]; + let mut rng = StdRng::seed_from_u64(109); + let values: Buffer = (0..N).map(|_| DISTINCT[rng.random_range(0..8)]).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// Full-precision uniform floats: ALP-RD habitat. +fn float_full_precision() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(110); + let values: Buffer = (0..N).map(|_| rng.random::()).collect(); + PrimitiveArray::new(values, Validity::NonNullable).into_array() +} + +/// 95% nulls over floats: null-dominated sparse float habitat. +fn float_mostly_null() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(111); + let mut validity: Vec = Vec::with_capacity(N); + let values: Buffer = (0..N) + .map(|_| { + let valid = rng.random_range(0..100) < 5; + validity.push(valid); + if valid { + rng.random::() * 100.0 + } else { + 0.0 + } + }) + .collect(); + PrimitiveArray::new( + values, + Validity::Array(BoolArray::from_iter(validity).into_array()), + ) + .into_array() +} + +/// High-cardinality strings with shared substructure (emails): FSST habitat. +fn string_fsst_structured() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(112); + let strings: Vec = (0..N) + .map(|_| { + format!( + "user{:06}@example{}.com", + rng.random_range(0..1_000_000), + rng.random_range(0..100) + ) + }) + .collect(); + VarBinViewArray::from_iter_str(strings.iter().map(String::as_str)).into_array() +} + +/// A dozen distinct strings: StringDict habitat. +fn string_low_cardinality() -> ArrayRef { + const DISTINCT: [&str; 12] = [ + "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", + "juliett", "kilo", "lima", + ]; + let mut rng = StdRng::seed_from_u64(113); + let strings: Vec> = (0..N) + .map(|_| Some(DISTINCT[rng.random_range(0..12)])) + .collect(); + VarBinViewArray::from_iter(strings, DType::Utf8(Nullability::NonNullable)).into_array() +} + +/// Low-cardinality binary blobs: BinaryDict habitat (Zstd habitat under `compact`). +fn binary_low_cardinality() -> ArrayRef { + const DISTINCT: [&[u8]; 5] = [ + &[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08], + &[0xDE, 0xAD, 0xBE, 0xEF], + &[0x00; 16], + &[0xFF, 0x00, 0xFF, 0x00, 0xFF, 0x00], + &[0x42; 12], + ]; + let mut rng = StdRng::seed_from_u64(114); + let blobs: Vec> = (0..N) + .map(|_| Some(DISTINCT[rng.random_range(0..5)])) + .collect(); + VarBinViewArray::from_iter(blobs, DType::Binary(Nullability::NonNullable)).into_array() +} + +/// Two-decimal-place decimals: DecimalScheme (byte-parts) habitat. +fn decimal_prices() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(115); + let values: Buffer = (0..N).map(|_| rng.random_range(0..10_000_000i64)).collect(); + DecimalArray::new(values, DecimalDType::new(12, 2), Validity::NonNullable).into_array() +} + +/// Near-monotone microsecond timestamps: TemporalScheme (datetime-parts) habitat. +fn temporal_timestamp_micros() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(116); + let mut value = 1_700_000_000_000_000i64; + let values: Buffer = (0..N) + .map(|_| { + value += rng.random_range(1_000..1_000_000); + value + }) + .collect(); + TemporalArray::new_timestamp( + PrimitiveArray::new(values, Validity::NonNullable).into_array(), + TimeUnit::Microseconds, + Some(Arc::from("UTC")), + ) + .into_array() +} + +/// Random booleans: no bool scheme is registered, pinning the "stays canonical" path. +fn bool_random() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(117); + BoolArray::from_iter((0..N).map(|_| rng.random::())).into_array() +} + +/// Struct of int/string/float fields: pins the structural recursion path. +fn struct_mixed() -> VortexResult { + Ok(StructArray::from_fields(&[ + ("id", int_arithmetic_sequence()), + ("category", string_low_cardinality()), + ("value", float_alp_prices()), + ])? + .into_array()) +} + +/// Variable-length lists of run-heavy ints: pins the list offsets/elements path. +fn list_of_int_runs() -> VortexResult { + let mut rng = StdRng::seed_from_u64(118); + let elements = int_runs(); + let mut offsets: Vec = Vec::with_capacity(N / 4 + 1); + let mut offset = 0i32; + offsets.push(offset); + while (offset as usize) < N { + offset = (offset + rng.random_range(1..8)).min(N as i32); + offsets.push(offset); + } + let offsets = PrimitiveArray::new(Buffer::copy_from(&offsets), Validity::NonNullable); + Ok(ListArray::try_new(elements, offsets.into_array(), Validity::NonNullable)?.into_array()) +} + +/// Excludes OnPair from the golden compressors: its dictionary training (upstream `onpair` +/// crate) iterates randomly-seeded `hashbrown` maps, so its compressed output — and therefore +/// its sampled estimate — differs run-to-run. A nondeterministic scheme cannot serve as a +/// golden baseline; excluding it keeps the remaining unstable schemes pinned. +#[cfg(feature = "unstable_encodings")] +fn without_onpair( + builder: vortex_btrblocks::BtrBlocksCompressorBuilder, +) -> vortex_btrblocks::BtrBlocksCompressorBuilder { + use vortex_btrblocks::SchemeExt; + use vortex_btrblocks::schemes::string::OnPairScheme; + + builder.exclude_schemes([OnPairScheme.id()]) +} + +#[cfg(not(feature = "unstable_encodings"))] +#[test] +fn golden_default() -> VortexResult<()> { + golden_corpus_snapshots("default", &BtrBlocksCompressor::default()) +} + +#[cfg(feature = "unstable_encodings")] +#[test] +fn golden_unstable() -> VortexResult<()> { + use vortex_btrblocks::BtrBlocksCompressorBuilder; + + golden_corpus_snapshots( + "unstable", + &without_onpair(BtrBlocksCompressorBuilder::default()).build(), + ) +} + +#[cfg(all(feature = "unstable_encodings", feature = "zstd", feature = "pco"))] +#[test] +fn golden_compact() -> VortexResult<()> { + use vortex_btrblocks::BtrBlocksCompressorBuilder; + + golden_corpus_snapshots( + "compact", + &without_onpair(BtrBlocksCompressorBuilder::default().with_compact()).build(), + ) +} diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__binary_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__compact__binary_low_cardinality.snap new file mode 100644 index 00000000000..d68474ce61c --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__binary_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: binary, len=16384, nbytes=315856 +root: vortex.dict(binary, len=16384) nbytes=6199 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.zstd(binary, len=5) nbytes=55 + metadata: nrows: 5, slice: 0..5 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__bool_random.snap b/vortex-btrblocks/tests/snapshots/golden__compact__bool_random.snap new file mode 100644 index 00000000000..d80e2fa4464 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__bool_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: bool, len=16384, nbytes=2048 +root: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__decimal_prices.snap b/vortex-btrblocks/tests/snapshots/golden__compact__decimal_prices.snap new file mode 100644 index 00000000000..780ef30a6b0 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__decimal_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: decimal(12,2), len=16384, nbytes=131072 +root: vortex.decimal_byte_parts(decimal(12,2), len=16384) nbytes=47666 + metadata: + msp: vortex.pco(i32, len=16384) nbytes=47666 + metadata: ptype: i32, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__float_alp_prices.snap b/vortex-btrblocks/tests/snapshots/golden__compact__float_alp_prices.snap new file mode 100644 index 00000000000..b1a3b41c4e7 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__float_alp_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=47699 + metadata: exponents: e: 14, f: 12 + encoded: vortex.pco(i64, len=16384) nbytes=47699 + metadata: ptype: i64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__float_full_precision.snap b/vortex-btrblocks/tests/snapshots/golden__compact__float_full_precision.snap new file mode 100644 index 00000000000..20fecf31e22 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__float_full_precision.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.pco(f64, len=16384) nbytes=110692 + metadata: ptype: f64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__float_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__compact__float_low_cardinality.snap new file mode 100644 index 00000000000..006f5a93b69 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__float_low_cardinality.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.dict(f64, len=16384) nbytes=6184 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.alp(f64, len=8) nbytes=40 + metadata: exponents: e: 16, f: 11 + encoded: vortex.pco(i64, len=8) nbytes=40 + metadata: ptype: i64, nrows: 8, slice: 0..8 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__float_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__compact__float_mostly_null.snap new file mode 100644 index 00000000000..134acccdc1c --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__float_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64?, len=16384, nbytes=133120 +root: vortex.sparse(f64?, len=16384) nbytes=7545 + metadata: fill_value: null + patch_indices: vortex.pco(u16, len=850) nbytes=636 + metadata: ptype: u16, nrows: 850, slice: 0..850 + patch_values: vortex.primitive(f64?, len=850) nbytes=6907 + metadata: ptype: f64 + validity: vortex.bool(bool, len=850) nbytes=107 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_arithmetic_sequence.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_arithmetic_sequence.snap new file mode 100644 index 00000000000..873f3655a1f --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_arithmetic_sequence.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_low_cardinality.snap new file mode 100644 index 00000000000..0cbe2e06814 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.dict(i64, len=16384) nbytes=6177 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.pco(i64, len=6) nbytes=33 + metadata: ptype: i64, nrows: 6, slice: 0..6 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter.snap new file mode 100644 index 00000000000..4a63aa0da71 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_monotone_jitter.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: vortex.pco(u64, len=16384) nbytes=15715 + metadata: ptype: u64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_mostly_null.snap new file mode 100644 index 00000000000..71e1c671a03 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_mostly_null.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32?, len=16384, nbytes=67584 +root: vortex.pco(i32?, len=16384) nbytes=3086 + metadata: ptype: i32, nrows: 16384, slice: 0..16384 + validity: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_negatives.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_negatives.snap new file mode 100644 index 00000000000..24949893190 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_negatives.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: fastlanes.for(i64, len=16384) nbytes=16384 + metadata: reference: -128i64 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=16384 + metadata: bit_width: 8, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_runs.snap new file mode 100644 index 00000000000..4ab2e46cead --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_runs.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32, len=16384, nbytes=65536 +root: vortex.zigzag(i32, len=16384) nbytes=2969 + metadata: + encoded: vortex.pco(u32, len=16384) nbytes=2969 + metadata: ptype: u32, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_sparse_outliers.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_sparse_outliers.snap new file mode 100644 index 00000000000..0303e0e4ef1 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_sparse_outliers.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.pco(i64, len=16384) nbytes=3817 + metadata: ptype: i64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__int_wide_random.snap b/vortex-btrblocks/tests/snapshots/golden__compact__int_wide_random.snap new file mode 100644 index 00000000000..90e1894f910 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__int_wide_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: vortex.primitive(u64, len=16384) nbytes=131072 + metadata: ptype: u64 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__list_of_int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__compact__list_of_int_runs.snap new file mode 100644 index 00000000000..88274daf4b1 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__list_of_int_runs.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: list(i32), len=4066, nbytes=81804 +root: vortex.list(list(i32), len=4066) nbytes=4434 + metadata: + elements: vortex.zigzag(i32, len=16384) nbytes=2969 + metadata: + encoded: vortex.pco(u32, len=16384) nbytes=2969 + metadata: ptype: u32, nrows: 16384, slice: 0..16384 + offsets: vortex.pco(u16, len=4067) nbytes=1465 + metadata: ptype: u16, nrows: 4067, slice: 0..4067 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__compact__string_fsst_structured.snap new file mode 100644 index 00000000000..49ad248087b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__string_fsst_structured.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=653785 +root: vortex.zstd(utf8, len=16384) nbytes=109119 + metadata: nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__string_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__compact__string_low_cardinality.snap new file mode 100644 index 00000000000..a65054ac178 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__string_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=262144 +root: vortex.dict(utf8, len=16384) nbytes=8287 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.zstd(utf8, len=12) nbytes=95 + metadata: nrows: 12, slice: 0..12 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__struct_mixed.snap b/vortex-btrblocks/tests/snapshots/golden__compact__struct_mixed.snap new file mode 100644 index 00000000000..83e34c583dc --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__struct_mixed.snap @@ -0,0 +1,19 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: {id=i64, category=utf8, value=f64}, len=16384, nbytes=524288 +root: vortex.struct({id=i64, category=utf8, value=f64}, len=16384) nbytes=55986 + metadata: + id: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 + category: vortex.dict(utf8, len=16384) nbytes=8287 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.zstd(utf8, len=12) nbytes=95 + metadata: nrows: 12, slice: 0..12 + value: vortex.alp(f64, len=16384) nbytes=47699 + metadata: exponents: e: 14, f: 12 + encoded: vortex.pco(i64, len=16384) nbytes=47699 + metadata: ptype: i64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__compact__temporal_timestamp_micros.snap b/vortex-btrblocks/tests/snapshots/golden__compact__temporal_timestamp_micros.snap new file mode 100644 index 00000000000..0422808ce29 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__compact__temporal_timestamp_micros.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: vortex.timestamp[µs, tz=UTC](i64), len=16384, nbytes=131072 +root: vortex.ext(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=40985 + metadata: + storage: vortex.pco(i64, len=16384) nbytes=40985 + metadata: ptype: i64, nrows: 16384, slice: 0..16384 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__binary_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__default__binary_low_cardinality.snap new file mode 100644 index 00000000000..56918c0d096 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__binary_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: binary, len=16384, nbytes=315856 +root: vortex.dict(binary, len=16384) nbytes=6240 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.varbinview(binary, len=5) nbytes=96 + metadata: diff --git a/vortex-btrblocks/tests/snapshots/golden__default__bool_random.snap b/vortex-btrblocks/tests/snapshots/golden__default__bool_random.snap new file mode 100644 index 00000000000..d80e2fa4464 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__bool_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: bool, len=16384, nbytes=2048 +root: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__decimal_prices.snap b/vortex-btrblocks/tests/snapshots/golden__default__decimal_prices.snap new file mode 100644 index 00000000000..f669755e4b1 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__decimal_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: decimal(12,2), len=16384, nbytes=131072 +root: vortex.decimal_byte_parts(decimal(12,2), len=16384) nbytes=49152 + metadata: + msp: fastlanes.bitpacked(i32, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__float_alp_prices.snap b/vortex-btrblocks/tests/snapshots/golden__default__float_alp_prices.snap new file mode 100644 index 00000000000..f70d90a301d --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__float_alp_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=49152 + metadata: exponents: e: 14, f: 12 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__float_full_precision.snap b/vortex-btrblocks/tests/snapshots/golden__default__float_full_precision.snap new file mode 100644 index 00000000000..487e7c7900b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__float_full_precision.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alprd(f64, len=16384) nbytes=112880 + metadata: right_bit_width: 52, patch_offset: 0 + left_parts: fastlanes.bitpacked(u16, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + right_parts: fastlanes.bitpacked(u64, len=16384) nbytes=106496 + metadata: bit_width: 52, offset: 0 + patch_indices: vortex.primitive(u16, len=60) nbytes=120 + metadata: ptype: u16 + patch_values: vortex.primitive(u16, len=60) nbytes=120 + metadata: ptype: u16 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__float_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__default__float_low_cardinality.snap new file mode 100644 index 00000000000..38d952262d6 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__float_low_cardinality.snap @@ -0,0 +1,19 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=10233 + metadata: exponents: e: 16, f: 12, patch_offset: 0 + encoded: vortex.dict(i64, len=16384) nbytes=6200 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.primitive(i64, len=7) nbytes=56 + metadata: ptype: i64 + patch_indices: vortex.primitive(u16, len=1996) nbytes=3992 + metadata: ptype: u16 + patch_values: vortex.constant(f64, len=1996) nbytes=9 + metadata: scalar: 1000.03125f64 + patch_chunk_offsets: vortex.primitive(u16, len=16) nbytes=32 + metadata: ptype: u16 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__float_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__default__float_mostly_null.snap new file mode 100644 index 00000000000..7b88d819472 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__float_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64?, len=16384, nbytes=133120 +root: vortex.sparse(f64?, len=16384) nbytes=8609 + metadata: fill_value: null + patch_indices: vortex.primitive(u16, len=850) nbytes=1700 + metadata: ptype: u16 + patch_values: vortex.primitive(f64?, len=850) nbytes=6907 + metadata: ptype: f64 + validity: vortex.bool(bool, len=850) nbytes=107 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_arithmetic_sequence.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_arithmetic_sequence.snap new file mode 100644 index 00000000000..873f3655a1f --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_arithmetic_sequence.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_low_cardinality.snap new file mode 100644 index 00000000000..8648200a138 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.dict(i64, len=16384) nbytes=6192 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.primitive(i64, len=6) nbytes=48 + metadata: ptype: i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap new file mode 100644 index 00000000000..4800aae289d --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_monotone_jitter.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: fastlanes.for(u64, len=16384) nbytes=49152 + metadata: reference: 1700000001036u64 + encoded: fastlanes.bitpacked(u64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_mostly_null.snap new file mode 100644 index 00000000000..73566369279 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32?, len=16384, nbytes=67584 +root: vortex.sparse(i32?, len=16384) nbytes=3031 + metadata: fill_value: null + patch_indices: vortex.primitive(u16, len=823) nbytes=1646 + metadata: ptype: u16 + patch_values: fastlanes.bitpacked(i32?, len=823) nbytes=1383 + metadata: bit_width: 10, offset: 0 + validity_child: vortex.bool(bool, len=823) nbytes=103 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_negatives.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_negatives.snap new file mode 100644 index 00000000000..24949893190 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_negatives.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: fastlanes.for(i64, len=16384) nbytes=16384 + metadata: reference: -128i64 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=16384 + metadata: bit_width: 8, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_runs.snap new file mode 100644 index 00000000000..271de8faf93 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_runs.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32, len=16384, nbytes=65536 +root: vortex.runend(i32, len=16384) nbytes=3968 + metadata: offset: 0 + ends: fastlanes.for(u16, len=1020) nbytes=1792 + metadata: reference: 13u16 + encoded: fastlanes.bitpacked(u16, len=1020) nbytes=1792 + metadata: bit_width: 14, offset: 0 + values: fastlanes.for(i32, len=1020) nbytes=2176 + metadata: reference: -49931i32 + encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 + metadata: bit_width: 17, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_sparse_outliers.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_sparse_outliers.snap new file mode 100644 index 00000000000..53a5d1e023c --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_sparse_outliers.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sparse(i64, len=16384) nbytes=5540 + metadata: fill_value: 1000000i64 + patch_indices: vortex.primitive(u16, len=848) nbytes=1696 + metadata: ptype: u16 + patch_values: fastlanes.for(i64, len=848) nbytes=3840 + metadata: reference: 1000830099i64 + encoded: fastlanes.bitpacked(i64, len=848) nbytes=3840 + metadata: bit_width: 30, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__int_wide_random.snap b/vortex-btrblocks/tests/snapshots/golden__default__int_wide_random.snap new file mode 100644 index 00000000000..90e1894f910 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__int_wide_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: vortex.primitive(u64, len=16384) nbytes=131072 + metadata: ptype: u64 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap new file mode 100644 index 00000000000..c9554add05a --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__list_of_int_runs.snap @@ -0,0 +1,25 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: list(i32), len=4066, nbytes=81804 +root: vortex.list(list(i32), len=4066) nbytes=11146 + metadata: + elements: vortex.runend(i32, len=16384) nbytes=3968 + metadata: offset: 0 + ends: fastlanes.for(u16, len=1020) nbytes=1792 + metadata: reference: 13u16 + encoded: fastlanes.bitpacked(u16, len=1020) nbytes=1792 + metadata: bit_width: 14, offset: 0 + values: fastlanes.for(i32, len=1020) nbytes=2176 + metadata: reference: -49931i32 + encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 + metadata: bit_width: 17, offset: 0 + offsets: fastlanes.bitpacked(u16, len=4067) nbytes=7178 + metadata: bit_width: 14, offset: 0 + patch_indices: vortex.primitive(u16, len=1) nbytes=2 + metadata: ptype: u16 + patch_values: vortex.constant(u16, len=1) nbytes=4 + metadata: scalar: 16384u16 + patch_chunk_offsets: vortex.primitive(u8, len=4) nbytes=4 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap new file mode 100644 index 00000000000..327f050b0d7 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__string_fsst_structured.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=653785 +root: vortex.fsst(utf8, len=16384) nbytes=151382 + metadata: len: 16384, nsymbols: 223 + uncompressed_lengths: vortex.sparse(u8, len=16384) nbytes=3154 + metadata: fill_value: 24u8 + patch_indices: vortex.primitive(u16, len=1575) nbytes=3150 + metadata: ptype: u16 + patch_values: vortex.constant(u8, len=1575) nbytes=2 + metadata: scalar: 23u8 + codes_offsets: fastlanes.bitpacked(u32, len=16385) nbytes=36992 + metadata: bit_width: 17, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__string_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__default__string_low_cardinality.snap new file mode 100644 index 00000000000..ec79989d46a --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__string_low_cardinality.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=262144 +root: vortex.dict(utf8, len=16384) nbytes=8373 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.fsst(utf8, len=12) nbytes=181 + metadata: len: 12, nsymbols: 9 + uncompressed_lengths: vortex.primitive(u8, len=12) nbytes=12 + metadata: ptype: u8 + codes_offsets: vortex.primitive(u8, len=13) nbytes=13 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__struct_mixed.snap b/vortex-btrblocks/tests/snapshots/golden__default__struct_mixed.snap new file mode 100644 index 00000000000..be8f48b779b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__struct_mixed.snap @@ -0,0 +1,23 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: {id=i64, category=utf8, value=f64}, len=16384, nbytes=524288 +root: vortex.struct({id=i64, category=utf8, value=f64}, len=16384) nbytes=57525 + metadata: + id: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 + category: vortex.dict(utf8, len=16384) nbytes=8373 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.fsst(utf8, len=12) nbytes=181 + metadata: len: 12, nsymbols: 9 + uncompressed_lengths: vortex.primitive(u8, len=12) nbytes=12 + metadata: ptype: u8 + codes_offsets: vortex.primitive(u8, len=13) nbytes=13 + metadata: ptype: u8 + value: vortex.alp(f64, len=16384) nbytes=49152 + metadata: exponents: e: 14, f: 12 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap b/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap new file mode 100644 index 00000000000..6f38e8e6f5d --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__default__temporal_timestamp_micros.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: vortex.timestamp[µs, tz=UTC](i64), len=16384, nbytes=131072 +root: vortex.ext(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=67584 + metadata: + storage: fastlanes.for(i64, len=16384) nbytes=67584 + metadata: reference: 1700000000891673i64 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=67584 + metadata: bit_width: 33, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__binary_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__binary_low_cardinality.snap new file mode 100644 index 00000000000..56918c0d096 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__binary_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: binary, len=16384, nbytes=315856 +root: vortex.dict(binary, len=16384) nbytes=6240 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.varbinview(binary, len=5) nbytes=96 + metadata: diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__bool_random.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__bool_random.snap new file mode 100644 index 00000000000..d80e2fa4464 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__bool_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: bool, len=16384, nbytes=2048 +root: vortex.bool(bool, len=16384) nbytes=2048 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__decimal_prices.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__decimal_prices.snap new file mode 100644 index 00000000000..f669755e4b1 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__decimal_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: decimal(12,2), len=16384, nbytes=131072 +root: vortex.decimal_byte_parts(decimal(12,2), len=16384) nbytes=49152 + metadata: + msp: fastlanes.bitpacked(i32, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__float_alp_prices.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__float_alp_prices.snap new file mode 100644 index 00000000000..f70d90a301d --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__float_alp_prices.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=49152 + metadata: exponents: e: 14, f: 12 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__float_full_precision.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__float_full_precision.snap new file mode 100644 index 00000000000..487e7c7900b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__float_full_precision.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alprd(f64, len=16384) nbytes=112880 + metadata: right_bit_width: 52, patch_offset: 0 + left_parts: fastlanes.bitpacked(u16, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + right_parts: fastlanes.bitpacked(u64, len=16384) nbytes=106496 + metadata: bit_width: 52, offset: 0 + patch_indices: vortex.primitive(u16, len=60) nbytes=120 + metadata: ptype: u16 + patch_values: vortex.primitive(u16, len=60) nbytes=120 + metadata: ptype: u16 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__float_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__float_low_cardinality.snap new file mode 100644 index 00000000000..38d952262d6 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__float_low_cardinality.snap @@ -0,0 +1,19 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64, len=16384, nbytes=131072 +root: vortex.alp(f64, len=16384) nbytes=10233 + metadata: exponents: e: 16, f: 12, patch_offset: 0 + encoded: vortex.dict(i64, len=16384) nbytes=6200 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.primitive(i64, len=7) nbytes=56 + metadata: ptype: i64 + patch_indices: vortex.primitive(u16, len=1996) nbytes=3992 + metadata: ptype: u16 + patch_values: vortex.constant(f64, len=1996) nbytes=9 + metadata: scalar: 1000.03125f64 + patch_chunk_offsets: vortex.primitive(u16, len=16) nbytes=32 + metadata: ptype: u16 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__float_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__float_mostly_null.snap new file mode 100644 index 00000000000..7b88d819472 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__float_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: f64?, len=16384, nbytes=133120 +root: vortex.sparse(f64?, len=16384) nbytes=8609 + metadata: fill_value: null + patch_indices: vortex.primitive(u16, len=850) nbytes=1700 + metadata: ptype: u16 + patch_values: vortex.primitive(f64?, len=850) nbytes=6907 + metadata: ptype: f64 + validity: vortex.bool(bool, len=850) nbytes=107 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_arithmetic_sequence.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_arithmetic_sequence.snap new file mode 100644 index 00000000000..873f3655a1f --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_arithmetic_sequence.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_low_cardinality.snap new file mode 100644 index 00000000000..8648200a138 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_low_cardinality.snap @@ -0,0 +1,11 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.dict(i64, len=16384) nbytes=6192 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=6144 + metadata: bit_width: 3, offset: 0 + values: vortex.primitive(i64, len=6) nbytes=48 + metadata: ptype: i64 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_monotone_jitter.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_monotone_jitter.snap new file mode 100644 index 00000000000..f1ded4c3e59 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_monotone_jitter.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: fastlanes.delta(u64, len=16384) nbytes=19840 + metadata: offset: 0 + bases: vortex.primitive(u64, len=256) nbytes=2048 + metadata: ptype: u64 + deltas: vortex.dict(u64, len=16384) nbytes=17792 + metadata: all_values_referenced: true + codes: vortex.primitive(u8, len=16384) nbytes=16384 + metadata: ptype: u8 + values: fastlanes.bitpacked(u64, len=201) nbytes=1408 + metadata: bit_width: 11, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_mostly_null.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_mostly_null.snap new file mode 100644 index 00000000000..73566369279 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_mostly_null.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32?, len=16384, nbytes=67584 +root: vortex.sparse(i32?, len=16384) nbytes=3031 + metadata: fill_value: null + patch_indices: vortex.primitive(u16, len=823) nbytes=1646 + metadata: ptype: u16 + patch_values: fastlanes.bitpacked(i32?, len=823) nbytes=1383 + metadata: bit_width: 10, offset: 0 + validity_child: vortex.bool(bool, len=823) nbytes=103 + metadata: offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_negatives.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_negatives.snap new file mode 100644 index 00000000000..24949893190 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_negatives.snap @@ -0,0 +1,9 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: fastlanes.for(i64, len=16384) nbytes=16384 + metadata: reference: -128i64 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=16384 + metadata: bit_width: 8, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_runs.snap new file mode 100644 index 00000000000..271de8faf93 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_runs.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i32, len=16384, nbytes=65536 +root: vortex.runend(i32, len=16384) nbytes=3968 + metadata: offset: 0 + ends: fastlanes.for(u16, len=1020) nbytes=1792 + metadata: reference: 13u16 + encoded: fastlanes.bitpacked(u16, len=1020) nbytes=1792 + metadata: bit_width: 14, offset: 0 + values: fastlanes.for(i32, len=1020) nbytes=2176 + metadata: reference: -49931i32 + encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 + metadata: bit_width: 17, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_sparse_outliers.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_sparse_outliers.snap new file mode 100644 index 00000000000..53a5d1e023c --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_sparse_outliers.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: i64, len=16384, nbytes=131072 +root: vortex.sparse(i64, len=16384) nbytes=5540 + metadata: fill_value: 1000000i64 + patch_indices: vortex.primitive(u16, len=848) nbytes=1696 + metadata: ptype: u16 + patch_values: fastlanes.for(i64, len=848) nbytes=3840 + metadata: reference: 1000830099i64 + encoded: fastlanes.bitpacked(i64, len=848) nbytes=3840 + metadata: bit_width: 30, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__int_wide_random.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__int_wide_random.snap new file mode 100644 index 00000000000..90e1894f910 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__int_wide_random.snap @@ -0,0 +1,7 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: u64, len=16384, nbytes=131072 +root: vortex.primitive(u64, len=16384) nbytes=131072 + metadata: ptype: u64 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap new file mode 100644 index 00000000000..c9554add05a --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__list_of_int_runs.snap @@ -0,0 +1,25 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: list(i32), len=4066, nbytes=81804 +root: vortex.list(list(i32), len=4066) nbytes=11146 + metadata: + elements: vortex.runend(i32, len=16384) nbytes=3968 + metadata: offset: 0 + ends: fastlanes.for(u16, len=1020) nbytes=1792 + metadata: reference: 13u16 + encoded: fastlanes.bitpacked(u16, len=1020) nbytes=1792 + metadata: bit_width: 14, offset: 0 + values: fastlanes.for(i32, len=1020) nbytes=2176 + metadata: reference: -49931i32 + encoded: fastlanes.bitpacked(i32, len=1020) nbytes=2176 + metadata: bit_width: 17, offset: 0 + offsets: fastlanes.bitpacked(u16, len=4067) nbytes=7178 + metadata: bit_width: 14, offset: 0 + patch_indices: vortex.primitive(u16, len=1) nbytes=2 + metadata: ptype: u16 + patch_values: vortex.constant(u16, len=1) nbytes=4 + metadata: scalar: 16384u16 + patch_chunk_offsets: vortex.primitive(u8, len=4) nbytes=4 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap new file mode 100644 index 00000000000..327f050b0d7 --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__string_fsst_structured.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=653785 +root: vortex.fsst(utf8, len=16384) nbytes=151382 + metadata: len: 16384, nsymbols: 223 + uncompressed_lengths: vortex.sparse(u8, len=16384) nbytes=3154 + metadata: fill_value: 24u8 + patch_indices: vortex.primitive(u16, len=1575) nbytes=3150 + metadata: ptype: u16 + patch_values: vortex.constant(u8, len=1575) nbytes=2 + metadata: scalar: 23u8 + codes_offsets: fastlanes.bitpacked(u32, len=16385) nbytes=36992 + metadata: bit_width: 17, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__string_low_cardinality.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__string_low_cardinality.snap new file mode 100644 index 00000000000..ec79989d46a --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__string_low_cardinality.snap @@ -0,0 +1,15 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: utf8, len=16384, nbytes=262144 +root: vortex.dict(utf8, len=16384) nbytes=8373 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.fsst(utf8, len=12) nbytes=181 + metadata: len: 12, nsymbols: 9 + uncompressed_lengths: vortex.primitive(u8, len=12) nbytes=12 + metadata: ptype: u8 + codes_offsets: vortex.primitive(u8, len=13) nbytes=13 + metadata: ptype: u8 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__struct_mixed.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__struct_mixed.snap new file mode 100644 index 00000000000..be8f48b779b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__struct_mixed.snap @@ -0,0 +1,23 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: {id=i64, category=utf8, value=f64}, len=16384, nbytes=524288 +root: vortex.struct({id=i64, category=utf8, value=f64}, len=16384) nbytes=57525 + metadata: + id: vortex.sequence(i64, len=16384) nbytes=0 + metadata: base: 10000i64, multiplier: 7i64 + category: vortex.dict(utf8, len=16384) nbytes=8373 + metadata: all_values_referenced: true + codes: fastlanes.bitpacked(u8, len=16384) nbytes=8192 + metadata: bit_width: 4, offset: 0 + values: vortex.fsst(utf8, len=12) nbytes=181 + metadata: len: 12, nsymbols: 9 + uncompressed_lengths: vortex.primitive(u8, len=12) nbytes=12 + metadata: ptype: u8 + codes_offsets: vortex.primitive(u8, len=13) nbytes=13 + metadata: ptype: u8 + value: vortex.alp(f64, len=16384) nbytes=49152 + metadata: exponents: e: 14, f: 12 + encoded: fastlanes.bitpacked(i64, len=16384) nbytes=49152 + metadata: bit_width: 24, offset: 0 diff --git a/vortex-btrblocks/tests/snapshots/golden__unstable__temporal_timestamp_micros.snap b/vortex-btrblocks/tests/snapshots/golden__unstable__temporal_timestamp_micros.snap new file mode 100644 index 00000000000..a763fa28f4b --- /dev/null +++ b/vortex-btrblocks/tests/snapshots/golden__unstable__temporal_timestamp_micros.snap @@ -0,0 +1,13 @@ +--- +source: vortex-btrblocks/tests/golden.rs +expression: rendered +--- +input: vortex.timestamp[µs, tz=UTC](i64), len=16384, nbytes=131072 +root: vortex.ext(vortex.timestamp[µs, tz=UTC](i64), len=16384) nbytes=43008 + metadata: + storage: fastlanes.delta(i64, len=16384) nbytes=43008 + metadata: offset: 0 + bases: vortex.primitive(i64, len=256) nbytes=2048 + metadata: ptype: i64 + deltas: fastlanes.bitpacked(i64, len=16384) nbytes=40960 + metadata: bit_width: 20, offset: 0 diff --git a/vortex-compressor/src/candidate.rs b/vortex-compressor/src/candidate.rs new file mode 100644 index 00000000000..8637862dc65 --- /dev/null +++ b/vortex-compressor/src/candidate.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Selection-time candidate bookkeeping. + +use vortex_array::ArrayRef; + +use crate::estimate::EstimateScore; +use crate::scheme::Scheme; +use crate::scheme::SchemeId; + +/// The mechanical facts the compressor knows about one (scheme, estimate) option during +/// scheme selection. +/// +/// A `Candidate` is the argument to [`CostModel::cost`]: it carries everything a model can +/// price without any new [`Scheme`] API. Candidates are built by the compressor during +/// selection and dropped when selection ends. +/// +/// [`CostModel::cost`]: crate::cost::CostModel::cost +#[derive(Debug)] +pub struct Candidate { + /// The scheme that produced this estimate. + pub scheme: &'static dyn Scheme, + + /// The ranked estimate: an estimated or sample-measured compression ratio, or the + /// zero-byte special case. + pub score: EstimateScore, + + /// Uncompressed size in bytes of the array under selection. + pub input_nbytes: u64, + + /// Number of values in the array under selection. + pub n_values: u64, + + /// The compressed sample array, when this estimate came from sampling. Its encoding tree + /// is the best available prediction of the full-array encoding tree. + pub sampled: Option, + + /// The cascade ancestry `(scheme_id, child_index)` of the selection site. + pub cascade: Vec<(SchemeId, usize)>, +} diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 727aa4ed98b..9e71fbbb615 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -3,6 +3,8 @@ //! Cascading array compression implementation. +use std::sync::Arc; + use vortex_array::ArrayRef; use vortex_array::ArraySlots; use vortex_array::Canonical; @@ -34,15 +36,19 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::builtins::IntDictScheme; +use crate::candidate::Candidate; use crate::constant; +use crate::cost::Cost; +use crate::cost::CostModel; +use crate::cost::SizeCost; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; use crate::estimate::DeferredEstimate; use crate::estimate::EstimateScore; use crate::estimate::EstimateVerdict; +use crate::estimate::SkipThreshold; use crate::estimate::WinnerEstimate; use crate::estimate::estimate_compression_ratio_with_sampling; -use crate::estimate::is_better_score; use crate::scheme::ChildSelection; use crate::scheme::DescendantExclusion; use crate::scheme::Scheme; @@ -90,10 +96,13 @@ pub struct CascadingCompressor { /// Descendant exclusion rules for the compressor's own cascading (e.g. excluding Dict from /// list offsets). root_exclusions: Vec, + + /// The cost model pricing candidates during scheme selection. + cost_model: Arc, } impl CascadingCompressor { - /// Creates a new compressor with the given schemes. + /// Creates a new compressor with the given schemes and the default [`SizeCost`] model. /// /// Root-level exclusion rules (e.g. excluding Dict from list offsets) are built /// automatically. @@ -107,9 +116,18 @@ impl CascadingCompressor { Self { schemes, root_exclusions, + cost_model: Arc::new(SizeCost), } } + /// Replaces the cost model used to price candidates during scheme selection. + /// + /// The default is [`SizeCost`], which maximizes estimated compression ratio. + pub fn with_cost_model(mut self, cost_model: Arc) -> Self { + self.cost_model = cost_model; + self + } + /// Compresses an array using cascading adaptive compression. /// /// First canonicalizes and compacts the array, then applies optimal compression schemes. @@ -342,7 +360,7 @@ impl CascadingCompressor { let actual_ratio = (after_nbytes != 0).then(|| before_nbytes as f64 / after_nbytes as f64); let accepted = after_nbytes < before_nbytes; - trace::record_winner_compress_result(after_nbytes, None, actual_ratio, accepted); + trace::record_winner_compress_result(after_nbytes, None, None, actual_ratio, accepted); return if accepted { Ok(compressed) @@ -383,6 +401,7 @@ impl CascadingCompressor { trace::record_winner_compress_result( after_nbytes, winner_estimate.trace_ratio(), + winner_estimate.trace_cost(), actual_ratio, accepted, ); @@ -397,16 +416,21 @@ impl CascadingCompressor { /// Calls [`expected_compression_ratio`] on each candidate and returns the winning scheme along /// with its resolved winner estimate, or `None` if no scheme beats the canonical encoding. /// + /// Each scored candidate is priced by the compressor's [`CostModel`]; the winner is the + /// candidate with the minimum cost, and only candidates pricing strictly below + /// [`CostModel::canonical_cost`] are eligible. + /// /// Selection runs in two passes. Pass 1 evaluates every immediate /// [`CompressionEstimate::Verdict`] and tracks the running best. [`Scheme`]s returning /// [`CompressionEstimate::Deferred`] are stashed for pass 2 so that we do not make any /// expensive computations if we don't have to. /// - /// Pass 2 evaluates the deferred work and, for each [`DeferredEstimate::Callback`], passes the - /// current best [`EstimateScore`] as an early-exit hint so the callback can return - /// [`EstimateVerdict::Skip`] without doing expensive work when it cannot beat the threshold. + /// Pass 2 evaluates the deferred work and, for each [`DeferredEstimate::Callback`], passes a + /// [`SkipThreshold`] wrapping the current best as an early-exit hint so the callback can + /// return [`EstimateVerdict::Skip`] without doing expensive work when it cannot win. /// - /// Ties are broken by registration order within each pass. + /// Ties are broken by registration order within each pass (displacing the running best + /// requires a strictly lower cost). /// /// [`expected_compression_ratio`]: Scheme::expected_compression_ratio fn choose_best_scheme( @@ -416,9 +440,24 @@ impl CascadingCompressor { compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult> { - let mut best: Option<(&'static dyn Scheme, EstimateScore)> = None; + let mut best: Option<(Cost, Candidate)> = None; let mut deferred: Vec<(&'static dyn Scheme, DeferredEstimate)> = Vec::new(); + let canonical_cost = self + .cost_model + .canonical_cost(data, data.array().len() as u64); + + let input_nbytes = data.array().nbytes(); + let n_values = data.array().len() as u64; + let new_candidate = |scheme, score, sampled| Candidate { + scheme, + score, + input_nbytes, + n_values, + sampled, + cascade: compress_ctx.cascade_history().to_vec(), + }; + // Pass 1: evaluate every immediate verdict. Stash deferred work for pass 2. { let _verdict_pass = trace::verdict_pass_span().entered(); @@ -430,9 +469,10 @@ impl CascadingCompressor { } CompressionEstimate::Verdict(EstimateVerdict::Ratio(ratio)) => { let score = EstimateScore::FiniteCompression(ratio); + let candidate = new_candidate(scheme, score, None); - if is_better_score(score, best.as_ref()) { - best = Some((scheme, score)); + if let Some(cost) = self.price(&candidate, canonical_cost, best.as_ref()) { + best = Some((cost, candidate)); } } CompressionEstimate::Deferred(deferred_estimate) => { @@ -446,22 +486,41 @@ impl CascadingCompressor { // short-circuit with `Skip` when they cannot beat it. for (scheme, deferred_estimate) in deferred { let _span = trace::scheme_eval_span(scheme.id()).entered(); - let threshold: Option = best.map(|(_, score)| score); + + // Model-side pruning: skip the deferred work outright when even this scheme's + // best case cannot strictly beat the best candidate so far. + if let Some((best_cost, _)) = best.as_ref() + && let Some(lower_bound) = self.cost_model.lower_bound(scheme.id(), data) + && lower_bound >= *best_cost + { + continue; + } + match deferred_estimate { DeferredEstimate::Sample => { - let score = estimate_compression_ratio_with_sampling( + let estimate = estimate_compression_ratio_with_sampling( self, scheme, data.array(), compress_ctx.clone(), exec_ctx, )?; + let candidate = new_candidate(scheme, estimate.score, Some(estimate.sampled)); - if is_better_score(score, best.as_ref()) { - best = Some((scheme, score)); + if let Some(cost) = self.price(&candidate, canonical_cost, best.as_ref()) { + best = Some((cost, candidate)); } } DeferredEstimate::Callback(callback) => { + let threshold = SkipThreshold::new( + best.as_ref() + .map(|(cost, candidate)| (*cost, candidate.score)), + Arc::clone(&self.cost_model), + scheme, + input_nbytes, + n_values, + compress_ctx.cascade_history().to_vec(), + ); match callback(self, data, threshold, compress_ctx.clone(), exec_ctx)? { EstimateVerdict::Skip => {} EstimateVerdict::AlwaysUse => { @@ -469,9 +528,12 @@ impl CascadingCompressor { } EstimateVerdict::Ratio(ratio) => { let score = EstimateScore::FiniteCompression(ratio); + let candidate = new_candidate(scheme, score, None); - if is_better_score(score, best.as_ref()) { - best = Some((scheme, score)); + if let Some(cost) = + self.price(&candidate, canonical_cost, best.as_ref()) + { + best = Some((cost, candidate)); } } } @@ -479,7 +541,31 @@ impl CascadingCompressor { } } - Ok(best.map(|(scheme, score)| (scheme, WinnerEstimate::Score(score)))) + // The sampled arrays carried by candidates are dropped here; only the winning scheme + // and its estimate survive selection. + Ok(best.map(|(cost, candidate)| { + ( + candidate.scheme, + WinnerEstimate::Score { + score: candidate.score, + cost, + }, + ) + })) + } + + /// Prices a candidate and returns its cost iff it becomes the new best: the model must + /// price it (`Some`), and the cost must be strictly below both the canonical baseline and + /// the best so far. Strict `<` preserves registration-order tie-breaking. + fn price( + &self, + candidate: &Candidate, + canonical_cost: Cost, + best: Option<&(Cost, Candidate)>, + ) -> Option { + let cost = self.cost_model.cost(candidate)?; + (cost < canonical_cost && best.is_none_or(|&(best_cost, _)| cost < best_cost)) + .then_some(cost) } // TODO(connor): Lots of room for optimization here. @@ -756,7 +842,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::AlwaysUse), + |_compressor, _data, _threshold, _ctx, _exec_ctx| Ok(EstimateVerdict::AlwaysUse), ))) } @@ -790,7 +876,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Skip), + |_compressor, _data, _threshold, _ctx, _exec_ctx| Ok(EstimateVerdict::Skip), ))) } @@ -824,7 +910,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Ratio(3.0)), + |_compressor, _data, _threshold, _ctx, _exec_ctx| Ok(EstimateVerdict::Ratio(3.0)), ))) } @@ -1014,7 +1100,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(2.0)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(2.0), .. })) if scheme.id() == DirectRatioScheme.id() )); Ok(()) @@ -1036,7 +1122,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(3.0)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(3.0), .. })) if scheme.id() == CallbackRatioScheme.id() )); Ok(()) @@ -1058,7 +1144,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(100.0)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(100.0), .. })) if scheme.id() == HugeRatioScheme.id() )); Ok(()) @@ -1080,7 +1166,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(100.0)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(100.0), .. })) if scheme.id() == HugeRatioScheme.id() )); Ok(()) @@ -1104,11 +1190,11 @@ mod tests { Ok(()) } - // Observer helper used by threshold-related tests. Captures the `best_so_far` value the - // compressor passes to its deferred callback. `OBSERVER_LOCK` serializes tests that share - // `OBSERVED_THRESHOLD` so they do not race. + // Observer helper used by threshold-related tests. Captures the threshold handle's + // ratio view as the compressor passes it to a deferred callback. `OBSERVER_LOCK` + // serializes tests that share `OBSERVED_THRESHOLD` so they do not race. static OBSERVER_LOCK: Mutex<()> = Mutex::new(()); - static OBSERVED_THRESHOLD: Mutex>> = Mutex::new(None); + static OBSERVED_THRESHOLD: Mutex>> = Mutex::new(None); #[derive(Debug)] struct ThresholdObservingScheme; @@ -1129,8 +1215,8 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, best_so_far, _ctx, _exec_ctx| { - *OBSERVED_THRESHOLD.lock() = Some(best_so_far); + |_compressor, _data, threshold, _ctx, _exec_ctx| { + *OBSERVED_THRESHOLD.lock() = Some(threshold.best_ratio()); Ok(EstimateVerdict::Skip) }, ))) @@ -1147,6 +1233,72 @@ mod tests { } } + /// Records the answers the compressor-built [`SkipThreshold`] gives at, just above, and + /// below the incumbent ratio. Shares `OBSERVER_LOCK` with the other observing tests. + static OBSERVED_CANNOT_WIN: Mutex> = Mutex::new(None); + + #[derive(Debug)] + struct CannotWinObservingScheme; + + impl Scheme for CannotWinObservingScheme { + fn scheme_name(&self) -> &'static str { + "test.cannot_win_observing" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, _data, threshold, _ctx, _exec_ctx| { + *OBSERVED_CANNOT_WIN.lock() = Some(( + threshold.best_case_ratio_cannot_win(2.0), + threshold.best_case_ratio_cannot_win(2.0f64.next_up()), + threshold.best_case_ratio_cannot_win(1.9), + )); + Ok(EstimateVerdict::Skip) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + /// The compressor-built threshold must reproduce the historical `max_ratio <= best` + /// skip rule exactly: a best case that ties or loses to the incumbent cannot win, one + /// that is strictly greater can. + #[test] + fn threshold_cannot_win_matches_ratio_rule() -> VortexResult<()> { + let _guard = OBSERVER_LOCK.lock(); + *OBSERVED_CANNOT_WIN.lock() = None; + + let compressor = + CascadingCompressor::new(vec![&DirectRatioScheme, &CannotWinObservingScheme]); + let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CannotWinObservingScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; + + // Incumbent ratio is 2.0: a tie cannot win, the next float up can, a loss cannot. + assert_eq!(*OBSERVED_CANNOT_WIN.lock(), Some((true, false, true))); + Ok(()) + } + #[derive(Debug)] struct CallbackMatchingRatioScheme; @@ -1166,7 +1318,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Ratio(2.0)), + |_compressor, _data, _threshold, _ctx, _exec_ctx| Ok(EstimateVerdict::Ratio(2.0)), ))) } @@ -1220,10 +1372,7 @@ mod tests { compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; let observed = *OBSERVED_THRESHOLD.lock(); - assert!(matches!( - observed, - Some(Some(EstimateScore::FiniteCompression(r))) if r == 2.0 - )); + assert_eq!(observed, Some(Some(2.0))); Ok(()) } @@ -1241,8 +1390,8 @@ mod tests { compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; - // The observing callback was invoked (outer `Some`) and `best_so_far` was `None` (inner - // `None`) because the zero-byte sample is never stored as the best. + // The observing callback was invoked (outer `Some`) and the threshold's ratio view + // was `None` (inner `None`) because the zero-byte sample is never stored as the best. let observed = *OBSERVED_THRESHOLD.lock(); assert_eq!(observed, Some(None)); Ok(()) @@ -1281,10 +1430,7 @@ mod tests { compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; let observed = *OBSERVED_THRESHOLD.lock(); - assert!(matches!( - observed, - Some(Some(EstimateScore::FiniteCompression(r))) if r == 3.0 - )); + assert_eq!(observed, Some(Some(3.0))); Ok(()) } @@ -1308,12 +1454,82 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(r)))) + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(r), .. })) if scheme.id() == DirectRatioScheme.id() && r == 2.0 )); Ok(()) } + /// Delegates pricing to [`SizeCost`] but bounds every scheme at a cost that can never + /// win, so all deferred work is pruned as soon as any best candidate exists. + #[derive(Debug)] + struct PruneAllDeferredModel; + + impl CostModel for PruneAllDeferredModel { + fn cost(&self, candidate: &Candidate) -> Option { + SizeCost.cost(candidate) + } + + fn canonical_cost(&self, data: &ArrayAndStats, n_values: u64) -> Cost { + SizeCost.canonical_cost(data, n_values) + } + + fn lower_bound(&self, _scheme: SchemeId, _data: &ArrayAndStats) -> Option { + Some(Cost::new(f64::MAX)) + } + } + + #[test] + fn lower_bound_prunes_deferred_work_behind_a_best() -> VortexResult<()> { + // `CallbackRatioScheme`'s deferred `Ratio(3.0)` would beat `DirectRatioScheme`'s + // immediate `Ratio(2.0)` (see `callback_ratio_competes_numerically`), but the model + // bounds it above the running best, so its deferred work is pruned. + let compressor = CascadingCompressor::new(vec![&DirectRatioScheme, &CallbackRatioScheme]) + .with_cost_model(Arc::new(PruneAllDeferredModel)); + let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CallbackRatioScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + let winner = compressor.choose_best_scheme( + &schemes, + &data, + CompressorContext::new(), + &mut exec_ctx, + )?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(2.0), .. })) + if scheme.id() == DirectRatioScheme.id() + )); + Ok(()) + } + + #[test] + fn lower_bound_never_prunes_without_a_best() -> VortexResult<()> { + // With no best candidate there is nothing to prune against, so the deferred + // callback still runs even under a prune-everything bound. + let compressor = CascadingCompressor::new(vec![&CallbackRatioScheme]) + .with_cost_model(Arc::new(PruneAllDeferredModel)); + let schemes: [&'static dyn Scheme; 1] = [&CallbackRatioScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + let winner = compressor.choose_best_scheme( + &schemes, + &data, + CompressorContext::new(), + &mut exec_ctx, + )?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Score { score: EstimateScore::FiniteCompression(3.0), .. })) + if scheme.id() == CallbackRatioScheme.id() + )); + Ok(()) + } + #[test] fn all_null_array_compresses_to_constant() -> VortexResult<()> { let array = PrimitiveArray::new( @@ -1356,14 +1572,16 @@ mod tests { // Before the fix this panicked with: // "this must be present since `DictScheme` declared that we need distinct values" let mut exec_ctx = SESSION.create_execution_ctx(); - let score = estimate_compression_ratio_with_sampling( + let estimate = estimate_compression_ratio_with_sampling( &compressor, &FloatDictScheme, &array, ctx, &mut exec_ctx, )?; - assert!(matches!(score, EstimateScore::FiniteCompression(ratio) if ratio.is_finite())); + assert!( + matches!(estimate.score, EstimateScore::FiniteCompression(ratio) if ratio.is_finite()) + ); Ok(()) } } diff --git a/vortex-compressor/src/cost/mod.rs b/vortex-compressor/src/cost/mod.rs new file mode 100644 index 00000000000..049659b045f --- /dev/null +++ b/vortex-compressor/src/cost/mod.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Pluggable cost models for scheme selection. +//! +//! A [`CostModel`] is the policy half of scheme selection: [`Scheme`]s produce mechanical +//! signals (estimated or sample-measured compression ratios, collected into a [`Candidate`]), +//! and the model prices each candidate. The compressor picks the candidate with the lowest +//! cost that (strictly) beats [`CostModel::canonical_cost`] — the price of leaving the array +//! in its canonical encoding. +//! +//! The default model is [`SizeCost`], which reproduces the compressor's historical +//! ratio-argmax selection bit-exactly. +//! +//! # What sits outside the model +//! +//! Two selection mechanisms are deliberately *not* routed through the cost model: +//! +//! - [`EstimateVerdict::AlwaysUse`] short-circuits selection entirely. It expresses semantic +//! normalization (e.g. decimal byte-parts, temporal decomposition), not a priced trade-off. +//! - The byte-acceptance gate: after the winning scheme compresses the full array, the result +//! is kept only if it is byte-wise smaller than its input. This is an axiom for **all** +//! models — compression never grows bytes. A model that prefers canonical (e.g. for speed) +//! expresses that by pricing every candidate at or above `canonical_cost`, so selection +//! returns no winner and the array stays canonical; the gate never forces a *bad* encoding, +//! only "no encoding". (The gate's `AnyScalarFn` carve-out is likewise semantic +//! denormalization, not cost.) +//! +//! # Determinism +//! +//! Cost models must be pure functions of the candidate and their own configuration: no +//! timing measurements, no I/O, no global state. The compressor's output must be a +//! deterministic function of its input and configuration. +//! +//! [`Scheme`]: crate::scheme::Scheme +//! [`EstimateVerdict::AlwaysUse`]: crate::estimate::EstimateVerdict::AlwaysUse + +mod size; + +use std::fmt::Debug; + +pub use size::SizeCost; + +pub use crate::candidate::Candidate; +use crate::scheme::SchemeId; +use crate::stats::ArrayAndStats; + +/// An opaque, totally ordered cost. Lower is better. +/// +/// Units are defined by each [`CostModel`], not by the framework: costs are only ever +/// compared *within* one model on one compressor instance, so cross-model comparability is +/// intentionally not provided. +/// +/// Values must be finite. `NaN`/infinite prices are not representable orderings; a model +/// that cannot price a candidate rejects it by returning `None` from [`CostModel::cost`]. +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] +pub struct Cost(f64); + +impl Cost { + /// Creates a cost from a finite value. + /// + /// # Panics + /// + /// Panics in debug builds if `value` is not finite. + pub fn new(value: f64) -> Self { + debug_assert!(value.is_finite(), "Cost must be finite, got {value}"); + Self(value) + } + + /// Returns the raw cost value. + pub fn value(self) -> f64 { + self.0 + } +} + +/// Prices [`Candidate`]s during scheme selection. +/// +/// Implementations must be deterministic pure functions of their inputs and cheap: `cost` is +/// called once per scored candidate per selection site, i.e. `O(schemes × cascade levels)` +/// times per compressed chunk. +/// +/// Selection semantics: the winner is the candidate with the minimum cost, requiring +/// strictly lower cost both to displace the running best (ties break by scheme registration +/// order) and to beat [`canonical_cost`]. Returning `None` from [`cost`] rejects the +/// candidate outright. +/// +/// See the [module docs](self) for the acceptance axioms that apply to all models. +/// +/// [`canonical_cost`]: CostModel::canonical_cost +/// [`cost`]: CostModel::cost +pub trait CostModel: Debug + Send + Sync + 'static { + /// Estimated cost of choosing this candidate. Lower is better; `None` rejects the + /// candidate. + fn cost(&self, candidate: &Candidate) -> Option; + + /// Cost of leaving the array canonical — the baseline every candidate must strictly + /// beat to be selected. + fn canonical_cost(&self, data: &ArrayAndStats, n_values: u64) -> Cost; + + /// A lower bound on the cost any candidate from `scheme` could possibly achieve on + /// `data`, or `None` (the default) if the model offers no bound. + /// + /// The compressor skips a scheme's deferred estimation work (sampling, callbacks) + /// outright when the bound is not strictly below the best cost observed so far, since + /// even the scheme's best case could not win selection. Models that can bound a + /// scheme's payoff from cheap facts (e.g. minimum plausible output bytes at zero decode + /// cost) can prune expensive sampling here; a sound bound never exceeds the cost of any + /// candidate the scheme could actually produce. + fn lower_bound(&self, scheme: SchemeId, data: &ArrayAndStats) -> Option { + let _ = (scheme, data); + None + } +} diff --git a/vortex-compressor/src/cost/size.rs b/vortex-compressor/src/cost/size.rs new file mode 100644 index 00000000000..00a71d6ad6f --- /dev/null +++ b/vortex-compressor/src/cost/size.rs @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The default, size-only cost model. + +use crate::candidate::Candidate; +use crate::cost::Cost; +use crate::cost::CostModel; +use crate::estimate::EstimateScore; +use crate::stats::ArrayAndStats; + +/// The default cost model: maximize estimated compression ratio. +/// +/// `SizeCost` reproduces the compressor's historical ratio-argmax selection **bit-exactly**: +/// +/// - A candidate is priced at `Cost(-ratio)` over the exact ratio signal the candidate +/// carries (an analytic estimate or a sample-measured `before / after`), so argmin cost is +/// argmax ratio with identical tie behavior. Pricing off the ratio rather than recomputing +/// bytes (`input / ratio`) matters: IEEE division could collapse strict ratio inequalities +/// into cost ties and silently flip winners via the registration-order tie-break. +/// - The canonical baseline is `Cost(-1.0)`, so `cost < canonical_cost` reproduces the +/// historical `ratio > 1.0` validity gate. +/// - Zero-byte sample results and non-finite or subnormal ratios are rejected (`None`), +/// reproducing the historical estimate-validity checks. +#[derive(Debug, Default, Clone, Copy)] +pub struct SizeCost; + +impl CostModel for SizeCost { + fn cost(&self, candidate: &Candidate) -> Option { + match candidate.score { + EstimateScore::FiniteCompression(ratio) => { + (ratio.is_finite() && !ratio.is_subnormal()).then(|| Cost::new(-ratio)) + } + EstimateScore::ZeroBytes => None, + } + } + + fn canonical_cost(&self, _data: &ArrayAndStats, _n_values: u64) -> Cost { + Cost::new(-1.0) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use vortex_array::ArrayRef; + use vortex_array::Canonical; + use vortex_array::ExecutionCtx; + use vortex_array::IntoArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::validity::Validity; + use vortex_buffer::buffer; + use vortex_error::VortexResult; + + use super::*; + use crate::CascadingCompressor; + use crate::ctx::CompressorContext; + use crate::estimate::CompressionEstimate; + use crate::estimate::EstimateVerdict; + use crate::scheme::Scheme; + use crate::stats::GenerateStatsOptions; + + #[derive(Debug)] + struct TestScheme; + + impl Scheme for TestScheme { + fn scheme_name(&self) -> &'static str { + "test.size_cost" + } + + fn matches(&self, _canonical: &Canonical) -> bool { + true + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::Skip) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + fn candidate(score: EstimateScore) -> Candidate { + Candidate { + scheme: &TestScheme, + score, + input_nbytes: 1024, + n_values: 256, + sampled: None, + cascade: Vec::new(), + } + } + + fn canonical_cost() -> Cost { + let array = PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array(); + let data = ArrayAndStats::new(array, GenerateStatsOptions::default()); + SizeCost.canonical_cost(&data, 4) + } + + /// The historical estimate-validity rule that `SizeCost` must reproduce: finite, + /// non-subnormal, and strictly better than canonical (`ratio > 1.0`). + fn legacy_is_valid(ratio: f64) -> bool { + ratio.is_finite() && !ratio.is_subnormal() && ratio > 1.0 + } + + /// Every ratio regime: clear winners, near-1.0 boundaries, exact ties, sub-1.0, + /// non-positive, and non-finite/subnormal garbage. + const RATIOS: &[f64] = &[ + 100.0, + 3.0, + 2.0, + 1.0 + f64::EPSILON, + 1.0, + 1.0 - f64::EPSILON, + 0.5, + 0.0, + -1.0, + f64::MIN_POSITIVE / 2.0, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + ]; + + /// `cost < canonical_cost` must reproduce `legacy_is_valid` exactly, with non-priceable + /// ratios rejected as `None`. + #[test] + fn validity_matches_ratio_gate() { + let canonical = canonical_cost(); + for &ratio in RATIOS { + let cost = SizeCost.cost(&candidate(EstimateScore::FiniteCompression(ratio))); + let valid = cost.is_some_and(|cost| cost < canonical); + assert_eq!( + valid, + legacy_is_valid(ratio), + "validity mismatch for ratio {ratio}" + ); + } + } + + /// For every ordered pair of ratios, the cost ordering must equal the (reversed) ratio + /// ordering the selector historically used: a challenger displaces the best iff its + /// ratio is strictly greater. Equal ratios must produce equal costs (ties keep the + /// incumbent via strict `<`). + #[test] + fn cost_ordering_matches_ratio_ordering() { + let valid: Vec = RATIOS + .iter() + .copied() + .filter(|&r| legacy_is_valid(r)) + .collect(); + for &best in &valid { + for &challenger in &valid { + let best_cost = SizeCost + .cost(&candidate(EstimateScore::FiniteCompression(best))) + .expect("valid ratio must be priceable"); + let challenger_cost = SizeCost + .cost(&candidate(EstimateScore::FiniteCompression(challenger))) + .expect("valid ratio must be priceable"); + + let displaces = challenger_cost < best_cost; + assert_eq!( + displaces, + challenger > best, + "ordering mismatch: challenger {challenger} vs best {best}" + ); + assert_eq!( + challenger_cost == best_cost, + challenger == best, + "tie mismatch: challenger {challenger} vs best {best}" + ); + } + } + } + + #[rstest] + #[case::nan(f64::NAN)] + #[case::infinite(f64::INFINITY)] + #[case::neg_infinite(f64::NEG_INFINITY)] + #[case::subnormal(f64::MIN_POSITIVE / 2.0)] + fn unpriceable_ratios_are_rejected(#[case] ratio: f64) { + assert!( + SizeCost + .cost(&candidate(EstimateScore::FiniteCompression(ratio))) + .is_none() + ); + } + + #[test] + fn zero_bytes_is_rejected() { + assert!( + SizeCost + .cost(&candidate(EstimateScore::ZeroBytes)) + .is_none() + ); + } + + /// Under `SizeCost`, [`SkipThreshold::best_case_ratio_cannot_win`] must reduce to + /// exactly the historical `max_ratio <= best_ratio` skip rule, including exact ties and + /// one-ulp boundaries; unpriceable best cases must never skip. + #[test] + fn skip_threshold_reduces_to_ratio_comparison() { + use std::sync::Arc; + + use crate::estimate::SkipThreshold; + + for &best_ratio in &[1.5, 2.0, 3.0, 60.8, 1024.0] { + let threshold = SkipThreshold::new( + Some(( + Cost::new(-best_ratio), + EstimateScore::FiniteCompression(best_ratio), + )), + Arc::new(SizeCost), + &TestScheme, + 1024, + 256, + Vec::new(), + ); + assert_eq!(threshold.best_ratio(), Some(best_ratio)); + + let max_ratios = [ + 0.5, + 1.0, + best_ratio.next_down(), + best_ratio, + best_ratio.next_up(), + best_ratio * 2.0, + ]; + for max_ratio in max_ratios { + assert_eq!( + threshold.best_case_ratio_cannot_win(max_ratio), + max_ratio <= best_ratio, + "mismatch for max_ratio {max_ratio} vs best {best_ratio}" + ); + } + + // Best cases SizeCost cannot price never skip: the callback proceeds and its + // real estimate is rejected later, exactly as before. + assert!(!threshold.best_case_ratio_cannot_win(f64::NAN)); + assert!(!threshold.best_case_ratio_cannot_win(f64::INFINITY)); + } + + // Without a best candidate there is no threshold to lose to. + let no_best = + SkipThreshold::new(None, Arc::new(SizeCost), &TestScheme, 1024, 256, Vec::new()); + assert!(!no_best.best_case_ratio_cannot_win(1e9)); + assert_eq!(no_best.best_ratio(), None); + } +} diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index 82d021353d2..df832f5d743 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -4,6 +4,7 @@ //! Compression ratio estimation types and sampling-based estimation. use std::fmt; +use std::sync::Arc; use vortex_array::ArrayRef; use vortex_array::Canonical; @@ -12,38 +13,129 @@ use vortex_array::IntoArray; use vortex_error::VortexResult; use crate::CascadingCompressor; +use crate::candidate::Candidate; +use crate::cost::Cost; +use crate::cost::CostModel; use crate::ctx::CompressorContext; use crate::sample::SAMPLE_SIZE; use crate::sample::sample; use crate::sample::sample_count_approx_one_percent; use crate::scheme::Scheme; use crate::scheme::SchemeExt; +use crate::scheme::SchemeId; use crate::stats::ArrayAndStats; use crate::trace; /// Closure type for [`DeferredEstimate::Callback`]. /// -/// The compressor calls this with the same arguments it would pass to sampling, plus the best -/// [`EstimateScore`] observed so far (if any). The closure must resolve directly to a terminal -/// [`EstimateVerdict`]. +/// The compressor calls this with the same arguments it would pass to sampling, plus a +/// [`SkipThreshold`] handle wrapping the best candidate observed so far. The closure must +/// resolve directly to a terminal [`EstimateVerdict`]. /// -/// The `best_so_far` threshold is an early-exit hint. If your scheme's maximum achievable -/// compression ratio is not strictly greater than -/// `best_so_far.and_then(EstimateScore::finite_ratio)`, you should return -/// [`EstimateVerdict::Skip`]. Returning a ratio equal to the threshold is permitted but will -/// lose to the prior best due to strict `>` tie-breaking in the selector. Use the threshold -/// only as an early-exit hint, never to perform additional work. +/// The threshold is an early-exit hint. If your scheme knows its maximum achievable +/// compression ratio, ask [`SkipThreshold::best_case_ratio_cannot_win`] before doing +/// expensive work and return [`EstimateVerdict::Skip`] when it answers `true`. Returning an +/// estimate that merely ties the threshold is permitted but will lose to the prior best, +/// since displacing the running best requires a strictly better (lower-cost) candidate. Use +/// the threshold only as an early-exit hint, never to perform additional work. #[rustfmt::skip] pub type EstimateFn = dyn FnOnce( &CascadingCompressor, &ArrayAndStats, - Option, + SkipThreshold, CompressorContext, &mut ExecutionCtx, ) -> VortexResult + Send + Sync; +/// Early-exit threshold handle passed to [`DeferredEstimate::Callback`] closures. +/// +/// Owned and constructed by the compressor during selection; wraps the best candidate +/// observed so far together with the compressor's [`CostModel`] and the selection-site +/// facts needed to price a hypothetical best-case candidate. Schemes keep their side of the +/// bargain — knowing their own best case — and ask the handle whether that best case could +/// still win. +pub struct SkipThreshold { + /// The best candidate so far: its cost and its ranked estimate. `None` when no + /// candidate has been stored yet. + best: Option<(Cost, EstimateScore)>, + + /// The compressor's cost model. + model: Arc, + + /// The scheme whose deferred estimate is being resolved. + scheme: &'static dyn Scheme, + + /// Uncompressed size in bytes of the array under selection. + input_nbytes: u64, + + /// Number of values in the array under selection. + n_values: u64, + + /// The cascade ancestry `(scheme_id, child_index)` of the selection site. + cascade: Vec<(SchemeId, usize)>, +} + +impl SkipThreshold { + /// Creates a threshold handle for `scheme` at a selection site. + /// + /// Normally the compressor builds these during selection. The constructor is public so + /// scheme implementors can unit-test their callbacks' skip decisions. + pub fn new( + best: Option<(Cost, EstimateScore)>, + model: Arc, + scheme: &'static dyn Scheme, + input_nbytes: u64, + n_values: u64, + cascade: Vec<(SchemeId, usize)>, + ) -> Self { + Self { + best, + model, + scheme, + input_nbytes, + n_values, + cascade, + } + } + + /// Returns `true` if a candidate achieving `max_ratio` — the best case the calling + /// scheme could possibly report — still could not displace the best candidate observed + /// so far, so the scheme should return [`EstimateVerdict::Skip`] without doing expensive + /// work. + /// + /// The handle prices a hypothetical candidate carrying `max_ratio` under the + /// compressor's cost model and compares it against the best cost so far; displacement + /// requires a strictly lower cost. Under the default [`SizeCost`] model this reduces to + /// exactly the historical `max_ratio <= best_ratio` skip. Returns `false` when there is + /// no best candidate yet or the best case cannot be priced. + /// + /// [`SizeCost`]: crate::cost::SizeCost + pub fn best_case_ratio_cannot_win(&self, max_ratio: f64) -> bool { + let Some((best_cost, _)) = self.best else { + return false; + }; + let best_case = Candidate { + scheme: self.scheme, + score: EstimateScore::FiniteCompression(max_ratio), + input_nbytes: self.input_nbytes, + n_values: self.n_values, + sampled: None, + cascade: self.cascade.clone(), + }; + self.model + .cost(&best_case) + .is_some_and(|cost| cost >= best_cost) + } + + /// The best finite compression ratio observed so far, if any — the pre-cost-model view + /// of the threshold, kept for callbacks that reason in ratio space. + pub fn best_ratio(&self) -> Option { + self.best.and_then(|(_, score)| score.finite_ratio()) + } +} + /// The result of a [`Scheme`]'s compression ratio estimation. /// /// This type is returned by [`Scheme::expected_compression_ratio`] to tell the compressor how @@ -96,9 +188,9 @@ pub enum DeferredEstimate { /// it cannot request more sampling or another deferred callback. /// /// The compressor evaluates all immediate [`CompressionEstimate::Verdict`] results before - /// invoking any deferred callback, and passes the best [`EstimateScore`] observed so far to - /// the callback. This lets the callback return [`EstimateVerdict::Skip`] without performing - /// expensive work when its maximum achievable ratio cannot beat the current best. See + /// invoking any deferred callback, and passes a [`SkipThreshold`] wrapping the best + /// candidate observed so far. This lets the callback return [`EstimateVerdict::Skip`] + /// without performing expensive work when its maximum achievable estimate cannot win. See /// [`EstimateFn`] for the full contract. Callback(Box), } @@ -137,27 +229,6 @@ impl EstimateScore { Self::ZeroBytes => None, } } - - /// Returns whether this estimate is eligible to compete. - fn is_valid(self) -> bool { - match self { - Self::FiniteCompression(ratio) => { - ratio.is_finite() && !ratio.is_subnormal() && ratio > 1.0 - } - Self::ZeroBytes => false, - } - } - - /// Returns whether this estimate beats another valid estimate. - fn beats(self, other: Self) -> bool { - match (self, other) { - (Self::ZeroBytes, _) => false, - (Self::FiniteCompression(_), Self::ZeroBytes) => true, - (Self::FiniteCompression(ratio), Self::FiniteCompression(best_ratio)) => { - ratio > best_ratio - } - } - } } /// Winner estimate carried from scheme selection into result tracing. @@ -165,8 +236,13 @@ impl EstimateScore { pub(super) enum WinnerEstimate { /// The scheme must be used immediately. AlwaysUse, - /// The scheme won by a ranked estimate. - Score(EstimateScore), + /// The scheme won by a ranked estimate, priced by the compressor's cost model. + Score { + /// The winning candidate's ranked estimate. + score: EstimateScore, + /// The winning candidate's cost under the compressor's cost model. + cost: Cost, + }, } impl WinnerEstimate { @@ -174,17 +250,28 @@ impl WinnerEstimate { pub(super) fn trace_ratio(self) -> Option { match self { Self::AlwaysUse => None, - Self::Score(score) => score.finite_ratio(), + Self::Score { score, .. } => score.finite_ratio(), + } + } + + /// Returns the traceable cost for the winning estimate. + pub(super) fn trace_cost(self) -> Option { + match self { + Self::AlwaysUse => None, + Self::Score { cost, .. } => Some(cost.value()), } } } -/// Returns `true` if `score` beats the current best estimate. -pub(super) fn is_better_score( - score: EstimateScore, - best: Option<&(&'static dyn Scheme, EstimateScore)>, -) -> bool { - score.is_valid() && best.is_none_or(|(_, best_score)| score.beats(*best_score)) +/// A sampling-based estimate: the ranked score together with the compressed sample array it +/// was measured on. +pub(crate) struct SampledEstimate { + /// The ranked estimate measured on the sample. + pub(crate) score: EstimateScore, + + /// The compressed sample array. Its encoding tree is the best available prediction of + /// the full-array encoding tree. + pub(crate) sampled: ArrayRef, } /// Estimates compression ratio by compressing a ~1% sample of the data. @@ -192,6 +279,9 @@ pub(super) fn is_better_score( /// Creates a new [`ArrayAndStats`] for the sample so that stats are generated from the sample, not /// the full array. /// +/// Returns the compressed sample alongside its score so the selection loop can keep the +/// sample's encoding tree for the winning candidate. +/// /// # Errors /// /// Returns an error if sample compression fails. @@ -201,7 +291,7 @@ pub(super) fn estimate_compression_ratio_with_sampling( array: &ArrayRef, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, -) -> VortexResult { +) -> VortexResult { let sample_array = if compress_ctx.is_sample() { array.clone() } else { @@ -232,7 +322,10 @@ pub(super) fn estimate_compression_ratio_with_sampling( trace::zero_byte_sample_result(scheme.id(), before); } - Ok(score) + Ok(SampledEstimate { + score, + sampled: compressed, + }) } impl fmt::Debug for DeferredEstimate { diff --git a/vortex-compressor/src/lib.rs b/vortex-compressor/src/lib.rs index 86df4fc4cba..b821c865202 100644 --- a/vortex-compressor/src/lib.rs +++ b/vortex-compressor/src/lib.rs @@ -63,11 +63,13 @@ //! with a short `jq` query. pub mod builtins; +pub mod cost; pub mod ctx; pub mod estimate; pub mod scheme; pub mod stats; +mod candidate; mod constant; mod sample; diff --git a/vortex-compressor/src/trace.rs b/vortex-compressor/src/trace.rs index 84027272b31..e6cdffed997 100644 --- a/vortex-compressor/src/trace.rs +++ b/vortex-compressor/src/trace.rs @@ -87,7 +87,7 @@ pub(super) fn zero_byte_sample_result(scheme: SchemeId, sampled_before: u64) { /// Builds a span covering the winning scheme's full-array compression. /// /// `scheme_chosen` and `input_nbytes` are known up front. `compressed_nbytes`, -/// `estimated_ratio`, `achieved_ratio`, and `accepted` are filled in by +/// `estimated_ratio`, `estimated_cost`, `achieved_ratio`, and `accepted` are filled in by /// [`record_winner_compress_result`] once the encode completes. #[inline] pub(super) fn winner_compress_span(scheme: SchemeId, before_nbytes: u64) -> tracing::Span { @@ -98,6 +98,7 @@ pub(super) fn winner_compress_span(scheme: SchemeId, before_nbytes: u64) -> trac input_nbytes = before_nbytes, compressed_nbytes = tracing::field::Empty, estimated_ratio = tracing::field::Empty, + estimated_cost = tracing::field::Empty, achieved_ratio = tracing::field::Empty, accepted = tracing::field::Empty, ) @@ -108,6 +109,7 @@ pub(super) fn winner_compress_span(scheme: SchemeId, before_nbytes: u64) -> trac pub(super) fn record_winner_compress_result( compressed_nbytes: u64, estimated_ratio: Option, + estimated_cost: Option, achieved_ratio: Option, accepted: bool, ) { @@ -116,6 +118,9 @@ pub(super) fn record_winner_compress_result( if let Some(r) = estimated_ratio { span.record("estimated_ratio", r); } + if let Some(c) = estimated_cost { + span.record("estimated_cost", c); + } if let Some(r) = achieved_ratio { span.record("achieved_ratio", r); }