Skip to content

Extract scheme-selection policy into a pluggable CostModel #8698

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
connortsui20 wants to merge 5 commits into from
Closed

Extract scheme-selection policy into a pluggable CostModel #8698

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ab15859
Introduce crate-private Candidate bookkeeping in scheme selection
claude Jul 9, 2026
b0075cc
Add cost module: Cost, CostModel trait, and the SizeCost default
claude Jul 9, 2026
81364c8
Wire scheme selection to the cost model; selection becomes argmin cost
claude Jul 9, 2026
33c9399
Make schemes produce candidate evidence for cost models
connortsui20 Jul 10, 2026
314bf6e
Organize candidate evaluation and cost model modules
connortsui20 Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions vortex-btrblocks/src/schemes/binary/zstd.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use vortex_array::ArrayRef;
use vortex_array::Canonical;
use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::DeferredEvaluation;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_error::VortexResult;

use crate::ArrayAndStats;
Expand All @@ -29,13 +29,13 @@ impl Scheme for ZstdScheme {
canonical.dtype().is_binary()
}

fn expected_compression_ratio(
fn evaluate(
&self,
_data: &ArrayAndStats,
_compress_ctx: CompressorContext,
_exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
CompressionEstimate::Deferred(DeferredEstimate::Sample)
) -> SchemeEvaluation {
SchemeEvaluation::Deferred(DeferredEvaluation::Sample)
}

fn compress(
Expand Down
10 changes: 5 additions & 5 deletions vortex-btrblocks/src/schemes/binary/zstd_buffers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use vortex_array::ArrayRef;
use vortex_array::Canonical;
use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::DeferredEvaluation;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_error::VortexResult;

use crate::ArrayAndStats;
Expand All @@ -29,13 +29,13 @@ impl Scheme for ZstdBuffersScheme {
canonical.dtype().is_binary()
}

fn expected_compression_ratio(
fn evaluate(
&self,
_data: &ArrayAndStats,
_compress_ctx: CompressorContext,
_exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
CompressionEstimate::Deferred(DeferredEstimate::Sample)
) -> SchemeEvaluation {
SchemeEvaluation::Deferred(DeferredEvaluation::Sample)
}

fn compress(
Expand Down
9 changes: 4 additions & 5 deletions vortex-btrblocks/src/schemes/decimal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ use vortex_array::arrays::DecimalArray;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::arrays::decimal::narrowed_decimal;
use vortex_array::dtype::DecimalType;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::EstimateVerdict;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_decimal_byte_parts::DecimalByteParts;
use vortex_error::VortexResult;

Expand Down Expand Up @@ -43,14 +42,14 @@ impl Scheme for DecimalScheme {
1
}

fn expected_compression_ratio(
fn evaluate(
&self,
_data: &ArrayAndStats,
_compress_ctx: CompressorContext,
_exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
) -> SchemeEvaluation {
// Decimal compression is almost always beneficial (narrowing + primitive compression).
CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse)
SchemeEvaluation::AlwaysUse
}

fn compress(
Expand Down
15 changes: 7 additions & 8 deletions vortex-btrblocks/src/schemes/float/alp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ use vortex_array::arrays::Patched;
use vortex_array::arrays::patched::use_experimental_patches;
use vortex_array::arrays::primitive::PrimitiveArrayExt;
use vortex_array::dtype::PType;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::EstimateVerdict;
use vortex_compressor::scheme::DeferredEvaluation;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_error::VortexResult;

use crate::ArrayAndStats;
Expand Down Expand Up @@ -45,24 +44,24 @@ impl Scheme for ALPScheme {
1
}

fn expected_compression_ratio(
fn evaluate(
&self,
data: &ArrayAndStats,
compress_ctx: CompressorContext,
_exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
) -> SchemeEvaluation {
// ALP encodes floats as integers. Without integer compression afterward, the encoded ints
// are the same size.
if compress_ctx.finished_cascading() {
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
return SchemeEvaluation::Skip;
}

// We don't support ALP for f16.
if data.array_as_primitive().ptype() == PType::F16 {
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
return SchemeEvaluation::Skip;
}

CompressionEstimate::Deferred(DeferredEstimate::Sample)
SchemeEvaluation::Deferred(DeferredEvaluation::Sample)
}

fn compress(
Expand Down
13 changes: 6 additions & 7 deletions vortex-btrblocks/src/schemes/float/alprd.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,8 @@ use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_array::arrays::primitive::PrimitiveArrayExt;
use vortex_array::dtype::PType;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::EstimateVerdict;
use vortex_compressor::scheme::DeferredEvaluation;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_error::VortexResult;
use vortex_error::vortex_panic;

Expand All @@ -37,18 +36,18 @@ impl Scheme for ALPRDScheme {
canonical.dtype().is_float()
}

fn expected_compression_ratio(
fn evaluate(
&self,
data: &ArrayAndStats,
_compress_ctx: CompressorContext,
_exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
) -> SchemeEvaluation {
// We don't support ALPRD for f16.
if data.array_as_primitive().ptype() == PType::F16 {
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
return SchemeEvaluation::Skip;
}

CompressionEstimate::Deferred(DeferredEstimate::Sample)
SchemeEvaluation::Deferred(DeferredEvaluation::Sample)
}

fn compress(
Expand Down
10 changes: 5 additions & 5 deletions vortex-btrblocks/src/schemes/float/pco.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use vortex_array::ArrayRef;
use vortex_array::Canonical;
use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::DeferredEvaluation;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_error::VortexResult;

use crate::ArrayAndStats;
Expand All @@ -29,13 +29,13 @@ impl Scheme for PcoScheme {
canonical.dtype().is_float()
}

fn expected_compression_ratio(
fn evaluate(
&self,
_data: &ArrayAndStats,
_compress_ctx: CompressorContext,
_exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
CompressionEstimate::Deferred(DeferredEstimate::Sample)
) -> SchemeEvaluation {
SchemeEvaluation::Deferred(DeferredEvaluation::Sample)
}

fn compress(
Expand Down
15 changes: 7 additions & 8 deletions vortex-btrblocks/src/schemes/float/rle.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@ use vortex_array::ArrayRef;
use vortex_array::Canonical;
use vortex_array::ExecutionCtx;
use vortex_compressor::scheme::AncestorExclusion;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::DeferredEvaluation;
use vortex_compressor::scheme::DescendantExclusion;
use vortex_compressor::scheme::EstimateVerdict;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_error::VortexResult;

use crate::ArrayAndStats;
Expand Down Expand Up @@ -48,22 +47,22 @@ impl Scheme for FloatRLEScheme {
rle_ancestor_exclusions()
}

fn expected_compression_ratio(
fn evaluate(
&self,
data: &ArrayAndStats,
compress_ctx: CompressorContext,
exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
) -> SchemeEvaluation {
// RLE is only useful when we cascade it with another encoding.
if compress_ctx.finished_cascading() {
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
return SchemeEvaluation::Skip;
}

if data.float_stats(exec_ctx).average_run_length() < RUN_LENGTH_THRESHOLD {
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
return SchemeEvaluation::Skip;
}

CompressionEstimate::Deferred(DeferredEstimate::Sample)
SchemeEvaluation::Deferred(DeferredEvaluation::Sample)
}

fn compress(
Expand Down
16 changes: 9 additions & 7 deletions vortex-btrblocks/src/schemes/float/sparse.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::arrays::primitive::PrimitiveArrayExt;
use vortex_compressor::scheme::CandidateEstimate;
use vortex_compressor::scheme::ChildSelection;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::DescendantExclusion;
use vortex_compressor::scheme::EstimateVerdict;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_error::VortexResult;
use vortex_sparse::Sparse;
use vortex_sparse::SparseExt as _;
Expand Down Expand Up @@ -52,28 +52,30 @@ impl Scheme for NullDominatedSparseScheme {
}]
}

fn expected_compression_ratio(
fn evaluate(
&self,
data: &ArrayAndStats,
_compress_ctx: CompressorContext,
exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
) -> SchemeEvaluation {
let len = data.array_len() as f64;
let stats = data.float_stats(exec_ctx);
let value_count = stats.value_count();

// All-null arrays should be compressed as constant instead anyways.
if value_count == 0 {
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
return SchemeEvaluation::Skip;
}

// If the majority (90%) of values is null, this will compress well.
if stats.null_count() as f64 / len > 0.9 {
return CompressionEstimate::Verdict(EstimateVerdict::Ratio(len / value_count as f64));
return SchemeEvaluation::Candidate(CandidateEstimate::from_compression_ratio(
len / value_count as f64,
));
}

// Otherwise we don't go this route.
CompressionEstimate::Verdict(EstimateVerdict::Skip)
SchemeEvaluation::Skip
}

fn compress(
Expand Down
13 changes: 6 additions & 7 deletions vortex-btrblocks/src/schemes/integer/bitpacking.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,8 @@ use vortex_array::IntoArray;
use vortex_array::arrays::Patched;
use vortex_array::arrays::patched::use_experimental_patches;
use vortex_array::arrays::primitive::PrimitiveArrayExt;
use vortex_compressor::scheme::CompressionEstimate;
use vortex_compressor::scheme::DeferredEstimate;
use vortex_compressor::scheme::EstimateVerdict;
use vortex_compressor::scheme::DeferredEvaluation;
use vortex_compressor::scheme::SchemeEvaluation;
use vortex_error::VortexResult;
use vortex_fastlanes::BitPacked;
use vortex_fastlanes::bitpack_compress::bit_width_histogram;
Expand All @@ -38,20 +37,20 @@ impl Scheme for BitPackingScheme {
canonical.dtype().is_int()
}

fn expected_compression_ratio(
fn evaluate(
&self,
data: &ArrayAndStats,
_compress_ctx: CompressorContext,
exec_ctx: &mut ExecutionCtx,
) -> CompressionEstimate {
) -> SchemeEvaluation {
let stats = data.integer_stats(exec_ctx);

// BitPacking only works for non-negative values.
if stats.erased().min_is_negative() {
return CompressionEstimate::Verdict(EstimateVerdict::Skip);
return SchemeEvaluation::Skip;
}

CompressionEstimate::Deferred(DeferredEstimate::Sample)
SchemeEvaluation::Deferred(DeferredEvaluation::Sample)
}

fn compress(
Expand Down
Loading
Loading