Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 55 additions & 2 deletions crates/paimon/src/table/data_evolution_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ use crate::spec::{
};
use crate::table::dedicated_format_file_writer::is_blob_or_video_file_name;
use crate::table::schema_manager::SchemaManager;
use crate::table::source::any_range_overlaps_file;
use crate::table::source::{any_range_overlaps_file, is_data_evolution_normal_file};
use crate::table::stats_filter::group_by_overlapping_row_id;
use crate::table::{ArrowRecordBatchStream, RESTEnv, RowRange};
use crate::{DataSplit, Error};
Expand Down Expand Up @@ -91,6 +91,26 @@ fn is_raw_convertible(files: &[DataFileMeta]) -> bool {
true
}

/// Whether the files of one row-id group are column slices of a single row range.
///
/// Every normal file must cover the same rows, and every dedicated (BLOB or
/// vector) file must lie inside that range. Only such a group is merged
/// column-wise; any other group holds files that were not written as slices of
/// each other.
pub(super) fn is_data_evolution_column_slice_group(files: &[DataFileMeta]) -> bool {
let mut normal_files = files
.iter()
.filter(|file| is_data_evolution_normal_file(file));
let Some((first, last)) = normal_files.next().and_then(DataFileMeta::row_id_range) else {
return false;
};
normal_files.all(|file| file.row_id_range() == Some((first, last)))
&& files.iter().all(|file| {
file.row_id_range()
.is_some_and(|(from, to)| from >= first && to <= last)
})
}

/// Split a reader input into independently readable row-id groups.
///
/// Native chunk shuffle can deliberately combine several disjoint groups in
Expand Down Expand Up @@ -551,7 +571,12 @@ impl DataEvolutionReader {
if remaining == Some(0) {
break 'splits;
}
if is_raw_convertible(&files) {
// Change events are physical file changes. Files of a streaming
// split that are not column slices of one row range are
// independent changes and are read as they are stored.
if is_raw_convertible(&files)
|| (split.is_streaming() && !is_data_evolution_column_slice_group(&files))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Apply physical-event semantics before scan pruning

Selecting raw events only here is too late: TableScan has already applied merged-row statistics and read-field pruning. On this PR's own two inserts (values 1..4) → compaction → value-only update to 50 fixture, the full timestamp window returns 8 physical events. With with_projection(&["id"]), its scan drops the update-only file and returns 4 rows; with with_filter(value = 1), latest-provider stats (min=max=50) drop the entire group and return zero instead of the original (1,1) event. Both are public-API reproductions on this exact head. Reading the original unfiltered/full-width plan through the same projected or filtered reader returns 8 or 1 respectively.

Please classify these physical-event groups before planner statistics/field pruning and use event-safe per-file pruning (preserving events whose projected fields are absent as NULL), or conservatively disable the merged-group optimizations for them. Add projection and predicate regressions alongside the two existing unfiltered window tests.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 373f19b: incremental non-column-slice groups now bypass merged statistics and projection pruning before the raw read, with projection and predicate regressions verified on the old and new heads.

{
for file_meta in files {
if remaining == Some(0) {
break 'splits;
Expand Down Expand Up @@ -4144,6 +4169,34 @@ mod tests {
);
}

#[test]
fn test_column_slice_groups_have_a_normal_file_covering_every_file() {
let slices = vec![
data_file("base.parquet", 0, 10, 1, Some(vec!["id"])),
data_file("partial.parquet", 0, 10, 2, Some(vec!["value"])),
data_file("payload.blob", 2, 5, 2, Some(vec!["payload"])),
];
assert!(is_data_evolution_column_slice_group(&slices));

// A compaction merged two inserts, then an update covered the result.
let compacted = vec![
data_file("base-0.parquet", 0, 2, 1, Some(vec!["id"])),
data_file("base-2.parquet", 2, 2, 1, Some(vec!["id"])),
data_file("update.parquet", 0, 4, 2, Some(vec!["value"])),
];
assert!(!is_data_evolution_column_slice_group(&compacted));

// A dedicated file outside the normal files' range is no slice of them.
let outside = vec![
data_file("base.parquet", 0, 10, 1, Some(vec!["id"])),
data_file("payload.blob", 8, 5, 2, Some(vec!["payload"])),
];
assert!(!is_data_evolution_column_slice_group(&outside));

let only_dedicated = vec![data_file("payload.blob", 0, 5, 1, Some(vec!["payload"]))];
assert!(!is_data_evolution_column_slice_group(&only_dedicated));
}

#[test]
fn test_prepared_merge_group_rejects_vector_only_split() {
// No normal anchor file -> DataInvalid.
Expand Down
53 changes: 33 additions & 20 deletions crates/paimon/src/table/table_scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

use super::bucket_filter::{compute_target_buckets, BucketSelector};
use super::chunk_shuffle::{chunk_shuffle_splits, ChunkShuffle};
use super::data_evolution_reader::is_data_evolution_column_slice_group;
use super::format_table_scan::FormatTableScan;
use super::global_index_result::GlobalIndexResult;
use super::global_index_scanner::RowRangeIndex;
Expand Down Expand Up @@ -3151,20 +3152,28 @@ impl<'a> PaimonTableScan<'a> {
scan_stats.data_evolution_groups_pruned_by_row_ranges +=
groups_pruned_by_row_ranges;

// Filter groups by merged stats before splitting.
// An incremental group that is not made of column slices holds
// independent physical events. Merged-row statistics and
// projection can discard events before the reader classifies it.
let physical_event_group = |group: &[DataFileMeta]| {
self.is_streaming() && !is_data_evolution_column_slice_group(group)
};

// Filter mergeable groups by merged stats before splitting.
let row_id_groups: Vec<Vec<DataFileMeta>> = if self.data_predicates.is_empty() {
row_id_groups
} else {
let before = row_id_groups.len();
let mut groups = Vec::with_capacity(before);
for group in row_id_groups {
if data_evolution_group_matches_predicates_for_table(
&self.table,
&group,
&self.data_predicates,
&mut stats_schema_cache,
)
.await
if physical_event_group(&group)
|| data_evolution_group_matches_predicates_for_table(
&self.table,
&group,
&self.data_predicates,
&mut stats_schema_cache,
)
.await
{
groups.push(group);
}
Expand All @@ -3179,18 +3188,22 @@ impl<'a> PaimonTableScan<'a> {
} else {
let mut pruned = Vec::with_capacity(row_id_groups.len());
for group in row_id_groups {
pruned.push(
prune_data_evolution_group_by_read_fields(
group,
read_field_ids,
deletion_vectors_enabled,
table_schema_id,
table_fields,
schema_manager,
&mut data_file_field_ids_cache,
)
.await?,
);
if physical_event_group(&group) {
pruned.push(group);
} else {
pruned.push(
prune_data_evolution_group_by_read_fields(
group,
read_field_ids,
deletion_vectors_enabled,
table_schema_id,
table_fields,
schema_manager,
&mut data_file_field_ids_cache,
)
.await?,
);
}
}
pruned
}
Expand Down
Loading
Loading