From 71160122cd2b0957f221461ae1213f9f0b977b6c Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Tue, 17 Nov 2026 09:48:06 +0800 Subject: [PATCH 1/6] fix(core): read data-evolution files of an incremental window that are not column slices as stored A window of an incremental-between-timestamp read can hold the files of two inserts and the file of an update of the rows a compaction merged from them. These are not column slices of one row range, and merging them failed the read with 'All data files in a field merge split should have the same row id range'. Read the files of such a streaming group as they are stored. Groups of ordinary reads and real column slices are unchanged. --- .../paimon/src/table/data_evolution_reader.rs | 57 ++++++- .../tests/data_evolution_window_test.rs | 153 ++++++++++++++++++ 2 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 crates/paimon/tests/data_evolution_window_test.rs diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 45ebb85b9..466ac12d2 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -38,7 +38,7 @@ use crate::spec::{ }; use crate::table::dedicated_format_file_writer::is_blob_or_video_file_name; use crate::table::schema_manager::SchemaManager; -use crate::table::source::any_range_overlaps_file; +use crate::table::source::{any_range_overlaps_file, is_data_evolution_normal_file}; use crate::table::stats_filter::group_by_overlapping_row_id; use crate::table::{ArrowRecordBatchStream, RESTEnv, RowRange}; use crate::{DataSplit, Error}; @@ -91,6 +91,26 @@ fn is_raw_convertible(files: &[DataFileMeta]) -> bool { true } +/// Whether the files of one row-id group are column slices of a single row range. +/// +/// Every normal file must cover the same rows, and every dedicated (BLOB or +/// vector) file must lie inside that range. Only such a group is merged +/// column-wise; any other group holds files that were not written as slices of +/// each other. +fn is_data_evolution_column_slice_group(files: &[DataFileMeta]) -> bool { + let mut normal_files = files + .iter() + .filter(|file| is_data_evolution_normal_file(file)); + let Some((first, last)) = normal_files.next().and_then(DataFileMeta::row_id_range) else { + return false; + }; + normal_files.all(|file| file.row_id_range() == Some((first, last))) + && files.iter().all(|file| { + file.row_id_range() + .is_some_and(|(from, to)| from >= first && to <= last) + }) +} + /// Split a reader input into independently readable row-id groups. /// /// Native chunk shuffle can deliberately combine several disjoint groups in @@ -551,7 +571,12 @@ impl DataEvolutionReader { if remaining == Some(0) { break 'splits; } - if is_raw_convertible(&files) { + // Change events are physical file changes. Files of a streaming + // split that are not column slices of one row range are + // independent changes and are read as they are stored. + if is_raw_convertible(&files) + || (split.is_streaming() && !is_data_evolution_column_slice_group(&files)) + { for file_meta in files { if remaining == Some(0) { break 'splits; @@ -4144,6 +4169,34 @@ mod tests { ); } + #[test] + fn test_column_slice_groups_have_a_normal_file_covering_every_file() { + let slices = vec![ + data_file("base.parquet", 0, 10, 1, Some(vec!["id"])), + data_file("partial.parquet", 0, 10, 2, Some(vec!["value"])), + data_file("payload.blob", 2, 5, 2, Some(vec!["payload"])), + ]; + assert!(is_data_evolution_column_slice_group(&slices)); + + // A compaction merged two inserts, then an update covered the result. + let compacted = vec![ + data_file("base-0.parquet", 0, 2, 1, Some(vec!["id"])), + data_file("base-2.parquet", 2, 2, 1, Some(vec!["id"])), + data_file("update.parquet", 0, 4, 2, Some(vec!["value"])), + ]; + assert!(!is_data_evolution_column_slice_group(&compacted)); + + // A dedicated file outside the normal files' range is no slice of them. + let outside = vec![ + data_file("base.parquet", 0, 10, 1, Some(vec!["id"])), + data_file("payload.blob", 8, 5, 2, Some(vec!["payload"])), + ]; + assert!(!is_data_evolution_column_slice_group(&outside)); + + let only_dedicated = vec![data_file("payload.blob", 0, 5, 1, Some(vec!["payload"]))]; + assert!(!is_data_evolution_column_slice_group(&only_dedicated)); + } + #[test] fn test_prepared_merge_group_rejects_vector_only_split() { // No normal anchor file -> DataInvalid. diff --git a/crates/paimon/tests/data_evolution_window_test.rs b/crates/paimon/tests/data_evolution_window_test.rs new file mode 100644 index 000000000..50fbcf83a --- /dev/null +++ b/crates/paimon/tests/data_evolution_window_test.rs @@ -0,0 +1,153 @@ +mod common; + +use arrow_array::{Array, Int32Array, Int64Array, RecordBatch}; +use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; +use futures::TryStreamExt; +use paimon::spec::{DataType, IntType, Schema, TableSchema}; +use std::collections::HashMap; +use std::sync::Arc; + +use common::incremental_helpers::{ + make_batch, memory_table, persist_table_schema, setup_dirs, write_batch, +}; + +fn rows(batches: &[RecordBatch]) -> Vec<(Option, Option)> { + let column = |batch: &RecordBatch, name: &str| { + let column = batch.column_by_name(name).unwrap(); + column + .as_any() + .downcast_ref::() + .unwrap() + .clone() + }; + let mut rows = Vec::new(); + for batch in batches { + let (ids, values) = (column(batch, "id"), column(batch, "value")); + for row in 0..batch.num_rows() { + rows.push(( + ids.is_valid(row).then(|| ids.value(row)), + values.is_valid(row).then(|| values.value(row)), + )); + } + } + rows.sort(); + rows +} + +/// Writes `value` of rows `0..values.len()` as a column slice of their files. +async fn update_values(table: &paimon::table::Table, values: Vec) { + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("_ROW_ID", ArrowDataType::Int64, false), + Field::new("value", ArrowDataType::Int32, false), + ])); + let row_ids = (0..values.len() as i64).collect::>(); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from(row_ids)), + Arc::new(Int32Array::from(values)), + ], + ) + .unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder + .new_data_evolution_writer(vec!["value".to_string()]) + .unwrap(); + writer.add_matched_batch(batch).unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + builder.new_commit().commit(messages).await.unwrap(); +} + +async fn compact(table: &paimon::table::Table) { + let compaction = table.new_compact_builder(); + let plan = compaction.new_planner().unwrap().plan().await.unwrap(); + let executor = compaction.new_executor().unwrap(); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + compaction + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); +} + +/// Reads the table as the changes of the window of every snapshot. +async fn read_window(table: &paimon::table::Table) -> paimon::Result> { + let options = HashMap::from([( + "incremental-between-timestamp".to_string(), + "0,9999999999999".to_string(), + )]); + let table = table.copy_with_time_travel(options).await?; + let builder = table.new_read_builder(); + let plan = builder.new_scan().plan().await?; + builder + .new_read()? + .to_arrow(plan.splits())? + .try_collect() + .await +} + +fn data_evolution_table(path: &str) -> (paimon::io::FileIO, paimon::table::Table) { + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("write-only", "true") + .option("compaction.min.file-num", "2") + .build() + .unwrap(); + memory_table(path, TableSchema::new(0, &schema)) +} + +/// A column update of rows an insert wrote is a slice of the insert's rows, and +/// the window of both merges them into the updated rows. +#[tokio::test] +async fn window_merges_a_column_update_with_the_insert_it_updates() { + let path = "memory:/data_evolution_window/slices"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + update_values(&table, vec![99, 20]).await; + + let batches = read_window(&table).await.unwrap(); + assert_eq!( + rows(&batches), + vec![(Some(1), Some(99)), (Some(2), Some(20))] + ); +} + +/// A compaction merges the files of two inserts into one, so an update of every +/// row covers that file and not the rows of either insert. Their files are not +/// column slices of one row range, and the window returns them as they are +/// stored instead of failing the read. +#[tokio::test] +async fn window_reads_files_that_are_not_column_slices_as_stored() { + let path = "memory:/data_evolution_window/compacted"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![1, 2])).await; + write_batch(&table, &make_batch(vec![3, 4], vec![3, 4])).await; + compact(&table).await; + update_values(&table, vec![50; 4]).await; + + let batches = read_window(&table).await.unwrap(); + assert_eq!( + rows(&batches), + vec![ + (None, Some(50)), + (None, Some(50)), + (None, Some(50)), + (None, Some(50)), + (Some(1), Some(1)), + (Some(2), Some(2)), + (Some(3), Some(3)), + (Some(4), Some(4)), + ] + ); +} From 6e456358047d36a5703934d6289f93363e019793 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Tue, 17 Nov 2026 12:00:59 +0800 Subject: [PATCH 2/6] [core] Add the license header to data_evolution_window_test.rs --- .../paimon/tests/data_evolution_window_test.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/paimon/tests/data_evolution_window_test.rs b/crates/paimon/tests/data_evolution_window_test.rs index 50fbcf83a..c38560de1 100644 --- a/crates/paimon/tests/data_evolution_window_test.rs +++ b/crates/paimon/tests/data_evolution_window_test.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod common; use arrow_array::{Array, Int32Array, Int64Array, RecordBatch}; From 373f19b24fb9a50e1f2c89d3e4a69d801563a49c Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Thu, 19 Nov 2026 12:20:55 +0800 Subject: [PATCH 3/6] fix(core): preserve physical events during incremental scan pruning --- .../paimon/src/table/data_evolution_reader.rs | 2 +- crates/paimon/src/table/table_scan.rs | 53 ++++++----- .../tests/data_evolution_window_test.rs | 87 ++++++++++++++++++- 3 files changed, 120 insertions(+), 22 deletions(-) diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 466ac12d2..e8c3ee065 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -97,7 +97,7 @@ fn is_raw_convertible(files: &[DataFileMeta]) -> bool { /// vector) file must lie inside that range. Only such a group is merged /// column-wise; any other group holds files that were not written as slices of /// each other. -fn is_data_evolution_column_slice_group(files: &[DataFileMeta]) -> bool { +pub(super) fn is_data_evolution_column_slice_group(files: &[DataFileMeta]) -> bool { let mut normal_files = files .iter() .filter(|file| is_data_evolution_normal_file(file)); diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 617967e0d..4e82c01eb 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -22,6 +22,7 @@ use super::bucket_filter::{compute_target_buckets, BucketSelector}; use super::chunk_shuffle::{chunk_shuffle_splits, ChunkShuffle}; +use super::data_evolution_reader::is_data_evolution_column_slice_group; use super::format_table_scan::FormatTableScan; use super::global_index_result::GlobalIndexResult; use super::global_index_scanner::RowRangeIndex; @@ -3151,20 +3152,28 @@ impl<'a> PaimonTableScan<'a> { scan_stats.data_evolution_groups_pruned_by_row_ranges += groups_pruned_by_row_ranges; - // Filter groups by merged stats before splitting. + // An incremental group that is not made of column slices holds + // independent physical events. Merged-row statistics and + // projection can discard events before the reader classifies it. + let physical_event_group = |group: &[DataFileMeta]| { + self.is_streaming() && !is_data_evolution_column_slice_group(group) + }; + + // Filter mergeable groups by merged stats before splitting. let row_id_groups: Vec> = if self.data_predicates.is_empty() { row_id_groups } else { let before = row_id_groups.len(); let mut groups = Vec::with_capacity(before); for group in row_id_groups { - if data_evolution_group_matches_predicates_for_table( - &self.table, - &group, - &self.data_predicates, - &mut stats_schema_cache, - ) - .await + if physical_event_group(&group) + || data_evolution_group_matches_predicates_for_table( + &self.table, + &group, + &self.data_predicates, + &mut stats_schema_cache, + ) + .await { groups.push(group); } @@ -3179,18 +3188,22 @@ impl<'a> PaimonTableScan<'a> { } else { let mut pruned = Vec::with_capacity(row_id_groups.len()); for group in row_id_groups { - pruned.push( - prune_data_evolution_group_by_read_fields( - group, - read_field_ids, - deletion_vectors_enabled, - table_schema_id, - table_fields, - schema_manager, - &mut data_file_field_ids_cache, - ) - .await?, - ); + if physical_event_group(&group) { + pruned.push(group); + } else { + pruned.push( + prune_data_evolution_group_by_read_fields( + group, + read_field_ids, + deletion_vectors_enabled, + table_schema_id, + table_fields, + schema_manager, + &mut data_file_field_ids_cache, + ) + .await?, + ); + } } pruned } diff --git a/crates/paimon/tests/data_evolution_window_test.rs b/crates/paimon/tests/data_evolution_window_test.rs index c38560de1..0a7385c4e 100644 --- a/crates/paimon/tests/data_evolution_window_test.rs +++ b/crates/paimon/tests/data_evolution_window_test.rs @@ -20,7 +20,7 @@ mod common; use arrow_array::{Array, Int32Array, Int64Array, RecordBatch}; use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; -use paimon::spec::{DataType, IntType, Schema, TableSchema}; +use paimon::spec::{DataType, Datum, IntType, PredicateBuilder, Schema, TableSchema}; use std::collections::HashMap; use std::sync::Arc; @@ -168,3 +168,88 @@ async fn window_reads_files_that_are_not_column_slices_as_stored() { ] ); } + +/// Projection pruning must not drop an update-only file from a physical-event +/// group: its absent projected fields are read as NULL like the other columns. +#[tokio::test] +async fn window_reads_physical_events_with_projection() { + let path = "memory:/data_evolution_window/projection"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![1, 2])).await; + write_batch(&table, &make_batch(vec![3, 4], vec![3, 4])).await; + compact(&table).await; + update_values(&table, vec![50; 4]).await; + + let options = HashMap::from([( + "incremental-between-timestamp".to_string(), + "0,9999999999999".to_string(), + )]); + let table = table.copy_with_time_travel(options).await.unwrap(); + let mut builder = table.new_read_builder(); + builder.with_projection(&["id"]).unwrap(); + let plan = builder.new_scan().plan().await.unwrap(); + let batches = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut ids = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .collect::>() + }) + .collect::>(); + ids.sort(); + assert_eq!( + ids, + vec![None, None, None, None, Some(1), Some(2), Some(3), Some(4),] + ); +} + +/// Merged latest-row statistics must not prune physical events whose stored +/// values satisfy the predicate. +#[tokio::test] +async fn window_filters_physical_events_by_stored_values() { + let path = "memory:/data_evolution_window/filter"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![1, 2])).await; + write_batch(&table, &make_batch(vec![3, 4], vec![3, 4])).await; + compact(&table).await; + update_values(&table, vec![50; 4]).await; + + let options = HashMap::from([( + "incremental-between-timestamp".to_string(), + "0,9999999999999".to_string(), + )]); + let table = table.copy_with_time_travel(options).await.unwrap(); + let mut builder = table.new_read_builder(); + builder.with_filter( + PredicateBuilder::new(table.schema().fields()) + .equal("value", Datum::Int(1)) + .unwrap(), + ); + let plan = builder.new_scan().plan().await.unwrap(); + let batches = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(rows(&batches), vec![(Some(1), Some(1))]); +} From fc616d553e2eb6a299c97aed021269339b3ecb49 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sun, 22 Nov 2026 00:27:55 +0800 Subject: [PATCH 4/6] fix(core): keep dedicated files with their incremental write events --- .../paimon/src/table/data_evolution_reader.rs | 105 +++++++++++++++++- crates/paimon/src/table/table_scan.rs | 22 +++- .../tests/data_evolution_window_test.rs | 105 ++++++++++++++++++ 3 files changed, 226 insertions(+), 6 deletions(-) diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index e8c3ee065..88f1d76b8 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -111,6 +111,66 @@ pub(super) fn is_data_evolution_column_slice_group(files: &[DataFileMeta]) -> bo }) } +/// Split overlapping files of a mixed-range window into readable events. +pub(super) fn incremental_event_groups( + files: &[DataFileMeta], +) -> crate::Result>>> { + let mixed = + |group: &[DataFileMeta]| group.len() > 1 && !is_data_evolution_column_slice_group(group); + if !mixed(files) { + return Ok(None); + } + let groups = group_by_overlapping_row_id(files.to_vec()); + if !groups.iter().any(|group| mixed(group)) { + return Ok(None); + } + let mut events = Vec::new(); + for group in groups { + if !mixed(&group) { + events.push(group); + } else if group.iter().all(is_data_evolution_normal_file) { + events.extend(group.into_iter().map(|file| vec![file])); + } else { + // APPEND files in a row-tracked table get the snapshot sequence + // at commit. Row IDs alone cannot distinguish a later update from + // the normal and independently rolled dedicated files of an insert. + let mut commits = std::collections::BTreeMap::>::new(); + for file in group { + if file.file_source != Some(0) || file.max_sequence_number <= 0 { + return Err(Error::Unsupported { + message: format!("Cannot identify the write of '{}' in a mixed-range data-evolution window", file.file_name), + }); + } + commits + .entry(file.max_sequence_number) + .or_default() + .push(file); + } + for files in commits.into_values() { + for event in group_by_overlapping_row_id(files) { + if event.len() > 1 + && (!is_data_evolution_column_slice_group(&event) + || event + .iter() + .filter(|file| is_data_evolution_normal_file(file)) + .count() + != 1 + || event + .iter() + .any(|file| file.min_sequence_number != file.max_sequence_number)) + { + return Err(Error::Unsupported { + message: format!("Cannot identify dedicated-file companions at row ID {:?} in a mixed-range data-evolution window", event[0].first_row_id), + }); + } + events.push(event); + } + } + } + } + Ok(Some(events)) +} + /// Split a reader input into independently readable row-id groups. /// /// Native chunk shuffle can deliberately combine several disjoint groups in @@ -477,7 +537,23 @@ impl DataEvolutionReader { if self.limit == Some(0) { return Ok(futures::stream::empty().boxed()); } - let splits: Vec = data_splits.to_vec(); + let mut splits = Vec::with_capacity(data_splits.len()); + for split in data_splits { + if split.is_streaming() && !split.raw_convertible() { + // Externally planned or chunk-shuffled splits may pack events + // back together. Row-ID based BLOB preselection must run on + // each event separately: different events can share a row ID. + if let Some(events) = incremental_event_groups(split.data_files())? { + splits.extend( + events + .into_iter() + .map(|files| split.for_data_evolution_files(files)), + ); + continue; + } + } + splits.push(split.clone()); + } if self.needs_managed_blob_preselection(&splits) { return Ok(try_stream! { @@ -571,11 +647,8 @@ impl DataEvolutionReader { if remaining == Some(0) { break 'splits; } - // Change events are physical file changes. Files of a streaming - // split that are not column slices of one row range are - // independent changes and are read as they are stored. if is_raw_convertible(&files) - || (split.is_streaming() && !is_data_evolution_column_slice_group(&files)) + || (split.is_streaming() && files.len() == 1) { for file_meta in files { if remaining == Some(0) { @@ -4197,6 +4270,28 @@ mod tests { assert!(!is_data_evolution_column_slice_group(&only_dedicated)); } + #[test] + fn ambiguous_mixed_range_blob_events_are_rejected() { + let mut files = vec![ + data_file("base.parquet", 0, 2, 1, Some(vec!["id"])), + data_file("payload.blob", 0, 2, 1, Some(vec!["payload"])), + data_file("update.parquet", 0, 4, 1, Some(vec!["value"])), + ]; + assert!(matches!( + incremental_event_groups(&files), + Err(Error::Unsupported { .. }) + )); + // A known commit still has two overlapping normal-file candidates. + for file in &mut files { + file.file_source = Some(0); + file.min_sequence_number = file.max_sequence_number; + } + assert!(matches!( + incremental_event_groups(&files), + Err(Error::Unsupported { .. }) + )); + } + #[test] fn test_prepared_merge_group_rejects_vector_only_split() { // No normal anchor file -> DataInvalid. diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 4e82c01eb..7381e257d 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -22,7 +22,9 @@ use super::bucket_filter::{compute_target_buckets, BucketSelector}; use super::chunk_shuffle::{chunk_shuffle_splits, ChunkShuffle}; -use super::data_evolution_reader::is_data_evolution_column_slice_group; +use super::data_evolution_reader::{ + incremental_event_groups, is_data_evolution_column_slice_group, +}; use super::format_table_scan::FormatTableScan; use super::global_index_result::GlobalIndexResult; use super::global_index_scanner::RowRangeIndex; @@ -3310,6 +3312,24 @@ impl<'a> PaimonTableScan<'a> { .collect() }; + let file_groups = if data_evolution_enabled && self.is_streaming() { + let mut events = Vec::with_capacity(file_groups.len()); + for group in file_groups { + if !group.raw_convertible { + if let Some(groups) = incremental_event_groups(&group.files)? { + events.extend(groups.into_iter().map(|files| SplitGroup { + raw_convertible: files.len() == 1, + files, + })); + continue; + } + } + events.push(group); + } + events + } else { + file_groups + }; for group in file_groups { let SplitGroup { files: file_group, diff --git a/crates/paimon/tests/data_evolution_window_test.rs b/crates/paimon/tests/data_evolution_window_test.rs index 0a7385c4e..cb93f73f3 100644 --- a/crates/paimon/tests/data_evolution_window_test.rs +++ b/crates/paimon/tests/data_evolution_window_test.rs @@ -253,3 +253,108 @@ async fn window_filters_physical_events_by_stored_values() { .unwrap(); assert_eq!(rows(&batches), vec![(Some(1), Some(1))]); } + +#[tokio::test] +async fn window_keeps_insert_payloads_with_their_rows_after_compaction() { + use arrow_array::{cast::AsArray, types::Int32Type}; + use arrow_array::{ArrayRef, LargeBinaryArray}; + use paimon::spec::BlobType; + + let path = "memory:/data_evolution_window/blob_insert"; + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .column("payload", DataType::Blob(BlobType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("write-only", "true") + .option("compaction.min.file-num", "2") + .build() + .unwrap(); + let (io, table) = memory_table(path, TableSchema::new(0, &schema)); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + for first in [1, 3] { + let ids = Int32Array::from(vec![first, first + 1]); + let payloads = [[first as u8], [(first + 1) as u8]]; + let batch = RecordBatch::try_from_iter([ + ("id", Arc::new(ids.clone()) as ArrayRef), + ("value", Arc::new(ids) as ArrayRef), + ( + "payload", + Arc::new(LargeBinaryArray::from_iter_values(payloads)) as ArrayRef, + ), + ]) + .unwrap(); + write_batch(&table, &batch).await; + } + compact(&table).await; + update_values(&table, vec![50; 4]).await; + + let window = table + .copy_with_time_travel(HashMap::from([( + "incremental-between-timestamp".into(), + "0,9999999999999".into(), + )])) + .await + .unwrap(); + for (filtered, chunked) in [(false, false), (true, false), (true, true)] { + let mut builder = window.new_read_builder(); + if filtered { + builder.with_filter( + PredicateBuilder::new(window.schema().fields()) + .equal("value", Datum::Int(1)) + .unwrap(), + ); + } + let scan = if chunked { + // The chunk carries multiple events for the same row ID. The + // filtered reader must fetch the BLOB from the matching event. + window + .new_read_builder() + .new_scan() + .with_chunk_shuffle(42, 4) + .unwrap() + } else { + builder.new_scan() + }; + let plan = scan.plan().await.unwrap(); + if !filtered { + assert_eq!( + plan.splits() + .iter() + .map(|split| split.merged_row_count()) + .sum::>(), + Some(8) + ); + } + let batches: Vec<_> = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut actual = Vec::new(); + for batch in batches { + let ids = batch.column(0).as_primitive::(); + let values = batch.column(1).as_primitive::(); + let payloads = batch.column(2).as_binary::(); + actual.extend( + ids.iter() + .zip(values.iter()) + .zip(payloads.iter()) + .map(|((id, value), payload)| (id, value, payload.map(<[u8]>::to_vec))), + ); + } + let mut expected = vec![(Some(1), Some(1), Some(vec![1]))]; + if !filtered { + expected.extend((2..=4).map(|i| (Some(i), Some(i), Some(vec![i as u8])))); + expected.extend(vec![(None, Some(50), None); 4]); + } + actual.sort(); + expected.sort(); + assert_eq!(actual, expected); + } +} From d9c96856edf5651ec9cd70fe2f177e30eebb6405 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sun, 22 Nov 2026 10:08:16 +0800 Subject: [PATCH 5/6] fix(core): preserve incremental events across selection and shuffle --- crates/paimon/src/table/chunk_shuffle.rs | 23 +++++ .../paimon/src/table/data_evolution_reader.rs | 50 ++++++---- crates/paimon/src/table/table_scan.rs | 13 ++- .../tests/data_evolution_window_test.rs | 95 ++++++++++++++++++- 4 files changed, 157 insertions(+), 24 deletions(-) diff --git a/crates/paimon/src/table/chunk_shuffle.rs b/crates/paimon/src/table/chunk_shuffle.rs index d426f649d..0ad1fd8e2 100644 --- a/crates/paimon/src/table/chunk_shuffle.rs +++ b/crates/paimon/src/table/chunk_shuffle.rs @@ -25,6 +25,7 @@ use std::collections::{BTreeMap, HashMap}; use crate::deletion_vector::{DeletionVector, DeletionVectorFactory}; use crate::spec::{BinaryRow, DataField, DataFileMeta, Datum}; +use crate::table::data_evolution_reader::split_incremental_event_groups; use crate::table::source::{data_evolution_anchor_file, is_data_evolution_normal_file}; use crate::table::stats_filter::group_by_overlapping_row_id; use crate::table::{merge_row_ranges, DataSplit, DataSplitBuilder, DeletionFile, RowRange, Table}; @@ -344,9 +345,15 @@ async fn evolution_chunks( .collect(); let files = group.files.iter().map(|input| input.file.clone()).collect(); let aligned_groups = group_by_overlapping_row_id(files); + let (aligned_groups, mixed_events) = if group.is_streaming { + split_incremental_event_groups(aligned_groups)? + } else { + (aligned_groups, false) + }; let mut chunks: Vec> = Vec::new(); let mut current = Vec::new(); let mut current_rows = 0; + let mut event_domains = BTreeMap::new(); for files in aligned_groups { let from = files @@ -415,10 +422,26 @@ async fn evolution_chunks( if current_rows == chunk_size { chunks.push(std::mem::take(&mut current)); current_rows = 0; + event_domains.clear(); } let Some(slice) = slicer.take(chunk_size - current_rows)? else { break; }; + if mixed_events { + // A split has one row-ID selection shared by all its files. + // Combining overlapping events could select the other write's + // rows or count two writes to the same row ID only once. + if event_domains + .range(..=to) + .next_back() + .is_some_and(|(_, end)| *end >= from) + { + return Err(crate::Error::Unsupported { + message: "chunk_shuffle cannot combine overlapping incremental events in one chunk".to_string(), + }); + } + event_domains.insert(from, to); + } current_rows += slice.live_rows; current.push(EvolutionSegment { files: inputs.clone(), diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 88f1d76b8..fba217619 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -115,14 +115,23 @@ pub(super) fn is_data_evolution_column_slice_group(files: &[DataFileMeta]) -> bo pub(super) fn incremental_event_groups( files: &[DataFileMeta], ) -> crate::Result>>> { - let mixed = - |group: &[DataFileMeta]| group.len() > 1 && !is_data_evolution_column_slice_group(group); - if !mixed(files) { + if files.len() <= 1 || is_data_evolution_column_slice_group(files) { return Ok(None); } - let groups = group_by_overlapping_row_id(files.to_vec()); + let (groups, mixed) = + split_incremental_event_groups(group_by_overlapping_row_id(files.to_vec()))?; + Ok(mixed.then_some(groups)) +} + +/// Preserve the event boundaries in a window; report whether any overlapping +/// row-ID group was split into distinct events. +pub(super) fn split_incremental_event_groups( + groups: Vec>, +) -> crate::Result<(Vec>, bool)> { + let mixed = + |group: &[DataFileMeta]| group.len() > 1 && !is_data_evolution_column_slice_group(group); if !groups.iter().any(|group| mixed(group)) { - return Ok(None); + return Ok((groups, false)); } let mut events = Vec::new(); for group in groups { @@ -148,16 +157,16 @@ pub(super) fn incremental_event_groups( } for files in commits.into_values() { for event in group_by_overlapping_row_id(files) { - if event.len() > 1 - && (!is_data_evolution_column_slice_group(&event) - || event - .iter() - .filter(|file| is_data_evolution_normal_file(file)) - .count() - != 1 - || event - .iter() - .any(|file| file.min_sequence_number != file.max_sequence_number)) + let anchors = event + .iter() + .filter(|file| is_data_evolution_normal_file(file)) + .count(); + if anchors != 1 + || (event.len() > 1 + && (!is_data_evolution_column_slice_group(&event) + || event.iter().any(|file| { + file.min_sequence_number != file.max_sequence_number + }))) { return Err(Error::Unsupported { message: format!("Cannot identify dedicated-file companions at row ID {:?} in a mixed-range data-evolution window", event[0].first_row_id), @@ -168,7 +177,7 @@ pub(super) fn incremental_event_groups( } } } - Ok(Some(events)) + Ok((events, true)) } /// Split a reader input into independently readable row-id groups. @@ -4290,6 +4299,15 @@ mod tests { incremental_event_groups(&files), Err(Error::Unsupported { .. }) )); + // Giving each file a separate commit leaves the BLOB without an anchor. + for (sequence, file) in (1_i64..).zip(&mut files) { + file.min_sequence_number = sequence; + file.max_sequence_number = sequence; + } + assert!(matches!( + incremental_event_groups(&files), + Err(Error::Unsupported { .. }) + )); } #[test] diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 7381e257d..b8e920b21 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -1330,6 +1330,8 @@ impl<'a> TableScan<'a> { /// keeps ordinary Rust integer calls ergonomic while allowing language /// bindings to preserve arbitrary-precision integer seeds. /// Only predicates on unmasked partition keys are supported. + /// In a data-evolution window, a chunk cannot combine distinct events + /// whose file ranges overlap: a split has one shared row-ID selection. pub fn with_chunk_shuffle(self, seed: impl ToString, chunk_size: u64) -> crate::Result { let config = ChunkShuffle::from_decimal_seed(&seed.to_string(), chunk_size)?; match self.0 { @@ -2929,10 +2931,13 @@ impl<'a> PaimonTableScan<'a> { scan_stats.record_split_planning(0, false); return Ok(Plan::new(Vec::new()).with_snapshot_id(snapshot.id())); } - // Positional scans count all candidate row IDs. Pruning a preceding - // manifest or file here would renumber the surviving rows; intersect - // explicit/global-index ranges after assigning positions instead. - let row_range_index = if data_evolution_enabled && self.row_position_selection().is_none() { + // Positional scans need the preceding row IDs. Incremental evolution + // scans need the whole overlap group to distinguish column slices from + // separate writes. Apply their selections to the planned groups below. + let row_range_index = if data_evolution_enabled + && !self.is_streaming() + && self.row_position_selection().is_none() + { manifest_row_ranges.clone().map(RowRangeIndex::create) } else { None diff --git a/crates/paimon/tests/data_evolution_window_test.rs b/crates/paimon/tests/data_evolution_window_test.rs index cb93f73f3..51418bea5 100644 --- a/crates/paimon/tests/data_evolution_window_test.rs +++ b/crates/paimon/tests/data_evolution_window_test.rs @@ -254,6 +254,49 @@ async fn window_filters_physical_events_by_stored_values() { assert_eq!(rows(&batches), vec![(Some(1), Some(1))]); } +#[tokio::test] +async fn row_id_selection_preserves_both_updates_in_a_mixed_window() { + use paimon::table::{IncrementalScanMode, RowRange}; + + let path = "memory:/data_evolution_window/selected_updates"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![1, 2])).await; + write_batch(&table, &make_batch(vec![3, 4], vec![3, 4])).await; + compact(&table).await; + update_values(&table, vec![50; 4]).await; + update_values(&table, vec![60; 4]).await; + + // The second insert makes these differently sized write events. Its rows + // fall outside the selection, but pruning it cannot turn the two updates + // into column slices of one event. + let mut builder = table.new_read_builder(); + builder.with_row_ranges(vec![RowRange::new(0, 1)]); + let plan = builder + .new_incremental_scan(IncrementalScanMode::Delta, 1, 5) + .plan_combined() + .await + .unwrap(); + let batches = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + rows(&batches), + vec![ + (None, Some(50)), + (None, Some(50)), + (None, Some(60)), + (None, Some(60)) + ] + ); +} + #[tokio::test] async fn window_keeps_insert_payloads_with_their_rows_after_compaction() { use arrow_array::{cast::AsArray, types::Int32Type}; @@ -298,7 +341,7 @@ async fn window_keeps_insert_payloads_with_their_rows_after_compaction() { )])) .await .unwrap(); - for (filtered, chunked) in [(false, false), (true, false), (true, true)] { + for (filtered, chunked) in [(false, false), (true, false), (true, true), (false, true)] { let mut builder = window.new_read_builder(); if filtered { builder.with_filter( @@ -308,8 +351,8 @@ async fn window_keeps_insert_payloads_with_their_rows_after_compaction() { ); } let scan = if chunked { - // The chunk carries multiple events for the same row ID. The - // filtered reader must fetch the BLOB from the matching event. + // Shuffle the complete window, then filter the planned chunks + // in the reader. Payloads must come from the matching write. window .new_read_builder() .new_scan() @@ -319,6 +362,28 @@ async fn window_keeps_insert_payloads_with_their_rows_after_compaction() { builder.new_scan() }; let plan = scan.plan().await.unwrap(); + if chunked && !filtered { + assert_eq!( + plan.splits() + .iter() + .map(|s| s.merged_row_count()) + .collect::>(), + vec![Some(4), Some(4)] + ); + for index in 0..2 { + let shard = window + .new_read_builder() + .new_scan() + .with_chunk_shuffle(42, 4) + .unwrap() + .with_shard(index, 2) + .unwrap() + .plan() + .await + .unwrap(); + assert_eq!(shard.splits(), &plan.splits()[index..=index]); + } + } if !filtered { assert_eq!( plan.splits() @@ -328,10 +393,19 @@ async fn window_keeps_insert_payloads_with_their_rows_after_compaction() { Some(8) ); } + // Read the same cross-language split frame that Python workers receive. + let splits = plan + .splits() + .iter() + .map(|split| { + paimon::table::DataSplit::deserialize_split_v1(&split.serialize_split_v1().unwrap()) + .unwrap() + }) + .collect::>(); let batches: Vec<_> = builder .new_read() .unwrap() - .to_arrow(plan.splits()) + .to_arrow(&splits) .unwrap() .try_collect() .await @@ -357,4 +431,17 @@ async fn window_keeps_insert_payloads_with_their_rows_after_compaction() { expected.sort(); assert_eq!(actual, expected); } + // Seven physical events cannot share one absolute row-ID selection: + // the same ID would have to select the insert and update independently. + let error = window + .new_read_builder() + .new_scan() + .with_chunk_shuffle(42, 7) + .unwrap() + .plan() + .await + .unwrap_err(); + assert!( + matches!(error, paimon::Error::Unsupported { message } if message.contains("overlapping incremental events")) + ); } From 157609351841a61bba97b7ca0a65cc8145e747e6 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Sun, 22 Nov 2026 10:14:28 +0800 Subject: [PATCH 6/6] fix(core): prune out-of-range files after incremental event grouping --- crates/paimon/src/table/table_scan.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index b8e920b21..4a75b558d 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -3110,6 +3110,11 @@ impl<'a> PaimonTableScan<'a> { .index_file_in_data_file_dir(); let mut data_file_field_ids_cache = DataFileFieldIdsCache::new(); + let event_row_index = if data_evolution_enabled && self.is_streaming() { + effective_row_ranges.clone().map(RowRangeIndex::create) + } else { + None + }; // Positional distribution precedes LIMIT. Building too few ordinary // append splits here could starve a later slice or shard. let can_push_down_limit = append_row_position_selection.is_none() @@ -3337,9 +3342,17 @@ impl<'a> PaimonTableScan<'a> { }; for group in file_groups { let SplitGroup { - files: file_group, + files: mut file_group, raw_convertible, } = group; + if let Some(index) = &event_row_index { + // The event boundaries are now fixed. Drop rolled BLOB or + // vector files outside the selection before creating splits. + file_group.retain(|file| data_file_overlaps_row_range_index(file, index)); + if file_group.is_empty() { + continue; + } + } let data_deletion_files = per_bucket_deletion_map.map(|per_bucket| { file_group .iter()