diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 45ebb85b9..e8c3ee065 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -38,7 +38,7 @@ use crate::spec::{ }; use crate::table::dedicated_format_file_writer::is_blob_or_video_file_name; use crate::table::schema_manager::SchemaManager; -use crate::table::source::any_range_overlaps_file; +use crate::table::source::{any_range_overlaps_file, is_data_evolution_normal_file}; use crate::table::stats_filter::group_by_overlapping_row_id; use crate::table::{ArrowRecordBatchStream, RESTEnv, RowRange}; use crate::{DataSplit, Error}; @@ -91,6 +91,26 @@ fn is_raw_convertible(files: &[DataFileMeta]) -> bool { true } +/// Whether the files of one row-id group are column slices of a single row range. +/// +/// Every normal file must cover the same rows, and every dedicated (BLOB or +/// vector) file must lie inside that range. Only such a group is merged +/// column-wise; any other group holds files that were not written as slices of +/// each other. +pub(super) fn is_data_evolution_column_slice_group(files: &[DataFileMeta]) -> bool { + let mut normal_files = files + .iter() + .filter(|file| is_data_evolution_normal_file(file)); + let Some((first, last)) = normal_files.next().and_then(DataFileMeta::row_id_range) else { + return false; + }; + normal_files.all(|file| file.row_id_range() == Some((first, last))) + && files.iter().all(|file| { + file.row_id_range() + .is_some_and(|(from, to)| from >= first && to <= last) + }) +} + /// Split a reader input into independently readable row-id groups. /// /// Native chunk shuffle can deliberately combine several disjoint groups in @@ -551,7 +571,12 @@ impl DataEvolutionReader { if remaining == Some(0) { break 'splits; } - if is_raw_convertible(&files) { + // Change events are physical file changes. Files of a streaming + // split that are not column slices of one row range are + // independent changes and are read as they are stored. + if is_raw_convertible(&files) + || (split.is_streaming() && !is_data_evolution_column_slice_group(&files)) + { for file_meta in files { if remaining == Some(0) { break 'splits; @@ -4144,6 +4169,34 @@ mod tests { ); } + #[test] + fn test_column_slice_groups_have_a_normal_file_covering_every_file() { + let slices = vec![ + data_file("base.parquet", 0, 10, 1, Some(vec!["id"])), + data_file("partial.parquet", 0, 10, 2, Some(vec!["value"])), + data_file("payload.blob", 2, 5, 2, Some(vec!["payload"])), + ]; + assert!(is_data_evolution_column_slice_group(&slices)); + + // A compaction merged two inserts, then an update covered the result. + let compacted = vec![ + data_file("base-0.parquet", 0, 2, 1, Some(vec!["id"])), + data_file("base-2.parquet", 2, 2, 1, Some(vec!["id"])), + data_file("update.parquet", 0, 4, 2, Some(vec!["value"])), + ]; + assert!(!is_data_evolution_column_slice_group(&compacted)); + + // A dedicated file outside the normal files' range is no slice of them. + let outside = vec![ + data_file("base.parquet", 0, 10, 1, Some(vec!["id"])), + data_file("payload.blob", 8, 5, 2, Some(vec!["payload"])), + ]; + assert!(!is_data_evolution_column_slice_group(&outside)); + + let only_dedicated = vec![data_file("payload.blob", 0, 5, 1, Some(vec!["payload"]))]; + assert!(!is_data_evolution_column_slice_group(&only_dedicated)); + } + #[test] fn test_prepared_merge_group_rejects_vector_only_split() { // No normal anchor file -> DataInvalid. diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 617967e0d..4e82c01eb 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -22,6 +22,7 @@ use super::bucket_filter::{compute_target_buckets, BucketSelector}; use super::chunk_shuffle::{chunk_shuffle_splits, ChunkShuffle}; +use super::data_evolution_reader::is_data_evolution_column_slice_group; use super::format_table_scan::FormatTableScan; use super::global_index_result::GlobalIndexResult; use super::global_index_scanner::RowRangeIndex; @@ -3151,20 +3152,28 @@ impl<'a> PaimonTableScan<'a> { scan_stats.data_evolution_groups_pruned_by_row_ranges += groups_pruned_by_row_ranges; - // Filter groups by merged stats before splitting. + // An incremental group that is not made of column slices holds + // independent physical events. Merged-row statistics and + // projection can discard events before the reader classifies it. + let physical_event_group = |group: &[DataFileMeta]| { + self.is_streaming() && !is_data_evolution_column_slice_group(group) + }; + + // Filter mergeable groups by merged stats before splitting. let row_id_groups: Vec> = if self.data_predicates.is_empty() { row_id_groups } else { let before = row_id_groups.len(); let mut groups = Vec::with_capacity(before); for group in row_id_groups { - if data_evolution_group_matches_predicates_for_table( - &self.table, - &group, - &self.data_predicates, - &mut stats_schema_cache, - ) - .await + if physical_event_group(&group) + || data_evolution_group_matches_predicates_for_table( + &self.table, + &group, + &self.data_predicates, + &mut stats_schema_cache, + ) + .await { groups.push(group); } @@ -3179,18 +3188,22 @@ impl<'a> PaimonTableScan<'a> { } else { let mut pruned = Vec::with_capacity(row_id_groups.len()); for group in row_id_groups { - pruned.push( - prune_data_evolution_group_by_read_fields( - group, - read_field_ids, - deletion_vectors_enabled, - table_schema_id, - table_fields, - schema_manager, - &mut data_file_field_ids_cache, - ) - .await?, - ); + if physical_event_group(&group) { + pruned.push(group); + } else { + pruned.push( + prune_data_evolution_group_by_read_fields( + group, + read_field_ids, + deletion_vectors_enabled, + table_schema_id, + table_fields, + schema_manager, + &mut data_file_field_ids_cache, + ) + .await?, + ); + } } pruned } diff --git a/crates/paimon/tests/data_evolution_window_test.rs b/crates/paimon/tests/data_evolution_window_test.rs new file mode 100644 index 000000000..0a7385c4e --- /dev/null +++ b/crates/paimon/tests/data_evolution_window_test.rs @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; + +use arrow_array::{Array, Int32Array, Int64Array, RecordBatch}; +use arrow_schema::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; +use futures::TryStreamExt; +use paimon::spec::{DataType, Datum, IntType, PredicateBuilder, Schema, TableSchema}; +use std::collections::HashMap; +use std::sync::Arc; + +use common::incremental_helpers::{ + make_batch, memory_table, persist_table_schema, setup_dirs, write_batch, +}; + +fn rows(batches: &[RecordBatch]) -> Vec<(Option, Option)> { + let column = |batch: &RecordBatch, name: &str| { + let column = batch.column_by_name(name).unwrap(); + column + .as_any() + .downcast_ref::() + .unwrap() + .clone() + }; + let mut rows = Vec::new(); + for batch in batches { + let (ids, values) = (column(batch, "id"), column(batch, "value")); + for row in 0..batch.num_rows() { + rows.push(( + ids.is_valid(row).then(|| ids.value(row)), + values.is_valid(row).then(|| values.value(row)), + )); + } + } + rows.sort(); + rows +} + +/// Writes `value` of rows `0..values.len()` as a column slice of their files. +async fn update_values(table: &paimon::table::Table, values: Vec) { + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("_ROW_ID", ArrowDataType::Int64, false), + Field::new("value", ArrowDataType::Int32, false), + ])); + let row_ids = (0..values.len() as i64).collect::>(); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from(row_ids)), + Arc::new(Int32Array::from(values)), + ], + ) + .unwrap(); + let builder = table.new_write_builder(); + let mut writer = builder + .new_data_evolution_writer(vec!["value".to_string()]) + .unwrap(); + writer.add_matched_batch(batch).unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + builder.new_commit().commit(messages).await.unwrap(); +} + +async fn compact(table: &paimon::table::Table) { + let compaction = table.new_compact_builder(); + let plan = compaction.new_planner().unwrap().plan().await.unwrap(); + let executor = compaction.new_executor().unwrap(); + let mut results = Vec::new(); + for task in plan.tasks() { + results.push(executor.execute(task).await.unwrap()); + } + compaction + .new_commit() + .unwrap() + .commit_compact(results) + .await + .unwrap(); +} + +/// Reads the table as the changes of the window of every snapshot. +async fn read_window(table: &paimon::table::Table) -> paimon::Result> { + let options = HashMap::from([( + "incremental-between-timestamp".to_string(), + "0,9999999999999".to_string(), + )]); + let table = table.copy_with_time_travel(options).await?; + let builder = table.new_read_builder(); + let plan = builder.new_scan().plan().await?; + builder + .new_read()? + .to_arrow(plan.splits())? + .try_collect() + .await +} + +fn data_evolution_table(path: &str) -> (paimon::io::FileIO, paimon::table::Table) { + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("write-only", "true") + .option("compaction.min.file-num", "2") + .build() + .unwrap(); + memory_table(path, TableSchema::new(0, &schema)) +} + +/// A column update of rows an insert wrote is a slice of the insert's rows, and +/// the window of both merges them into the updated rows. +#[tokio::test] +async fn window_merges_a_column_update_with_the_insert_it_updates() { + let path = "memory:/data_evolution_window/slices"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![10, 20])).await; + update_values(&table, vec![99, 20]).await; + + let batches = read_window(&table).await.unwrap(); + assert_eq!( + rows(&batches), + vec![(Some(1), Some(99)), (Some(2), Some(20))] + ); +} + +/// A compaction merges the files of two inserts into one, so an update of every +/// row covers that file and not the rows of either insert. Their files are not +/// column slices of one row range, and the window returns them as they are +/// stored instead of failing the read. +#[tokio::test] +async fn window_reads_files_that_are_not_column_slices_as_stored() { + let path = "memory:/data_evolution_window/compacted"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![1, 2])).await; + write_batch(&table, &make_batch(vec![3, 4], vec![3, 4])).await; + compact(&table).await; + update_values(&table, vec![50; 4]).await; + + let batches = read_window(&table).await.unwrap(); + assert_eq!( + rows(&batches), + vec![ + (None, Some(50)), + (None, Some(50)), + (None, Some(50)), + (None, Some(50)), + (Some(1), Some(1)), + (Some(2), Some(2)), + (Some(3), Some(3)), + (Some(4), Some(4)), + ] + ); +} + +/// Projection pruning must not drop an update-only file from a physical-event +/// group: its absent projected fields are read as NULL like the other columns. +#[tokio::test] +async fn window_reads_physical_events_with_projection() { + let path = "memory:/data_evolution_window/projection"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![1, 2])).await; + write_batch(&table, &make_batch(vec![3, 4], vec![3, 4])).await; + compact(&table).await; + update_values(&table, vec![50; 4]).await; + + let options = HashMap::from([( + "incremental-between-timestamp".to_string(), + "0,9999999999999".to_string(), + )]); + let table = table.copy_with_time_travel(options).await.unwrap(); + let mut builder = table.new_read_builder(); + builder.with_projection(&["id"]).unwrap(); + let plan = builder.new_scan().plan().await.unwrap(); + let batches = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut ids = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .collect::>() + }) + .collect::>(); + ids.sort(); + assert_eq!( + ids, + vec![None, None, None, None, Some(1), Some(2), Some(3), Some(4),] + ); +} + +/// Merged latest-row statistics must not prune physical events whose stored +/// values satisfy the predicate. +#[tokio::test] +async fn window_filters_physical_events_by_stored_values() { + let path = "memory:/data_evolution_window/filter"; + let (io, table) = data_evolution_table(path); + setup_dirs(&io, path).await; + persist_table_schema(&io, path, table.schema()).await; + write_batch(&table, &make_batch(vec![1, 2], vec![1, 2])).await; + write_batch(&table, &make_batch(vec![3, 4], vec![3, 4])).await; + compact(&table).await; + update_values(&table, vec![50; 4]).await; + + let options = HashMap::from([( + "incremental-between-timestamp".to_string(), + "0,9999999999999".to_string(), + )]); + let table = table.copy_with_time_travel(options).await.unwrap(); + let mut builder = table.new_read_builder(); + builder.with_filter( + PredicateBuilder::new(table.schema().fields()) + .equal("value", Datum::Int(1)) + .unwrap(), + ); + let plan = builder.new_scan().plan().await.unwrap(); + let batches = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(rows(&batches), vec![(Some(1), Some(1))]); +}