diff --git a/crates/paimon/src/arrow/format/mod.rs b/crates/paimon/src/arrow/format/mod.rs index 1edc54631..bc558992e 100644 --- a/crates/paimon/src/arrow/format/mod.rs +++ b/crates/paimon/src/arrow/format/mod.rs @@ -484,7 +484,7 @@ pub(crate) fn create_format_reader_with_budget( Ok(configure_format_reader(reader, fields)) } -fn configure_format_reader( +pub(crate) fn configure_format_reader( reader: Box, fields: FormatReadFields<'_>, ) -> ConfiguredFormatReader { diff --git a/crates/paimon/src/arrow/format/parquet.rs b/crates/paimon/src/arrow/format/parquet.rs index 794f7e597..acd214726 100644 --- a/crates/paimon/src/arrow/format/parquet.rs +++ b/crates/paimon/src/arrow/format/parquet.rs @@ -48,13 +48,14 @@ use parquet::arrow::async_reader::{AsyncFileReader, MetadataFetch}; use parquet::arrow::{AsyncArrowWriter, ParquetRecordBatchStreamBuilder, ProjectionMask}; use parquet::basic::{Compression, ZstdLevel}; use parquet::file::metadata::{ - KeyValue, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData, + ColumnChunkMetaData, KeyValue, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, + RowGroupMetaData, }; use parquet::file::page_index::column_index::ColumnIndexMetaData; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use parquet::file::statistics::Statistics as ParquetStatistics; use std::cmp::Ordering; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ops::Range; use std::sync::{Arc, Mutex}; use tokio::sync::{mpsc, oneshot}; @@ -65,6 +66,7 @@ pub(crate) struct ParquetFormatReader { read_budget: Option>, page_index_enabled: bool, metadata_cache_enabled: bool, + metadata: Option, } impl Default for ParquetFormatReader { @@ -73,6 +75,7 @@ impl Default for ParquetFormatReader { read_budget: None, page_index_enabled: true, metadata_cache_enabled: true, + metadata: None, } } } @@ -88,9 +91,85 @@ impl ParquetFormatReader { page_index_enabled: core_options.parquet_filter_column_index_enabled()?, // Format-table files may be replaced in place. metadata_cache_enabled: !core_options.is_format_table(), + metadata: None, }) } + pub(crate) fn with_metadata(mut self, metadata: ArrowReaderMetadata) -> Self { + self.metadata = Some(metadata); + self + } + + fn reader_options(&self, has_predicate: bool, has_selection: bool) -> ArrowReaderOptions { + let mut options = ArrowReaderOptions::new(); + if self.page_index_enabled { + if has_predicate { + options = options.with_column_index_policy(PageIndexPolicy::Optional); + } + if has_predicate || has_selection { + options = options.with_offset_index_policy(PageIndexPolicy::Optional); + } + } + options + } + + pub(crate) async fn load_selection_metadata( + &self, + reader: Box, + file_size: u64, + has_predicates: bool, + ) -> parquet::errors::Result { + let mut reader = ArrowFileReader::new(file_size, reader.into()) + .with_metadata_cache_enabled(self.metadata_cache_enabled); + let options = self.reader_options(has_predicates, true); + ArrowReaderMetadata::load_async(&mut reader, options).await + } + + pub(crate) fn candidate_row_selection( + &self, + metadata: &ArrowReaderMetadata, + mut predicates: Vec, + file_fields: &[DataField], + ) -> Option { + // A cast can change the unit of a timestamp or the scale of a decimal bound. + let mut referenced = HashSet::new(); + predicates.retain(|predicate| { + referenced.clear(); + predicate.collect_leaf_field_indices(&mut referenced); + referenced.iter().all(|&index| { + let Some(field) = file_fields.get(index) else { + return false; + }; + let Ok(physical) = metadata.schema().field_with_name(field.name()) else { + return false; + }; + crate::arrow::paimon_type_to_arrow(field.data_type()) + .is_ok_and(|expected| physical.data_type() == &expected) + }) + }); + self.predicate_selection(metadata.metadata(), &predicates, file_fields) + } + + fn predicate_selection( + &self, + metadata: &ParquetMetaData, + predicates: &[Predicate], + file_fields: &[DataField], + ) -> Option { + let groups = predicate_row_group_selectors(metadata.row_groups(), predicates, file_fields); + let all_groups_skipped = groups + .as_ref() + .is_some_and(|groups| groups.iter().all(|group| group.skip)); + if self.page_index_enabled && !all_groups_skipped { + if let Some(pages) = + build_predicate_page_selection(metadata, predicates, file_fields, groups.as_deref()) + { + return Some(pages); + } + } + groups.map(RowSelection::from) + } + #[cfg(test)] fn with_read_budget(read_budget: Arc) -> Self { Self { @@ -145,19 +224,13 @@ pub(crate) async fn parquet_granules( source: None, }); } - let Some(offset_index) = metadata - .offset_index() - .filter(|index| index.len() == metadata.row_groups().len()) - else { + let Some(offset_index) = metadata.offset_index() else { return Ok((row_group_granules(&metadata, &columns), false)); }; let mut granules = Vec::new(); let mut cursor = 0i64; for (row_group, indexes) in metadata.row_groups().iter().zip(offset_index) { - let Ok(row_count) = usize::try_from(row_group.num_rows()) else { - return Ok((row_group_granules(&metadata, &columns), false)); - }; let Some(leaf_pages) = columns .iter() .map(|index| indexes.get(*index).map(|index| index.page_locations())) @@ -168,8 +241,7 @@ pub(crate) async fn parquet_granules( let pages = leaf_pages[0]; if pages.is_empty() || leaf_pages.iter().any(|other| { - !page_boundaries_valid(other, row_count) - || other.len() != pages.len() + other.len() != pages.len() || other .iter() .zip(pages) @@ -186,25 +258,13 @@ pub(crate) async fn parquet_granules( for (column_index, pages) in columns.iter().zip(&leaf_pages) { let column = row_group.column(*column_index); let (column_start, _) = column.byte_range(); - if let Some(first) = pages.first() { - let Ok(first_page) = u64::try_from(first.offset) else { - return Ok((row_group_granules(&metadata, &columns), false)); - }; - if column_start < first_page { - byte_ranges.push(column_start..first_page); - } + let first_page = pages[0].offset as u64; + if column_start < first_page { + byte_ranges.push(column_start..first_page); } let page = &pages[page_index]; - let (Ok(start), Ok(length)) = ( - u64::try_from(page.offset), - u64::try_from(page.compressed_page_size), - ) else { - return Ok((row_group_granules(&metadata, &columns), false)); - }; - let Some(end) = start.checked_add(length).filter(|end| *end <= file_size) else { - return Ok((row_group_granules(&metadata, &columns), false)); - }; - byte_ranges.push(start..end); + let start = page.offset as u64; + byte_ranges.push(start..start + page.compressed_page_size as u64); } granules.push(ParquetGranule { first_row: cursor + page.first_row_index, @@ -997,7 +1057,7 @@ impl FormatFileReader for ParquetFormatReader { row_selection: Option>, ) -> crate::Result { let shared_reader: Arc = reader.into(); - let arrow_file_reader = ArrowFileReader::new(file_size, Arc::clone(&shared_reader)) + let mut arrow_file_reader = ArrowFileReader::new(file_size, Arc::clone(&shared_reader)) .with_metadata_cache_enabled(self.metadata_cache_enabled); let empty_predicates = Vec::new(); @@ -1010,22 +1070,16 @@ impl FormatFileReader for ParquetFormatReader { .as_ref() .is_some_and(|factory| factory.supports_row_group_pruning()); - // Predicates need both indexes for page-stat pruning. Row selection only - // needs OffsetIndex so arrow-rs can avoid fetching unselected pages. - let mut arrow_options = ArrowReaderOptions::new(); - if self.page_index_enabled { - if !preds.is_empty() { - arrow_options = arrow_options.with_column_index_policy(PageIndexPolicy::Optional); - } - if !preds.is_empty() || row_selection.is_some() { - arrow_options = arrow_options.with_offset_index_policy(PageIndexPolicy::Optional); + let arrow_options = self.reader_options(!preds.is_empty(), row_selection.is_some()); + let metadata = match &self.metadata { + Some(metadata) => metadata.clone(), + None => { + ArrowReaderMetadata::load_async(&mut arrow_file_reader, arrow_options.clone()) + .await? } - } - let mut batch_stream_builder = ParquetRecordBatchStreamBuilder::new_with_options( - arrow_file_reader, - arrow_options.clone(), - ) - .await?; + }; + let mut batch_stream_builder = + ParquetRecordBatchStreamBuilder::new_with_metadata(arrow_file_reader, metadata); // Variant columns read only for extraction keep their metadata dictionary-encoded. let variant_schema = dictionary_variant_metadata_schema(batch_stream_builder.schema(), read_fields); @@ -1042,24 +1096,11 @@ impl FormatFileReader for ParquetFormatReader { let parquet_schema = batch_stream_builder.parquet_schema().clone(); - // Determine whether the Arrow row filter enforces every pushed-down - // predicate exactly. It only accepts top-level supported leaves; `Or`, - // `Not`, and unsupported leaves fall through to conservative pruning - // only, which is NOT exact. When any predicate is not fully enforced we - // add a row-level residual backstop over the decoded batch. - // - // Fast path (the common case, including all-AND-of-simple-leaves): every - // predicate is fully enforced, so we decode exactly `read_fields`, skip - // the residual pass entirely, and return the stream as before — zero - // added overhead. let all_enforced = preds .iter() .all(|p| predicate_fully_enforced_by_row_filter(&parquet_schema, p, file_fields)); - // Residual branch must decode the predicate columns too, or the residual - // pass could not see a predicate on a non-projected column (Gap A). The - // projection MASK here (which columns to DECODE) is distinct from - // `DataFileReader`'s by-name output projection applied downstream. + // The residual needs its predicate columns even when the caller projects them out. let scan_fields: Vec = if all_enforced { read_fields.to_vec() } else { @@ -1159,22 +1200,8 @@ impl FormatFileReader for ParquetFormatReader { batch_stream_builder.with_row_filter(ParquetRowFilter::new(decoder_predicates)); } - let predicate_row_selection = build_predicate_row_selection( - batch_stream_builder.metadata().row_groups(), - preds, - file_fields, - )?; - let mut combined_selection = predicate_row_selection; - - // Page-level selection. Returns `None` when ColumnIndex / OffsetIndex are - // absent (page index not loaded, older files, writer without page index) - // or when no page could be skipped, so intersecting is a no-op then. - let page_selection = if self.page_index_enabled { - build_predicate_page_selection(batch_stream_builder.metadata(), preds, file_fields)? - } else { - None - }; - combined_selection = intersect_optional_row_selections(combined_selection, page_selection); + let mut combined_selection = + self.predicate_selection(batch_stream_builder.metadata(), preds, file_fields); if let Some(ref ranges) = row_selection { let range_selection = @@ -2157,32 +2184,43 @@ struct ParquetRowGroupStats<'a> { column_indices: &'a [Option], } +impl ParquetRowGroupStats<'_> { + fn column_statistics(&self, index: usize) -> Option<&ParquetStatistics> { + let column_index = self.column_indices.get(index).copied().flatten()?; + let column = self.row_group.columns().get(column_index)?; + let stats = column.statistics()?; + let rows = self.row_group.num_rows(); + if column.num_values() != rows { + return None; + } + if let Some(nulls) = stats.null_count_opt() { + let rows = u64::try_from(rows).ok()?; + if nulls > rows + || (nulls == rows + && (stats.min_bytes_opt().is_some() || stats.max_bytes_opt().is_some())) + { + return None; + } + } + Some(stats) + } +} + impl StatsAccessor for ParquetRowGroupStats<'_> { fn row_count(&self) -> i64 { self.row_group.num_rows() } fn null_count(&self, index: usize) -> Option { - let _ = index; - None + i64::try_from(self.column_statistics(index)?.null_count_opt()?).ok() } fn min_value(&self, index: usize, data_type: &DataType) -> Option { - let column_index = self.column_indices.get(index).copied().flatten()?; - parquet_stats_to_datum( - self.row_group.column(column_index).statistics()?, - data_type, - true, - ) + parquet_stats_to_datum(self.column_statistics(index)?, data_type, true) } fn max_value(&self, index: usize, data_type: &DataType) -> Option { - let column_index = self.column_indices.get(index).copied().flatten()?; - parquet_stats_to_datum( - self.row_group.column(column_index).statistics()?, - data_type, - false, - ) + parquet_stats_to_datum(self.column_statistics(index)?, data_type, false) } fn supports_in_min_max_pruning(&self) -> bool { @@ -2190,17 +2228,15 @@ impl StatsAccessor for ParquetRowGroupStats<'_> { } } -fn build_predicate_row_selection( +fn predicate_row_group_selectors( row_groups: &[RowGroupMetaData], predicates: &[Predicate], file_fields: &[DataField], -) -> crate::Result> { +) -> Option> { if predicates.is_empty() || row_groups.is_empty() { - return Ok(None); + return None; } - // Predicates have already been remapped to file-level indices by the caller - // (remap_predicates_to_file in reader.rs), so we use an identity mapping here. let identity_mapping: Vec> = (0..file_fields.len()).map(Some).collect(); let column_indices = build_row_group_column_indices(row_groups[0].columns(), file_fields); let mut selectors = Vec::with_capacity(row_groups.len()); @@ -2223,15 +2259,11 @@ fn build_predicate_row_selection( }); } - if all_selected { - Ok(None) - } else { - Ok(Some(selectors.into())) - } + (!all_selected).then_some(selectors) } fn build_row_group_column_indices( - columns: &[parquet::file::metadata::ColumnChunkMetaData], + columns: &[ColumnChunkMetaData], file_fields: &[DataField], ) -> Vec> { let mut by_root_name: HashMap<&str, Option> = HashMap::new(); @@ -2548,21 +2580,36 @@ fn binary_table_stats_from_datums( // Page-index (ColumnIndex / OffsetIndex) pruning // --------------------------------------------------------------------------- -/// Stats view over one data page of a **single** column, backed by the Parquet -/// ColumnIndex. Plugs into the same [`StatsAccessor`] evaluator as row-group -/// pruning so both layers share identical fail-open semantics. -/// -/// Only `target_index` is exposed; every other column reports no stats so its -/// leaves fail open. Parquet pages are laid out per column chunk (different -/// columns may have different page counts and boundaries), so each column must -/// be pruned against its own page layout — see [`build_predicate_page_selection`]. struct ParquetPageStats<'a> { - /// File-field index this page belongs to. target_index: usize, - /// ColumnIndex of the target column for the current row group. column_index: &'a ColumnIndexMetaData, page_idx: usize, page_row_count: i64, + null_count: Option, +} + +impl<'a> ParquetPageStats<'a> { + fn new( + target_index: usize, + column_index: &'a ColumnIndexMetaData, + page_idx: usize, + page_row_count: i64, + ) -> Option { + let all_null = column_index.is_null_page(page_idx); + let null_count = column_index.null_count(page_idx); + if null_count.is_some_and(|count| { + !(0..=page_row_count).contains(&count) || (count == page_row_count) != all_null + }) { + return None; + } + Some(Self { + target_index, + column_index, + page_idx, + page_row_count, + null_count: null_count.or(all_null.then_some(page_row_count)), + }) + } } impl StatsAccessor for ParquetPageStats<'_> { @@ -2574,7 +2621,7 @@ impl StatsAccessor for ParquetPageStats<'_> { if index != self.target_index { return None; } - self.column_index.null_count(self.page_idx) + self.null_count } fn min_value(&self, index: usize, data_type: &DataType) -> Option { @@ -2596,11 +2643,6 @@ impl StatsAccessor for ParquetPageStats<'_> { } } -/// Decode a per-page min/max from a [`ColumnIndexMetaData`] into a [`Datum`]. -/// -/// Returns `None` (fail-open: keep the page) for null pages, missing values, or -/// any type that the footer-side path also excludes (decimals, sub-millisecond -/// timestamps). fn page_index_value_to_datum( column_index: &ColumnIndexMetaData, page_idx: usize, @@ -2684,55 +2726,33 @@ fn page_index_value_to_datum( } } -/// Build a page-granular [`RowSelection`] from the Parquet ColumnIndex / -/// OffsetIndex by pruning each predicate column against **its own** page layout, -/// then intersecting the per-column selections. -/// -/// Parquet splits pages per column chunk, so different columns in a row group -/// may have different page counts and boundaries. Each column is therefore -/// evaluated over its own `OffsetIndex` pages, exposing only that column to the -/// stats evaluator (every other column fails open). Because the top-level -/// predicate list is a conjunction, intersecting the per-column selections is -/// sound. -/// -/// Returns `None` when the page index is absent or when no page could be skipped -/// (so the caller leaves the coarser row-group selection untouched). A page is -/// kept whenever its stats are unavailable or the predicate cannot be decided -/// from them — pruning never drops a page it is unsure about. +/// Pages can start at different rows in each column. Intersect the row positions +/// that survive each column's statistics within the remaining row groups. +/// `None` leaves the row-group selection unchanged. fn build_predicate_page_selection( metadata: &ParquetMetaData, predicates: &[Predicate], file_fields: &[DataField], -) -> crate::Result> { - if predicates.is_empty() { - return Ok(None); - } - let (Some(column_index), Some(offset_index)) = - (metadata.column_index(), metadata.offset_index()) - else { - return Ok(None); - }; + group_selectors: Option<&[RowSelector]>, +) -> Option { + let column_index = metadata.column_index()?; + let offset_index = metadata.offset_index()?; let row_groups = metadata.row_groups(); - if row_groups.is_empty() { - return Ok(None); + if predicates.is_empty() || row_groups.is_empty() { + return None; } - // Predicates are already remapped to file-level indices by the caller, so an - // identity mapping suffices (same convention as row-group pruning). let identity_mapping: Vec> = (0..file_fields.len()).map(Some).collect(); let column_lookup = build_row_group_column_indices(row_groups[0].columns(), file_fields); let total_rows: usize = row_groups.iter().map(|rg| rg.num_rows() as usize).sum(); - let mut referenced_fields = Vec::new(); + let mut referenced_fields = HashSet::new(); for predicate in predicates { - collect_leaf_field_indices(predicate, &mut referenced_fields); + predicate.collect_leaf_field_indices(&mut referenced_fields); } - referenced_fields.sort_unstable(); - referenced_fields.dedup(); let mut combined: Option = None; for field_index in referenced_fields { - // The column must resolve to a single parquet column present in the file. let Some(parquet_col) = column_lookup.get(field_index).copied().flatten() else { continue; }; @@ -2745,9 +2765,10 @@ fn build_predicate_page_selection( let rg_rows = row_group.num_rows() as usize; let base = rg_base; rg_base += rg_rows; + if group_selectors.is_some_and(|groups| groups[rg_idx].skip) { + continue; + } - // Missing page index for this column/row group: keep the whole group - // (row-group pruning still applied). let (Some(col_index), Some(col_offset)) = ( column_index.get(rg_idx).and_then(|rg| rg.get(parquet_col)), offset_index.get(rg_idx).and_then(|rg| rg.get(parquet_col)), @@ -2756,45 +2777,44 @@ fn build_predicate_page_selection( continue; }; let pages = col_offset.page_locations(); - // Fail open when the column index is absent for this chunk - // (`ColumnIndexMetaData::NONE` — its accessors panic rather than - // return `None`), when the two indexes disagree on the page count, - // or when the page row boundaries are malformed. Either way we - // cannot safely map page stats to row ranges. if pages.is_empty() || matches!(col_index, ColumnIndexMetaData::NONE) || col_index.num_pages() as usize != pages.len() - || !page_boundaries_valid(pages, rg_rows) { ranges.push(base..base + rg_rows); continue; } + let mut selected_start = 0; for (page_idx, page) in pages.iter().enumerate() { let page_start = page.first_row_index as usize; let page_end = pages .get(page_idx + 1) .map_or(rg_rows, |next| next.first_row_index as usize); - if page_end <= page_start { - continue; - } - let stats = ParquetPageStats { - target_index: field_index, - column_index: col_index, + let stats = ParquetPageStats::new( + field_index, + col_index, page_idx, - page_row_count: (page_end - page_start) as i64, - }; - if predicates_may_match_with_schema( - predicates, - &stats, - &identity_mapping, - file_fields, - ) { - ranges.push(base + page_start..base + page_end); - } else { + (page_end - page_start) as i64, + ); + if stats.is_some_and(|stats| { + !predicates_may_match_with_schema( + predicates, + &stats, + &identity_mapping, + file_fields, + ) + }) { + if selected_start < page_start { + ranges.push(base + selected_start..base + page_start); + } + selected_start = page_end; any_skipped = true; } } + if selected_start < rg_rows { + ranges.push(base + selected_start..base + rg_rows); + } } if !any_skipped { @@ -2807,51 +2827,41 @@ fn build_predicate_page_selection( }); } - Ok(combined) -} - -/// Collect the file-field indices referenced by leaf predicates (recursing -/// through compound nodes). Indices are already file-level (see caller). -fn collect_leaf_field_indices(predicate: &Predicate, out: &mut Vec) { - match predicate { - Predicate::NestedLeaf { index, .. } => out.push(*index), - Predicate::Leaf { index, .. } => out.push(*index), - Predicate::And(children) | Predicate::Or(children) => { - for child in children { - collect_leaf_field_indices(child, out); - } - } - Predicate::Not(child) => collect_leaf_field_indices(child, out), - Predicate::AlwaysTrue | Predicate::AlwaysFalse => {} - } + combined } -/// Validate that an OffsetIndex's page row boundaries are well-formed for a row -/// group of `rg_rows` rows: the first page starts at row 0, `first_row_index` -/// is strictly increasing, and every value stays within `[0, rg_rows]`. -/// -/// Malformed metadata (negative, non-monotonic, or out-of-range boundaries) -/// would otherwise produce invalid row ranges — huge values from `i64 as usize` -/// underflow, or ranges past the row group — that panic -/// `RowSelection::from_consecutive_ranges`. Callers fail open when this returns -/// `false`. -fn page_boundaries_valid( +fn page_locations_valid( pages: &[parquet::file::page_index::offset_index::PageLocation], - rg_rows: usize, + row_count: i64, + column: &ColumnChunkMetaData, + file_size: u64, ) -> bool { - let rg_rows = rg_rows as i64; - let mut prev: i64 = -1; - for (page_idx, page) in pages.iter().enumerate() { - let first_row = page.first_row_index; - if page_idx == 0 && first_row != 0 { - return false; - } - if first_row <= prev || first_row > rg_rows { - return false; - } - prev = first_row; + let Some(first) = pages.first() else { + return row_count == 0; + }; + let data_start = column.data_page_offset(); + let chunk_start = column.dictionary_page_offset().unwrap_or(data_start); + let Some(chunk_end) = chunk_start.checked_add(column.compressed_size()) else { + return false; + }; + if !(0..=data_start).contains(&chunk_start) + || !u64::try_from(chunk_end).is_ok_and(|end| end <= file_size) + || first.first_row_index != 0 + || first.offset != data_start + { + return false; } - true + pages.iter().enumerate().all(|(index, page)| { + let (next_row, next_byte) = pages.get(index + 1).map_or((row_count, chunk_end), |next| { + (next.first_row_index, next.offset) + }); + page.first_row_index < next_row + && page.compressed_page_size > 0 + && page + .offset + .checked_add(i64::from(page.compressed_page_size)) + .is_some_and(|end| end <= next_byte) + }) } // --------------------------------------------------------------------------- @@ -2868,7 +2878,14 @@ fn parquet_stats_to_datum( } else { stats.max_is_exact() }; - if !exact { + // Legacy byte-array bounds can use signed byte ordering. + if !exact + || (stats.is_min_max_deprecated() + && matches!( + stats, + ParquetStatistics::ByteArray(_) | ParquetStatistics::FixedLenByteArray(_) + )) + { return None; } @@ -3169,16 +3186,6 @@ fn page_index_policy_tag(policy: PageIndexPolicy) -> u8 { } } -/// ArrowFileReader is a wrapper around a FileRead that impls parquets AsyncFileReader. -/// -/// # TODO -/// -/// [ParquetObjectReader](https://docs.rs/parquet/latest/src/parquet/arrow/async_reader/store.rs.html#64) -/// contains the following hints to speed up metadata loading, similar to iceberg, we can consider adding them to this struct: -/// -/// - `metadata_size_hint`: Provide a hint as to the size of the parquet file's footer. -/// - `preload_column_index`: Load the Column Index as part of [`Self::get_metadata`]. -/// - `preload_offset_index`: Load the Offset Index as part of [`Self::get_metadata`]. struct ArrowFileReader { file_size: u64, r: Arc, @@ -3249,7 +3256,34 @@ impl ArrowFileReader { if let Some(policy) = offset_index_policy { reader = reader.with_offset_index_policy(policy); } - Ok(Arc::new(reader.load_and_finish(self, file_size).await?)) + let mut metadata = reader.load_and_finish(self, file_size).await?; + if let Some(index) = metadata.offset_index() { + // Arrow also uses these offsets to decode selections from other column providers. + let valid = index.len() == metadata.num_row_groups() + && index + .iter() + .zip(metadata.row_groups()) + .all(|(columns, group)| { + columns.len() == group.num_columns() + && columns.iter().zip(group.columns()).all(|(index, column)| { + page_locations_valid( + index.page_locations(), + group.num_rows(), + column, + file_size, + ) + }) + }); + if !valid { + if offset_index_policy == Some(PageIndexPolicy::Required) { + return Err(parquet::errors::ParquetError::General( + "Invalid page locations in Parquet OffsetIndex".into(), + )); + } + metadata = metadata.into_builder().set_offset_index(None).build(); + } + } + Ok(Arc::new(metadata)) } } @@ -5941,32 +5975,29 @@ mod tests { vec![Datum::Int(-1), Datum::Int(30)], )]; let selection = - super::build_predicate_row_selection(metadata.row_groups(), &outside, &fields) - .unwrap() + super::predicate_row_group_selectors(metadata.row_groups(), &outside, &fields) .expect("all row groups should be skipped"); - assert_eq!(selection.row_count(), 0); + assert_eq!(RowSelection::from(selection).row_count(), 0); let overlapping = vec![id_leaf( PredicateOperator::In, vec![Datum::Int(5), Datum::Int(30)], )]; let selection = - super::build_predicate_row_selection(metadata.row_groups(), &overlapping, &fields) - .unwrap() + super::predicate_row_group_selectors(metadata.row_groups(), &overlapping, &fields) .expect("only the first row group should be kept"); - assert_eq!(selection.row_count(), 10); + assert_eq!(RowSelection::from(selection).row_count(), 10); } #[tokio::test] - async fn test_row_group_selection_in_fails_open_on_unusable_stats() { + async fn test_row_group_selection_fails_open_on_unusable_stats() { let fields = vec![int_field("id"), int_field("value")]; let predicates = vec![id_leaf(PredicateOperator::In, vec![Datum::Int(100)])]; let bytes = write_multi_row_group_parquet(10, 10, EnabledStatistics::None, false).await; let metadata = load_metadata_with_page_index(&bytes, false); let selection = - super::build_predicate_row_selection(metadata.row_groups(), &predicates, &fields) - .unwrap(); + super::predicate_row_group_selectors(metadata.row_groups(), &predicates, &fields); assert!(selection.is_none(), "missing stats must fail open"); let bytes = write_multi_row_group_parquet(10, 10, EnabledStatistics::Chunk, false).await; @@ -5986,23 +6017,104 @@ mod tests { .build() .unwrap(); damaged_row_group.columns_mut()[0] = damaged_id_column; - let selection = - super::build_predicate_row_selection(&[damaged_row_group], &predicates, &fields) - .unwrap(); - assert!(selection.is_none(), "inverted stats must fail open"); + for predicate in [ + predicates[0].clone(), + id_leaf(PredicateOperator::Eq, vec![Datum::Int(5)]), + id_leaf( + PredicateOperator::Between, + vec![Datum::Int(3), Datum::Int(8)], + ), + ] { + let selection = super::predicate_row_group_selectors( + std::slice::from_ref(&damaged_row_group), + &[predicate], + &fields, + ); + assert!(selection.is_none(), "inverted stats must fail open"); + } let varchar_type = DataType::VarChar(VarCharType::new(20).unwrap()); let varchar_fields = vec![DataField::new(0, "id".to_string(), varchar_type)]; let varchar_predicates = vec![PredicateBuilder::new(&varchar_fields) .is_in("id", vec![Datum::String("100".to_string())]) .unwrap()]; - let selection = super::build_predicate_row_selection( + let selection = super::predicate_row_group_selectors( metadata.row_groups(), &varchar_predicates, &varchar_fields, - ) - .unwrap(); + ); assert!(selection.is_none(), "incomparable stats must fail open"); + + let binary = DataType::VarBinary(crate::spec::VarBinaryType::new(2).unwrap()); + for deprecated in [true, false] { + let min = vec![1, 255]; + let max = vec![2, 0]; + let stats = [ + ParquetStatistics::byte_array( + Some(min.clone().into()), + Some(max.clone().into()), + None, + Some(0), + deprecated, + ), + ParquetStatistics::fixed_len_byte_array( + Some(min.clone().into()), + Some(max.clone().into()), + None, + Some(0), + deprecated, + ), + ]; + for stats in stats { + for (is_min, bound) in [(true, &min), (false, &max)] { + assert_eq!( + super::parquet_stats_to_datum(&stats, &binary, is_min), + (!deprecated).then(|| Datum::Bytes(bound.clone())) + ); + } + } + } + } + + #[tokio::test] + async fn row_group_null_predicates_require_consistent_counts_for_flat_columns() { + let bytes = write_multi_row_group_parquet(10, 10, EnabledStatistics::Chunk, true).await; + let metadata = load_metadata_with_page_index(&bytes, false); + let fields = vec![int_field("id"), int_field("value")]; + for (values, nulls, min, max, may_have_nulls, may_have_values) in [ + (10, Some(0), Some(0), Some(9), false, true), + (10, Some(3), Some(0), Some(9), true, true), + (10, Some(10), None, None, true, false), + (10, None, None, None, true, true), + (10, Some(11), None, None, true, true), + (10, Some(10), Some(0), Some(9), true, true), + (9, Some(0), Some(0), Some(8), true, true), + ] { + let mut group = metadata.row_group(0).clone(); + group.columns_mut()[0] = group + .column(0) + .clone() + .into_builder() + .set_num_values(values) + .set_statistics(ParquetStatistics::int32(min, max, None, nulls, false)) + .build() + .unwrap(); + for (op, keep) in [ + (PredicateOperator::IsNull, may_have_nulls), + (PredicateOperator::IsNotNull, may_have_values), + ] { + let selection = super::predicate_row_group_selectors( + std::slice::from_ref(&group), + &[id_leaf(op, vec![])], + &fields, + ); + assert_eq!( + selection.is_none_or(|groups| !groups[0].skip), + keep, + "{op:?}, values={values}, nulls={nulls:?}, min={min:?}, max={max:?}" + ); + } + } } // ----------------------------------------------------------------------- @@ -7219,90 +7331,38 @@ mod tests { } #[tokio::test] - async fn test_page_selection_eq_keeps_only_matching_page() { - // 80 rows / 10 per page = 8 pages; page p covers ids [p*10, p*10+10). + async fn page_selection_keeps_candidate_row_positions() { let bytes = write_multi_page_parquet(10, 80).await; let metadata = load_metadata_with_page_index(&bytes, true); + let without_indexes = load_metadata_with_page_index(&bytes, false); let fields = vec![int_field("id"), int_field("value")]; - // Eq(35) falls only in page 3 ([30, 40)). - let predicates = vec![id_leaf(PredicateOperator::Eq, vec![Datum::Int(35)])]; - let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields) - .unwrap() - .expect("a page should be skipped"); - assert_eq!(sel.row_count(), 10); - } - - #[tokio::test] - async fn test_page_selection_eq_outside_all_pages_skips_everything() { - let bytes = write_multi_page_parquet(10, 80).await; - let metadata = load_metadata_with_page_index(&bytes, true); - let fields = vec![int_field("id"), int_field("value")]; - - // 1000 is past every page's max (79). - let predicates = vec![id_leaf(PredicateOperator::Eq, vec![Datum::Int(1000)])]; - let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields) - .unwrap() - .expect("all pages should be skipped"); - assert_eq!(sel.row_count(), 0); - } - - #[tokio::test] - async fn test_page_selection_in_keeps_only_matching_page() { - let bytes = write_multi_page_parquet(10, 80).await; - let metadata = load_metadata_with_page_index(&bytes, true); - let fields = vec![int_field("id"), int_field("value")]; - - let predicates = vec![id_leaf( - PredicateOperator::In, - vec![Datum::Int(35), Datum::Int(1000)], - )]; - let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields) - .unwrap() - .expect("non-matching pages should be skipped"); - assert_eq!(sel.row_count(), 10); - } - - #[tokio::test] - async fn test_page_selection_range_keeps_overlapping_pages() { - let bytes = write_multi_page_parquet(10, 80).await; - let metadata = load_metadata_with_page_index(&bytes, true); - let fields = vec![int_field("id"), int_field("value")]; - - // Lt(25) overlaps pages 0 ([0,10)), 1 ([10,20)), 2 ([20,30)) — 30 rows. - let predicates = vec![id_leaf(PredicateOperator::Lt, vec![Datum::Int(25)])]; - let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields) - .unwrap() - .expect("some pages should be skipped"); - assert_eq!(sel.row_count(), 30); - } - - #[tokio::test] - async fn test_page_selection_neq_falls_open() { - let bytes = write_multi_page_parquet(10, 80).await; - let metadata = load_metadata_with_page_index(&bytes, true); - let fields = vec![int_field("id"), int_field("value")]; - - // NotEq is conservative under stats: every page holds other values, so - // no page can be excluded → helper returns None (selection unchanged). - let predicates = vec![id_leaf(PredicateOperator::NotEq, vec![Datum::Int(35)])]; - let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields).unwrap(); - assert!(sel.is_none(), "NotEq must not skip any page (got {sel:?})"); - } - - #[tokio::test] - async fn test_page_selection_returns_none_without_page_index() { - let bytes = write_multi_page_parquet(10, 80).await; - // Metadata parsed without the page index → helper must fall open. - let metadata = load_metadata_with_page_index(&bytes, false); - let fields = vec![int_field("id"), int_field("value")]; - - let predicates = vec![id_leaf(PredicateOperator::Eq, vec![Datum::Int(35)])]; - let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields).unwrap(); - assert!( - sel.is_none(), - "missing page index must fall open (got {sel:?})" - ); + for (op, values, expected) in [ + (PredicateOperator::Eq, vec![35], Some((30..40).collect())), + (PredicateOperator::Eq, vec![1000], Some(vec![])), + ( + PredicateOperator::In, + vec![35, 75, 1000], + Some((30..40).chain(70..80).collect()), + ), + (PredicateOperator::Lt, vec![25], Some((0..30).collect())), + (PredicateOperator::NotEq, vec![35], None), + ] { + let predicate = id_leaf(op, values.into_iter().map(Datum::Int).collect()); + let predicates = std::slice::from_ref(&predicate); + let selection = + super::build_predicate_page_selection(&metadata, predicates, &fields, None); + assert_eq!( + selection.as_ref().map(selected_rows), + expected, + "{predicate:?}" + ); + assert!( + super::build_predicate_page_selection(&without_indexes, predicates, &fields, None) + .is_none(), + "{predicate:?}" + ); + } } /// Expand a [`RowSelection`] into the set of selected 0-based row indices. @@ -7364,8 +7424,7 @@ mod tests { op: PredicateOperator::Eq, literals: vec![Datum::Int(350)], }]; - let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields) - .unwrap() + let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields, None) .expect("value pages should be prunable"); // The matching row (index 35) must survive; pruning is still effective @@ -7379,6 +7438,37 @@ mod tests { ); } + #[test] + fn page_stats_require_consistent_null_counts() { + use super::{ParquetPageStats, StatsAccessor}; + use parquet::file::metadata::ColumnIndexBuilder; + + for (all_null, count, expected) in [ + (true, 10, Some(10)), + (false, 0, Some(0)), + (false, 3, Some(3)), + (true, 0, None), + (false, 10, None), + (false, -1, None), + (false, 11, None), + ] { + let mut index = ColumnIndexBuilder::new(parquet::basic::Type::INT32); + index.append( + all_null, + 0_i32.to_le_bytes().to_vec(), + 9_i32.to_le_bytes().to_vec(), + count, + ); + let index = index.build().unwrap(); + let stats = ParquetPageStats::new(0, &index, 0, 10); + assert_eq!( + stats.map(|stats| stats.null_count(0).unwrap()), + expected, + "all_null={all_null}, count={count}" + ); + } + } + #[tokio::test] async fn test_page_selection_fails_open_on_missing_column_index() { // A column with only chunk-level statistics has no ColumnIndex @@ -7420,7 +7510,7 @@ mod tests { op: PredicateOperator::Eq, literals: vec![Datum::Int(350)], }]; - let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields).unwrap(); + let sel = super::build_predicate_page_selection(&metadata, &predicates, &fields, None); assert!( sel.is_none(), "column without a ColumnIndex must fall open (got {sel:?})" @@ -8089,39 +8179,41 @@ mod tests { assert_eq!(names, vec!["b", "c", "d"]); } - #[test] - fn test_page_boundaries_valid() { + #[tokio::test] + async fn test_page_locations_valid() { use parquet::file::page_index::offset_index::PageLocation; - let page = |first_row_index: i64| PageLocation { - offset: 0, - compressed_page_size: 0, - first_row_index, - }; - - // Well-formed: starts at 0, strictly increasing, within [0, rg_rows]. - assert!(super::page_boundaries_valid( - &[page(0), page(10), page(20)], - 30 - )); - // Single page starting at 0. - assert!(super::page_boundaries_valid(&[page(0)], 10)); - - // First page not at row 0. - assert!(!super::page_boundaries_valid(&[page(5), page(10)], 30)); - // Negative first_row_index (would cast to a huge usize). - assert!(!super::page_boundaries_valid(&[page(0), page(-1)], 30)); - // Non-monotonic. - assert!(!super::page_boundaries_valid( - &[page(0), page(20), page(10)], - 30 - )); - // Duplicate (not strictly increasing). - assert!(!super::page_boundaries_valid( - &[page(0), page(10), page(10)], - 30 - )); - // Beyond the row group. - assert!(!super::page_boundaries_valid(&[page(0), page(40)], 30)); + let bytes = write_multi_row_group_parquet(30, 30, EnabledStatistics::Chunk, true).await; + let metadata = load_metadata_with_page_index(&bytes, false); + let column = metadata.row_group(0).column(0); + let file_size = bytes.len() as u64; + for (starts, rows, valid) in [ + (&[0, 10, 20][..], 30, true), + (&[0], 1, true), + (&[], 0, true), + (&[], 30, false), + (&[0], 0, false), + (&[5, 10], 30, false), + (&[0, -1], 30, false), + (&[0, 20, 10], 30, false), + (&[0, 10, 10], 30, false), + (&[0, 30], 30, false), + (&[0, 40], 30, false), + ] { + let pages = starts + .iter() + .enumerate() + .map(|(index, &first_row_index)| PageLocation { + offset: column.data_page_offset() + index as i64, + compressed_page_size: 1, + first_row_index, + }) + .collect::>(); + assert_eq!( + super::page_locations_valid(&pages, rows, column, file_size), + valid, + "{starts:?}, rows={rows}" + ); + } } // ----------------------------------------------------------------------- diff --git a/crates/paimon/src/predicate_stats.rs b/crates/paimon/src/predicate_stats.rs index 06b53c3e1..9cadaa34c 100644 --- a/crates/paimon/src/predicate_stats.rs +++ b/crates/paimon/src/predicate_stats.rs @@ -41,6 +41,23 @@ pub(crate) fn predicates_may_match_with_schema( }) } +fn ordered_bounds( + stats: &T, + index: usize, + stats_data_type: &DataType, + predicate_data_type: &DataType, +) -> Option<(Datum, Datum)> { + let min = coerce_stats_datum_for_predicate( + stats.min_value(index, stats_data_type)?, + predicate_data_type, + )?; + let max = coerce_stats_datum_for_predicate( + stats.max_value(index, stats_data_type)?, + predicate_data_type, + )?; + (min <= max).then_some((min, max)) +} + pub(crate) fn data_leaf_may_match( index: usize, stats_data_type: &DataType, @@ -116,34 +133,16 @@ pub(crate) fn data_leaf_may_match( None => return true, }; - let min_value = match stats - .min_value(index, stats_data_type) - .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type)) - { - Some(value) => value, - None => return true, - }; - let max_value = match stats - .max_value(index, stats_data_type) - .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type)) - { - Some(value) => value, - None => return true, + let Some((min_value, max_value)) = + ordered_bounds(stats, index, stats_data_type, predicate_data_type) + else { + return true; }; - match op { - PredicateOperator::In => { - if !matches!( - min_value.partial_cmp(&max_value), - Some(Ordering::Less | Ordering::Equal) - ) { - return true; - } - literals.iter().any(|literal| { - !matches!(literal.partial_cmp(&min_value), Some(Ordering::Less)) - && !matches!(literal.partial_cmp(&max_value), Some(Ordering::Greater)) - }) - } + PredicateOperator::In => literals.iter().any(|literal| { + !matches!(literal.partial_cmp(&min_value), Some(Ordering::Less)) + && !matches!(literal.partial_cmp(&max_value), Some(Ordering::Greater)) + }), PredicateOperator::Eq => { !matches!(literal.partial_cmp(&min_value), Some(Ordering::Less)) && !matches!(literal.partial_cmp(&max_value), Some(Ordering::Greater)) @@ -284,48 +283,24 @@ pub(crate) fn data_leaf_must_match( match op { PredicateOperator::IsNull => return null_count == Some(row_count), PredicateOperator::IsNotNull => return null_count == Some(0), - _ => { - if null_count != Some(0) { - return false; - } - } + _ if null_count != Some(0) => return false, + _ => {} } - - let min_value = match stats - .min_value(index, stats_data_type) - .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type)) - { - Some(value) => value, - None => return false, - }; - let max_value = match stats - .max_value(index, stats_data_type) - .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type)) - { - Some(value) => value, - None => return false, - }; - if !matches!( - min_value.partial_cmp(&max_value), - Some(Ordering::Less | Ordering::Equal) - ) { + // Floating-point bounds can omit NaNs even when the null count is zero. + if matches!(stats_data_type, DataType::Float(_) | DataType::Double(_)) { return false; } + let Some((min_value, max_value)) = + ordered_bounds(stats, index, stats_data_type, predicate_data_type) + else { + return false; + }; + match op { - // `must_match` asserts *every* non-null row equals the literal, which - // `min == max == literal` only proves when the bounds cover every row. - // For FLOAT/DOUBLE a NaN row is left out of min/max without being counted - // null (see `equality_exclusion_is_sound`), so `[1.0, NaN]` reports - // `min == max == 1.0` yet not every row equals 1.0 -- claiming must_match - // there makes `NOT(= 1.0)` wrongly prune the file and drop the NaN row. - // The dual `may_match` NotEq/NotIn arms already carry this guard. - PredicateOperator::Eq => { - equality_exclusion_is_sound(&min_value) - && literals - .first() - .is_some_and(|literal| min_value == *literal && max_value == *literal) - } + PredicateOperator::Eq => literals + .first() + .is_some_and(|literal| min_value == *literal && max_value == *literal), PredicateOperator::NotEq => literals.first().is_some_and(|literal| { matches!(literal.partial_cmp(&min_value), Some(Ordering::Less)) || matches!(literal.partial_cmp(&max_value), Some(Ordering::Greater)) @@ -348,11 +323,7 @@ pub(crate) fn data_leaf_must_match( Some(Ordering::Greater | Ordering::Equal) ) }), - PredicateOperator::In => { - equality_exclusion_is_sound(&min_value) - && min_value == max_value - && literals.contains(&min_value) - } + PredicateOperator::In => min_value == max_value && literals.contains(&min_value), PredicateOperator::Between => { let (Some(low), Some(high)) = (literals.first(), literals.get(1)) else { return false; @@ -455,21 +426,11 @@ fn between_may_match( let (Some(low), Some(high)) = (literals.first(), literals.get(1)) else { return true; }; - let min_value = match stats - .min_value(index, stats_data_type) - .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type)) - { - Some(value) => value, - None => return true, - }; - let max_value = match stats - .max_value(index, stats_data_type) - .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type)) - { - Some(value) => value, - None => return true, + let Some((min_value, max_value)) = + ordered_bounds(stats, index, stats_data_type, predicate_data_type) + else { + return true; }; - let max_ge_low = !matches!(max_value.partial_cmp(low), Some(Ordering::Less)); let min_le_high = !matches!(min_value.partial_cmp(high), Some(Ordering::Greater)); let overlaps = max_ge_low && min_le_high; @@ -578,7 +539,7 @@ fn predicate_must_match_with_schema( } } -fn stats_evolution_is_safe(source: &DataType, target: &DataType) -> bool { +pub(crate) fn stats_evolution_is_safe(source: &DataType, target: &DataType) -> bool { // Java's CastExecutors.castLiteralsWithEvolution permits unchanged types // and integer widening. Narrowing can wrap, and timestamp precision changes // alter values even when their Arrow storage types are identical. Neither @@ -1019,13 +980,7 @@ mod tests { } #[test] - fn not_eq_and_not_in_on_float_keep_files_that_may_hold_nan() { - // A Rust-written FLOAT/DOUBLE column `[1.0, NaN]` reports `min == max == - // 1.0` with `null_count == 0` (the writer leaves NaN out of min/max - // without counting it null). `must_match(Eq 1.0)` must NOT claim every - // row equals 1.0 -- the NaN row does not -- so `NOT(v = 1.0)` and - // `NOT(v IN (1.0))` must KEEP the file (the NaN row satisfies `<> 1.0`). - // Mirrors the guard already on the `may_match` NotEq/NotIn arms. + fn negated_float_predicates_keep_files_that_may_contain_nan() { let dt = DataType::Double(DoubleType::new()); let fields = vec![DataField::new(0, "v".to_string(), dt.clone())]; let mapping = vec![Some(0)]; @@ -1036,17 +991,22 @@ mod tests { max: Some(Datum::Double(1.0)), }; - for op in [PredicateOperator::Eq, PredicateOperator::In] { + for (op, values) in [ + (PredicateOperator::Eq, vec![1.0]), + (PredicateOperator::In, vec![1.0]), + (PredicateOperator::Lt, vec![2.0]), + (PredicateOperator::Between, vec![0.0, 2.0]), + ] { let not_pred = Predicate::negate(Predicate::Leaf { column: "v".to_string(), index: 0, data_type: dt.clone(), op, - literals: vec![Datum::Double(1.0)], + literals: values.into_iter().map(Datum::Double).collect(), }); assert!( predicates_may_match_with_schema(&[not_pred], &stats, &mapping, &fields), - "NOT({op:?} 1.0) must keep a float file whose min==max==1.0 (may hold NaN)" + "NOT({op:?}) must retain the possible NaN row" ); } } diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index f59c1d7fa..c83da97b3 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -876,16 +876,23 @@ impl<'a> CoreOptions<'a> { .unwrap_or(false) } + /// Use the winning column providers' Parquet statistics to restrict merged reads. + pub fn data_evolution_merged_read_stats_pushdown_enabled(&self) -> crate::Result { + self.boolean_option("data-evolution.merged-read.stats-pushdown.enabled", true) + } + /// Whether normal Data Evolution files also write an aligned ROW sidecar. pub fn data_evolution_row_sidecar_enabled(&self) -> crate::Result { - match self.options.get("data-evolution.row-sidecar.enabled") { - None => Ok(false), + self.boolean_option("data-evolution.row-sidecar.enabled", false) + } + + fn boolean_option(&self, key: &str, default: bool) -> crate::Result { + match self.options.get(key) { + None => Ok(default), Some(value) if value.eq_ignore_ascii_case("true") => Ok(true), Some(value) if value.eq_ignore_ascii_case("false") => Ok(false), Some(value) => Err(crate::Error::ConfigInvalid { - message: format!( - "data-evolution.row-sidecar.enabled must be true or false, got {value}" - ), + message: format!("{key} must be true or false, got {value}"), }), } } diff --git a/crates/paimon/src/table/compact/data_evolution/vector.rs b/crates/paimon/src/table/compact/data_evolution/vector.rs index 4ef7eeb22..b75e5cf6f 100644 --- a/crates/paimon/src/table/compact/data_evolution/vector.rs +++ b/crates/paimon/src/table/compact/data_evolution/vector.rs @@ -358,7 +358,8 @@ pub(super) async fn execute( first, last, resources.clone(), - )?; + ) + .await?; let mut writer = DataFileWriter::new( table.file_io().clone(), table.location().into(), diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index 39dc38fe0..88342c8d6 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -16,13 +16,14 @@ // under the License. mod blob_fallback; +mod candidate_ranges; mod vector_versions; use super::blob_resolver::BlobReadLimiter; use super::data_file_reader::{ - append_null_row_id_column, attach_row_id, expand_selected_row_ids, - file_index_selection_to_local_ranges, insert_column_at, project_residual_output, - DataFileReadTiming, DataFileReader, + dv_to_non_deleted_ranges, file_index_selection_to_local_ranges, insert_column_at, + project_residual_output, row_id_column_for_batch, row_ids_from_ranges, to_local_row_ranges, + DataFileReadTiming, DataFileReader, FileReadSelection, }; use super::read_limit::take_limited_batch; use crate::arrow::format::blob::DEFAULT_BLOB_READ_PARALLELISM; @@ -38,6 +39,7 @@ use crate::spec::{ ROW_ID_FIELD_NAME, }; use crate::table::dedicated_format_file_writer::is_blob_or_video_file_name; +use crate::table::row_id_predicate::intersect_sorted_ranges; use crate::table::schema_manager::SchemaManager; use crate::table::source::any_range_overlaps_file; use crate::table::stats_filter::group_by_overlapping_row_id; @@ -46,6 +48,7 @@ use crate::{DataSplit, Error}; use arrow_array::{Array, Int64Array, LargeBinaryArray, RecordBatch}; use async_stream::try_stream; use futures::{StreamExt, TryStreamExt}; +use parquet::arrow::arrow_reader::ArrowReaderMetadata; use roaring::RoaringBitmap; use std::collections::{HashMap, HashSet}; use std::future::Future; @@ -72,24 +75,15 @@ fn is_raw_convertible(files: &[DataFileMeta]) -> bool { if files.len() <= 1 { return true; } - // If all files have first_row_id and their row_id ranges don't overlap, they're independent. - if files.iter().any(|f| f.first_row_id.is_none()) { - return false; - } - let mut ranges: Vec<(i64, i64)> = files + let Some(mut ranges) = files .iter() - .map(|f| { - let start = f.first_row_id.unwrap(); - (start, start + f.row_count) - }) - .collect(); - ranges.sort_by_key(|r| r.0); - for w in ranges.windows(2) { - if w[0].1 > w[1].0 { - return false; - } - } - true + .map(DataFileMeta::row_id_range) + .collect::>>() + else { + return false; + }; + ranges.sort_unstable(); + ranges.windows(2).all(|pair| pair[0].1 < pair[1].0) } /// Split a reader input into independently readable row-id groups. @@ -108,7 +102,7 @@ fn reader_file_groups(files: &[DataFileMeta]) -> Vec> { /// Reuse the field merger without an anchor or deletion mask. The compactor /// has already checked contiguous physical coverage for every written field. -pub(super) fn read_vector_compaction( +pub(super) async fn read_vector_compaction( table: &crate::table::Table, split: &DataSplit, fields: Vec, @@ -138,13 +132,14 @@ pub(super) fn read_vector_compaction( .with_table_options(table.schema().options().clone()) .with_mosaic_prefetch(super::table_read::configured_mosaic_prefetch(table)?) .with_parquet_read_budget(Some(budget)); + let group = PreparedMergeGroup { + files: normalize_merge_group(split.data_files().to_vec())?, + logical_row_count: last - first + 1, + first_row_id: first, + }; reader.merge_files_by_columns( split, - &PreparedMergeGroup { - files: normalize_merge_group(split.data_files().to_vec())?, - logical_row_count: last - first + 1, - first_row_id: first, - }, + reader.plan_sources(&group, false).await?, None, None, usize::try_from(last - first + 1).map_err(|_| Error::DataInvalid { @@ -293,7 +288,10 @@ impl DataEvolutionReader { output_schema, output_read_type: read_type, wide_output_schema, - predicates, + predicates: predicates + .into_iter() + .flat_map(Predicate::split_and) + .collect(), blob_as_descriptor, blob_descriptor_fields, blob_view_fields, @@ -542,23 +540,15 @@ impl DataEvolutionReader { let filter_before_blob_resolution = self.can_filter_before_blob_resolution(blob_view_lookup.is_some(), &descriptor_fields); - // A raw-convertible split consists only of independent files: every - // row-id segment has a single column provider, so predicates can be - // pushed into its file reader. Keep the exact residual below as a - // format-independent backstop. - // - // Positional `_ROW_ID` attachment needs the unfiltered physical row - // stream, and predicates on resolved BLOB/BLOB-view values must run - // after that transformation. Both cases deliberately stay on the - // residual-only path. - let push_down_raw_predicates = !self.predicates.is_empty() - && self.row_id_index.is_none() - && filter_before_blob_resolution; - // A bitmap selection preserves physical positions, including when - // `_ROW_ID` needs attaching before the residual predicate runs. - let read_file_index = !self.predicates.is_empty() - && filter_before_blob_resolution - && CoreOptions::new(&self.table_options).file_index_read_enabled(); + let has_data_filter = !self.predicates.is_empty() && filter_before_blob_resolution; + // Decoder filters discard positions needed to attach _ROW_ID. + let push_down_raw_predicates = has_data_filter && self.row_id_index.is_none(); + let options = CoreOptions::new(&self.table_options); + let read_file_index = has_data_filter && options.file_index_read_enabled(); + let read_merged_stats = options.data_evolution_merged_read_stats_pushdown_enabled()? + && has_data_filter + && options.row_tracking_enabled() + && options.bucket() == -1; // A managed BLOB file fetches payloads as its batch is decoded. // Keep batches small and restrict predicate-free file selections // to the remaining output quota below. @@ -651,7 +641,7 @@ impl DataEvolutionReader { .collect::>(); effective_row_ranges = Some(match effective_row_ranges { Some(ref ranges) => - intersect_local_ranges(ranges, &index_ranges), + intersect_sorted_ranges(ranges, &index_ranges), None => index_ranges, }); } @@ -688,26 +678,18 @@ impl DataEvolutionReader { } } - let selected_row_ids = if self.row_id_index.is_some() && has_row_id { - selected_absolute_row_ranges_for_file( - file_meta.first_row_id.unwrap(), - file_meta.row_count, - effective_row_ranges.as_deref(), - deletion_vector.as_deref(), - )? - .map(|ranges| { - expand_selected_row_ids( - file_meta.first_row_id.unwrap(), + let mut row_ids = match (self.row_id_index, file_meta.first_row_id) { + (Some(_), Some(first)) => { + let ranges = selected_absolute_row_ranges_for_file( + first, file_meta.row_count, - &ranges, - ) - }) - } else { - None + effective_row_ranges.as_deref(), + deletion_vector.as_deref(), + )?; + Some(row_ids_from_ranges(first, ranges)) + } + _ => None, }; - let file_base_row_id = file_meta.first_row_id.unwrap_or(0); - let mut row_id_cursor = file_base_row_id; - let mut row_id_offset: usize = 0; let mut stream = raw_file_reader.clone() .with_nested_field_enabled( @@ -715,29 +697,19 @@ impl DataEvolutionReader { .data_evolution_nested_field_enabled() || file_schema_nested_enabled, ) - .read_single_file_stream_with_requested_ranges( + .read_single_file_stream_with_row_id_ranges( &split, file_meta, data_fields, deletion_vector, - effective_row_ranges, - sidecar_row_ranges, + FileReadSelection::new(effective_row_ranges, sidecar_row_ranges), )?; while remaining != Some(0) { let Some(batch) = stream.next().await else { break }; let batch = batch?; - let num_rows = batch.num_rows(); let batch = if let Some(idx) = self.row_id_index { - if !has_row_id { - append_null_row_id_column(batch, idx, &self.wide_output_schema)? - } else if let Some(ref ids) = selected_row_ids { - attach_row_id(batch, idx, ids, &mut row_id_offset, &self.wide_output_schema)? - } else { - let row_ids: Vec = (row_id_cursor..row_id_cursor + num_rows as i64).collect(); - row_id_cursor += num_rows as i64; - let array: Arc = Arc::new(Int64Array::from(row_ids)); - insert_column_at(batch, array, idx, &self.wide_output_schema)? - } + let ids = row_id_column_for_batch(row_ids.as_mut(), batch.num_rows())?; + insert_column_at(batch, ids, idx, &self.wide_output_schema)? } else { batch }; @@ -770,7 +742,7 @@ impl DataEvolutionReader { ) }).collect::>(); indexed_row_ranges = Some(match indexed_row_ranges { - Some(ref ranges) => intersect_local_ranges(ranges, &index_ranges), + Some(ref ranges) => intersect_sorted_ranges(ranges, &index_ranges), None => index_ranges, }); } @@ -781,7 +753,7 @@ impl DataEvolutionReader { if indexed_row_ranges.as_ref().is_some_and(Vec::is_empty) { continue; } - let anchor_deletion_vector = read_anchor_deletion_vector( + let mut anchor_deletion_vector = read_anchor_deletion_vector( &self.file_io, &split, &prepared_group.files, @@ -795,16 +767,16 @@ impl DataEvolutionReader { .as_ref() .map(|ctx| ctx.deletion_vector.as_ref()), )?; - // Every provider shares the DV-aware bitmap. Without - // a selective bitmap, only caller ranges choose ROW. - // LIMIT never changes sidecar eligibility. + if selected_ranges.as_ref().is_some_and(Vec::is_empty) { + continue; + } let sidecar_row_ranges = if has_bitmap_selection { selected_ranges.clone() } else { indexed_row_ranges.clone() }; - let effective_row_ranges = if self.predicates.is_empty() { - if let Some(left) = remaining { + let mut effective_row_ranges = match remaining { + Some(left) if self.predicates.is_empty() => { selected_ranges = Some(prefix_row_ranges( selected_ranges, prepared_group.first_row_id, @@ -812,34 +784,41 @@ impl DataEvolutionReader { left, )); selected_ranges.clone() - } else { - indexed_row_ranges.clone() } - } else { - indexed_row_ranges + _ => indexed_row_ranges, }; + let mut source_plan = self + .plan_sources(&prepared_group, effective_row_ranges.is_some()) + .await?; + if read_merged_stats && !source_plan.nested_enabled { + if let Some(live) = self + .candidate_ranges( + &split, + &mut source_plan, + selected_ranges.as_deref(), + sidecar_row_ranges.as_deref(), + ) + .await + { + if live.is_empty() { + continue; + } + effective_row_ranges = Some(live.clone()); + selected_ranges = Some(live); + // The shared selection already excludes deleted rows from every source. + anchor_deletion_vector = None; + } + } let expected_output_rows = match selected_ranges.as_ref() { Some(ranges) => ranges.iter().map(|r| r.count() as usize).sum(), None => prepared_group.logical_row_count as usize, }; - let selected_row_ids = if self.row_id_index.is_some() { - selected_ranges.as_ref().map(|ranges| { - expand_selected_row_ids( - prepared_group.first_row_id, - prepared_group.logical_row_count, - ranges, - ) - }) - } else { - None - }; - let mut row_id_cursor = prepared_group.first_row_id; - let mut row_id_offset: usize = 0; + let mut row_ids = row_ids_from_ranges(prepared_group.first_row_id, selected_ranges); let mut merge_stream = self.merge_files_by_columns( &split, - &prepared_group, + source_plan, effective_row_ranges, sidecar_row_ranges, expected_output_rows, @@ -848,16 +827,9 @@ impl DataEvolutionReader { while remaining != Some(0) { let Some(batch) = merge_stream.next().await else { break }; let batch = batch?; - let num_rows = batch.num_rows(); let batch = if let Some(idx) = self.row_id_index { - if let Some(ref ids) = selected_row_ids { - attach_row_id(batch, idx, ids, &mut row_id_offset, &self.wide_output_schema)? - } else { - let row_ids: Vec = (row_id_cursor..row_id_cursor + num_rows as i64).collect(); - row_id_cursor += num_rows as i64; - let array: Arc = Arc::new(Int64Array::from(row_ids)); - insert_column_at(batch, array, idx, &self.wide_output_schema)? - } + let ids = row_id_column_for_batch(Some(&mut row_ids), batch.num_rows())?; + insert_column_at(batch, ids, idx, &self.wide_output_schema)? } else { batch }; @@ -1151,21 +1123,48 @@ impl DataEvolutionReader { Ok(result) } - /// Merge multiple logical sources column-wise for data evolution. - /// - /// Normal partial-column files remain one source per file. Rolling `.blob` - /// files are first grouped into a logical BlobBunch source per field, then - /// source streams are merged by projected field position. + async fn plan_sources( + &self, + group: &PreparedMergeGroup, + row_id_pushdown: bool, + ) -> crate::Result { + let (infos, historical_nested) = load_file_infos( + &self.schema_manager, + self.table_schema_id, + &self.table_fields, + &group.files, + ) + .await?; + let nested_enabled = CoreOptions::new(&self.table_options) + .data_evolution_nested_field_enabled() + || historical_nested; + let mut blob_inline_fields = self.blob_descriptor_fields.clone(); + blob_inline_fields.extend(self.blob_view_fields.iter().cloned()); + build_source_plan_with_row_id_pushdown( + group, + &infos, + &self.wide_file_read_type, + &blob_inline_fields, + SourcePlanReadContext { + table_fields: &self.table_fields, + partition_fields: &self.partition_fields, + row_id_pushdown, + nested_enabled, + }, + ) + } + + /// Advance all column sources over the same row positions even when their batch boundaries differ. fn merge_files_by_columns( &self, split: &DataSplit, - prepared_group: &PreparedMergeGroup, + source_plan: SourcePlan, row_ranges: Option>, requested_row_ranges: Option>, expected_output_rows: usize, anchor_deletion_vector: Option, ) -> crate::Result { - if prepared_group.files.is_empty() { + if source_plan.sources.is_empty() { return Ok(futures::stream::empty().boxed()); } @@ -1173,7 +1172,6 @@ impl DataEvolutionReader { let schema_manager = self.schema_manager.clone(); let table_schema_id = self.table_schema_id; let split = split.clone(); - let prepared_group = prepared_group.clone(); // The merge plan reads the WIDE field set so widened predicate columns // are materialized for the residual filter. Blob descriptor/view // resolution is intentionally deferred to `finish_wide_batch`, where @@ -1182,8 +1180,6 @@ impl DataEvolutionReader { let read_type = self.wide_file_read_type.clone(); let table_fields = self.table_fields.clone(); let partition_fields = self.partition_fields.clone(); - let mut blob_inline_fields = self.blob_descriptor_fields.clone(); - blob_inline_fields.extend(self.blob_view_fields.iter().cloned()); let blob_as_descriptor = self.blob_as_descriptor; let blob_parallelism = self.blob_parallelism; let batch_size = self.effective_batch_size(); @@ -1191,36 +1187,12 @@ impl DataEvolutionReader { let table_options = Arc::clone(&self.table_options); let mosaic_prefetch = self.mosaic_prefetch; let read_timing = self.read_timing.clone(); - let anchor_deletion_vector = anchor_deletion_vector.clone(); // Match the input cap so a LIMIT cannot resolve payloads for rows that // are only going to be discarded from the merge output. let merge_batch_size = batch_size.unwrap_or(1024).clamp(1, 1024); let target_schema = crate::arrow::build_read_arrow_schema(&read_type)?; Ok(try_stream! { - let (file_infos, file_schema_nested_enabled) = load_file_infos( - &schema_manager, - table_schema_id, - &table_fields, - &prepared_group.files, - ) - .await?; - let nested_enabled = CoreOptions::new(&table_options) - .data_evolution_nested_field_enabled() - || file_schema_nested_enabled; - let source_plan = build_source_plan_with_row_id_pushdown( - &prepared_group, - &file_infos, - &read_type, - &blob_inline_fields, - SourcePlanReadContext { - table_fields: &table_fields, - partition_fields: &partition_fields, - row_id_pushdown: row_ranges.is_some(), - nested_enabled, - }, - )?; - let active_source_indices: Vec = source_plan .sources .iter() @@ -1295,7 +1267,7 @@ impl DataEvolutionReader { blob_parallelism, source_parquet_read_budget.clone(), Arc::clone(&table_options), - nested_enabled, + source_plan.nested_enabled, mosaic_prefetch, read_timing.clone(), anchor_deletion_vector.as_ref(), @@ -2036,16 +2008,22 @@ fn open_source_stream( ) } FieldSource::DataFile { - file, data_fields, .. + file, + data_fields, + parquet_metadata, + .. } => { let deletion_vector = shifted_deletion_vector_for_file(file, anchor_deletion_vector)?; - file_reader.read_single_file_stream_with_requested_ranges( + file_reader.read_single_file_stream_with_row_id_ranges( split, file.as_ref().clone(), data_fields.clone(), deletion_vector, - row_ranges, - requested_row_ranges, + FileReadSelection { + rows: row_ranges, + sidecar_rows: requested_row_ranges, + parquet_metadata: parquet_metadata.clone(), + }, ) } FieldSource::BlobBunch { bunch, .. } => { @@ -2341,25 +2319,14 @@ fn selected_absolute_row_ranges_for_file( } let mut local_ranges = if let Some(dv) = deletion_vector { - non_deleted_local_ranges(row_count, dv) + dv_to_non_deleted_ranges(dv, row_count) } else { vec![RowRange::new(0, row_count - 1)] }; if let Some(ranges) = row_ranges { - let selected = crate::table::merge_row_ranges( - ranges - .iter() - .filter_map(|range| { - range - .intersect_inclusive(first_row_id, first_row_id + row_count - 1) - .map(|range| { - RowRange::new(range.from() - first_row_id, range.to() - first_row_id) - }) - }) - .collect(), - ); - local_ranges = intersect_local_ranges(&local_ranges, &selected); + let selected = to_local_row_ranges(ranges, first_row_id, row_count); + local_ranges = intersect_sorted_ranges(&local_ranges, &selected); } let absolute = local_ranges @@ -2381,7 +2348,7 @@ fn prefix_row_ranges( if row_count == 0 { Vec::new() } else { - vec![RowRange::new(first_row_id, first_row_id + row_count - 1)] + vec![RowRange::new(first_row_id, first_row_id + (row_count - 1))] } }); let mut left = limit; @@ -2395,7 +2362,7 @@ fn prefix_row_ranges( } prefix.push(RowRange::new( range.from(), - range.from() + i64::try_from(take).unwrap_or(i64::MAX) - 1, + range.from() + (i64::try_from(take).unwrap_or(i64::MAX) - 1), )); left -= take; if left == 0 { @@ -2405,43 +2372,6 @@ fn prefix_row_ranges( prefix } -fn non_deleted_local_ranges(row_count: i64, deletion_vector: &DeletionVector) -> Vec { - let mut ranges = Vec::new(); - let mut cursor = 0i64; - for (start, end) in deletion_vector.deleted_ranges() { - let (start, end) = (i64::from(start), i64::from(end)); - if start >= row_count { - break; - } - if start > cursor { - ranges.push(RowRange::new(cursor, start - 1)); - } - cursor = end + 1; - } - if cursor < row_count { - ranges.push(RowRange::new(cursor, row_count - 1)); - } - ranges -} - -fn intersect_local_ranges(left: &[RowRange], right: &[RowRange]) -> Vec { - let mut result = Vec::new(); - let (mut i, mut j) = (0usize, 0usize); - while i < left.len() && j < right.len() { - let from = left[i].from().max(right[j].from()); - let to = left[i].to().min(right[j].to()); - if from <= to { - result.push(RowRange::new(from, to)); - } - if left[i].to() < right[j].to() { - i += 1; - } else { - j += 1; - } - } - result -} - #[derive(Debug, Clone)] struct PreparedMergeGroup { files: Vec, @@ -2613,6 +2543,7 @@ struct SourcePlan { sources: Vec, column_plan: Vec>, nested_plan: Vec>, + nested_enabled: bool, } /// Logical inputs needed while selecting physical column providers. @@ -2793,6 +2724,7 @@ fn build_source_plan_with_row_id_pushdown( file: Box::new(file.clone()), data_fields: info.data_fields.clone(), read_fields: Vec::new(), + parquet_metadata: None, }); // Java rowTypeWithRowTracking adds this metadata provider even // when write_cols omits the physical sequence column. The newest @@ -2949,6 +2881,7 @@ fn build_source_plan_with_row_id_pushdown( sources, column_plan, nested_plan, + nested_enabled, }) } @@ -2979,6 +2912,7 @@ enum FieldSource { file: Box, data_fields: Option>, read_fields: Vec, + parquet_metadata: Option, }, VectorBunch { bunch: VectorBunch, @@ -3865,6 +3799,10 @@ mod tests { vec![RowRange::new(10, 12)] ); assert!(prefix_row_ranges(None, 10, 4, 0).is_empty()); + assert_eq!( + prefix_row_ranges(None, i64::MAX - 3, 4, 4), + [RowRange::new(i64::MAX - 3, i64::MAX)] + ); } #[test] @@ -8433,7 +8371,7 @@ mod tests { file } - async fn write_schema_file(table: &Table, schema: &TableSchema) { + pub(super) async fn write_schema_file(table: &Table, schema: &TableSchema) { let path = table.schema_manager().schema_path(schema.id()); let dir = path.rsplit_once('/').map(|(dir, _)| dir).unwrap(); table.file_io().mkdirs(dir).await.unwrap(); @@ -8447,7 +8385,7 @@ mod tests { .unwrap(); } - async fn write_test_deletion_file( + pub(super) async fn write_test_deletion_file( file_io: &crate::io::FileIO, path: &str, deleted_rows: &[u32], @@ -9148,60 +9086,6 @@ mod tests { } } - /// _ROW_ID + predicate, raw branch: surviving rows keep their ORIGINAL row - /// ids (ids are attached before the residual filter), not renumbered ones. - #[tokio::test] - async fn test_evolution_read_row_id_with_predicate_raw_branch() { - let tempdir = tempdir().unwrap(); - let table_path = local_file_path(tempdir.path()); - let bucket_dir = tempdir.path().join("bucket-0"); - fs::create_dir_all(&bucket_dir).unwrap(); - - let parquet_path = bucket_dir.join("data.parquet"); - write_int_parquet_file(&parquet_path, vec![("id", vec![1, 2, 3, 4])], None); - - let table = two_col_evolution_table(table_path); - let split = DataSplitBuilder::new() - .with_snapshot(1) - .with_partition(BinaryRow::new(0)) - .with_bucket(0) - .with_bucket_path(local_file_path(&bucket_dir)) - .with_total_buckets(1) - .with_data_files(vec![data_file_meta_with_path( - "data.parquet", - 100, - 4, - 1, - parquet_path.metadata().unwrap().len() as i64, - Some(vec!["id"]), - )]) - .build() - .unwrap(); - - let pb = PredicateBuilder::new(table.schema().fields()); - let predicate = pb.greater_or_equal("id", Datum::Int(3)).unwrap(); - - let mut builder = table.new_read_builder(); - builder - .with_projection(&["id", crate::spec::ROW_ID_FIELD_NAME]) - .unwrap() - .with_filter(predicate); - let read = builder.new_read().unwrap(); - let batches = read - .to_arrow(&[split]) - .unwrap() - .try_collect::>() - .await - .unwrap(); - - assert_eq!(collect_int_values(&batches, "id"), vec![3, 4]); - // Original ids 102, 103 — NOT renumbered to 100, 101. - assert_eq!( - collect_long_values(&batches, crate::spec::ROW_ID_FIELD_NAME), - vec![102, 103] - ); - } - #[tokio::test] async fn test_evolution_read_applies_row_id_residual_to_the_row_ids() { let tempdir = tempdir().unwrap(); @@ -9394,68 +9278,94 @@ mod tests { ); } - /// _ROW_ID + predicate, merge branch: same guarantee across a column merge. #[tokio::test] - async fn test_evolution_read_row_id_with_predicate_merge_branch() { + async fn filtered_raw_and_merged_groups_keep_their_row_ids() { let tempdir = tempdir().unwrap(); - let table_path = local_file_path(tempdir.path()); let bucket_dir = tempdir.path().join("bucket-0"); fs::create_dir_all(&bucket_dir).unwrap(); - - let id_path = bucket_dir.join("id.parquet"); - write_int_parquet_file(&id_path, vec![("id", vec![1, 2, 3, 4])], None); - let value_path = bucket_dir.join("value.parquet"); - write_int_parquet_file(&value_path, vec![("value", vec![5, 20, 30, 40])], None); - - let table = two_col_evolution_table(table_path); + let table = two_col_evolution_table(local_file_path(tempdir.path())); + let mut files = Vec::new(); + for (name, first, version, columns) in [ + ("id.parquet", 100, 1, vec![("id", vec![1, 2, 3, 4])]), + ( + "value.parquet", + 100, + 2, + vec![("value", vec![5, 20, 30, 40])], + ), + ( + "raw.parquet", + i64::MAX - 3, + 1, + vec![("id", vec![5, 6, 7, 8]), ("value", vec![0, 40, 0, 40])], + ), + ] { + let path = bucket_dir.join(name); + let names = columns.iter().map(|(name, _)| *name).collect(); + write_int_parquet_file(&path, columns, Some(2)); + files.push(data_file_meta_with_path( + name, + first, + 4, + version, + path.metadata().unwrap().len() as i64, + Some(names), + )); + } let split = DataSplitBuilder::new() .with_snapshot(1) .with_partition(BinaryRow::new(0)) .with_bucket(0) .with_bucket_path(local_file_path(&bucket_dir)) .with_total_buckets(1) - .with_data_files(vec![ - data_file_meta_with_path( - "id.parquet", - 100, - 4, - 1, - id_path.metadata().unwrap().len() as i64, - Some(vec!["id"]), - ), - data_file_meta_with_path( - "value.parquet", - 100, - 4, - 2, - value_path.metadata().unwrap().len() as i64, - Some(vec!["value"]), - ), - ]) + .with_data_files(files) .build() .unwrap(); - let pb = PredicateBuilder::new(table.schema().fields()); - let predicate = pb.equal("value", Datum::Int(40)).unwrap(); - - let mut builder = table.new_read_builder(); - builder - .with_projection(&["id", crate::spec::ROW_ID_FIELD_NAME]) - .unwrap() - .with_filter(predicate); - let read = builder.new_read().unwrap(); - let batches = read - .to_arrow(&[split]) - .unwrap() - .try_collect::>() - .await - .unwrap(); - - assert_eq!(collect_int_values(&batches, "id"), vec![4]); - assert_eq!( - collect_long_values(&batches, crate::spec::ROW_ID_FIELD_NAME), - vec![103] - ); + let sparse = split.clone().with_selected_row_ranges(vec![ + RowRange::new(102, 103), + RowRange::new(i64::MAX - 2, i64::MAX - 2), + RowRange::new(i64::MAX, i64::MAX), + ]); + for enabled in [true, false] { + let table = table.copy_with_options(HashMap::from([ + ( + "data-evolution.merged-read.stats-pushdown.enabled".into(), + enabled.to_string(), + ), + ("read.batch-size".into(), "2".into()), + ])); + for (split, predicate, ids, row_ids) in [ + ( + &split, + pb.equal("value", Datum::Int(40)).unwrap(), + vec![4, 6, 8], + vec![103, i64::MAX - 2, i64::MAX], + ), + ( + &sparse, + pb.greater_or_equal("id", Datum::Int(3)).unwrap(), + vec![3, 4, 6, 8], + vec![102, 103, i64::MAX - 2, i64::MAX], + ), + ] { + let mut builder = table.new_read_builder(); + builder + .with_projection(&["id", ROW_ID_FIELD_NAME]) + .unwrap() + .with_filter(predicate); + let batches = builder + .new_read() + .unwrap() + .to_arrow(std::slice::from_ref(split)) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(collect_int_values(&batches, "id"), ids); + assert_eq!(collect_long_values(&batches, ROW_ID_FIELD_NAME), row_ids); + } + } } /// A predicate on a column that one file group lacks (null-filled): diff --git a/crates/paimon/src/table/data_evolution_reader/candidate_ranges.rs b/crates/paimon/src/table/data_evolution_reader/candidate_ranges.rs new file mode 100644 index 000000000..3a7411df2 --- /dev/null +++ b/crates/paimon/src/table/data_evolution_reader/candidate_ranges.rs @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{DataEvolutionReader, FieldSource, SourcePlan}; +use crate::arrow::filtering::build_field_mapping; +use crate::arrow::format::parquet::ParquetFormatReader; +use crate::predicate_stats::stats_evolution_is_safe; +use crate::spec::{CoreOptions, DataFileMeta, DataType, Predicate}; +use crate::table::data_file_reader::{open_file_reader, to_local_row_ranges}; +use crate::table::row_id_predicate::intersect_sorted_ranges; +use crate::table::row_sidecar; +use crate::table::RowRange; +use crate::{DataSplit, Error}; +use parquet::arrow::arrow_reader::{ArrowReaderMetadata, RowSelection}; +use std::collections::HashSet; + +#[cfg(test)] +#[path = "candidate_ranges_tests.rs"] +mod tests; + +impl DataEvolutionReader { + /// Logical row IDs that may match in the winning providers. An empty list + /// skips the group and `None` leaves its selection unchanged. + pub(super) async fn candidate_ranges( + &self, + split: &DataSplit, + plan: &mut SourcePlan, + selected_ranges: Option<&[RowRange]>, + sidecar_ranges: Option<&[RowRange]>, + ) -> Option> { + let FieldSource::DataFile { file, .. } = plan.sources.first()? else { + return None; + }; + let (first, last) = file.row_id_range()?; + self.compute_candidate_ranges(split, plan, first, selected_ranges, sidecar_ranges) + .await + .unwrap_or_else(|error| { + log::warn!( + "Parquet statistics pruning failed for row IDs {first}..={last} in {}. Using the original row selection ({error})", + split.bucket_path() + ); + None + }) + } + + async fn compute_candidate_ranges( + &self, + split: &DataSplit, + plan: &mut SourcePlan, + first_row_id: i64, + selected_ranges: Option<&[RowRange]>, + sidecar_ranges: Option<&[RowRange]>, + ) -> crate::Result>> { + for source in &plan.sources { + if !self.can_use_source_stats(source, sidecar_ranges)? { + return Ok(None); + } + } + let reader = ParquetFormatReader::with_options(&self.table_options, None)?; + let mut candidates: Option> = None; + let mut sources_without_filters = Vec::new(); + for source in &mut plan.sources { + let FieldSource::DataFile { + file, + data_fields, + read_fields, + parquet_metadata, + } = source + else { + return Ok(None); + }; + if read_fields.is_empty() { + continue; + } + let file_fields = data_fields.as_deref().unwrap_or(&self.table_fields); + let winning_ids: HashSet<_> = read_fields.iter().map(|field| field.id()).collect(); + let mut mapping = build_field_mapping(&self.table_fields, file_fields); + for (index, field) in mapping.iter_mut().zip(&self.table_fields) { + *index = index.filter(|&i| { + winning_ids.contains(&field.id()) + && supports_stats(field.data_type()) + && !self.partition_fields.iter().any(|p| p.id() == field.id()) + && stats_evolution_is_safe(file_fields[i].data_type(), field.data_type()) + }); + } + // Remap a whole conjunct so an OR or NOT spanning providers stays residual. + let predicates: Vec<_> = self + .predicates + .iter() + .filter_map(|p| p.remap_field_index(&mapping)) + .filter(|p| !matches!(p, Predicate::AlwaysTrue)) + .collect(); + if predicates.is_empty() { + sources_without_filters.push((file, parquet_metadata)); + continue; + } + let metadata = self + .load_source_metadata(split, file, &reader, true) + .await?; + let selection = reader.candidate_row_selection(&metadata, predicates, file_fields); + *parquet_metadata = Some(metadata); + if let Some(selection) = selection { + let ranges = absolute_row_ranges(first_row_id, &selection); + let live = match candidates.as_deref().or(selected_ranges) { + Some(previous) => intersect_sorted_ranges(previous, &ranges), + None => ranges, + }; + if live.is_empty() { + return Ok(Some(live)); + } + candidates = Some(live); + } + } + let Some(candidates) = candidates else { + return Ok(None); + }; + if selected_ranges == Some(candidates.as_slice()) { + return Ok(None); + } + // A newly narrowed selection could hide a short file from an unfiltered source. + for (file, metadata) in sources_without_filters { + *metadata = Some( + self.load_source_metadata(split, file, &reader, false) + .await?, + ); + } + Ok(Some(candidates)) + } + + async fn load_source_metadata( + &self, + split: &DataSplit, + file: &DataFileMeta, + reader: &ParquetFormatReader, + has_predicates: bool, + ) -> crate::Result { + let path = split.data_file_path(file); + let size = u64::try_from(file.file_size).map_err(|_| Error::DataInvalid { + message: format!("Negative file size {} for {path}", file.file_size), + source: None, + })?; + let input = open_file_reader(&self.file_io, &path, size, self.read_timing.as_ref()).await?; + let metadata = reader + .load_selection_metadata(input, size, has_predicates) + .await + .map_err(|error| { + Error::from_parquet_with_context( + error, + &format!("Failed to read pruning metadata for {path}"), + ) + })?; + let parquet = metadata.metadata(); + let footer_rows = parquet.file_metadata().num_rows(); + let group_rows = parquet.row_groups().iter().try_fold(0usize, |sum, group| { + sum.checked_add(usize::try_from(group.num_rows()).ok()?) + }); + if footer_rows != file.row_count + || group_rows.is_none() + || group_rows != usize::try_from(file.row_count).ok() + { + let group_rows = group_rows.map_or_else(|| "invalid".into(), |rows| rows.to_string()); + return Err(Error::DataInvalid { + message: format!( + "Inconsistent row counts for {path} (manifest {}, footer {footer_rows}, row groups {group_rows})", + file.row_count + ), + source: None, + }); + } + Ok(metadata) + } + + fn can_use_source_stats( + &self, + source: &FieldSource, + sidecar_ranges: Option<&[RowRange]>, + ) -> crate::Result { + let FieldSource::DataFile { + file, + data_fields, + read_fields, + .. + } = source + else { + return Ok(false); + }; + let path = file.external_path.as_deref().unwrap_or(&file.file_name); + let fields = data_fields.as_deref().unwrap_or(&self.table_fields); + let nested_write = file + .write_cols + .iter() + .flatten() + .any(|col| col.contains('.') && !fields.iter().any(|f| f.name() == col)); + if nested_write + || !path + .rsplit_once('.') + .is_some_and(|(_, ext)| ext.eq_ignore_ascii_case("parquet")) + { + return Ok(false); + } + + // A sparse read can use a ROW sidecar even when the primary is missing. + let Some(ranges) = sidecar_ranges else { + return Ok(true); + }; + let options = CoreOptions::new(&self.table_options); + if read_fields.is_empty() + || !file.extra_files.iter().any(|name| name.ends_with(".row")) + || !row_sidecar::supports_read_type( + read_fields, + self.blob_as_descriptor || options.blob_as_descriptor(), + ) + { + return Ok(true); + } + let Some((first, _)) = file.row_id_range() else { + return Ok(false); + }; + let local = to_local_row_ranges(ranges, first, file.row_count); + Ok(row_sidecar::selected_sidecar(file, Some(&local), &options)?.is_none()) + } +} + +fn absolute_row_ranges(first: i64, selection: &RowSelection) -> Vec { + let mut ranges = Vec::new(); + let mut offset = 0; + for selector in selection.iter() { + let end = offset + selector.row_count as i64; + if !selector.skip { + ranges.push(RowRange::new(first + offset, first + (end - 1))); + } + offset = end; + } + ranges +} + +fn supports_stats(ty: &DataType) -> bool { + match ty { + // Parquet has no seconds timestamp annotation. An Arrow TIMESTAMP(0) + // may store seconds as INT64, while short timestamp bounds use millis. + DataType::Timestamp(ts) => ts.precision() > 0, + DataType::LocalZonedTimestamp(ts) => ts.precision() > 0, + DataType::Boolean(_) + | DataType::TinyInt(_) + | DataType::SmallInt(_) + | DataType::Int(_) + | DataType::BigInt(_) + | DataType::Date(_) + | DataType::Char(_) + | DataType::VarChar(_) + | DataType::Binary(_) + | DataType::VarBinary(_) + | DataType::Decimal(_) + | DataType::Time(_) => true, + // Float bounds can omit NaNs, and complex types need field-specific statistics. + _ => false, + } +} diff --git a/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs b/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs new file mode 100644 index 000000000..593f84b2b --- /dev/null +++ b/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs @@ -0,0 +1,904 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::super::tests::{data_file_meta_with_path, write_schema_file, write_test_deletion_file}; +use crate::catalog::Identifier; +use crate::common::CatalogOptions; +use crate::io::FileIOBuilder; +use crate::spec::{ + BinaryRow, DataFileMeta, DataType, Datum, IntType, Predicate, PredicateBuilder, Schema, + SchemaChange, TableSchema, +}; +use crate::table::data_file_index_writer::FileIndexOptions; +use crate::table::data_file_reader::DataFileReadTiming; +use crate::table::{DataSplit, DataSplitBuilder, RowRange, Table}; +use crate::Error; +use arrow_array::{ArrayRef, Int32Array, Int64Array, RecordBatch}; +use futures::TryStreamExt; +use parquet::arrow::ArrowWriter; +use parquet::file::page_index::offset_index::PageLocation; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const FIRST: i64 = 100_000; +const OPTION: &str = "data-evolution.merged-read.stats-pushdown.enabled"; + +async fn write_file( + table: &Table, + name: &str, + version: i64, + columns: Vec<(&str, ArrayRef)>, + group_rows: usize, + page_rows: usize, + statistics: EnabledStatistics, +) -> DataFileMeta { + let names = columns.iter().map(|(name, _)| *name).collect::>(); + let batch = RecordBatch::try_from_iter(columns).unwrap(); + let properties = WriterProperties::builder() + .set_dictionary_enabled(false) + .set_statistics_enabled(statistics) + .set_offset_index_disabled(statistics != EnabledStatistics::Page) + .set_max_row_group_row_count(Some(group_rows)) + .set_data_page_row_count_limit(page_rows) + .set_write_batch_size(page_rows) + .build(); + let mut writer = ArrowWriter::try_new(Vec::new(), batch.schema(), Some(properties)).unwrap(); + writer.write(&batch).unwrap(); + let bytes = writer.into_inner().unwrap(); + let file = data_file_meta_with_path( + name, + FIRST, + batch.num_rows() as i64, + version, + bytes.len() as i64, + Some(names), + ); + table + .file_io() + .new_output(&format!("{}/bucket-0/{name}", table.location())) + .unwrap() + .write(bytes.into()) + .await + .unwrap(); + file +} + +fn split_builder(table: &Table, files: Vec) -> DataSplitBuilder { + DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(format!("{}/bucket-0", table.location())) + .with_total_buckets(1) + .with_data_files(files) +} + +async fn fixture() -> (Table, DataSplit) { + fixture_with_stats(4096, EnabledStatistics::Page, 64).await +} + +async fn fixture_with_stats( + rows: i32, + statistics: EnabledStatistics, + score_page_rows: usize, +) -> (Table, DataSplit) { + let int = DataType::Int(IntType::new()); + let schema = TableSchema::new( + 0, + &Schema::builder() + .column("slot", int.clone()) + .column("id", int.clone()) + .column("score", int.clone()) + .option("data-evolution.enabled", "true") + .option("row-tracking.enabled", "true") + .build() + .unwrap(), + ); + let table = Table::new( + FileIOBuilder::new("memory") + .with_prop(CatalogOptions::FILE_FORMAT_METADATA_CACHE_MAX_SIZE, "0") + .build() + .unwrap(), + Identifier::new("default", "candidates"), + "memory:/candidates".into(), + schema.clone(), + None, + ); + write_schema_file(&table, &schema).await; + let base = write_file( + &table, + "base.parquet", + 1, + vec![ + ("id", Arc::new(Int32Array::from_iter_values(0..rows))), + ("slot", Arc::new(Int32Array::from_value(7, rows as usize))), + ("score", Arc::new(Int32Array::from_value(-1, rows as usize))), + ], + 733, + 37, + EnabledStatistics::Page, + ) + .await; + let obsolete = write_file( + &table, + "obsolete.parquet", + 2, + vec![("score", Arc::new(Int32Array::from_value(-2, rows as usize)))], + 317, + 17, + EnabledStatistics::Page, + ) + .await; + let score = write_file( + &table, + "score.parquet", + 3, + vec![( + "score", + Arc::new(Int32Array::from_iter( + (0..rows).map(|i| (i % 257 != 0).then_some(i)), + )), + )], + 1024, + score_page_rows, + statistics, + ) + .await; + let current = schema + .apply_changes(vec![ + SchemaChange::rename_column("score".into(), "rating".into()), + SchemaChange::drop_column("slot".into()), + SchemaChange::add_column("slot".into(), int), + ]) + .unwrap(); + let split = split_builder(&table, vec![base, obsolete, score]) + .build() + .unwrap(); + ( + table.copy_with_resolved_schema(current, "main").unwrap(), + split, + ) +} + +struct Measurement { + ids: Vec, + data_bytes: u64, + total_bytes: u64, + separate_index_reads: usize, + source_rows: u64, + elapsed: Duration, +} + +async fn read_batches( + table: &Table, + split: &DataSplit, + predicate: &Predicate, + limit: Option, + timing: Arc, +) -> crate::Result> { + let mut builder = table.new_read_builder(); + builder.with_filter(predicate.clone()); + if table.schema().core_options().row_tracking_enabled() { + builder.with_projection(&["id", "_ROW_ID"])?; + } else { + builder.with_projection(&["id"])?; + } + if let Some(limit) = limit { + builder.with_limit(limit); + } + builder + .new_read()? + .with_data_file_read_timing(timing) + .to_arrow(std::slice::from_ref(split))? + .try_collect() + .await +} + +fn checked_ids(table: &Table, split: &DataSplit, batches: &[RecordBatch]) -> Vec { + let first = split.data_files()[0].first_row_id.unwrap(); + let mut ids = Vec::new(); + for batch in batches { + let id = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + if table.schema().core_options().row_tracking_enabled() { + let row_id = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + assert_eq!(row_id.value(i), first + i64::from(id.value(i))); + } + } + ids.extend_from_slice(id.values()); + } + ids +} + +async fn scan( + table: &Table, + split: &DataSplit, + predicate: &Predicate, + limit: Option, +) -> Measurement { + let timing = Arc::new(DataFileReadTiming::default()); + let start = Instant::now(); + let batches = read_batches(table, split, predicate, limit, timing.clone()) + .await + .unwrap(); + let elapsed = start.elapsed(); + let ids = checked_ids(table, split, &batches); + let reads = std::mem::take(&mut *timing.read_ranges.lock().unwrap()); + let mut data_bytes = 0; + let mut total_bytes = 0; + let mut separate_index_reads = 0; + for file in split.data_files() { + let path = split.data_file_path(file); + if !matches!(file.file_name.as_str(), "base.parquet" | "score.parquet") { + assert!(!reads.contains_key(path.as_str()), "opened unused {path}"); + continue; + } + let Some(ranges) = reads.get(path.as_str()) else { + assert!(ids.is_empty()); + continue; + }; + let bytes = table + .file_io() + .new_input(&path) + .unwrap() + .read() + .await + .unwrap(); + let metadata = parquet::file::metadata::ParquetMetaDataReader::new() + .parse_and_finish(&bytes) + .unwrap(); + let data_end = metadata + .row_groups() + .iter() + .flat_map(|rg| rg.columns()) + .map(|col| { + let (start, length) = col.byte_range(); + start + length + }) + .max() + .unwrap(); + let mut footer_reads = 0; + let mut index_reads = 0; + for range in ranges { + let size = range.end - range.start; + total_bytes += size; + if range.end <= data_end { + data_bytes += size; + } else if range.end == file.file_size as u64 { + footer_reads += 1; + } else { + assert!( + range.start >= data_end, + "unexpected read {range:?} for {path}" + ); + index_reads += 1; + } + } + assert_eq!(footer_reads, 1, "footer fetches for {path}"); + assert!(index_reads <= 1, "repeated index fetch for {path}"); + separate_index_reads += index_reads; + } + Measurement { + ids, + data_bytes, + total_bytes, + separate_index_reads, + source_rows: timing.source_rows.load(Ordering::Relaxed), + elapsed, + } +} + +fn disabled(table: &Table) -> Table { + table.copy_with_options(HashMap::from([(OPTION.into(), "false".into())])) +} + +async fn assert_candidates( + table: &Table, + split: &DataSplit, + predicate: &Predicate, + expected_ids: &[i32], + expected_source_rows: Option, +) { + let baseline = scan(&disabled(table), split, predicate, None).await; + let pruned = scan(table, split, predicate, None).await; + assert_eq!(baseline.ids, expected_ids, "{predicate:?}"); + assert_eq!(pruned.ids, baseline.ids, "{predicate:?}"); + if let Some(rows) = expected_source_rows { + assert_eq!(pruned.source_rows, rows, "{predicate:?}"); + } +} + +#[tokio::test] +async fn merged_candidates_share_pages_and_fall_back_to_row_groups_or_full_reads() { + for (rows, stats, page_rows, expected_rows) in [ + (32_768, EnabledStatistics::Page, 1, 1), + (4096, EnabledStatistics::Chunk, 64, 1024), + (4096, EnabledStatistics::None, 64, 4096), + ] { + let (table, split) = fixture_with_stats(rows, stats, page_rows).await; + let predicate = PredicateBuilder::new(table.schema().fields()) + .equal("rating", Datum::Int(2117)) + .unwrap(); + let on = scan(&table, &split, &predicate, None).await; + let off = scan(&disabled(&table), &split, &predicate, None).await; + assert_eq!(on.ids, [2117]); + assert_eq!(on.ids, off.ids); + assert_eq!(on.source_rows, 2 * expected_rows); + assert_eq!(off.source_rows, 2 * rows as u64); + assert_eq!(off.separate_index_reads, 0); + assert_eq!( + on.separate_index_reads, + usize::from(stats == EnabledStatistics::Page) + ); + if stats == EnabledStatistics::Page { + let invalid = + table.copy_with_options(HashMap::from([(OPTION.into(), "invalid".into())])); + assert!(matches!( + read_batches(&invalid, &split, &predicate, None, Default::default()).await, + Err(Error::ConfigInvalid { message }) if message.contains(OPTION) + )); + let row_groups_only = table.copy_with_options(HashMap::from([( + "parquet.filter.columnindex.enabled".into(), + "false".into(), + )])); + let result = scan(&row_groups_only, &split, &predicate, None).await; + assert_eq!(result.ids, off.ids); + assert_eq!(result.source_rows, 2 * 1024); + + let null_id = PredicateBuilder::new(table.schema().fields()) + .is_null("id") + .unwrap(); + let on = scan(&row_groups_only, &split, &null_id, None).await; + let off = scan(&disabled(&row_groups_only), &split, &null_id, None).await; + assert!(off.ids.is_empty()); + assert_eq!(on.ids, off.ids); + assert_eq!(on.source_rows, 0); + assert_eq!(off.source_rows, rows as u64); + } + if stats != EnabledStatistics::None { + assert!(on.total_bytes < off.total_bytes); + assert!( + on.data_bytes < off.data_bytes, + "{} >= {}", + on.data_bytes, + off.data_bytes + ); + } + } +} + +#[tokio::test] +async fn merged_candidates_use_winning_ids_and_preserve_boolean_residuals() { + let (table, split) = fixture().await; + let mut files = split.data_files().to_vec(); + for file in &mut files { + file.first_row_id = Some(i64::MAX - (file.row_count - 1)); + } + let split = split_builder(&table, files).build().unwrap(); + let p = PredicateBuilder::new(table.schema().fields()); + let id = |n| p.equal("id", Datum::Int(n)).unwrap(); + let rating = |n| p.equal("rating", Datum::Int(n)).unwrap(); + for (predicate, expected, candidate_rows) in [ + ( + Predicate::Not(Box::new( + p.less_or_equal("rating", Datum::Int(4094)).unwrap(), + )), + vec![4095], + Some(64), + ), + ( + Predicate::or(vec![rating(201), id(3011)]), + vec![201, 3011], + Some(4096), + ), + ( + Predicate::or(vec![rating(207), rating(3001)]), + vec![207, 3001], + Some(128), + ), + ( + Predicate::and(vec![id(2117), rating(2117)]), + vec![2117], + Some(20), + ), + ( + Predicate::and(vec![p.is_null("slot").unwrap(), rating(3001)]), + vec![3001], + Some(64), + ), + ( + Predicate::and(vec![ + p.is_null("rating").unwrap(), + p.between("id", Datum::Int(400), Datum::Int(800)).unwrap(), + ]), + vec![514, 771], + None, + ), + ( + Predicate::and(vec![ + p.is_in("id", vec![Datum::Int(12), Datum::Int(13)]).unwrap(), + Predicate::Not(Box::new(Predicate::and(vec![id(12), rating(13)]))), + ]), + vec![12, 13], + None, + ), + (Predicate::and(vec![id(222), rating(3011)]), vec![], Some(0)), + ( + Predicate::or(vec![ + p.greater_or_equal("rating", Datum::Int(0)).unwrap(), + p.is_null("rating").unwrap(), + ]), + (0..4096).collect(), + Some(4096), + ), + ] { + assert_candidates( + &table, + &split, + &predicate, + &expected, + candidate_rows.map(|rows| 2 * rows), + ) + .await; + } +} + +#[tokio::test] +async fn merged_candidates_intersect_bitmap_hits_requested_ranges_and_deletions_before_limit() { + let (table, split) = fixture().await; + let table = table.copy_with_options(HashMap::from([("read.batch-size".into(), "2".into())])); + let dv = + write_test_deletion_file(table.file_io(), "memory:/candidates/dv", &[2002, 2004]).await; + let mut files = split.data_files().to_vec(); + let index_options = HashMap::from([("file-index.bitmap.columns".into(), "id".into())]); + let mut index = FileIndexOptions::parse(&index_options, table.schema().fields()) + .unwrap() + .unwrap() + .create_writer() + .unwrap(); + let ids = RecordBatch::try_from_iter([( + "id", + Arc::new(Int32Array::from_iter_values(0..4096)) as ArrayRef, + )]) + .unwrap(); + index.write(&ids).unwrap(); + files[0].embedded_index = Some(index.serialize().unwrap().to_vec()); + let split = split_builder(&table, files) + .with_data_deletion_files(vec![Some(dv), None, None]) + .with_row_ranges(vec![ + RowRange::new(FIRST + 2010, FIRST + 2020), + RowRange::new(FIRST + 2002, FIRST + 2007), + RowRange::new(FIRST + 1000, FIRST + 1001), + RowRange::new(FIRST + 3000, FIRST + 3001), + ]) + .build() + .unwrap(); + let p = PredicateBuilder::new(table.schema().fields()); + let indexed_ids = [ + 1000, 2000, 2002, 2003, 2004, 2006, 2007, 2010, 2012, 2018, 3001, + ]; + let predicate = Predicate::and(vec![ + p.between("rating", Datum::Int(2000), Datum::Int(2015)) + .unwrap(), + p.is_in("id", indexed_ids.map(Datum::Int).to_vec()).unwrap(), + ]); + let deleted = split + .clone() + .with_selected_row_ranges(vec![RowRange::new(FIRST + 2002, FIRST + 2002)]); + let without_bitmap = table.copy_with_options(HashMap::from([( + "file-index.read.enabled".into(), + "false".into(), + )])); + for (table, candidates) in [(&table, 6), (&disabled(&table), 8), (&without_bitmap, 15)] { + let full = scan(table, &split, &predicate, None).await; + assert_eq!(full.ids, [2003, 2006, 2007, 2010, 2012]); + assert_eq!(full.source_rows, 2 * candidates); + assert_eq!( + scan(table, &split, &predicate, Some(3)).await.ids, + [2003, 2006, 2007] + ); + let result = scan(table, &deleted, &predicate, None).await; + assert!(result.ids.is_empty()); + assert_eq!(result.total_bytes, 0); + } +} + +#[tokio::test] +async fn merged_candidates_use_bounds_only_for_order_preserving_schema_changes() { + use crate::spec::{BigIntType, TinyIntType}; + + let (table, split) = fixture().await; + for (ty, value, expected, candidates) in [ + ( + DataType::BigInt(BigIntType::new()), + Datum::Long(2117), + vec![2117], + 64, + ), + ( + DataType::TinyInt(TinyIntType::new()), + Datum::TinyInt(-1), + (255..4096).step_by(256).collect(), + 4096, + ), + ] { + let schema = table + .schema() + .apply_changes(vec![SchemaChange::update_column_type("rating".into(), ty)]) + .unwrap(); + let table = table.copy_with_resolved_schema(schema, "main").unwrap(); + let predicate = PredicateBuilder::new(table.schema().fields()) + .equal("rating", value) + .unwrap(); + assert_candidates(&table, &split, &predicate, &expected, Some(2 * candidates)).await; + } +} + +#[tokio::test] +async fn merged_candidates_require_row_tracking_unbucketed_tables_and_flat_parquet_groups() { + use crate::spec::{BlobType, FloatType, VectorType}; + + let (table, split) = fixture().await; + let mut cases = Vec::new(); + for (key, value) in [ + ("row-tracking.enabled", "false"), + ("bucket", "4"), + ("data-evolution.nested-field.enabled", "true"), + ] { + let table = table.copy_with_options(HashMap::from([(key.into(), value.into())])); + cases.push((table, split.clone())); + } + for (name, ty) in [ + ("extra.orc", DataType::Int(IntType::new())), + ("extra.blob", DataType::Blob(BlobType::new())), + ( + "extra.vector.parquet", + DataType::Vector( + VectorType::try_new(true, 2, DataType::Float(FloatType::new())).unwrap(), + ), + ), + ] { + let schema = table + .schema() + .apply_changes(vec![SchemaChange::add_column("extra".into(), ty)]) + .unwrap(); + let table = table.copy_with_resolved_schema(schema, "main").unwrap(); + let mut extra = data_file_meta_with_path(name, FIRST, 4096, 4, 1, Some(vec!["extra"])); + extra.schema_id = table.schema().id(); + let mut files = split.data_files().to_vec(); + files.push(extra); + let split = split_builder(&table, files).build().unwrap(); + cases.push((table, split)); + } + for (table, split) in cases { + let p = PredicateBuilder::new(table.schema().fields()) + .equal("rating", Datum::Int(2117)) + .unwrap(); + assert_candidates(&table, &split, &p, &[2117], Some(2 * 4096)).await; + } +} + +#[tokio::test] +async fn merged_candidates_require_compatible_physical_statistics() { + use crate::spec::{FloatType, TimestampType}; + use arrow_array::{ + Float32Array, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampSecondArray, + }; + + let (table, split) = fixture().await; + let timestamp = |precision| DataType::Timestamp(TimestampType::new(precision).unwrap()); + let instant = Datum::Timestamp { + millis: 2_117_000, + nanos: 0, + }; + let seconds = 0..4096; + let millis = TimestampMillisecondArray::from_iter_values(seconds.clone().map(|s| s * 1000)); + let micros = + TimestampMicrosecondArray::from_iter_values(seconds.clone().map(|s| s * 1_000_000)); + let floats = Float32Array::from_iter_values(seconds.clone().map(|i| { + if i == 0 { + f32::NAN + } else { + i as f32 + } + })); + let seconds = TimestampSecondArray::from_iter_values(seconds); + for (ty, column, value, eq_rows, or_rows) in [ + ( + timestamp(3), + Arc::new(millis) as ArrayRef, + instant.clone(), + 64, + 2048, + ), + (timestamp(3), Arc::new(micros), instant.clone(), 4096, 4096), + (timestamp(0), Arc::new(seconds), instant, 4096, 4096), + ( + DataType::Float(FloatType::new()), + Arc::new(floats), + Datum::Float(2117.0), + 4096, + 4096, + ), + ] { + let schema = table + .schema() + .apply_changes(vec![ + SchemaChange::drop_column("rating".into()), + SchemaChange::add_column("rating".into(), ty), + SchemaChange::add_column("rank".into(), DataType::Int(IntType::new())), + ]) + .unwrap(); + let table = table.copy_with_resolved_schema(schema, "main").unwrap(); + let mut file = write_file( + &table, + "score.parquet", + 4, + vec![ + ("rating", column), + ("rank", Arc::new(Int32Array::from_iter_values(0..4096))), + ], + 1024, + 64, + EnabledStatistics::Page, + ) + .await; + file.schema_id = table.schema().id(); + let split = split_builder(&table, vec![split.data_files()[0].clone(), file]) + .build() + .unwrap(); + let p = PredicateBuilder::new(table.schema().fields()); + let rating = p.equal("rating", value).unwrap(); + let rank = |value| p.equal("rank", Datum::Int(value)).unwrap(); + for (predicate, expected, candidates) in [ + (rating.clone(), vec![2117], eq_rows), + ( + Predicate::and(vec![rating.clone(), rank(2117)]), + vec![2117], + 64, + ), + ( + Predicate::or(vec![rating, rank(15)]), + vec![15, 2117], + or_rows, + ), + ] { + assert_candidates(&table, &split, &predicate, &expected, Some(2 * candidates)).await; + } + } +} + +#[tokio::test] +async fn merged_candidates_require_consistent_source_row_counts() { + let (table, split) = fixture().await; + let off = disabled(&table); + let p = PredicateBuilder::new(table.schema().fields()) + .equal("rating", Datum::Int(-10)) + .unwrap(); + let mut files = split.data_files().to_vec(); + for file in &mut files { + file.row_count += 1; + } + let wrong_count = split_builder(&table, files).build().unwrap(); + for table in [&table, &off] { + let err = read_batches(table, &wrong_count, &p, None, Default::default()) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("produced 4096 rows but expected 4097"), + "{err}" + ); + } + let mut short = write_file( + &table, + "short.parquet", + 1, + vec![("id", Arc::new(Int32Array::from_iter_values(0..3000)))], + 733, + 37, + EnabledStatistics::Page, + ) + .await; + short.row_count = 4096; + let mut files = split.data_files().to_vec(); + files[0] = short; + let split = split_builder(&table, files).build().unwrap(); + let p = PredicateBuilder::new(table.schema().fields()) + .equal("rating", Datum::Int(2117)) + .unwrap(); + for table in [&off, &table] { + let err = read_batches(table, &split, &p, None, Default::default()) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("exhausted at different row counts"), + "{err}" + ); + } +} + +fn rewrite_page_locations( + bytes: &bytes::Bytes, + edit: impl FnOnce(&mut [PageLocation], i64), +) -> Vec { + use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader, ParquetMetaDataWriter}; + use parquet::file::writer::TrackedWrite; + use std::io::Write; + + let metadata = ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(bytes) + .unwrap(); + let group = metadata.row_groups().last().unwrap(); + let row_count = group.num_rows(); + let (start, len) = group.columns().last().unwrap().byte_range(); + let mut builder = metadata.into_builder(); + let mut offsets = builder.take_offset_index().unwrap(); + edit( + &mut offsets.last_mut().unwrap()[0].page_locations, + row_count, + ); + let metadata = builder.set_offset_index(Some(offsets)).build(); + let mut output = Vec::new(); + let mut writer = TrackedWrite::new(&mut output); + writer.write_all(&bytes[..(start + len) as usize]).unwrap(); + ParquetMetaDataWriter::new_with_tracked(writer, &metadata) + .finish() + .unwrap(); + output +} + +#[tokio::test] +async fn merged_candidates_keep_normal_read_behavior_for_damaged_files() { + let (table, split) = fixture().await; + let off = disabled(&table); + let p = PredicateBuilder::new(table.schema().fields()) + .equal("rating", Datum::Int(2117)) + .unwrap(); + let read = |table| read_batches(table, &split, &p, None, Default::default()); + let io = table.file_io(); + let last = PredicateBuilder::new(table.schema().fields()) + .equal("rating", Datum::Int(4095)) + .unwrap(); + for name in ["score.parquet", "base.parquet"] { + let file_index = split + .data_files() + .iter() + .position(|f| f.file_name == name) + .unwrap(); + let path = split.data_file_path(&split.data_files()[file_index]); + let output = io.new_output(&path).unwrap(); + let bytes = io.new_input(&path).unwrap().read().await.unwrap(); + for damaged_bytes in [ + rewrite_page_locations(&bytes, |pages, rows| { + pages.last_mut().unwrap().first_row_index = rows; + }), + rewrite_page_locations(&bytes, |pages, _| { + pages.last_mut().unwrap().offset = pages[0].offset; + }), + rewrite_page_locations(&bytes, |pages, _| { + pages.last_mut().unwrap().compressed_page_size += 1; + }), + ] { + let mut files = split.data_files().to_vec(); + files[file_index].file_size = damaged_bytes.len() as i64; + let damaged = split_builder(&table, files).build().unwrap(); + output.write(damaged_bytes.into()).await.unwrap(); + let rows = if name == "score.parquet" { 1024 } else { 64 }; + assert_candidates(&table, &damaged, &last, &[4095], Some(2 * rows)).await; + let last_rows = + damaged.with_selected_row_ranges(vec![RowRange::new(FIRST + 4094, FIRST + 4095)]); + assert_candidates(&table, &last_rows, &last, &[4095], Some(2 * 2)).await; + } + + let metadata = parquet::file::metadata::ParquetMetaDataReader::new() + .parse_and_finish(&bytes) + .unwrap(); + let column = metadata.row_group(0).column(0); + let (offset, len) = if name == "score.parquet" { + (column.column_index_offset(), column.column_index_length()) + } else { + (column.offset_index_offset(), column.offset_index_length()) + }; + let start = offset.unwrap() as usize; + let mut corrupt = bytes.to_vec(); + corrupt[start..start + len.unwrap() as usize].fill(0xff); + output.write(corrupt.into()).await.unwrap(); + for table in [&off, &table] { + assert_eq!( + checked_ids(table, &split, &read(table).await.unwrap()), + [2117], + "{name}" + ); + } + let mut corrupt_footer = bytes.to_vec(); + *corrupt_footer.last_mut().unwrap() = 0; + output.write(corrupt_footer.into()).await.unwrap(); + assert_eq!( + read(&table).await.unwrap_err().to_string(), + read(&off).await.unwrap_err().to_string(), + "{name}" + ); + io.delete_file(&path).await.unwrap(); + assert_eq!( + read(&table).await.unwrap_err().to_string(), + read(&off).await.unwrap_err().to_string(), + "{name}" + ); + if name == "base.parquet" { + let outside = split + .clone() + .with_selected_row_ranges(vec![RowRange::new(FIRST + 3000, FIRST + 3001)]); + let result = scan(&table, &outside, &p, None).await; + assert!(result.ids.is_empty()); + assert_eq!(result.data_bytes, 0); + } + output.write(bytes).await.unwrap(); + } +} + +#[tokio::test] +#[ignore = "run with --ignored --nocapture to measure cold-metadata merged reads"] +async fn benchmark_merged_candidates() { + let rows = 262_144; + let (table, split) = fixture_with_stats(rows, EnabledStatistics::Page, 64).await; + let off = disabled(&table); + let p = PredicateBuilder::new(table.schema().fields()); + for (name, predicate, expected_ids) in [ + ( + "selective", + p.equal("rating", Datum::Int(2117)).unwrap(), + vec![2117], + ), + ( + "nonselective", + Predicate::or(vec![ + p.greater_or_equal("rating", Datum::Int(0)).unwrap(), + p.is_null("rating").unwrap(), + ]), + (0..rows).collect(), + ), + ] { + let mut times = [Vec::new(), Vec::new()]; + for i in 0..7 { + let order = if i % 2 == 0 { [0, 1] } else { [1, 0] }; + for mode in order { + let result = scan([&off, &table][mode], &split, &predicate, None).await; + assert_eq!(result.ids, expected_ids); + times[mode].push(result.elapsed); + if i == 6 { + times[mode].sort_unstable(); + println!("{name} enabled={} rows={} source_rows={} data_bytes={} total_bytes={} median={:?}", + mode == 1, result.ids.len(), result.source_rows, result.data_bytes, result.total_bytes, times[mode][3]); + } + } + } + } +} diff --git a/crates/paimon/src/table/data_evolution_reader/vector_versions.rs b/crates/paimon/src/table/data_evolution_reader/vector_versions.rs index 87c5fe08d..642409d5c 100644 --- a/crates/paimon/src/table/data_evolution_reader/vector_versions.rs +++ b/crates/paimon/src/table/data_evolution_reader/vector_versions.rs @@ -24,11 +24,12 @@ use async_stream::try_stream; use futures::StreamExt; use super::{ - intersect_local_ranges, is_vector_store_file_name, vector_file_row_range, vector_format_suffix, - PreparedMergeGroup, ResolvedFileInfo, + is_vector_store_file_name, vector_file_row_range, vector_format_suffix, PreparedMergeGroup, + ResolvedFileInfo, }; use crate::spec::{DataField, DataFileMeta}; use crate::table::data_file_reader::DataFileReader; +use crate::table::row_id_predicate::intersect_sorted_ranges; use crate::table::{ArrowRecordBatchStream, RowRange}; use crate::{DataSplit, Error}; @@ -198,7 +199,7 @@ pub(super) fn read( previous_version = Some((file.max_sequence_number, range.to())); let winning = crate::table::source::exclude_row_ranges(std::slice::from_ref(&range), &covered); - let ranges = intersect_local_ranges(&winning, &selected); + let ranges = intersect_sorted_ranges(&winning, &selected); if !ranges.is_empty() { let index = file_reads.len(); reads.extend(ranges.iter().map(|range| (range.clone(), Some(index)))); diff --git a/crates/paimon/src/table/data_file_reader.rs b/crates/paimon/src/table/data_file_reader.rs index 4fd25de50..9115631cd 100644 --- a/crates/paimon/src/table/data_file_reader.rs +++ b/crates/paimon/src/table/data_file_reader.rs @@ -30,6 +30,7 @@ use crate::io::{FileIO, FileRead}; use crate::spec::{ is_variant_extraction_row_type, DataField, DataFileMeta, DataType, Predicate, ROW_ID_FIELD_NAME, }; +use crate::table::row_id_predicate::intersect_sorted_ranges; use crate::table::schema_manager::SchemaManager; use crate::table::ArrowRecordBatchStream; use crate::table::RowRange; @@ -38,6 +39,7 @@ use arrow_array::{Array, Int64Array, RecordBatch}; use async_stream::try_stream; use futures::StreamExt; +use parquet::arrow::arrow_reader::ArrowReaderMetadata; use roaring::RoaringBitmap; use std::collections::HashMap; use std::ops::Range; @@ -56,6 +58,10 @@ pub(crate) struct DataFileReadTiming { file_schema_open_nanos: AtomicU64, first_batch_wait_nanos: AtomicU64, remaining_batch_wait_nanos: AtomicU64, + #[cfg(test)] + pub(super) read_ranges: std::sync::Mutex, Vec>>>, + #[cfg(test)] + pub(super) source_rows: AtomicU64, } impl DataFileReadTiming { @@ -102,6 +108,30 @@ impl DataFileReadTiming { struct TimedFileRead { inner: Box, timing: Arc, + #[cfg(test)] + path: Arc, +} + +pub(super) async fn open_file_reader( + file_io: &FileIO, + path: &str, + size: u64, + timing: Option<&Arc>, +) -> crate::Result> { + let input = file_io.new_input(path)?; + let start = timing.map(|_| Instant::now()); + let reader = input.reader_with_size(size).await?; + if let (Some(timing), Some(start)) = (timing, start) { + timing.add_file_read(start.elapsed()); + Ok(Box::new(TimedFileRead { + inner: Box::new(reader), + timing: Arc::clone(timing), + #[cfg(test)] + path: path.into(), + })) + } else { + Ok(Box::new(reader)) + } } #[async_trait::async_trait] @@ -114,6 +144,14 @@ impl FileRead for TimedFileRead { } async fn read(&self, range: Range) -> crate::Result { + #[cfg(test)] + self.timing + .read_ranges + .lock() + .unwrap() + .entry(self.path.clone()) + .or_default() + .push(range.clone()); let start = Instant::now(); let result = self.inner.read(range).await; self.timing.add_file_read(start.elapsed()); @@ -135,6 +173,23 @@ impl FileRead for TimedFileRead { type DataFieldsExtractor = dyn Fn(&[DataField]) -> crate::Result> + Send + Sync; +pub(super) struct FileReadSelection { + pub(super) rows: Option>, + /// Caller ranges or live bitmap hits used to choose a ROW sidecar before Parquet pruning or LIMIT. + pub(super) sidecar_rows: Option>, + pub(super) parquet_metadata: Option, +} + +impl FileReadSelection { + pub(super) fn new(rows: Option>, sidecar_rows: Option>) -> Self { + Self { + rows, + sidecar_rows, + parquet_metadata: None, + } + } +} + /// Reads data files through their format-specific readers. #[derive(Clone)] pub(crate) struct DataFileReader { @@ -410,13 +465,12 @@ impl DataFileReader { selected_ranges }; - let mut stream = reader.read_single_file_stream_with_selection( + let mut stream = reader.read_single_file_stream_with_local_selection( &split, file_meta, data_fields, None, - row_selection, - sidecar_ranges, + FileReadSelection::new(row_selection, sidecar_ranges), )?; while let Some(batch) = stream.next().await { yield batch?; @@ -494,50 +548,44 @@ impl DataFileReader { dv: Option>, row_ranges: Option>, ) -> crate::Result { - self.read_single_file_stream_with_requested_ranges( + self.read_single_file_stream_with_row_id_ranges( split, file_meta, data_fields, dv, - row_ranges.clone(), - row_ranges, + FileReadSelection::new(row_ranges.clone(), row_ranges), ) } - /// The caller supplies sidecar candidates separately from physical ranges: - /// a selective bitmap includes DV filtering, while LIMIT never affects it. - pub(super) fn read_single_file_stream_with_requested_ranges( + pub(super) fn read_single_file_stream_with_row_id_ranges( &self, split: &DataSplit, file_meta: DataFileMeta, data_fields: Option>, dv: Option>, - row_ranges: Option>, - requested_ranges: Option>, + selection: FileReadSelection, ) -> crate::Result { - let local_ranges = row_ranges.as_ref().map(|ranges| { + let to_local = |ranges: &[RowRange]| { to_local_row_ranges( ranges, file_meta.first_row_id.unwrap_or(0), file_meta.row_count, ) - }); + }; + let local_ranges = selection.rows.as_deref().map(to_local); let row_selection = merge_row_selection(file_meta.row_count, dv.as_deref(), local_ranges.as_deref()); - let sidecar_ranges = requested_ranges.as_ref().map(|ranges| { - to_local_row_ranges( - ranges, - file_meta.first_row_id.unwrap_or(0), - file_meta.row_count, - ) - }); - self.read_single_file_stream_with_selection( + let sidecar_ranges = selection.sidecar_rows.as_deref().map(to_local); + self.read_single_file_stream_with_local_selection( split, file_meta, data_fields, None, - row_selection, - sidecar_ranges, + FileReadSelection { + rows: row_selection, + sidecar_rows: sidecar_ranges, + ..selection + }, ) } @@ -558,25 +606,28 @@ impl DataFileReader { dv.as_deref(), local_row_ranges.as_deref(), ); - self.read_single_file_stream_with_selection( + self.read_single_file_stream_with_local_selection( split, file_meta, data_fields, Some(data_schema_fields), - row_selection, - local_row_ranges, + FileReadSelection::new(row_selection, local_row_ranges), ) } - fn read_single_file_stream_with_selection( + fn read_single_file_stream_with_local_selection( &self, split: &DataSplit, file_meta: DataFileMeta, data_fields: Option>, data_schema_fields: Option>, - row_selection: Option>, - sidecar_ranges: Option>, + selection: FileReadSelection, ) -> crate::Result { + let FileReadSelection { + rows: row_selection, + sidecar_rows: sidecar_ranges, + parquet_metadata, + } = selection; if row_selection.as_ref().is_some_and(Vec::is_empty) { return Ok(futures::stream::empty().boxed()); } @@ -708,18 +759,29 @@ impl DataFileReader { let projected_read_fields = read_data_fields(&data_schema_fields, &read_type, self.nested_field_enabled)?; let path_to_read = split.data_file_path(&file_meta); - let configured_reader = create_format_reader_with_budget( - &path_to_read, - blob_as_descriptor, - FormatReadFields { - data_schema: &data_schema_fields, - projected: &projected_read_fields, - }, - &table_options, - parquet_read_budget, - blob_parallelism, - mosaic_prefetch, - )?; + let fields = FormatReadFields { + data_schema: &data_schema_fields, + projected: &projected_read_fields, + }; + let configured_reader = match parquet_metadata { + Some(metadata) => { + let reader = crate::arrow::format::parquet::ParquetFormatReader::with_options( + &table_options, + parquet_read_budget, + )? + .with_metadata(metadata); + crate::arrow::format::configure_format_reader(Box::new(reader), fields) + } + None => create_format_reader_with_budget( + &path_to_read, + blob_as_descriptor, + fields, + &table_options, + parquet_read_budget, + blob_parallelism, + mosaic_prefetch, + )?, + }; // Remap predicates from table-level to file-level indices. let file_predicates = if post_read_filter { None @@ -767,19 +829,7 @@ impl DataFileReader { let index_mapping = create_index_mapping(&read_type, &source_fields); let source_fields = Some(source_fields); - let input_file = file_io.new_input(&read_path)?; - let open_start = read_timing.as_ref().map(|_| Instant::now()); - let file_reader = input_file.reader_with_size(read_size).await?; - if let (Some(timing), Some(start)) = (read_timing.as_ref(), open_start) { - timing.add_file_read(start.elapsed()); - } - let file_reader: Box = match read_timing.as_ref() { - Some(timing) => Box::new(TimedFileRead { - inner: Box::new(file_reader), - timing: Arc::clone(timing), - }), - None => Box::new(file_reader), - }; + let file_reader = open_file_reader(&file_io, &read_path, read_size, read_timing.as_ref()).await?; let is_parquet = read_path.to_ascii_lowercase().ends_with(".parquet"); let target_schema = if read_path != path_to_read && read_path.ends_with(".row") { let names = read_type.iter().filter(|field| matches!(field.data_type(), DataType::Blob(_))) @@ -788,13 +838,11 @@ impl DataFileReader { } else { target_schema }; - let selected_row_ids = selected_row_ids_for_read( + let mut row_ids = selected_row_ids_for_read( projects_row_id, file_meta.first_row_id, row_selection.as_deref(), ); - let mut row_id_cursor = file_meta.first_row_id.unwrap_or(0); - let mut row_id_offset = 0usize; let mut batch_stream = format_reader.read_batch_stream( file_reader, @@ -833,19 +881,17 @@ impl DataFileReader { first_batch = false; let batch = batch?; let num_rows = batch.num_rows(); + #[cfg(test)] + if let Some(timing) = read_timing.as_ref() { + timing.source_rows.fetch_add(num_rows as u64, Ordering::Relaxed); + } let batch_schema = batch.schema(); // Build output columns using index mapping (field-ID-based) or by name. let mut columns: Vec> = Vec::with_capacity(target_schema.fields().len()); for (i, target_field) in target_schema.fields().iter().enumerate() { if target_field.name() == ROW_ID_FIELD_NAME { - columns.push(row_id_column_for_batch( - file_meta.first_row_id, - num_rows, - &mut row_id_cursor, - selected_row_ids.as_deref(), - &mut row_id_offset, - )?); + columns.push(row_id_column_for_batch(row_ids.as_mut(), num_rows)?); continue; } @@ -1471,22 +1517,24 @@ fn data_schema_fields_for_file( /// `first_row_id` is the coordinate base selected by the table's read path: /// stable row ID for row-tracked tables, or cumulative split-local physical /// offset for raw tables without row tracking. -fn to_local_row_ranges( +pub(super) fn to_local_row_ranges( row_ranges: &[RowRange], first_row_id: i64, row_count: i64, ) -> Vec { - let file_end = first_row_id + row_count - 1; + if row_count <= 0 { + return Vec::new(); + } + let file_end = first_row_id.saturating_add(row_count - 1); crate::table::merge_row_ranges( row_ranges .iter() .filter_map(|r| { - if r.to() < first_row_id || r.from() > file_end { - return None; - } - let local_from = (r.from() - first_row_id).max(0); - let local_to = (r.to() - first_row_id).min(row_count - 1); - Some(RowRange::new(local_from, local_to)) + let overlap = r.intersect_inclusive(first_row_id, file_end)?; + Some(RowRange::new( + overlap.from() - first_row_id, + overlap.to() - first_row_id, + )) }) .collect(), ) @@ -1595,7 +1643,7 @@ fn ranges_cover_all_rows(ranges: &[RowRange], row_count: i64) -> bool { } /// Convert a DeletionVector into sorted non-deleted inclusive RowRanges. -fn dv_to_non_deleted_ranges(dv: &DeletionVector, row_count: i64) -> Vec { +pub(super) fn dv_to_non_deleted_ranges(dv: &DeletionVector, row_count: i64) -> Vec { let mut result = Vec::new(); let mut cursor: i64 = 0; for (start, end) in dv.deleted_ranges() { @@ -1614,134 +1662,43 @@ fn dv_to_non_deleted_ranges(dv: &DeletionVector, row_count: i64) -> Vec Vec { - let mut result = Vec::new(); - let (mut i, mut j) = (0, 0); - while i < a.len() && j < b.len() { - let from = a[i].from().max(b[j].from()); - let to = a[i].to().min(b[j].to()); - if from <= to { - result.push(RowRange::new(from, to)); - } - if a[i].to() < b[j].to() { - i += 1; - } else { - j += 1; - } - } - result -} - -/// Expand row_ranges into a flat sequence of selected row IDs for a file. -/// Intended for per-batch _ROW_ID attachment — callers should not pass -/// whole-file ranges with millions of rows, as this allocates a Vec -/// proportional to the selected range size. -pub(super) fn expand_selected_row_ids( +pub(super) fn row_ids_from_ranges( first_row_id: i64, - row_count: i64, - row_ranges: &[RowRange], -) -> Vec { - if row_count == 0 { - return Vec::new(); - } - let file_end = first_row_id + row_count - 1; - let mut ids = Vec::new(); - for r in row_ranges { - let from = r.from().max(first_row_id); - let to = r.to().min(file_end); - for id in from..=to { - ids.push(id); - } - } - ids -} - -fn expand_local_selected_row_ids(first_row_id: i64, local_ranges: &[RowRange]) -> Vec { - let mut ids = Vec::new(); - for range in local_ranges { - for local_id in range.from()..=range.to() { - ids.push(first_row_id + local_id); - } - } - ids + ranges: Option>, +) -> impl Iterator { + let contiguous = ranges.is_none().then_some(first_row_id..=i64::MAX); + let selected = ranges.into_iter().flatten().map(|r| r.from()..=r.to()); + selected.chain(contiguous).flatten() } fn selected_row_ids_for_read( projects_row_id: bool, first_row_id: Option, - row_selection: Option<&[RowRange]>, -) -> Option> { - if !projects_row_id { - return None; - } - match (first_row_id, row_selection) { - (Some(first_row_id), Some(ranges)) => { - Some(expand_local_selected_row_ids(first_row_id, ranges)) - } - _ => None, - } + local_ranges: Option<&[RowRange]>, +) -> Option> { + let first = first_row_id.filter(|_| projects_row_id)?; + let positions = row_ids_from_ranges(0, local_ranges.map(|ranges| ranges.to_vec())); + Some(positions.map_while(move |position| first.checked_add(position))) } -fn row_id_column_for_batch( - first_row_id: Option, +pub(super) fn row_id_column_for_batch( + row_ids: Option<&mut impl Iterator>, num_rows: usize, - row_id_cursor: &mut i64, - selected_row_ids: Option<&[i64]>, - row_id_offset: &mut usize, ) -> crate::Result> { - let Some(_) = first_row_id else { + let Some(row_ids) = row_ids else { return Ok(Arc::new(Int64Array::new_null(num_rows))); }; - - if let Some(selected_row_ids) = selected_row_ids { - let end = *row_id_offset + num_rows; - if end > selected_row_ids.len() { - return Err(Error::UnexpectedError { - message: format!( - "Row ID offset out of bounds: need {}..{} but selected_row_ids has {} entries", - *row_id_offset, - end, - selected_row_ids.len() - ), - source: None, - }); - } - let batch_ids = &selected_row_ids[*row_id_offset..end]; - *row_id_offset = end; - return Ok(Arc::new(Int64Array::from(batch_ids.to_vec()))); - } - - let start = *row_id_cursor; - let end = start + num_rows as i64; - *row_id_cursor = end; - Ok(Arc::new(Int64Array::from((start..end).collect::>()))) -} - -pub(super) fn attach_row_id( - batch: RecordBatch, - row_id_index: usize, - selected_row_ids: &[i64], - row_id_offset: &mut usize, - output_schema: &Arc, -) -> crate::Result { - let num_rows = batch.num_rows(); - let end = *row_id_offset + num_rows; - if end > selected_row_ids.len() { - return Err(Error::UnexpectedError { + let ids = Int64Array::from_iter_values(row_ids.take(num_rows)); + if ids.len() != num_rows { + return Err(Error::DataInvalid { message: format!( - "Row ID offset out of bounds: need {}..{} but selected_row_ids has {} entries", - *row_id_offset, - end, - selected_row_ids.len() + "Batch has {num_rows} rows but only {} selected row IDs remain", + ids.len() ), source: None, }); } - let batch_ids = &selected_row_ids[*row_id_offset..end]; - *row_id_offset = end; - let array: Arc = Arc::new(Int64Array::from(batch_ids.to_vec())); - insert_column_at(batch, array, row_id_index, output_schema) + Ok(Arc::new(ids)) } pub(super) fn insert_column_at( @@ -1767,16 +1724,6 @@ pub(super) fn insert_column_at( }) } -/// Append a null `_ROW_ID` column for files without `first_row_id`. -pub(super) fn append_null_row_id_column( - batch: RecordBatch, - insert_index: usize, - output_schema: &Arc, -) -> crate::Result { - let array: Arc = Arc::new(Int64Array::new_null(batch.num_rows())); - insert_column_at(batch, array, insert_index, output_schema) -} - #[cfg(test)] mod row_tests { use super::*; @@ -2799,6 +2746,12 @@ mod tests { to_local_row_ranges(&[RowRange::new(103, 104)], 100, 6), vec![RowRange::new(3, 4)] ); + let all_ids = [RowRange::new(i64::MIN, i64::MAX)]; + assert_eq!( + to_local_row_ranges(&all_ids, i64::MAX - 5, 6), + vec![RowRange::new(0, 5)] + ); + assert!(to_local_row_ranges(&all_ids, 100, 0).is_empty()); } #[test] @@ -2830,6 +2783,7 @@ mod tests { let timed = TimedFileRead { inner: Box::new(reader), timing: Arc::new(DataFileReadTiming::default()), + path: "memory:/timed-file".into(), }; assert!(std::ptr::eq( @@ -2924,20 +2878,29 @@ mod tests { } #[test] - fn selected_row_ids_are_built_only_when_projected() { + fn projected_row_ids_follow_the_selection_one_batch_at_a_time() { let huge_selection = [RowRange::new(0, i64::from(i32::MAX))]; + assert!(selected_row_ids_for_read(false, Some(10), Some(&huge_selection)).is_none()); assert_eq!( - selected_row_ids_for_read(false, Some(10), Some(&huge_selection)), - None + selected_row_ids_for_read(true, Some(10), Some(&huge_selection)) + .unwrap() + .take(3) + .collect::>(), + [10, 11, 12] ); assert_eq!( selected_row_ids_for_read( true, Some(10), Some(&[RowRange::new(1, 2), RowRange::new(4, 4)]), - ), - Some(vec![11, 12, 14]) + ) + .unwrap() + .collect::>(), + [11, 12, 14] ); + let mut ids = selected_row_ids_for_read(true, Some(i64::MAX), None).unwrap(); + assert_eq!(row_id_column_for_batch(Some(&mut ids), 1).unwrap().len(), 1); + assert!(row_id_column_for_batch(Some(&mut ids), 1).is_err()); } #[test] diff --git a/crates/paimon/src/table/row_sidecar.rs b/crates/paimon/src/table/row_sidecar.rs index a337b9c9d..9f4bfd968 100644 --- a/crates/paimon/src/table/row_sidecar.rs +++ b/crates/paimon/src/table/row_sidecar.rs @@ -184,7 +184,7 @@ pub(super) fn propagate_blob_data(schema: &SchemaRef, batch: &RecordBatch) -> Sc /// Exactly one ROW extra file is required, as in Java. Do not guess between /// multiple sidecars or enable one merely because deletion vectors prune rows. -fn selected_sidecar<'a>( +pub(super) fn selected_sidecar<'a>( file: &'a DataFileMeta, local_ranges: Option<&[RowRange]>, options: &CoreOptions<'_>, diff --git a/crates/paimon/tests/data_evolution_bitmap_merge_test.rs b/crates/paimon/tests/data_evolution_bitmap_merge_test.rs index 1118cddd0..097f5fcf9 100644 --- a/crates/paimon/tests/data_evolution_bitmap_merge_test.rs +++ b/crates/paimon/tests/data_evolution_bitmap_merge_test.rs @@ -383,10 +383,10 @@ async fn missing_latest_bitmap_keeps_primary_and_does_not_reuse_stale_index() { let table = table("missing_latest_index").await; append(&table, -20, 20).await; let mut messages = append(&table, 0, 100).await; - let unindexed_writer = table.copy_with_options(HashMap::from([( - "file-index.bitmap.columns".into(), - "id,a".into(), - )])); + let unindexed_writer = table.copy_with_options(HashMap::from([ + ("file-index.bitmap.columns".into(), "id,a".into()), + ("parquet.page.row.count.limit".into(), "8".into()), + ])); let updated = (0..100) .map(|id| match id { 5 => 999, @@ -433,6 +433,29 @@ async fn missing_latest_bitmap_keeps_primary_and_does_not_reuse_stale_index() { ) .await; assert_payload(&batch, &[5, 42], false); + + let predicate = Predicate::and(vec![ + predicates + .is_in("id", [5, 42, 43, 55, 90].map(Datum::Int).to_vec()) + .unwrap(), + predicates + .between("b", Datum::Int(40), Datum::Int(60)) + .unwrap(), + ]); + for enabled in [true, false] { + let table = table.copy_with_options(HashMap::from([ + ( + "data-evolution.row-sidecar.max-selected-rows".into(), + "1".into(), + ), + ( + "data-evolution.merged-read.stats-pushdown.enabled".into(), + enabled.to_string(), + ), + ])); + let batch = query(&table, predicate.clone(), &["payload", "_ROW_ID"]).await; + assert_payload(&batch, &[43, 55], false); + } } #[tokio::test] diff --git a/docs/src/benchmark.md b/docs/src/benchmark.md index 59f64813a..51f921294 100644 --- a/docs/src/benchmark.md +++ b/docs/src/benchmark.md @@ -316,3 +316,34 @@ The optimization is selected automatically for FAST-mode global searches with disjoint index ranges and no raw sources or deletion exclusions. Other paths and boundary ties retain exhaustive evaluation. The benchmark uses an internal reference reader to compare against exhaustive candidate collection without adding a table option. + +## Data Evolution merged Parquet reads + +Run the selective and full-range filters over the same 262,144 logical rows +with merged-read statistics pushdown enabled and disabled. + +```bash +cargo +1.98.0 test --locked -p paimon --lib benchmark_merged_candidates -- --ignored --nocapture +``` + +The fixture uses in-memory files, different page and row-group boundaries for +the two current column providers, and a disabled shared metadata cache. The +benchmark counts rows returned by the Parquet readers to the merge and bytes +requested for column data. Total bytes also include the footer and index +prefetch. Scan times are medians of seven runs with alternating execution order. + +One run on an Apple M4 Pro with Rust 1.98.0 in the default test profile +produced the following results. The selective filter returned one row in both +modes. The full-range filter returned all 262,144 rows. + +| Filter | Pushdown | Rows returned to merge | Data bytes | Total bytes | Median scan time | +| --- | --- | ---: | ---: | ---: | ---: | +| Selective | Off | 524,288 | 2,342,012 | 3,390,588 | 61.8 ms | +| Selective | On | 128 | 756 | 1,049,332 | 13.0 ms | +| Full range | Off | 524,288 | 2,342,012 | 3,390,588 | 74.7 ms | +| Full range | On | 524,288 | 2,342,012 | 3,390,588 | 78.0 ms | + +Each current provider required one footer fetch per scan. The median full-range +scan took 4.4% longer with pushdown enabled in this run. The counting-I/O test +also checks that planning and decoding share a single index fetch when the +page index falls outside the 512 KiB footer prefetch. diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 0af935696..507c3fd9f 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -163,6 +163,34 @@ Parquet Page Index pruning is enabled by default. Set this may reduce request count while increasing bytes read. The option does not change Page Index generation. +### Merged Data Evolution reads + +On row-tracked Data Evolution append tables with `bucket=-1`, merged reads use +Parquet statistics by default when ordinary column files cover the same row-ID +range. The reader splits filters at `AND` and uses a file's statistics when +that file supplies every field in a condition. It uses field IDs and file +versions to find the current provider even after a rename, and can widen +integer bounds to the current type. It evaluates an `OR` or `NOT` spanning +files after the merge. + +Row-group bounds, NULL counts and page indexes narrow the candidate row IDs. +The reader intersects them with requested ranges and file-index hits, excludes +deleted rows, and reads the same positions from every contributing file. It +reuses Parquet metadata and applies the full filter to the reconstructed rows +before `LIMIT`, even when files have different page boundaries. + +If a page index is missing, its row positions are inconsistent, or its byte +ranges overlap or extend outside the column chunk, the reader can still prune +using row-group statistics. Before reading a narrowed, nonempty selection, it +checks contributing files' row counts against the manifest. A mismatch restores +the original read path. +Dedicated BLOB or vector files, nested-field evolution and ROW sidecar reads +follow their usual read paths. + +Set `data-evolution.merged-read.stats-pushdown.enabled=false` to disable this +optimization. For selective and full-range scan results, see the +[merged Parquet read benchmark](benchmark.md#data-evolution-merged-parquet-reads). + ## Mosaic File Format Mosaic data file reads and writes are available for ordinary Paimon tables. Set