diff --git a/.github/.rat-excludes b/.github/.rat-excludes index b4d9af5ce..2c4f7cfbd 100644 --- a/.github/.rat-excludes +++ b/.github/.rat-excludes @@ -9,11 +9,6 @@ asan_symbolize\.py fix_includes\.py lint_exclusions\.txt iwyu_tool\.py -arrow\.diff -jieba\.diff -orc\.diff -oss_sdk_v2\.diff -vortex\.diff licenses/* .codespell_ignore .gitignore diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 1149fed13..dffebe650 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -1502,7 +1502,7 @@ macro(build_vortex) # Vortex's C API can only read a whole file already in memory or a path it resolves itself, # neither of which can go through paimon's filesystem. `callback_io.rs` adds an entry point # backed by host callbacks; it is maintained in this repository and injected into the Vortex - # source tree, and the patch only declares the module. + # source tree. The patch declares the module. set(VORTEX_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/vortex.diff") set(VORTEX_CALLBACK_IO_SRC "${CMAKE_SOURCE_DIR}/crates/vortex_callback_io/callback_io.rs") diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index f42eea640..aeaf5d3fe 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index e7523add27..e079a1ad41 100644 --- a/cpp/cmake_modules/BuildUtils.cmake diff --git a/cmake_modules/jieba.diff b/cmake_modules/jieba.diff index d74c3f605..3f8f27781 100644 --- a/cmake_modules/jieba.diff +++ b/cmake_modules/jieba.diff @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/include/cppjieba/KeywordExtractor.hpp b/include/cppjieba/KeywordExtractor.hpp index 24b2c40..c7c6a94 100644 --- a/include/cppjieba/KeywordExtractor.hpp diff --git a/cmake_modules/orc.diff b/cmake_modules/orc.diff index 88742dbdc..ca10ce4ef 100644 --- a/cmake_modules/orc.diff +++ b/cmake_modules/orc.diff @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/c++/include/orc/MemoryPool.hh b/c++/include/orc/MemoryPool.hh index a914e5f26..efe1d4933 100644 --- a/c++/include/orc/MemoryPool.hh diff --git a/cmake_modules/oss_sdk_v2.diff b/cmake_modules/oss_sdk_v2.diff index 1418d92f6..8636895d7 100644 --- a/cmake_modules/oss_sdk_v2.diff +++ b/cmake_modules/oss_sdk_v2.diff @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/sdk/src/internal/ClientImplBase.cpp b/sdk/src/internal/ClientImplBase.cpp --- a/sdk/src/internal/ClientImplBase.cpp +++ b/sdk/src/internal/ClientImplBase.cpp diff --git a/cmake_modules/vortex.diff b/cmake_modules/vortex.diff index 26aca1037..5ac9f18a3 100644 --- a/cmake_modules/vortex.diff +++ b/cmake_modules/vortex.diff @@ -1,9 +1,500 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/vortex-ffi/src/lib.rs b/vortex-ffi/src/lib.rs +index f4ce314c07373fa5b89f9225da1e6162ce956443..af526729ba3672eb1d507e54ebf14c750c83f8e1 100644 --- a/vortex-ffi/src/lib.rs +++ b/vortex-ffi/src/lib.rs -@@ -8,4 +8,5 @@ +@@ -6,6 +6,7 @@ + //! Native interface to Vortex arrays, types, files and streams. + mod array; +mod callback_io; mod array_iterator; mod binary; mod data_source; +diff --git a/vortex-ffi/src/scan.rs b/vortex-ffi/src/scan.rs +index 921c4f506211b299751f38645fdbde6155c9ad86..8c4560d2d5c88b8479a4a8b47b692924860d415b 100644 +--- a/vortex-ffi/src/scan.rs ++++ b/vortex-ffi/src/scan.rs +@@ -25,8 +25,11 @@ use vortex::buffer::Buffer; + use vortex::error::VortexResult; + use vortex::error::vortex_bail; + use vortex::error::vortex_ensure; ++use vortex::expr::merge; ++use vortex::expr::pack; + use vortex::expr::root; + use vortex::io::runtime::BlockingRuntime; ++use vortex::layout::layouts::row_idx::row_idx; + use vortex::layout::scan::arrow::RecordBatchIteratorAdapter; + use vortex::scan::DataSourceScan; + use vortex::scan::Partition; +@@ -112,6 +115,9 @@ pub struct vx_scan_options { + pub limit: u64, + /// If true, return in storage order. + pub ordered: bool, ++ /// If true, append a `_row_idx` UInt64 column carrying each returned row's original physical ++ /// row index, stable across `filter` and `selection`. The host strips it before use. ++ pub include_row_idx: bool, + } + + #[repr(C)] +@@ -180,7 +186,14 @@ fn scan_request(opts: *const vx_scan_options) -> VortexResult { + let limit = (opts.limit != 0).then_some(opts.limit); + + Ok(ScanRequest { +- projection, ++ // paimon-cpp: when include_row_idx is set, append a hidden `_row_idx` column so the host can ++ // recover each surviving row's original physical index after `filter`/`selection` drop rows. ++ // row_idx() is substituted with the real index array during the scan; the host strips it. ++ projection: if opts.include_row_idx { ++ merge([projection, pack([("_row_idx", row_idx())], false.into())]) ++ } else { ++ projection ++ }, + filter, + row_range, + selection, +@@ -353,7 +366,9 @@ pub unsafe extern "C-unwind" fn vx_partition_scan_arrow( + let array_stream = partition.execute()?; + let dtype = array_stream.dtype(); + +- let schema = dtype.to_arrow_schema()?; ++ // paimon-cpp: export standard (non-view) string/binary types so the C++ reader skips its ++ // per-row view->standard rebuild; execute_arrow casts to this target internally. ++ let schema = crate::callback_io::normalize_arrow_schema_views(dtype.to_arrow_schema()?); + let schema = Arc::new(schema); + let target = Field::new_struct("", schema.fields().clone(), false); + +diff --git a/vortex-ffi/src/dtype.rs b/vortex-ffi/src/dtype.rs +index c74fa71b6edf43710d446a6e97a5ca936428ad67..d06a40f63b6c53fd359c0eecfeae9cd352c60bd7 100644 +--- a/vortex-ffi/src/dtype.rs ++++ b/vortex-ffi/src/dtype.rs +@@ -344,7 +344,10 @@ pub unsafe extern "C-unwind" fn vx_dtype_to_arrow_schema( + ) -> c_int { + try_or(err, 1, || { + let dtype = vx_dtype::as_ref(dtype); +- let arrow_schema = dtype.to_arrow_schema()?; ++ // paimon-cpp: report standard (non-view) string/binary types so the exported schema matches ++ // the data that vx_partition_scan_arrow now produces (see callback_io normalize). ++ let arrow_schema = ++ crate::callback_io::normalize_arrow_schema_views(dtype.to_arrow_schema()?); + let arrow_schema = FFI_ArrowSchema::try_from(&arrow_schema)?; + unsafe { ptr::write(schema, arrow_schema) }; + Ok(0) +diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h +index 26c7b409cd982abce17fbb41a131814b7613018c..75da718b9ae99bdf4420893374af862ddb501782 100644 +--- a/vortex-ffi/cinclude/vortex.h ++++ b/vortex-ffi/cinclude/vortex.h +@@ -622,6 +622,11 @@ typedef struct { + * If true, return in storage order. + */ + bool ordered; ++ /** ++ * If true, append a `_row_idx` UInt64 column carrying each returned row's original physical ++ * row index, stable across filter and selection. The host strips it before use. ++ */ ++ bool include_row_idx; + } vx_scan_options; + + #ifdef __cplusplus +diff --git a/vortex-ffi/Cargo.toml b/vortex-ffi/Cargo.toml +index 6407f97fe46303f6dc9bd6b7f1f0f44392f5e534..80b62d2b5b8446bbac6379ac2384fd23c216e241 100644 +--- a/vortex-ffi/Cargo.toml ++++ b/vortex-ffi/Cargo.toml +@@ -25,6 +25,7 @@ arrow-schema = { workspace = true } + async-fs = { workspace = true } + futures = { workspace = true } + itertools = { workspace = true } ++jiff = { workspace = true } + mimalloc = { workspace = true, optional = true } + object_store = { workspace = true, features = ["aws", "azure", "gcp"] } + paste = { workspace = true } +diff --git a/Cargo.lock b/Cargo.lock +index fd66ecc567062fe122290bdb541bb3fa4420babe..55a0bdc4ad1c9ccaf85d19946f01465c25ef76a8 100644 +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -9747,6 +9747,7 @@ dependencies = [ + "cbindgen", + "futures", + "itertools 0.14.0", ++ "jiff", + "mimalloc", + "object_store 0.13.2", + "paste", +diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs +index d6070ac1a4db34380f8aca8b463e8d513f5d1757..72159ee38465ede1d09489c0074d666908f0e4d5 100644 +--- a/vortex-array/src/executor.rs ++++ b/vortex-array/src/executor.rs +@@ -26,7 +26,6 @@ use vortex_error::VortexResult; + use vortex_error::vortex_bail; + use vortex_error::vortex_ensure; + use vortex_error::vortex_panic; +-use vortex_session::Ref; + use vortex_session::SessionExt; + use vortex_session::VortexSession; + +@@ -424,14 +423,9 @@ impl Executable for ArrayRef { + } + } + +- let tmp_session = ctx.session().clone(); +- let kernels = tmp_session.get_opt::(); +- + for (slot_idx, slot) in array.slots().iter().enumerate() { + let Some(child) = slot else { continue }; +- if let Some(executed_parent) = +- execute_parent_for_child(&array, child, slot_idx, kernels.as_ref(), ctx)? +- { ++ if let Some(executed_parent) = execute_parent_for_child(&array, child, slot_idx, ctx)? { + ctx.log(format_args!( + "execute_parent: slot[{}]({}) rewrote {} -> {}", + slot_idx, +@@ -542,13 +536,15 @@ fn execute_parent_for_child( + parent: &ArrayRef, + child: &ArrayRef, + slot_idx: usize, +- kernels: Option<&Ref>, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { +- if let Some(kernels) = kernels +- && let Some(plugins) = +- kernels.find_execute_parent(parent.encoding_id(), child.encoding_id()) +- { ++ // Session refs hold a DashMap shard lock. Release it before invoking kernels or encodings, ++ // which may lazily initialize another session variable on the same shard. ++ let plugins = ctx ++ .session() ++ .get_opt::() ++ .and_then(|kernels| kernels.find_execute_parent(parent.encoding_id(), child.encoding_id())); ++ if let Some(plugins) = plugins { + for plugin in plugins.as_ref() { + if let Some(result) = plugin(child, parent, slot_idx, ctx)? { + return Ok(Some(result)); +@@ -561,14 +557,9 @@ fn execute_parent_for_child( + + /// Try execute_parent on each occupied slot of the array. + fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult> { +- let tmp_session = ctx.session().clone(); +- let kernels = tmp_session.get_opt::(); +- + for (slot_idx, slot) in array.slots().iter().enumerate() { + let Some(child) = slot else { continue }; +- if let Some(executed_parent) = +- execute_parent_for_child(array, child, slot_idx, kernels.as_ref(), ctx)? +- { ++ if let Some(executed_parent) = execute_parent_for_child(array, child, slot_idx, ctx)? { + ctx.log(format_args!( + "execute_parent: slot[{}]({}) rewrote {} -> {}", + slot_idx, +diff --git a/vortex-buffer/src/trusted_len.rs b/vortex-buffer/src/trusted_len.rs +index 13cf25d0546428dfc719e077b696d352242a0b9e..f8ce4fc04d68085c5aba75494e98a447f13826b4 100644 +--- a/vortex-buffer/src/trusted_len.rs ++++ b/vortex-buffer/src/trusted_len.rs +@@ -32,6 +32,11 @@ impl Iterator for TrustedLenAdapter { + + #[inline] + fn next(&mut self) -> Option { ++ #[cfg(not(debug_assertions))] ++ { ++ self.inner.next() ++ } ++ #[cfg(debug_assertions)] + match self.inner.next() { + None => { + #[cfg(debug_assertions)] +diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs +index e8bf95e14cdd3faed26d9fcc464da8276b523001..a002d7926eded2941863f94968329345e9d075eb 100644 +--- a/vortex-array/src/arrow/executor/list.rs ++++ b/vortex-array/src/arrow/executor/list.rs +@@ -12,26 +12,33 @@ use vortex_buffer::BufferMut; + use vortex_error::VortexExpect; + use vortex_error::VortexResult; + use vortex_error::vortex_ensure; ++use vortex_error::vortex_err; + + use crate::ArrayRef; + use crate::Canonical; + use crate::ExecutionCtx; ++use crate::IntoArray; + use crate::arrays::Chunked; ++use crate::arrays::ChunkedArray; + use crate::arrays::List; + use crate::arrays::ListArray; + use crate::arrays::ListView; + use crate::arrays::ListViewArray; ++use crate::arrays::PrimitiveArray; + use crate::arrays::chunked::ChunkedArrayExt; + use crate::arrays::list::ListArrayExt; + use crate::arrays::listview::ListViewArrayExt; + use crate::arrays::listview::ListViewDataParts; + use crate::arrays::listview::ListViewRebuildMode; ++use crate::arrays::listview::list_view_from_list; + use crate::arrow::executor::validity::to_arrow_null_buffer; + use crate::arrow::session::ArrowSessionExt; + use crate::builtins::ArrayBuiltins; + use crate::dtype::DType; + use crate::dtype::NativePType; + use crate::dtype::Nullability; ++use crate::dtype::PType; ++use crate::validity::Validity; + + #[allow(rustdoc::broken_intra_doc_links)] + /// Convert a Vortex VarBinArray into an Arrow [`GenericListArray`](arrow_array:array::GenericListArray). +@@ -42,7 +49,15 @@ pub(super) fn to_arrow_list( + ) -> VortexResult { + // If the Vortex array is already in List format, we can directly convert it. + if let Some(array) = array.as_opt::() { +- return list_to_list::(&array.into_owned(), elements_field, ctx); ++ let array = array.into_owned(); ++ // NULL list slots may own non-empty child ranges. They are not logically referenced ++ // and must not be expanded even when they lie between two valid lists. ++ let array = if array.validity()?.execute_no_nulls(array.len(), ctx)? { ++ array ++ } else { ++ compact_list_view_elements(&list_view_from_list(array, ctx)?, ctx)? ++ }; ++ return list_to_list::(&array, elements_field, ctx); + } + + // Converting each chunk individually, then using the fast concat logic from arrow +@@ -56,30 +71,16 @@ pub(super) fn to_arrow_list( + return Ok(arrow_select::concat::concat(&refs)?); + } + +- // If the Vortex array is a ListViewArray, rebuild to ZCTL if needed and convert. ++ // If the Vortex array is a ListViewArray, select only its referenced elements. + let array = match array.try_downcast::() { +- Ok(array) => { +- let zctl = if array.is_zero_copy_to_list() { +- array +- } else { +- array.rebuild(ListViewRebuildMode::MakeZeroCopyToList, ctx)? +- }; +- return list_view_zctl::(zctl, elements_field, ctx); +- } ++ Ok(array) => return list_view_to_list::(array, elements_field, ctx), + Err(a) => a, + }; + +- // Otherwise, we execute the array to become a ListViewArray, then rebuild to ZCTL. +- // Note: arrow_cast::cast supports ListView → List (apache/arrow-rs#8735), but it +- // unconditionally uses take. Our rebuild uses a heuristic that picks list-by-list +- // for large lists, which avoids materializing a large index buffer. ++ // Otherwise, execute to ListView but keep shared children lazy until the referenced ranges ++ // have been selected. Rebuilding non-ZCTL views first can canonicalize unreferenced children. + let list_view = array.execute::(ctx)?; +- let zctl = if list_view.is_zero_copy_to_list() { +- list_view +- } else { +- list_view.rebuild(ListViewRebuildMode::MakeZeroCopyToList, ctx)? +- }; +- list_view_zctl::(zctl, elements_field, ctx) ++ list_view_to_list::(list_view, elements_field, ctx) + } + + #[allow(rustdoc::broken_intra_doc_links)] +@@ -89,6 +90,11 @@ fn list_to_list( + elements_field: &FieldRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult { ++ // Slicing a Vortex List retains its complete elements array. Trim before exporting children ++ // so a bounded parent batch cannot materialize unreferenced string/binary values. Resetting ++ // before narrowing offsets also avoids overflow from a large unused leading prefix. ++ let array = array.reset_offsets(false, ctx)?; ++ + // We must cast the offsets to the required offset type. + let offsets = array + .offsets() +@@ -119,16 +125,22 @@ fn list_to_list( + ))) + } + +-fn list_view_zctl( ++fn list_view_to_list( + array: ListViewArray, + elements_field: &FieldRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult { +- assert!(array.is_zero_copy_to_list()); ++ if !array.is_zero_copy_to_list() || !array.validity()?.execute_no_nulls(array.len(), ctx)? { ++ return list_to_list::( ++ &compact_list_view_elements(&array, ctx)?, ++ elements_field, ++ ctx, ++ ); ++ } + + if array.is_empty() { + let elements = ctx.session().clone().arrow().execute_arrow( +- array.elements().clone(), ++ array.elements().slice(0..0)?, + Some(elements_field.as_ref()), + ctx, + )?; +@@ -140,6 +152,10 @@ fn list_view_zctl( + ))); + } + ++ // ZCTL views have contiguous referenced elements, but a sliced view can still retain the ++ // complete native chunk. Trim and rebase before converting children or narrowing offsets. ++ let array = array.rebuild(ListViewRebuildMode::TrimElements, ctx)?; ++ + let ListViewDataParts { + elements, + offsets, +@@ -200,6 +216,59 @@ fn list_view_zctl( + ))) + } + ++// Assemble only referenced slices in output order. In particular, a take of arbitrary encoded ++// children may execute the entire values array before selecting indices; a ListView rebuild may ++// do the same. Keeping the ranges as chunks lets recursive Arrow export materialize each slice. ++fn compact_list_view_elements( ++ array: &ListViewArray, ++ ctx: &mut ExecutionCtx, ++) -> VortexResult { ++ let validity = array.validity()?; ++ let valid = validity.execute_mask(array.len(), ctx)?; ++ let offsets = array ++ .offsets() ++ .cast(DType::Primitive(PType::U64, Nullability::NonNullable))? ++ .execute::(ctx)? ++ .to_buffer::(); ++ let sizes = array ++ .sizes() ++ .cast(DType::Primitive(PType::U64, Nullability::NonNullable))? ++ .execute::(ctx)? ++ .to_buffer::(); ++ ++ let mut chunks = Vec::with_capacity(array.len()); ++ let mut list_offsets = BufferMut::::with_capacity(array.len() + 1); ++ let mut n_elements = 0u64; ++ list_offsets.push(n_elements); ++ for index in 0..array.len() { ++ if valid.value(index) && sizes[index] != 0 { ++ let start = usize::try_from(offsets[index]) ++ .map_err(|error| vortex_err!("List offset out of range: {error}"))?; ++ let size = usize::try_from(sizes[index]) ++ .map_err(|error| vortex_err!("List size out of range: {error}"))?; ++ let end = start ++ .checked_add(size) ++ .ok_or_else(|| vortex_err!("List element range overflow"))?; ++ n_elements = n_elements ++ .checked_add(sizes[index]) ++ .ok_or_else(|| vortex_err!("List output element count overflow"))?; ++ chunks.push(array.elements().slice(start..end)?); ++ } ++ list_offsets.push(n_elements); ++ } ++ ++ let elements = if chunks.is_empty() { ++ Canonical::empty(array.elements().dtype()).into_array() ++ } else { ++ ChunkedArray::try_new(chunks, array.elements().dtype().clone())?.into_array() ++ }; ++ ListArray::try_new( ++ elements, ++ PrimitiveArray::new(list_offsets.freeze(), Validity::NonNullable).into_array(), ++ validity, ++ ) ++} ++ + #[cfg(test)] + mod tests { + use std::sync::Arc; +diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs +index 002b4b1e9029a2fa0cf2b7044fe7ee2db062bb82..e9cd0e843443951a8022223db86c5f8935cae496 100644 +--- a/vortex-layout/src/layouts/dict/reader.rs ++++ b/vortex-layout/src/layouts/dict/reader.rs +@@ -45,8 +45,8 @@ pub struct DictReader { + values_len: usize, + /// Cached dict values array + values_array: OnceLock, +- /// Cache of expression evaluation results on the values array by expression +- values_evals: DashMap, ++ /// Expression results are only cached for filtering, not payload projection. ++ values_evals: OnceLock>, + + values: LayoutReaderRef, + codes: LayoutReaderRef, +@@ -135,12 +135,13 @@ impl DictReader { + // shouldn't. + // TODO(joe): fixme + ++ let values_evals = self.values_evals.get_or_init(DashMap::default); + // Check cache first with read-only lock +- if let Some(fut) = self.values_evals.get(&expr) { ++ if let Some(fut) = values_evals.get(&expr) { + return fut.clone(); + } + +- self.values_evals ++ values_evals + .entry(expr.clone()) + .or_insert_with(|| { + self.values_array_uncanonical() +diff --git a/vortex-ffi/src/session.rs b/vortex-ffi/src/session.rs +index 2af3d99744475fcbe94b060f3700cb75c417c946..305262ec98fb4f87677bcc68eb2dcc23cf74ac52 100644 +--- a/vortex-ffi/src/session.rs ++++ b/vortex-ffi/src/session.rs +@@ -1,7 +1,10 @@ + // SPDX-License-Identifier: Apache-2.0 + // SPDX-FileCopyrightText: Copyright the Vortex contributors + ++use std::sync::LazyLock; ++ + use vortex::VortexSessionDefault; ++use vortex::array::arrow::ArrowSessionExt; + use vortex::error::VortexResult; + use vortex::error::vortex_ensure; + use vortex::io::runtime::BlockingRuntime; +@@ -17,6 +20,34 @@ box_wrapper!( + vx_session + ); + ++// This session is private to the Paimon callback reader. Public/configurable ++// sessions must remain fresh: VortexSession::clone shares mutable SessionVars. ++// Single-file callback opens keep file metadata, caches and execution contexts ++// outside this default session. Initialize the only lazy read-path registry ++// before publishing it, and never configure/register against these handles. ++static PAIMON_READ_SESSION: LazyLock = LazyLock::new(|| { ++ let session = VortexSession::default().with_handle(RUNTIME.handle()); ++ drop(session.arrow()); ++ session ++}); ++ ++/// Create an owned handle to Paimon's shared default callback-reader session. ++/// ++/// This private integration entry point does not change the isolation of ++/// [`vx_session_new`] or [`vx_session_new_with`]. Free the owned handle with ++/// [`vx_session_free`]; other reader handles remain valid. ++/// ++/// # Safety ++/// ++/// The handle is only for Paimon's fixed-runtime, single-file callback read and ++/// Arrow export paths. The caller must not configure it, register plugins, change ++/// session variables/allocators/runtime, or pass it to a writer or multi-file ++/// source. Those operations require a fresh [`vx_session_new`] session. ++#[unsafe(no_mangle)] ++pub unsafe extern "C-unwind" fn vx_paimon_read_session_new() -> *mut vx_session { ++ vx_session::new(PAIMON_READ_SESSION.clone()) ++} ++ + /// Create an FFI session from a configured default session. + pub fn vx_session_new_with( + configure: impl FnOnce(VortexSession) -> VortexSession, diff --git a/crates/vortex_callback_io/callback_io.rs b/crates/vortex_callback_io/callback_io.rs index d3d2faadc..3c824cb0f 100644 --- a/crates/vortex_callback_io/callback_io.rs +++ b/crates/vortex_callback_io/callback_io.rs @@ -14,6 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(non_camel_case_types)] + //! Callback I/O for Vortex using Paimon's `InputStream` and `OutputStream`. //! //! Copied into `vortex-ffi/src/` at build time; see `cmake_modules/vortex.diff`. @@ -22,10 +24,19 @@ //! Sink entry points are adapted from Vortex's `vortex-ffi/src/sink.rs` //! (SPDX-License-Identifier: Apache-2.0, Copyright the Vortex contributors). +use std::ffi::CStr; +use std::ffi::c_char; +use std::ffi::c_int; use std::ffi::c_void; use std::io; +use std::ptr::write; use std::sync::Arc; +use arrow_array::RecordBatch; +use arrow_array::cast::AsArray; +use arrow_array::ffi_stream::FFI_ArrowArrayStream; +use arrow_schema::ArrowError; +use arrow_schema::Field; use futures::FutureExt; use futures::SinkExt; use futures::TryStreamExt; @@ -33,15 +44,22 @@ use futures::channel::mpsc; use futures::channel::mpsc::Sender; use futures::future::BoxFuture; use vortex::array::ArrayRef; +use vortex::array::VortexSessionExecute; +use vortex::array::arrow::ArrowSessionExt; use vortex::array::buffer::BufferHandle; use vortex::array::stream::ArrayStreamAdapter; use vortex::buffer::Alignment; use vortex::buffer::ByteBufferMut; use vortex::dtype::DType; +use vortex::dtype::Nullability; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_ensure; use vortex::error::vortex_err; +use vortex::extension::datetime::Date; +use vortex::extension::datetime::TemporalMetadata; +use vortex::extension::datetime::TimeUnit; +use vortex::extension::datetime::Timestamp; use vortex::file::OpenOptionsSessionExt; use vortex::file::WriteOptionsSessionExt; use vortex::file::WriteStrategyBuilder; @@ -54,6 +72,10 @@ use vortex::io::runtime::BlockingRuntime; use vortex::io::runtime::Handle; use vortex::io::runtime::Task; use vortex::io::session::RuntimeSessionExt; +use vortex::layout::scan::arrow::RecordBatchIteratorAdapter; +use vortex::scalar::PValue; +use vortex::scalar::Scalar; +use vortex::scalar::ScalarValue; use crate::RUNTIME; use crate::array::vx_array; @@ -62,6 +84,9 @@ use crate::dtype::vx_dtype; use crate::error::try_or; use crate::error::try_or_default; use crate::error::vx_error; +use crate::scalar::vx_scalar; +use crate::scan::VxPartitionScan; +use crate::scan::vx_partition; use crate::session::vx_session; /// Fill `length` bytes starting at `offset` into `dst`. @@ -143,6 +168,7 @@ struct CallbackReadAt { len: u64, handle: Handle, concurrency: usize, + coalesce_config: CoalesceConfig, } /// Maximum concurrent host reads per file, also bounding blocking-pool usage. @@ -151,7 +177,7 @@ const DEFAULT_CONCURRENCY: usize = 16; impl VortexReadAt for CallbackReadAt { fn coalesce_config(&self) -> Option { // Coalesce reads to reduce remote I/O and blocking-pool overhead. - Some(CoalesceConfig::object_storage()) + Some(self.coalesce_config) } fn concurrency(&self) -> usize { @@ -218,6 +244,7 @@ unsafe fn data_source_new_callback( session: *const vx_session, callbacks: vx_input_callbacks, size: u64, + coalesce_config: CoalesceConfig, ) -> VortexResult<*const vx_data_source> { // Take ownership before validation so errors also release the context. let target = Arc::new(CallbackTarget { @@ -226,6 +253,19 @@ unsafe fn data_source_new_callback( }); vortex_ensure!(!session.is_null()); + vortex_ensure!( + coalesce_config.max_size > 0, + "coalesced max_size must be positive" + ); + // Vortex adds the distance to file offsets, which are bounded by i64::MAX in Paimon. + vortex_ensure!( + coalesce_config.max_size <= i64::MAX as u64, + "coalesced max_size must not exceed INT64_MAX" + ); + vortex_ensure!( + coalesce_config.distance <= coalesce_config.max_size, + "coalesced max_gap must not exceed max_size" + ); let read_at_fn = callbacks .read_at_fn .ok_or_else(|| vortex_err!("vx_input_callbacks.read_at_fn is required"))?; @@ -237,32 +277,43 @@ unsafe fn data_source_new_callback( len: size, handle: session.handle(), concurrency: DEFAULT_CONCURRENCY, + coalesce_config, }); let file = RUNTIME.block_on(async { session.open_options().open(reader).await })?; Ok(vx_data_source::new(file.data_source()?)) } -/// Create a data source that reads through host callbacks. +/// Create a data source that reads through host callbacks with explicit I/O coalescing. /// -/// `size` is the total file length in bytes. Takes ownership of the host context, -/// which is released when the data source is freed or this call fails. +/// `size` is the total file length in bytes. `max_size` must be in `1..=i64::MAX`, and `max_gap` +/// must not exceed it. A smaller gap avoids reading unrelated bytes between sparse requests. /// -/// On error, returns NULL and sets "err". +/// The callbacks and their context are owned by the returned data source and released when +/// it is freed. They are consumed and released even when this call fails, including invalid +/// coalescing options. On error, returns NULL and sets `err`. /// /// # Safety /// /// `session` must be a valid `vx_session`. `callbacks.read_at_fn` must be non-null, thread-safe, -/// and valid for as long as the data source lives. +/// and valid for as long as the data source lives. `callbacks.ctx` must be valid for the supplied +/// callbacks until `release_fn` is called. `err` must be null or point to a null `*mut vx_error`. #[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_data_source_new_callback( +pub unsafe extern "C-unwind" fn vx_data_source_new_callback_with_coalesce( session: *const vx_session, callbacks: vx_input_callbacks, size: u64, + max_gap: u64, + max_size: u64, err: *mut *mut vx_error, ) -> *const vx_data_source { try_or(err, std::ptr::null(), || unsafe { - data_source_new_callback(session, callbacks, size) + data_source_new_callback( + session, + callbacks, + size, + CoalesceConfig::new(max_gap, max_size), + ) }) } @@ -474,6 +525,273 @@ pub unsafe extern "C-unwind" fn vx_callback_sink_abort(sink: *mut vx_callback_si drop(unsafe { Box::from_raw(sink) }); } +/// Retain the compressed input chunk until its row slices have been consumed. In particular, +/// never materialize all of a dictionary's repeated values before applying the host's row limit. +struct BoundedArrayIterator { + input: I, + current: Option, + offset: usize, + batch_size: usize, + finished: bool, +} + +impl>> BoundedArrayIterator { + fn new(input: I, batch_size: usize) -> VortexResult { + vortex_ensure!(batch_size > 0, "Vortex Arrow batch size must be positive"); + Ok(Self { + input, + current: None, + offset: 0, + batch_size, + finished: false, + }) + } +} + +impl>> Iterator for BoundedArrayIterator { + type Item = VortexResult; + + fn next(&mut self) -> Option { + if self.finished { + return None; + } + loop { + if let Some(chunk) = &self.current { + // Subtract first so even a caller-provided usize::MAX row limit cannot overflow. + let length = self.batch_size.min(chunk.len() - self.offset); + let end = self.offset + length; + let sliced = chunk.slice(self.offset..end); + self.offset = end; + if end == chunk.len() || sliced.is_err() { + self.current = None; + self.offset = 0; + } + if sliced.is_err() { + self.finished = true; + } + return Some(sliced); + } + match self.input.next() { + Some(Ok(chunk)) if chunk.is_empty() => continue, + Some(Ok(chunk)) => self.current = Some(chunk), + Some(Err(error)) => { + self.finished = true; + return Some(Err(error)); + } + None => { + self.finished = true; + return None; + } + } + } + } +} + +/// Scan a partition to standard Arrow arrays, slicing compressed chunks before conversion. +/// +/// Each exported batch contains at most `batch_size` rows. Conversion is lazy: requesting the +/// next Arrow batch converts only its slice, not every slice of the current native chunk. +/// This is a row limit, not a byte limit: each batch must still fit Arrow's Utf8/Binary offsets. +/// +/// A non-null `partition` is consumed on success and failure, including a zero batch size. The +/// caller must release a successful `stream` through the Arrow C stream release callback. +/// +/// # Safety +/// +/// `partition` must be null or a valid, unconsumed partition handle. `session` must be null or +/// a valid session handle. `stream` must be null or point to writable, uninitialized/released storage. +/// On failure the stream storage is unchanged; error ownership follows the other Vortex FFI calls. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_partition_scan_arrow_with_batch_size( + session: *const vx_session, + partition: *mut vx_partition, + batch_size: usize, + stream: *mut FFI_ArrowArrayStream, + err: *mut *mut vx_error, +) -> c_int { + try_or(err, 1, || { + vortex_ensure!(!partition.is_null(), "Vortex partition is null"); + // Take ownership before validating the other arguments, so callers never need to guess + // whether an invalid request has already consumed the partition. + let partition = match *vx_partition::into_box(partition) { + VxPartitionScan::Pending(partition) => partition, + _ => vortex_bail!("Vortex partition is already being consumed"), + }; + vortex_ensure!(batch_size > 0, "Vortex Arrow batch size must be positive"); + vortex_ensure!(!session.is_null(), "Vortex session is null"); + vortex_ensure!(!stream.is_null(), "Vortex Arrow stream output is null"); + let array_stream = partition.execute()?; + let schema = Arc::new(normalize_arrow_schema_views( + array_stream.dtype().to_arrow_schema()?, + )); + let target = Field::new_struct("", schema.fields().clone(), false); + let session = vx_session::as_ref(session).clone(); + let mut chunks = + BoundedArrayIterator::new(RUNTIME.block_on_stream(array_stream), batch_size)?; + let mut failed = false; + let iter = std::iter::from_fn(move || { + if failed { + return None; + } + let chunk = chunks.next()?; + let result = (|| -> VortexResult { + let mut ctx = session.create_execution_ctx(); + let arrow = session + .arrow() + .execute_arrow(chunk?, Some(&target), &mut ctx)?; + Ok(RecordBatch::from(arrow.as_struct().clone())) + })(); + failed = result.is_err(); + Some(result.map_err(|error| ArrowError::ExternalError(Box::new(error)))) + }); + let reader = RecordBatchIteratorAdapter::new(iter, schema); + unsafe { write(stream, FFI_ArrowArrayStream::new(Box::new(reader))) }; + Ok(0) + }) +} + +/// Export standard Arrow string/binary types recursively so the declared schema matches the +/// buffer layout expected by Paimon's C++ reader. +pub(crate) fn normalize_arrow_schema_views(schema: arrow_schema::Schema) -> arrow_schema::Schema { + let fields = schema + .fields() + .iter() + .cloned() + .map(normalize_field_views) + .collect::(); + arrow_schema::Schema::new_with_metadata(fields, schema.metadata) +} + +fn normalize_field_views(field: arrow_schema::FieldRef) -> arrow_schema::FieldRef { + Arc::new( + field + .as_ref() + .clone() + .with_data_type(normalize_dtype_views(field.data_type())), + ) +} + +fn normalize_dtype_views(dtype: &arrow_schema::DataType) -> arrow_schema::DataType { + use arrow_schema::DataType; + match dtype { + DataType::Utf8View => DataType::Utf8, + DataType::BinaryView => DataType::Binary, + DataType::Struct(fields) => { + DataType::Struct(fields.iter().cloned().map(normalize_field_views).collect()) + } + DataType::List(field) => DataType::List(normalize_field_views(Arc::clone(field))), + DataType::LargeList(field) => DataType::LargeList(normalize_field_views(Arc::clone(field))), + DataType::FixedSizeList(field, size) => { + DataType::FixedSizeList(normalize_field_views(Arc::clone(field)), *size) + } + other => other.clone(), + } +} + +/// Time unit of a timestamp scalar literal. Mirrored in `vortex_ffi.h`; the discriminants are an +/// ABI contract, so do not reorder or renumber them. +#[repr(C)] +#[allow(non_camel_case_types, dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum vx_time_unit { + /// Seconds since the Unix epoch. + VX_TIME_UNIT_SECOND = 0, + /// Milliseconds since the Unix epoch. + VX_TIME_UNIT_MILLISECOND = 1, + /// Microseconds since the Unix epoch. + VX_TIME_UNIT_MICROSECOND = 2, + /// Nanoseconds since the Unix epoch. + VX_TIME_UNIT_NANOSECOND = 3, +} + +/// Create a timestamp scalar literal: `value` units since the Unix epoch under `unit`. +/// +/// `timezone` is a null-terminated UTF-8 IANA time-zone name, or null for a local (wall-clock) +/// timestamp. Vortex's own C ABI has no timestamp/date scalar constructor, so paimon-cpp adds these +/// two here to push down temporal predicates. Returns null without an error when a valid value +/// exceeds the native temporal range; other failures return null and populate `err`. +/// +/// # Safety +/// +/// `timezone`, when not null, must point to a valid null-terminated UTF-8 string that outlives this +/// call. `err` must be null or point to a null `*mut vx_error`. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_scalar_new_timestamp( + value: i64, + unit: vx_time_unit, + timezone: *const c_char, + is_nullable: bool, + err: *mut *mut vx_error, +) -> *mut vx_scalar { + try_or(err, std::ptr::null_mut(), || { + let time_unit = match unit { + vx_time_unit::VX_TIME_UNIT_SECOND => TimeUnit::Seconds, + vx_time_unit::VX_TIME_UNIT_MILLISECOND => TimeUnit::Milliseconds, + vx_time_unit::VX_TIME_UNIT_MICROSECOND => TimeUnit::Microseconds, + vx_time_unit::VX_TIME_UNIT_NANOSECOND => TimeUnit::Nanoseconds, + }; + let tz = if timezone.is_null() { + None + } else { + let name = unsafe { CStr::from_ptr(timezone) } + .to_str() + .map_err(|e| vortex_err!("timezone is not valid utf8: {e}"))?; + Some(Arc::from(name)) + }; + // Check timezone errors independently of the value, including values outside the native + // range. A malformed timezone must not be mistaken for an unsupported constant. + if tz.is_some() { + TemporalMetadata::Timestamp(&TimeUnit::Seconds, &tz).to_jiff(0)?; + } + // Timestamp::unpack_native uses Jiff Timestamp even without a timezone. Check both its + // Span setters and epoch range without introducing a timezone-database lookup. + let Ok(span) = time_unit.to_jiff_span(value) else { + return Ok(std::ptr::null_mut()); + }; + if jiff::Timestamp::UNIX_EPOCH.checked_add(span).is_err() { + return Ok(std::ptr::null_mut()); + } + let dtype = DType::Extension( + Timestamp::new_with_tz(time_unit, tz, Nullability::from(is_nullable)).erased(), + ); + Ok(vx_scalar::new(Scalar::try_new( + dtype, + Some(ScalarValue::Primitive(PValue::I64(value))), + )?)) + }) +} + +/// Create a date scalar literal: `days` days since the Unix epoch (1970-01-01). +/// Returns null without an error when the value exceeds the native civil-date range; other +/// failures return null and populate `err`. +/// +/// # Safety +/// +/// `err` must be null or point to a null `*mut vx_error`. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_scalar_new_date( + days: i32, + is_nullable: bool, + err: *mut *mut vx_error, +) -> *mut vx_scalar { + try_or(err, std::ptr::null_mut(), || { + // The fixed Days unit and epoch are valid; only a native range limit can fail here. + // Date's storage accepts all i32 values, but formatting needs a representable civil date. + if TemporalMetadata::Date(&TimeUnit::Days) + .to_jiff(i64::from(days)) + .is_err() + { + return Ok(std::ptr::null_mut()); + } + let dtype = + DType::Extension(Date::new(TimeUnit::Days, Nullability::from(is_nullable)).erased()); + Ok(vx_scalar::new(Scalar::try_new( + dtype, + Some(ScalarValue::Primitive(PValue::I32(days))), + )?)) + }) +} + #[cfg(test)] #[cfg(unix)] mod tests { @@ -518,7 +836,7 @@ mod tests { dst: *mut u8, length: usize, ) -> i32 { - // SAFETY: `ctx` is the `FileContext` handed to `vx_data_source_new_callback`, and `dst` + // SAFETY: `ctx` is the `FileContext` handed to the callback data source, and `dst` // covers `length` bytes, per the callback contract. let context = unsafe { &*ctx.cast::() }; let buffer = unsafe { std::slice::from_raw_parts_mut(dst, length) }; @@ -544,16 +862,12 @@ mod tests { unsafe { let session = vx_session_new(); - let vx_dtype_ptr = vx_dtype::new(Arc::new(dtype.clone())); + let vx_dtype_ptr = vx_dtype::new(Arc::new(dtype)); let mut error = ptr::null_mut(); let c_path = std::ffi::CString::new(path.clone()).unwrap(); - let sink = vx_array_sink_open_file( - session, - c_path.as_ptr(), - vx_dtype_ptr, - &raw mut error, - ); + let sink = + vx_array_sink_open_file(session, c_path.as_ptr(), vx_dtype_ptr, &raw mut error); assert!(error.is_null()); let array = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); let vx_array_ptr = vx_array::new(Arc::new(array.into_array())); @@ -573,7 +887,15 @@ mod tests { release_fn: Some(file_release), }; - let data_source = vx_data_source_new_callback(session, callbacks, size, &raw mut error); + let config = CoalesceConfig::object_storage(); + let data_source = vx_data_source_new_callback_with_coalesce( + session, + callbacks, + size, + config.distance, + config.max_size, + &raw mut error, + ); assert!(error.is_null()); assert!(!data_source.is_null()); @@ -605,7 +927,15 @@ mod tests { release_fn: Some(mark_released), }; - let data_source = vx_data_source_new_callback(session, callbacks, 0, &raw mut error); + let config = CoalesceConfig::object_storage(); + let data_source = vx_data_source_new_callback_with_coalesce( + session, + callbacks, + 0, + config.distance, + config.max_size, + &raw mut error, + ); assert!(data_source.is_null()); assert!(!error.is_null()); vx_error_free(error); @@ -644,7 +974,9 @@ mod tests { // SAFETY: as above; `dst` covers `length` writable bytes. let holder = unsafe { &*ctx.cast::>() }; let data = holder.data.lock().unwrap(); - let start = offset as usize; + let Ok(start) = usize::try_from(offset) else { + return -1; + }; let Some(end) = start.checked_add(length) else { return -1; }; @@ -668,7 +1000,7 @@ mod tests { unsafe { let session = vx_session_new(); - let vx_dtype_ptr = vx_dtype::new(Arc::new(dtype.clone())); + let vx_dtype_ptr = vx_dtype::new(Arc::new(dtype)); let mut error = ptr::null_mut(); let out_ctx = Box::into_raw(Box::new(Arc::clone(&shared))); @@ -699,7 +1031,8 @@ mod tests { assert!(size > 0, "the sink must have written through the callback"); let in_ctx = Box::into_raw(Box::new(Arc::clone(&shared))); - let data_source = vx_data_source_new_callback( + let config = CoalesceConfig::object_storage(); + let data_source = vx_data_source_new_callback_with_coalesce( session, vx_input_callbacks { ctx: in_ctx.cast::(), @@ -707,6 +1040,8 @@ mod tests { release_fn: Some(buffer_release), }, size, + config.distance, + config.max_size, &raw mut error, ); assert!(error.is_null()); diff --git a/docs/source/api/file_format.rst b/docs/source/api/file_format.rst index f90e07f22..0ec5643b7 100644 --- a/docs/source/api/file_format.rst +++ b/docs/source/api/file_format.rst @@ -55,3 +55,97 @@ Interface .. doxygenclass:: paimon::WrittenFileStatsProvider :members: :undoc-members: + + +.. _cpp-api-vortex-read-options: + +Vortex Read Options +=================== + +Pass Vortex read options through ``ReadContextBuilder::AddOption`` at runtime. +Read options override table options for each key; a key absent from both uses +its default. Changes take effect when a new reader is created and do not change +existing readers. + +Predicate Pushdown +------------------ + +``vortex.read.in-pushdown.max-values`` sets the maximum list length eligible for +native IN/NOT IN pushdown. It accepts a decimal integer from ``0`` to +``4294967295`` (``UINT32_MAX``), with a default of ``32``. Negative, malformed or +out-of-range values are rejected. The limit applies independently to each IN or +NOT IN predicate and counts the original list entries, including duplicates. + +A positive value permits lists of up to that length, subject to type and value +safety checks. ``0`` disables only IN/NOT IN pushdown. A list exceeding the limit +is not pushed down and is never truncated. Vortex also skips native pushdown for +empty IN/NOT IN lists. ``TableRead`` rejects IN/NOT IN predicates containing a +``NULL`` literal during predicate validation. + +An ``AND`` expression can push down its eligible children. An ``OR`` expression +can be pushed down when each direct child yields a native candidate expression. +For example, Vortex can use ``A OR C`` as a candidate filter for +``(A AND B) OR C`` when ``A`` and ``C`` are eligible and ``B`` requires residual +filtering. + +Increasing the membership limit does not bypass these fallback conditions: + +- ``FLOAT``/``DOUBLE`` comparisons and IN/NOT IN are not pushed down, including + predicates with finite constants, because native NaN ordering differs from + Paimon's comparisons. +- Timestamp constants must convert to the file's timestamp unit without precision + loss or integer overflow and remain within the supported native temporal range. + This checks the predicate constant; stored values are not truncated. +- Timestamp value predicates require an empty stored timezone or the exact + timezone label ``UTC``. Other stored labels fall back, independently of the + timezone requested by the reader. +- Decimal constants must normalize to the file's precision and scale without + rounding or overflow. + +``IS NULL`` and ``IS NOT NULL`` remain eligible for floating-point and timestamp +fields. If the file contains a top-level user column named ``_row_idx``, however, +the reader disables all native predicate filtering, even when that column is not +projected. Projection and bitmap selection remain available. + +The Vortex adapter does not deduplicate lists or compress consecutive integer +values into ranges. The query engine is responsible for these optimizations. +To allow up to 128 entries per IN/NOT IN predicate: + +.. code-block:: cpp + + read_context_builder.AddOption("vortex.read.in-pushdown.max-values", "128"); + +Native pushdown may return candidate rows requiring further filtering. For exact +results, retain the full residual predicate: enable ``EnablePredicateFilter(true)`` +with ``TableRead``, or apply equivalent filtering in the caller. This also applies +when an IN/NOT IN predicate exceeds the configured limit. + +I/O Coalescing +-------------- + +``vortex.read.io-coalesce.max-gap`` and ``vortex.read.io-coalesce.max-size`` +accept decimal byte counts without unit suffixes, up to +``9223372036854775807`` (``INT64_MAX``). Their defaults are ``1048576`` +(1 MiB) and ``16777216`` (16 MiB), respectively. ``max-gap`` must be non-negative, +``max-size`` must be positive, and ``max-gap`` must not exceed ``max-size``. +The limits are validated when the reader opens its data source. + +``max-size`` is a threshold for merging requested ranges. An individual original +request may exceed it, so it does not impose a hard read-size or memory limit. +For example, configure a 64 KiB gap and a 4 MiB merging threshold: + +.. code-block:: cpp + + read_context_builder.AddOption("vortex.read.io-coalesce.max-gap", "65536"); + read_context_builder.AddOption("vortex.read.io-coalesce.max-size", "4194304"); + +Vortex Timestamp Storage +======================== + +For newly written ``TIMESTAMP_LTZ`` fields, the writer normalizes non-empty +timezone metadata to ``UTC``, including timestamps nested in supported struct, +list and fixed-size-list types. Epoch values and timestamp units are unchanged; +the represented instants are not shifted. Timestamp fields without a timezone +retain empty timezone metadata. This changes the observable stored file schema; +existing files are not rewritten. Value predicates on existing timestamp fields +with other stored timezone labels follow the fallback rules above. diff --git a/src/paimon/format/vortex/CMakeLists.txt b/src/paimon/format/vortex/CMakeLists.txt index a03a5842b..99ed3e62a 100644 --- a/src/paimon/format/vortex/CMakeLists.txt +++ b/src/paimon/format/vortex/CMakeLists.txt @@ -25,6 +25,7 @@ if(PAIMON_ENABLE_VORTEX) vortex_format_writer.cpp vortex_ffi_util.cpp vortex_io_callbacks.cpp + vortex_predicate_converter.cpp vortex_stats_extractor.cpp vortex_writer_builder.cpp) @@ -51,6 +52,7 @@ if(PAIMON_ENABLE_VORTEX) add_paimon_test(vortex_format_test SOURCES vortex_file_format_test.cpp + vortex_predicate_converter_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static diff --git a/src/paimon/format/vortex/vortex_defs.h b/src/paimon/format/vortex/vortex_defs.h new file mode 100644 index 000000000..ed713bb89 --- /dev/null +++ b/src/paimon/format/vortex/vortex_defs.h @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +namespace paimon::vortex { + +inline constexpr char kVortexReadInPushdownMaxValues[] = "vortex.read.in-pushdown.max-values"; +inline constexpr uint32_t kDefaultVortexReadInPushdownMaxValues = 32; + +/// Limits applied independently to each IN or NOT IN predicate. +struct VortexPredicateOptions { + /// Maximum number of input literals, including duplicates; zero disables membership pushdown. + uint32_t max_in_values = kDefaultVortexReadInPushdownMaxValues; +}; + +inline constexpr char kVortexReadIoCoalesceMaxGap[] = "vortex.read.io-coalesce.max-gap"; +inline constexpr char kVortexReadIoCoalesceMaxSize[] = "vortex.read.io-coalesce.max-size"; + +// Preserve Vortex's object-storage defaults when no explicit values are supplied. +inline constexpr uint64_t kDefaultVortexReadIoCoalesceMaxGap = 1ULL << 20; +inline constexpr uint64_t kDefaultVortexReadIoCoalesceMaxSize = 16ULL << 20; + +/// I/O range coalescing limits, measured in bytes. +struct VortexIoCoalesceOptions { + /// Maximum gap between ranges considered for merging. + uint64_t max_gap = kDefaultVortexReadIoCoalesceMaxGap; + /// Maximum merged span; a single original request may exceed this threshold. + uint64_t max_size = kDefaultVortexReadIoCoalesceMaxSize; +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_ffi.h b/src/paimon/format/vortex/vortex_ffi.h index d573d1279..cb2e646fd 100644 --- a/src/paimon/format/vortex/vortex_ffi.h +++ b/src/paimon/format/vortex/vortex_ffi.h @@ -49,6 +49,11 @@ extern "C" { // crates/vortex_callback_io/callback_io.rs); keep in sync with the Rust definitions. extern "C" { +/// Create an owned handle to the shared, fixed-runtime Paimon callback-reader session. +/// Free it with vx_session_free. Do not configure/register against this session or use it +/// for writers/multi-file sources; those require the fresh vx_session_new API. +vx_session* vx_paimon_read_session_new(); + /// Fill `length` bytes starting at `offset` into `dst`. Returns 0 on success and non-zero on /// failure; a partial read must be reported as a failure. Only the status code crosses the /// boundary, so the callback is expected to keep its error detail on `ctx`. @@ -70,11 +75,21 @@ struct vx_input_callbacks { vx_release_fn release_fn; }; -/// Create a data source that reads through `callbacks`. `size` is the total file length in bytes. -/// Returns null and sets `err` on failure. -const vx_data_source* vx_data_source_new_callback(const vx_session* session, - vx_input_callbacks callbacks, uint64_t size, - vx_error** err); +/// Create a callback data source with explicit I/O coalescing. `size` is the total file length +/// in bytes. `max_size` must be in [1, INT64_MAX] and `max_gap` must not exceed it. Returns null +/// and sets `err` on failure. The callback context is released exactly once, even on error. +const vx_data_source* vx_data_source_new_callback_with_coalesce(const vx_session* session, + vx_input_callbacks callbacks, + uint64_t size, uint64_t max_gap, + uint64_t max_size, vx_error** err); + +/// Export standard Arrow batches with at most `batch_size` rows, slicing encoded arrays before +/// materializing string/binary data. `batch_size` must be positive. Consumes `partition` on both +/// success and error; do not free or reuse it afterwards. Returns 0 on success, or 1 and sets +/// `err`. +int vx_partition_scan_arrow_with_batch_size(const vx_session* session, vx_partition* partition, + size_t batch_size, FFI_ArrowArrayStream* stream, + vx_error** err); /// Append `length` bytes from `src` to the host sink. Returns 0 on success and non-zero on failure; /// a partial write must be reported as a failure. Writes for one sink are sequential, never @@ -112,4 +127,24 @@ void vx_callback_sink_close(vx_callback_sink* sink, vx_error** error_out); /// Discard the sink without writing a footer, leaving whatever the host received invalid. void vx_callback_sink_abort(vx_callback_sink* sink); + +/// TODO: use `vx_scalar_new_extension(dtype, storage, err)` interface when update vortex version. +/// Timestamp literal unit. Values must match vx_time_unit in callback_io.rs. +enum vx_time_unit { + VX_TIME_UNIT_SECOND = 0, + VX_TIME_UNIT_MILLISECOND = 1, + VX_TIME_UNIT_MICROSECOND = 2, + VX_TIME_UNIT_NANOSECOND = 3, +}; + +/// Create a timestamp literal in `unit` since the Unix epoch. `timezone` is an IANA zone name, +/// or null for wall-clock time. Returns null without an error when a valid value exceeds the +/// native temporal range; other failures return null and set `err`. Free with vx_scalar_free. +vx_scalar* vx_scalar_new_timestamp(int64_t value, vx_time_unit unit, const char* timezone, + bool is_nullable, vx_error** err); + +/// Create a date scalar literal: `days` days since the Unix epoch (1970-01-01). Returns null +/// without an error outside the native civil-date range; other failures return null and set `err`. +/// Free the result with `vx_scalar_free`. +vx_scalar* vx_scalar_new_date(int32_t days, bool is_nullable, vx_error** err); } diff --git a/src/paimon/format/vortex/vortex_ffi_util.h b/src/paimon/format/vortex/vortex_ffi_util.h index 6172b3175..ad91430f1 100644 --- a/src/paimon/format/vortex/vortex_ffi_util.h +++ b/src/paimon/format/vortex/vortex_ffi_util.h @@ -32,13 +32,26 @@ namespace paimon::vortex { /// out-parameter. Status VortexFfiError(const std::string& operation, vx_error* error); +/// Stateless deleter bound to a Vortex C release function. +template +struct VxDeleter { + void operator()(T* ptr) const noexcept { + FreeFn(ptr); + } +}; + +template +using VxUniquePtr = std::unique_ptr>; + // Owning RAII aliases for the Vortex C handles; each frees its handle on scope exit. The element // type matches the constness returned by the corresponding constructor function. -using VxSessionPtr = std::unique_ptr; -using VxDataSourcePtr = std::unique_ptr; -using VxScanPtr = std::unique_ptr; -using VxPartitionPtr = std::unique_ptr; -using VxDtypePtr = std::unique_ptr; -using VxArrayPtr = std::unique_ptr; +using VxSessionPtr = VxUniquePtr; +using VxDataSourcePtr = VxUniquePtr; +using VxScanPtr = VxUniquePtr; +using VxPartitionPtr = VxUniquePtr; +using VxDtypePtr = VxUniquePtr; +using VxArrayPtr = VxUniquePtr; +using VxExpressionPtr = VxUniquePtr; +using VxScalarPtr = VxUniquePtr; } // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.cpp b/src/paimon/format/vortex/vortex_file_batch_reader.cpp index e171589e5..f930a6cde 100644 --- a/src/paimon/format/vortex/vortex_file_batch_reader.cpp +++ b/src/paimon/format/vortex/vortex_file_batch_reader.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include "arrow/api.h" #include "arrow/c/bridge.h" @@ -35,146 +36,202 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/math.h" +#include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/vortex/vortex_ffi_util.h" +#include "paimon/format/vortex/vortex_predicate_converter.h" #include "paimon/fs/file_system.h" +#include "paimon/utils/roaring_bitmap32.h" namespace paimon::vortex { namespace { -// Vortex's Arrow export represents strings and binaries with the view types (StringView / -// BinaryView). paimon schemas use the standard string/binary types, so map the view types back, -// recursing through nested types. Types that contain no views are returned unchanged. -std::shared_ptr NormalizeViewType(const std::shared_ptr& type) { - switch (type->id()) { - case arrow::Type::STRING_VIEW: - return arrow::utf8(); - case arrow::Type::BINARY_VIEW: - return arrow::binary(); +// Must match the physical row index column appended by vortex-ffi scan.rs. +constexpr char kRowIdxColumn[] = "_row_idx"; + +// Validate field references without changing read-side types or metadata. Type conversion +// remains the responsibility of AlignToReadSchema. +Status ValidateReadFieldReferences(const std::shared_ptr& file_type, + const std::shared_ptr& read_type) { + if (file_type->id() != read_type->id()) { + return Status::OK(); + } + switch (read_type->id()) { case arrow::Type::STRUCT: { - arrow::FieldVector fields; - fields.reserve(type->num_fields()); - for (const std::shared_ptr& field : type->fields()) { - fields.push_back(field->WithType(NormalizeViewType(field->type()))); + PAIMON_ASSIGN_OR_RAISE(std::vector projection, + ArrowUtils::CreateProjection(arrow::schema(file_type->fields()), + read_type->fields())); + for (int32_t i = 0; i < read_type->num_fields(); ++i) { + PAIMON_RETURN_NOT_OK(ValidateReadFieldReferences( + file_type->field(projection[i])->type(), read_type->field(i)->type())); } - return arrow::struct_(fields); + return Status::OK(); } case arrow::Type::LIST: - return arrow::list(type->field(0)->WithType(NormalizeViewType(type->field(0)->type()))); case arrow::Type::FIXED_SIZE_LIST: - return arrow::fixed_size_list( - type->field(0)->WithType(NormalizeViewType(type->field(0)->type())), - checked_cast(*type).list_size()); + // Container field names (for example item/element) need not match. + return ValidateReadFieldReferences(file_type->field(0)->type(), + read_type->field(0)->type()); + case arrow::Type::MAP: { + const auto& file_map = checked_cast(*file_type); + const auto& read_map = checked_cast(*read_type); + PAIMON_RETURN_NOT_OK( + ValidateReadFieldReferences(file_map.key_type(), read_map.key_type())); + return ValidateReadFieldReferences(file_map.item_type(), read_map.item_type()); + } default: - return type; + return Status::OK(); } } -std::shared_ptr NormalizeViewSchema(const std::shared_ptr& schema) { - arrow::FieldVector fields; - fields.reserve(schema->num_fields()); - for (const std::shared_ptr& field : schema->fields()) { - fields.push_back(field->WithType(NormalizeViewType(field->type()))); - } - return arrow::schema(fields, schema->metadata()); -} +} // namespace -// Rebuilds `array` with every StringView/BinaryView column (which Vortex's Arrow export produces) -// converted to the standard string/binary type. Arrow 17 has no cast kernel between the view and -// standard types, so the leaves are rebuilt with builders and the parents reconstructed, recursing -// through structs and lists. Arrays with no view-typed data are returned unchanged. `array` must be -// offset-normalized (offset 0) so the rebuilt parents and children stay consistent. -Result> NormalizeViewArray(const std::shared_ptr& array, - arrow::MemoryPool* pool) { - // TODO(zhouyc-ali): avoid the per-row copy by exporting standard utf8/binary - switch (array->type_id()) { - case arrow::Type::STRING_VIEW: { - const auto& view = checked_cast(*array); - arrow::StringBuilder builder(pool); - for (int64_t i = 0; i < view.length(); ++i) { - if (view.IsNull(i)) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); - } else { - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(view.GetView(i))); - } - } - std::shared_ptr out; - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&out)); - return out; +Result VortexFileBatchReader::SelectionState::Create( + const std::optional& bitmap, uint64_t total_rows) { + SelectionState state; + state.has_selection_ = bitmap.has_value(); + state.selected_row_count_ = total_rows; + if (bitmap) { + const int32_t cardinality = bitmap->Cardinality(); + if (cardinality < 0 || static_cast(cardinality) > total_rows) { + return Status::Invalid("Vortex selection cardinality exceeds the file row count"); } - case arrow::Type::BINARY_VIEW: { - const auto& view = checked_cast(*array); - arrow::BinaryBuilder builder(pool); - for (int64_t i = 0; i < view.length(); ++i) { - if (view.IsNull(i)) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); - } else { - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(view.GetView(i))); - } - } - std::shared_ptr out; - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&out)); - return out; + state.selected_row_count_ = static_cast(cardinality); + state.excludes_ = state.selected_row_count_ > total_rows / 2; + + // Seek to the first invalid row without enumerating a dense selection. Bitmap values + // above INT32_MAX appear as negative int32_t values and remain invalid for this reader. + const uint64_t max_bitmap_row = std::numeric_limits::max(); + const auto first_invalid_row = static_cast(std::min(total_rows, max_bitmap_row)); + const auto end = bitmap->End(); + auto invalid_row = bitmap->EqualOrLarger(first_invalid_row); + if (total_rows > max_bitmap_row && invalid_row != end && + *invalid_row == std::numeric_limits::max()) { + ++invalid_row; } - case arrow::Type::STRUCT: { - const auto& struct_array = checked_cast(*array); - arrow::ArrayVector children; - arrow::FieldVector fields; - children.reserve(struct_array.num_fields()); - fields.reserve(struct_array.num_fields()); - bool changed = false; - for (int32_t i = 0; i < struct_array.num_fields(); ++i) { - const std::shared_ptr& child = struct_array.field(i); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_child, - NormalizeViewArray(child, pool)); - changed = changed || normalized_child.get() != child.get(); - fields.push_back(array->type()->field(i)->WithType(normalized_child->type())); - children.push_back(std::move(normalized_child)); + if (invalid_row != end) { + return Status::Invalid(fmt::format("Vortex selected row {} is out of range {}", + static_cast(*invalid_row), total_rows)); + } + if (state.selected_row_count_ == total_rows) { + return state; + } + + state.indices_.reserve(static_cast( + state.excludes_ ? total_rows - state.selected_row_count_ : state.selected_row_count_)); + if (state.excludes_) { + RoaringBitmap32 excluded; + excluded.AddRange(0, static_cast(std::min(total_rows, max_bitmap_row))); + if (total_rows > max_bitmap_row) { + excluded.Add(std::numeric_limits::max()); } - if (!changed) { - return array; + excluded -= *bitmap; + const auto excluded_end = excluded.End(); + for (auto iter = excluded.Begin(); iter != excluded_end; ++iter) { + state.indices_.push_back(static_cast(*iter)); } - return std::make_shared( - arrow::struct_(fields), struct_array.length(), children, struct_array.null_bitmap(), - struct_array.null_count(), struct_array.offset()); - } - case arrow::Type::LIST: { - const auto& list = checked_cast(*array); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr values, - NormalizeViewArray(list.values(), pool)); - if (values.get() == list.values().get()) { - return array; + // Rows beyond the bitmap's nonnegative int32_t domain are all excluded. + for (uint64_t row = max_bitmap_row + 1; row < total_rows; ++row) { + state.indices_.push_back(row); } - return std::make_shared( - arrow::list(array->type()->field(0)->WithType(values->type())), list.length(), - list.value_offsets(), values, list.null_bitmap(), list.null_count(), list.offset()); + } else { + for (auto iter = bitmap->Begin(); iter != end; ++iter) { + state.indices_.push_back(static_cast(*iter)); + } + } + } + return state; +} + +void VortexFileBatchReader::SelectionState::ApplyToScan(vx_scan_options* options) const { + // A dense bitmap sends excluded rows; all-selected uses the default INCLUDE_ALL mode. + if (has_selection_ && !indices_.empty()) { + options->selection.idx = indices_.data(); + options->selection.idx_len = indices_.size(); + options->selection.include = + excludes_ ? VX_SELECTION_EXCLUDE_RANGE : VX_SELECTION_INCLUDE_RANGE; + } +} + +Result VortexFileBatchReader::SelectionState::GetPhysicalRowId( + uint64_t selected_rank) const { + // Ordered scans emit selected rows in physical order, regardless of which side is stored. + if (has_selection_) { + if (selected_rank >= selected_row_count_) { + return Status::Invalid(fmt::format("emitted row offset {} exceeds selection size {}", + selected_rank, selected_row_count_)); + } + if (!excludes_) { + return indices_[selected_rank]; } - case arrow::Type::FIXED_SIZE_LIST: { - const auto& list = checked_cast(*array); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr values, - NormalizeViewArray(list.values(), pool)); - if (values.get() == list.values().get()) { - return array; + // For excluded row i, excluded[i] - i is the number of selected rows before it. + // Count exclusions at or before this selected rank without expanding a row mapping. + size_t begin = 0; + size_t end = indices_.size(); + while (begin < end) { + const size_t middle = begin + (end - begin) / 2; + if (indices_[middle] - middle <= selected_rank) { + begin = middle + 1; + } else { + end = middle; } - const auto& fsl_type = checked_cast(*array->type()); - return std::make_shared( - arrow::fixed_size_list(array->type()->field(0)->WithType(values->type()), - fsl_type.list_size()), - list.length(), values, list.null_bitmap(), list.null_count(), list.offset()); } - default: - return array; + return selected_rank + begin; } + return selected_rank; } -} // namespace +Status VortexFileBatchReader::RowPositionState::CaptureRowIndices( + const std::shared_ptr& array) { + if (has_filter_) { + // Capture physical row indices before AlignToReadSchema removes the hidden column. + const auto& struct_type = checked_cast(*array->type()); + const int32_t row_idx_pos = struct_type.num_fields() - 1; + if (row_idx_pos < 0 || struct_type.field(row_idx_pos)->name() != kRowIdxColumn) { + return Status::Invalid("Vortex scan did not append the _row_idx column"); + } + previous_batch_row_idx_ = + checked_cast(*array).field(row_idx_pos); + } + return Status::OK(); +} + +void VortexFileBatchReader::RowPositionState::RecordBatch(uint64_t row_count) { + previous_first_row_ = rows_emitted_; + previous_batch_row_count_ = row_count; + rows_emitted_ += row_count; +} + +Result VortexFileBatchReader::RowPositionState::GetPhysicalRowId( + uint64_t batch_row_id, const SelectionState& selection) const { + if (previous_batch_row_count_ == 0) { + return Status::Invalid(previous_first_row_ == std::numeric_limits::max() + ? "no Vortex batch has been read yet" + : "last Vortex batch was EOF"); + } + if (batch_row_id >= previous_batch_row_count_) { + return Status::Invalid(fmt::format("batch row id {} is out of range {}", batch_row_id, + previous_batch_row_count_)); + } + // The hidden column accounts for both filtering and selection; emitted offsets do not. + if (has_filter_) { + if (previous_batch_row_idx_ == nullptr || + batch_row_id >= static_cast(previous_batch_row_idx_->length())) { + return Status::Invalid( + fmt::format("Vortex filter row index unavailable for row {}", batch_row_id)); + } + return checked_cast(*previous_batch_row_idx_) + .Value(static_cast(batch_row_id)); + } + return selection.GetPhysicalRowId(previous_first_row_ + batch_row_id); +} VortexFileBatchReader::VortexFileBatchReader( const std::shared_ptr& input, int32_t batch_size, std::shared_ptr input_context, VxSessionPtr session, VxDataSourcePtr data_source, VxScanPtr scan, const std::shared_ptr& file_schema, - const std::shared_ptr& struct_type, uint64_t total_rows, + uint64_t total_rows, const VortexPredicateOptions& predicate_options, const std::shared_ptr& arrow_pool) : input_(input), batch_size_(batch_size), @@ -182,14 +239,16 @@ VortexFileBatchReader::VortexFileBatchReader( session_(std::move(session)), data_source_(std::move(data_source)), scan_(std::move(scan)), - file_schema_(NormalizeViewSchema(file_schema)), - struct_type_(struct_type), + file_schema_(file_schema), total_rows_(total_rows), arrow_pool_(arrow_pool), + predicate_options_(predicate_options), metrics_(std::make_shared()) {} Result> VortexFileBatchReader::Create( const std::shared_ptr& input, int32_t batch_size, + const VortexPredicateOptions& predicate_options, + const VortexIoCoalesceOptions& coalesce_options, const std::shared_ptr& arrow_pool) { if (input == nullptr || arrow_pool == nullptr || batch_size <= 0) { return Status::Invalid( @@ -200,15 +259,14 @@ Result> VortexFileBatchReader::Create( auto length = static_cast(signed_length); auto input_context = std::make_shared(input); - VxSessionPtr session(vx_session_new(), vx_session_free); + VxSessionPtr session(vx_paimon_read_session_new()); if (session == nullptr) { return Status::IOError("failed to create Vortex session"); } vx_error* error = nullptr; - VxDataSourcePtr data_source( - vx_data_source_new_callback(session.get(), VortexInputContext::MakeCallbacks(input_context), - length, &error), - vx_data_source_free); + VxDataSourcePtr data_source(vx_data_source_new_callback_with_coalesce( + session.get(), VortexInputContext::MakeCallbacks(input_context), length, + coalesce_options.max_gap, coalesce_options.max_size, &error)); if (data_source == nullptr) { return VortexCallbackError("open Vortex callback data source", error, input_context->GetCallbackStatus()); @@ -229,9 +287,6 @@ Result> VortexFileBatchReader::Create( } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, arrow::ImportSchema(&ffi_schema)); - // Built from the raw view-typed schema on purpose: NextBatch imports each scanned batch as this - // exact type, while the constructor normalizes its own copy kept as file_schema_. - std::shared_ptr struct_type = arrow::struct_(file_schema->fields()); vx_estimate row_count{}; vx_data_source_get_row_count(data_source.get(), &row_count); @@ -244,31 +299,70 @@ Result> VortexFileBatchReader::Create( // scan may be consumed only once. return std::unique_ptr(new VortexFileBatchReader( input, batch_size, std::move(input_context), std::move(session), std::move(data_source), - VxScanPtr(nullptr, vx_scan_free), file_schema, struct_type, total_rows, arrow_pool)); + VxScanPtr(), file_schema, total_rows, predicate_options, arrow_pool)); } VortexFileBatchReader::~VortexFileBatchReader() { CloseInternal(); } +Status VortexFileBatchReader::CreateScan() { + vx_error* error = nullptr; + // Scan in storage order: Vortex defaults to ordered=false and may emit chunks out of order, + // which would misalign the physical row positions NextBatch assigns. + vx_scan_options options{}; + options.ordered = true; + // The scan clones the projection, so these expressions need only outlive scan creation. + VxExpressionPtr root; + VxExpressionPtr projection; + std::vector name_ptrs; + if (read_schema_ != nullptr) { + name_ptrs.reserve(read_schema_->num_fields()); + // The schema owns these names; Vortex copies them when creating the projection. + for (const std::shared_ptr& field : read_schema_->fields()) { + name_ptrs.push_back(field->name().c_str()); + } + root = VxExpressionPtr(vx_expression_root()); + // The FFI requires non-null names even for an explicit zero-column projection. + const char* empty_name = nullptr; + const char* const* names = name_ptrs.empty() ? &empty_name : name_ptrs.data(); + projection = VxExpressionPtr(vx_expression_select(names, name_ptrs.size(), root.get())); + options.projection = projection.get(); + } + selection_.ApplyToScan(&options); + // Filtering drops rows; include_row_idx preserves their original physical positions. + // The scan clones the filter, so it can be freed after scan creation. + VxExpressionPtr filter; + // SetReadSchema validates requested fields against the file schema, so checking the + // file alone also protects the hidden index from collisions with requested fields. + const bool has_row_idx_column = !file_schema_->GetAllFieldIndices(kRowIdxColumn).empty(); + if (!has_row_idx_column) { + PAIMON_ASSIGN_OR_RAISE(filter, VortexPredicateConverter::Convert( + predicate_filter_, file_schema_, predicate_options_)); + } + if (filter != nullptr) { + options.filter = filter.get(); + options.include_row_idx = true; + row_position_.EnableFilterRowIds(); + } + VxScanPtr scan(vx_data_source_scan(data_source_.get(), &options, /*estimate=*/nullptr, &error)); + if (scan == nullptr) { + return VortexCallbackError("create Vortex scan", error, + input_context_->GetCallbackStatus()); + } + scan_ = std::move(scan); + return Status::OK(); +} + Result VortexFileBatchReader::OpenNextPartitionStream() { + if (selection_.IsEmpty()) { + return false; + } if (scan_ == nullptr) { - vx_error* error = nullptr; - // Scan in storage order: Vortex defaults to ordered=false and may emit chunks out of order, - // which would misalign the physical row positions NextBatch assigns. - vx_scan_options options{}; // zero-init: all columns, no filter/row-range/selection/limit - options.ordered = true; - VxScanPtr scan( - vx_data_source_scan(data_source_.get(), &options, /*estimate=*/nullptr, &error), - vx_scan_free); - if (scan == nullptr) { - return VortexCallbackError("create Vortex scan", error, - input_context_->GetCallbackStatus()); - } - scan_ = std::move(scan); + PAIMON_RETURN_NOT_OK(CreateScan()); } vx_error* error = nullptr; - VxPartitionPtr partition(vx_scan_next_partition(scan_.get(), &error), vx_partition_free); + VxPartitionPtr partition(vx_scan_next_partition(scan_.get(), &error)); if (error != nullptr) { return VortexCallbackError("advance Vortex scan partition", error, input_context_->GetCallbackStatus()); @@ -276,18 +370,39 @@ Result VortexFileBatchReader::OpenNextPartitionStream() { if (partition == nullptr) { return false; // Scan exhausted. } - // vx_partition_scan_arrow consumes the partition (even on error), so release ownership here. + // Arrow stream export consumes the partition (even on error), so release ownership here. ::ArrowArrayStream stream{}; error = nullptr; - if (vx_partition_scan_arrow(session_.get(), partition.release(), &stream, &error) != 0) { + if (vx_partition_scan_arrow_with_batch_size(session_.get(), partition.release(), + static_cast(batch_size_), &stream, + &error) != 0) { return VortexCallbackError("scan Vortex partition to Arrow", error, input_context_->GetCallbackStatus()); } current_stream_ = stream; stream_active_ = true; + PAIMON_RETURN_NOT_OK(ImportCurrentStreamSchema()); return true; } +Status VortexFileBatchReader::ImportCurrentStreamSchema() { + // Import batches with the scan's actual output schema (projected columns, standard types). + ::ArrowSchema ffi_schema = {}; + if (current_stream_.get_schema(¤t_stream_, &ffi_schema) != 0) { + const char* message = current_stream_.get_last_error(¤t_stream_); + std::string message_copy = message == nullptr ? "unknown" : message; + Status callback_status = input_context_->GetCallbackStatus(); + ReleaseStream(); + if (!callback_status.ok()) { + return callback_status.WithMessage("read Vortex stream schema", ": ", + callback_status.message()); + } + return Status::IOError("Vortex Arrow stream get_schema failed: ", message_copy); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(current_stream_schema_, arrow::ImportSchema(&ffi_schema)); + return Status::OK(); +} + Result> VortexFileBatchReader::ReadNextArray() { while (true) { if (!stream_active_) { @@ -317,8 +432,9 @@ Result> VortexFileBatchReader::ReadNextArray() { ReleaseStream(); // Partition exhausted; try the next one. continue; } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, - arrow::ImportArray(&ffi_array, struct_type_)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr array, + arrow::ImportArray(&ffi_array, arrow::struct_(current_stream_schema_->fields()))); if (array->length() == 0) { continue; // Skip empty batches. } @@ -331,7 +447,6 @@ Result> VortexFileBatchReader::AlignToReadSchema( if (read_schema_ == nullptr) { return array; } - // TODO(zhouyc-ali): push top-level column pruning into the Vortex scan std::shared_ptr read_type = arrow::struct_(read_schema_->fields()); // Vortex reports the file's zone and paimon asks for the reader's; the instants are the same. return NestedProjectionUtils::AlignArrayToReadType( @@ -348,7 +463,7 @@ Result VortexFileBatchReader::NextBatch() { current_batch_offset_ = 0; } if (current_batch_ == nullptr) { - previous_batch_row_count_ = 0; + row_position_.SetEof(); return BatchReader::MakeEofBatch(); } @@ -358,16 +473,11 @@ Result VortexFileBatchReader::NextBatch() { current_batch_->Slice(current_batch_offset_, row_count); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_array, ArrowUtils::NormalizeArrayOffsets(sliced_array, arrow_pool_.get())); - // Vortex exports strings/binaries as Arrow view types; convert to the standard ones (no cast - // kernel in Arrow 17) after offset normalization, so the array is offset-0. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output_array, - NormalizeViewArray(normalized_array, arrow_pool_.get())); + PAIMON_RETURN_NOT_OK(row_position_.CaptureRowIndices(normalized_array)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr projected_array, - AlignToReadSchema(output_array)); + AlignToReadSchema(normalized_array)); - previous_first_row_ = rows_emitted_; - previous_batch_row_count_ = static_cast(row_count); - rows_emitted_ += static_cast(row_count); + row_position_.RecordBatch(static_cast(row_count)); current_batch_offset_ += row_count; auto ffi_array = std::make_unique<::ArrowArray>(); @@ -387,38 +497,35 @@ Result> VortexFileBatchReader::GetFileSchema() co Status VortexFileBatchReader::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { + if (closed_) { + return Status::Invalid("Vortex reader is closed"); + } if (read_schema == nullptr) { return Status::Invalid("Vortex read schema is nullptr"); } - // TODO(zhouyc-ali): push the row selection into the Vortex scan - (void)selection_bitmap; - // No pushdown: the whole file is scanned and reading restarts from row 0, so drop the - // single-use scan; the next read re-creates it. + PAIMON_ASSIGN_OR_RAISE(SelectionState selection, + SelectionState::Create(selection_bitmap, total_rows_)); + // Leave the active scan and its row mapping unchanged if validation or import fails. PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr imported_read_schema, arrow::ImportSchema(read_schema)); + PAIMON_RETURN_NOT_OK( + ArrowSchemaValidator::ValidateNoRedundantFields(imported_read_schema->fields())); + PAIMON_RETURN_NOT_OK(ValidateReadFieldReferences( + arrow::struct_(file_schema_->fields()), arrow::struct_(imported_read_schema->fields()))); + // Recreate the single-use scan on the next read with the new projection and predicates. read_schema_ = std::move(imported_read_schema); predicate_filter_ = std::dynamic_pointer_cast(predicate); + selection_ = std::move(selection); ReleaseStream(); current_batch_.reset(); current_batch_offset_ = 0; scan_.reset(); - rows_emitted_ = 0; - previous_first_row_ = std::numeric_limits::max(); - previous_batch_row_count_ = 0; + row_position_.Reset(); return Status::OK(); } Result VortexFileBatchReader::GetPreviousBatchFileRowId(uint64_t batch_row_id) const { - if (previous_batch_row_count_ == 0) { - return Status::Invalid(previous_first_row_ == std::numeric_limits::max() - ? "no Vortex batch has been read yet" - : "last Vortex batch was EOF"); - } - if (batch_row_id >= previous_batch_row_count_) { - return Status::Invalid(fmt::format("batch row id {} is out of range {}", batch_row_id, - previous_batch_row_count_)); - } - return previous_first_row_ + batch_row_id; + return row_position_.GetPhysicalRowId(batch_row_id, selection_); } Result VortexFileBatchReader::GetNumberOfRows() const { @@ -437,6 +544,7 @@ void VortexFileBatchReader::ReleaseStream() { if (stream_active_) { ArrowArrayStreamRelease(¤t_stream_); current_stream_ = ::ArrowArrayStream{}; + current_stream_schema_.reset(); stream_active_ = false; } } @@ -450,6 +558,8 @@ void VortexFileBatchReader::CloseInternal() { ReleaseStream(); current_batch_.reset(); scan_.reset(); + selection_ = SelectionState{}; + row_position_.Reset(); data_source_.reset(); session_.reset(); input_context_.reset(); diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.h b/src/paimon/format/vortex/vortex_file_batch_reader.h index 949a9ae45..8d4dcbbb2 100644 --- a/src/paimon/format/vortex/vortex_file_batch_reader.h +++ b/src/paimon/format/vortex/vortex_file_batch_reader.h @@ -22,11 +22,14 @@ #include #include #include +#include #include "arrow/type_fwd.h" +#include "paimon/format/vortex/vortex_defs.h" #include "paimon/format/vortex/vortex_ffi.h" #include "paimon/format/vortex/vortex_ffi_util.h" #include "paimon/format/vortex/vortex_io_callbacks.h" +#include "paimon/format/vortex/vortex_predicate_converter.h" #include "paimon/reader/file_batch_reader.h" #include "paimon/result.h" @@ -50,12 +53,16 @@ class VortexFileBatchReader : public FileBatchReader { public: static Result> Create( const std::shared_ptr& input, int32_t batch_size, + const VortexPredicateOptions& predicate_options, + const VortexIoCoalesceOptions& coalesce_options, const std::shared_ptr& arrow_pool); ~VortexFileBatchReader() override; Result NextBatch() override; Result> GetFileSchema() const override; + /// Read fields must be unique and present in the file, including nested projections. + /// Schema validation failures leave the active scan and row positions unchanged. Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) override; Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override; @@ -63,23 +70,76 @@ class VortexFileBatchReader : public FileBatchReader { std::shared_ptr GetReaderMetrics() const override; void Close() override; bool SupportPreciseBitmapSelection() const override { - return false; + return true; } private: + // Bitmap selection owns the smaller side and maps selected ranks to physical rows. + class SelectionState { + public: + static Result Create(const std::optional& bitmap, + uint64_t total_rows); + + bool IsEmpty() const { + return has_selection_ && selected_row_count_ == 0; + } + uint64_t SelectedRowCount() const { + return selected_row_count_; + } + uint64_t IndexCount() const { + return indices_.size(); + } + void ApplyToScan(vx_scan_options* options) const; + Result GetPhysicalRowId(uint64_t selected_rank) const; + + private: + bool has_selection_ = false; + bool excludes_ = false; + uint64_t selected_row_count_ = 0; + std::vector indices_; + }; + + // Track output positions and retain native row ids when a pushed filter removes rows. + class RowPositionState { + public: + void Reset() { + *this = RowPositionState{}; + } + void EnableFilterRowIds() { + has_filter_ = true; + } + void SetEof() { + previous_batch_row_count_ = 0; + } + Status CaptureRowIndices(const std::shared_ptr& array); + void RecordBatch(uint64_t row_count); + Result GetPhysicalRowId(uint64_t batch_row_id, + const SelectionState& selection) const; + + private: + uint64_t rows_emitted_ = 0; + uint64_t previous_first_row_ = std::numeric_limits::max(); + uint64_t previous_batch_row_count_ = 0; + bool has_filter_ = false; + std::shared_ptr previous_batch_row_idx_; + }; + VortexFileBatchReader(const std::shared_ptr& input, int32_t batch_size, std::shared_ptr input_context, VxSessionPtr session, VxDataSourcePtr data_source, VxScanPtr scan, - const std::shared_ptr& file_schema, - const std::shared_ptr& struct_type, uint64_t total_rows, + const std::shared_ptr& file_schema, uint64_t total_rows, + const VortexPredicateOptions& predicate_options, const std::shared_ptr& arrow_pool); /// Pull the next Arrow array from the current partition stream, opening the next partition's /// stream when the current one is exhausted. Returns nullptr at end of scan. Result> ReadNextArray(); - /// Open the next partition and fill `current_stream_` via `vx_partition_scan_arrow`. + /// Create the single-use scan with the current projection, selection and predicate. + Status CreateScan(); + /// Open the next partition and fill `current_stream_` with bounded Arrow batches. /// Returns false when there are no more partitions. Result OpenNextPartitionStream(); + Status ImportCurrentStreamSchema(); /// Reshape a scanned batch to `read_schema_`, selecting columns by name and recursing into /// nested fields. Returns `array` unchanged when SetReadSchema has not been called. Result> AlignToReadSchema( @@ -96,30 +156,26 @@ class VortexFileBatchReader : public FileBatchReader { VxSessionPtr session_; VxDataSourcePtr data_source_; VxScanPtr scan_; - // The file schema with Vortex's string/binary view types normalized to the standard types; this - // is what GetFileSchema() reports to paimon. + // Full file schema with standard Arrow types, as reported by GetFileSchema(). std::shared_ptr file_schema_; - // The RAW view-typed struct that Vortex's Arrow export produces. NextBatch imports each scanned - // batch as this type (so it must match the C export's buffer layout), then NormalizeViewArray - // rewrites it to file_schema_'s types. Keeping this view-typed is load-bearing: normalizing it - // would make ImportArray misread the buffers. - std::shared_ptr struct_type_; // The schema NextBatch must output, set by SetReadSchema (the projected read schema). Null // until SetReadSchema is called, in which case the full file schema is returned. std::shared_ptr read_schema_; uint64_t total_rows_ = 0; ::ArrowArrayStream current_stream_{}; + // Actual scan output schema, including projection and any hidden physical row index. + std::shared_ptr current_stream_schema_; bool stream_active_ = false; std::shared_ptr current_batch_; int64_t current_batch_offset_ = 0; - uint64_t rows_emitted_ = 0; - uint64_t previous_first_row_ = std::numeric_limits::max(); - uint64_t previous_batch_row_count_ = 0; + SelectionState selection_; + RowPositionState row_position_; std::shared_ptr arrow_pool_; - std::shared_ptr predicate_filter_; // from SetReadSchema; not pushed down + VortexPredicateOptions predicate_options_; + std::shared_ptr predicate_filter_; std::shared_ptr metrics_; bool closed_ = false; }; diff --git a/src/paimon/format/vortex/vortex_file_format.cpp b/src/paimon/format/vortex/vortex_file_format.cpp index 9d85b2fb7..39b670db6 100644 --- a/src/paimon/format/vortex/vortex_file_format.cpp +++ b/src/paimon/format/vortex/vortex_file_format.cpp @@ -21,6 +21,9 @@ #include "arrow/c/bridge.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/format/vortex/vortex_defs.h" +#include "paimon/format/vortex/vortex_predicate_converter.h" #include "paimon/format/vortex/vortex_reader_builder.h" #include "paimon/format/vortex/vortex_stats_extractor.h" #include "paimon/format/vortex/vortex_writer_builder.h" @@ -29,7 +32,19 @@ namespace paimon::vortex { Result> VortexFileFormat::CreateReaderBuilder( int32_t batch_size) const { - return std::make_unique(batch_size); + VortexPredicateOptions predicate_options; + PAIMON_ASSIGN_OR_RAISE( + predicate_options.max_in_values, + OptionsUtils::GetValueFromMap(options_, kVortexReadInPushdownMaxValues, + kDefaultVortexReadInPushdownMaxValues)); + VortexIoCoalesceOptions coalesce_options; + PAIMON_ASSIGN_OR_RAISE(coalesce_options.max_gap, OptionsUtils::GetValueFromMap( + options_, kVortexReadIoCoalesceMaxGap, + kDefaultVortexReadIoCoalesceMaxGap)); + PAIMON_ASSIGN_OR_RAISE(coalesce_options.max_size, OptionsUtils::GetValueFromMap( + options_, kVortexReadIoCoalesceMaxSize, + kDefaultVortexReadIoCoalesceMaxSize)); + return std::make_unique(batch_size, predicate_options, coalesce_options); } Result> VortexFileFormat::CreateWriterBuilder( diff --git a/src/paimon/format/vortex/vortex_file_format_test.cpp b/src/paimon/format/vortex/vortex_file_format_test.cpp index cb383f741..12912cbad 100644 --- a/src/paimon/format/vortex/vortex_file_format_test.cpp +++ b/src/paimon/format/vortex/vortex_file_format_test.cpp @@ -19,15 +19,21 @@ // Round-trip unit tests for the Vortex file format: write, then read back via vortex-ffi. +#include #include +#include #include #include #include #include +#include +#include #include #include +#include #include #include +#include #include #include @@ -35,29 +41,39 @@ #include "arrow/array/concatenate.h" #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" +#include "arrow/util/key_value_metadata.h" +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/predicate/predicate_filter.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" #include "paimon/format/column_stats.h" #include "paimon/format/file_format.h" #include "paimon/format/file_format_factory.h" #include "paimon/format/format_stats_extractor.h" #include "paimon/format/format_writer.h" #include "paimon/format/reader_builder.h" +#include "paimon/format/vortex/vortex_defs.h" +#include "paimon/format/vortex/vortex_file_batch_reader.h" #include "paimon/format/vortex/vortex_io_callbacks.h" #include "paimon/format/writer_builder.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/reader/batch_reader.h" #include "paimon/reader/file_batch_reader.h" #include "paimon/testing/utils/io_exception_helper.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/utils/roaring_bitmap32.h" namespace paimon::vortex::test { @@ -157,6 +173,153 @@ class VortexFileFormatTest : public ::testing::Test { } } + Result> CreatePredicateReader( + const std::string& path, const std::shared_ptr& schema, + const std::shared_ptr& predicate, const std::string& max_values) const { + std::map options = {{"file.format", "vortex"}}; + if (!max_values.empty()) { + options.emplace(kVortexReadInPushdownMaxValues, max_values); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format, + FileFormatFactory::Get("vortex", options)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr builder, + format->CreateReaderBuilder(/*batch_size=*/2)); + builder->WithMemoryPool(pool_); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, file_system_->Open(path)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, builder->Build(input)); + ::ArrowSchema ffi_schema = {}; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema)); + PAIMON_RETURN_NOT_OK( + reader->SetReadSchema(&ffi_schema, predicate, /*selection_bitmap=*/std::nullopt)); + return reader; + } + + // This is the native file reader, before the SDK residual predicate. A declined membership + // predicate intentionally returns a superset, including null rows, for the caller to filter. + Result> CollectNativeRowIds( + std::unique_ptr reader) const { + std::vector row_ids; + while (true) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + PAIMON_RETURN_NOT_OK(paimon::test::ReadResultCollector::CheckBatchOffset(batch)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr array, + arrow::ImportArray(batch.first.get(), batch.second.get())); + for (int64_t i = 0; i < array->length(); ++i) { + PAIMON_ASSIGN_OR_RAISE(uint64_t row_id, + reader->GetPreviousBatchFileRowId(static_cast(i))); + row_ids.push_back(row_id); + } + } + return row_ids; + } + + // Verify both native candidates and the real SDK residual predicate against an independent + // expected result. Every returned field is also checked against the source physical row. + void AssertNativeAndResidualRows(std::unique_ptr reader, + const std::shared_ptr& predicate, + const std::shared_ptr& source, + const std::vector& expected_candidates, + const std::vector& expected_matches) const { + const auto residual = std::dynamic_pointer_cast(predicate); + ASSERT_NE(residual, nullptr); + const auto& source_rows = checked_cast(*source); + std::vector candidates; + std::vector matches; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadNextBatch(reader.get())); + if (array == nullptr) { + break; + } + ASSERT_TRUE(array->ValidateFull().ok()); + const auto& rows = checked_cast(*array); + ASSERT_EQ(rows.num_fields(), source_rows.num_fields()); + ASSERT_OK_AND_ASSIGN(std::vector mask, residual->Test(*array, arrow_pool_.get())); + ASSERT_EQ(mask.size(), static_cast(array->length())); + for (int64_t i = 0; i < array->length(); ++i) { + ASSERT_OK_AND_ASSIGN(uint64_t row_id, + reader->GetPreviousBatchFileRowId(static_cast(i))); + ASSERT_LT(row_id, static_cast(source->length())); + candidates.push_back(row_id); + for (int32_t field = 0; field < rows.num_fields(); ++field) { + const auto actual = rows.field(field)->GetScalar(i); + const auto expected = + source_rows.field(field)->GetScalar(static_cast(row_id)); + ASSERT_TRUE(actual.ok()); + ASSERT_TRUE(expected.ok()); + ASSERT_TRUE(actual.ValueOrDie()->Equals(*expected.ValueOrDie())); + } + if (mask[i] != 0) { + matches.push_back(row_id); + } + } + } + ASSERT_EQ(candidates, expected_candidates); + ASSERT_EQ(matches, expected_matches); + } + + static std::shared_ptr ReaderTestSchema() { + return arrow::schema( + {arrow::field("id", arrow::int32(), false), arrow::field("name", arrow::utf8())}); + } + + Result WriteReaderTestFile() const { + const std::shared_ptr schema = ReaderTestSchema(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr data, + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(schema->fields()), + R"([[0,"zero"],[1,null],[2,"two"],[3,"three"],[4,null],[5,"five"]])")); + const std::string path = PathUtil::JoinPath(directory_->Str(), "reader.vortex"); + PAIMON_RETURN_NOT_OK(WriteFile(path, schema, data, /*batch_size=*/2)); + return path; + } + + Result> CreateReader(const std::string& path, + int32_t batch_size = 2) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr builder, + format_->CreateReaderBuilder(batch_size)); + builder->WithMemoryPool(pool_); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, file_system_->Open(path)); + return builder->Build(input); + } + + Status ConfigureReader(FileBatchReader* reader, const std::shared_ptr& schema, + const std::shared_ptr& predicate = nullptr, + const std::optional& selection = std::nullopt) const { + ::ArrowSchema ffi_schema = {}; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema)); + ScopeGuard release_schema([&ffi_schema]() { + if (ffi_schema.release != nullptr) { + ffi_schema.release(&ffi_schema); + } + }); + return reader->SetReadSchema(&ffi_schema, predicate, selection); + } + + void AssertSelectionState(const FileBatchReader* reader, uint64_t expected_rows, + uint64_t expected_indices) const { + const auto& selection_state = + checked_cast(reader)->selection_; + ASSERT_EQ(selection_state.SelectedRowCount(), expected_rows); + ASSERT_EQ(selection_state.IndexCount(), expected_indices); + } + + Result> ReadNextBatch(FileBatchReader* reader) const { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + return std::shared_ptr(); + } + PAIMON_RETURN_NOT_OK(paimon::test::ReadResultCollector::CheckBatchOffset(batch)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr array, + arrow::ImportArray(batch.first.get(), batch.second.get())); + return array; + } + protected: std::shared_ptr format_; std::shared_ptr file_system_; @@ -204,6 +367,59 @@ class RecordingInputStream : public InputStream { int close_count_ = 0; }; +// Record callback I/O without serializing the underlying reads. +class ReadAtTrackingInputStream : public InputStream { + public: + explicit ReadAtTrackingInputStream(std::shared_ptr input) + : input_(std::move(input)) {} + + Status Seek(int64_t offset, SeekOrigin origin) override { + return input_->Seek(offset, origin); + } + Result GetPos() const override { + return input_->GetPos(); + } + Result Read(char* buffer, int64_t size) override { + return input_->Read(buffer, size); + } + Result Read(char* buffer, int64_t size, int64_t offset) override { + RecordRead(offset, size); + return input_->Read(buffer, size, offset); + } + void ReadAsync(char* buffer, int64_t size, int64_t offset, + std::function&& callback) override { + RecordRead(offset, size); + input_->ReadAsync(buffer, size, offset, std::move(callback)); + } + Result GetUri() const override { + return input_->GetUri(); + } + Result Length() const override { + return input_->Length(); + } + Status Close() override { + return input_->Close(); + } + void ClearReadAtRanges() { + std::lock_guard lock(mutex_); + read_at_ranges_.clear(); + } + std::vector> GetReadAtRanges() const { + std::lock_guard lock(mutex_); + return read_at_ranges_; + } + + private: + void RecordRead(int64_t offset, int64_t size) { + std::lock_guard lock(mutex_); + read_at_ranges_.emplace_back(offset, size); + } + + std::shared_ptr input_; + mutable std::mutex mutex_; + std::vector> read_at_ranges_; +}; + } // namespace // The stream must stay open while a read callback still owns the context, and close exactly once @@ -317,9 +533,8 @@ TEST_F(VortexFileFormatTest, WriterBuilderRejectsUnsupportedTypes) { writer_builder->WithMemoryPool(pool_); ASSERT_OK_AND_ASSIGN(std::shared_ptr output, file_system_->Create(path, /*overwrite=*/true)); - Result> writer = writer_builder->Build(output, "zstd"); - ASSERT_FALSE(writer.ok()); - ASSERT_TRUE(writer.status().IsInvalid()) << writer.status().ToString(); + ASSERT_NOK_WITH_MSG(writer_builder->Build(output, "zstd"), + "Vortex file format does not support type MAP"); } } @@ -361,6 +576,144 @@ TEST_F(VortexFileFormatTest, ReadRetypesStoredTimestampTimezone) { ASSERT_TRUE(values.IsNull(1)); } +// Java's LTZ schema and predicate literals use UTC. Normalize only the timezone metadata; +// changing the integer timestamps here would change the instants being stored. +TEST_F(VortexFileFormatTest, WriterNormalizesTimestampTimezonesAcrossPrecisions) { + const std::vector units = { + arrow::TimeUnit::SECOND, arrow::TimeUnit::MILLI, arrow::TimeUnit::MICRO, + arrow::TimeUnit::NANO}; + const std::string json = + R"([[-1000001,-1000001,-1000001],[-1,-1,-1],[null,null,0],[0,0,0],[1,1,1],[123456789,123456789,123456789]])"; + const std::vector zones = {"Asia/Shanghai", "America/New_York"}; + for (const std::string& zone : zones) { + for (arrow::TimeUnit::type unit : units) { + SCOPED_TRACE(fmt::format("zone={}, unit={}", zone, static_cast(unit))); + const auto make_schema = [unit](const std::string& timezone) { + const auto metadata = arrow::key_value_metadata({"timestamp-tag"}, {"retained"}); + return arrow::schema( + {arrow::field("plain", arrow::timestamp(unit), true, metadata), + arrow::field("ltz", arrow::timestamp(unit, timezone), true, metadata), + arrow::field("utc", arrow::timestamp(unit, "UTC"), false, metadata)}, + metadata); + }; + const auto schema = make_schema(zone); + const auto original_schema = make_schema(zone); + const auto utc_schema = make_schema("UTC"); + std::shared_ptr written = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json) + .ValueOrDie(); + std::shared_ptr expected_utc = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(utc_schema->fields()), + json) + .ValueOrDie(); + const std::string path = + PathUtil::JoinPath(directory_->Str(), fmt::format("timezone-{}-{}.vortex", + zone.substr(zone.find('/') + 1), + static_cast(unit))); + ASSERT_OK(WriteFile(path, schema, written, /*batch_size=*/2)); + ASSERT_TRUE(schema->Equals(*original_schema, /*check_metadata=*/true)); + ASSERT_TRUE(written->type()->Equals(arrow::struct_(original_schema->fields()), + /*check_metadata=*/true)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> file_schema, + reader->GetFileSchema()); + std::shared_ptr stored_schema = + arrow::ImportSchema(file_schema.get()).ValueOrDie(); + ASSERT_TRUE(stored_schema->Equals(*utc_schema)) << stored_schema->ToString(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual_utc, + ReadFile(path, utc_schema, /*batch_size=*/2, + /*expected_row_count=*/6)); + ASSERT_TRUE(actual_utc->Equals(expected_utc)); + ASSERT_TRUE(actual_utc->type()->Equals(expected_utc->type(), /*check_metadata=*/true)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual_local, + ReadFile(path, schema, /*batch_size=*/3, + /*expected_row_count=*/6)); + ASSERT_TRUE(actual_local->Equals(written)); + ASSERT_TRUE(actual_local->type()->Equals(written->type(), /*check_metadata=*/true)); + } + } +} + +TEST_F(VortexFileFormatTest, WriterNormalizesNestedTimestampTimezones) { + const auto make_schema = [](bool utc) { + const auto metadata = arrow::key_value_metadata({"nested-tag"}, {"retained"}); + const auto local_millis = + arrow::timestamp(arrow::TimeUnit::MILLI, utc ? "UTC" : "Asia/Shanghai"); + const auto local_nanos = + arrow::timestamp(arrow::TimeUnit::NANO, utc ? "UTC" : "America/New_York"); + const auto local_seconds = + arrow::timestamp(arrow::TimeUnit::SECOND, utc ? "UTC" : "America/New_York"); + return arrow::schema( + {arrow::field( + "r", + arrow::struct_( + {arrow::field("ts", local_millis, false, metadata), + arrow::field("plain", arrow::timestamp(arrow::TimeUnit::MICRO), true, + metadata), + arrow::field("l", + arrow::list(arrow::field("item", local_nanos, true, metadata)), + true, metadata)}), + false, metadata), + arrow::field( + "f", + arrow::fixed_size_list(arrow::field("item", local_seconds, true, metadata), 2), + true, metadata)}, + metadata); + }; + const auto schema = make_schema(/*utc=*/false); + const auto original_schema = make_schema(/*utc=*/false); + const auto utc_schema = make_schema(/*utc=*/true); + const std::string json = R"([ + [{"ts":-1,"plain":-1000001,"l":[-1,null,123456789]},[-1,0]], + [{"ts":-2,"plain":null,"l":null},null], + [{"ts":0,"plain":null,"l":[]},[1,null]], + [{"ts":1,"plain":1,"l":[0]},[null,-1000001]] + ])"; + std::shared_ptr written = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json) + .ValueOrDie(); + std::shared_ptr expected_utc = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(utc_schema->fields()), json) + .ValueOrDie(); + const std::string path = PathUtil::JoinPath(directory_->Str(), "nested-timezones.vortex"); + ASSERT_OK(WriteFile(path, schema, written, /*batch_size=*/2)); + ASSERT_TRUE(schema->Equals(*original_schema, /*check_metadata=*/true)); + ASSERT_TRUE(written->type()->Equals(arrow::struct_(original_schema->fields()), + /*check_metadata=*/true)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> file_schema, reader->GetFileSchema()); + std::shared_ptr stored_schema = + arrow::ImportSchema(file_schema.get()).ValueOrDie(); + ASSERT_TRUE(stored_schema->Equals(*utc_schema)) << stored_schema->ToString(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual_utc, + ReadFile(path, utc_schema, /*batch_size=*/3, /*expected_row_count=*/4)); + ASSERT_TRUE(actual_utc->Equals(expected_utc)); + ASSERT_TRUE(actual_utc->type()->Equals(expected_utc->type(), /*check_metadata=*/true)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual_local, + ReadFile(path, schema, /*batch_size=*/1, /*expected_row_count=*/4)); + ASSERT_TRUE(actual_local->Equals(written)); + ASSERT_TRUE(actual_local->type()->Equals(written->type(), /*check_metadata=*/true)); +} + +// Use the native file reader so an SDK residual filter cannot hide declined LTZ pushdown. +TEST_F(VortexFileFormatTest, NormalizedTimestampTimezoneSupportsNativePredicates) { + const auto schema = arrow::schema( + {arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"))}); + std::shared_ptr written = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), + R"([[-1001],[-1],[null],[0],[1],[1000]])") + .ValueOrDie(); + const std::string path = PathUtil::JoinPath(directory_->Str(), "timezone-predicates.vortex"); + ASSERT_OK(WriteFile(path, schema, written, /*batch_size=*/2)); + const auto predicate = PredicateBuilder::Equal(0, "ltz", FieldType::TIMESTAMP, + Literal(Timestamp::FromEpochMillis(-1))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + CreatePredicateReader(path, schema, predicate, /*max_values=*/"")); + AssertNativeAndResidualRows(std::move(reader), predicate, written, {1}, {1}); +} + TEST_F(VortexFileFormatTest, WriteThenRead) { std::string path = PathUtil::JoinPath(directory_->Str(), "data.vortex"); arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false), @@ -429,6 +782,90 @@ TEST_F(VortexFileFormatTest, WriteThenReadListOfString) { AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5}); } +// NULL parent lists must export correctly even when a bounded batch has no child ranges. +TEST_F(VortexFileFormatTest, NestedListNullAndEmptyBatchesRoundTrip) { + struct NestedListCase { + std::string name; + std::shared_ptr type; + std::vector> values; + }; + const std::vector cases = { + {"list-list-int", + arrow::list(arrow::list(arrow::int32())), + {{"null-only", R"([[null]])"}, + {"all-null", R"([[null],[null]])"}, + {"empty-only", R"([[[]]])"}, + {"empty-inner", R"([[[[]]]])"}, + {"nullable-inner", R"([[[null,[],[null],[1,null,2]]]])"}, + {"mixed", R"([[null],[[]],[[[]]],[[null]],[[[1,2],[],null,[3]]],[null]])"}}}, + {"list-list-string", + arrow::list(arrow::list(arrow::utf8())), + {{"null-only", R"([[null]])"}, + {"all-null", R"([[null],[null]])"}, + {"empty-only", R"([[[]]])"}, + {"empty-inner", R"([[[[]]]])"}, + {"nullable-inner", R"([[[null,[],[null],["",null,"one"]]]])"}, + {"mixed", + R"([[null],[[]],[[[]]],[[null]],[[["one","a-long-string-beyond-the-inline-view-limit"],[],null,["three",null]]],[null]])"}}}, + {"list-struct-list-string", + arrow::list(arrow::struct_({arrow::field("tags", arrow::list(arrow::utf8()))})), + {{"null-only", R"([[null]])"}, + {"all-null", R"([[null],[null]])"}, + {"empty-only", R"([[[]]])"}, + {"empty-inner", R"([[[{"tags":[]}]]])"}, + {"nullable-inner", R"([[[null,{"tags":null},{"tags":[]},{"tags":[null,""]}]]])"}, + {"mixed", + R"([[null],[[]],[[null]],[[{"tags":[]}]],[[{"tags":["one","a-long-string-beyond-the-inline-view-limit",null]},{"tags":null},null]],[null]])"}}}}; + for (const NestedListCase& test_case : cases) { + const auto schema = arrow::schema({arrow::field("nested", test_case.type)}); + for (const auto& [name, json] : test_case.values) { + SCOPED_TRACE(fmt::format("type={}, case={}", test_case.name, name)); + const auto source_result = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json); + ASSERT_TRUE(source_result.ok()) << source_result.status().ToString(); + const std::shared_ptr source = source_result.ValueOrDie(); + ASSERT_TRUE(source->ValidateFull().ok()); + const std::string path = PathUtil::JoinPath( + directory_->Str(), fmt::format("{}-{}.vortex", test_case.name, name)); + // Keep mixed rows in one input batch so the reader slices shared child elements. + ASSERT_OK(WriteFile(path, schema, source, /*batch_size=*/16)); + for (int32_t batch_size : {1, 2}) { + SCOPED_TRACE(fmt::format("batch_size={}", batch_size)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + CreateReader(path, batch_size)); + ASSERT_OK(ConfigureReader(reader.get(), schema)); + ASSERT_OK_AND_ASSIGN(uint64_t total_rows, reader->GetNumberOfRows()); + ASSERT_EQ(total_rows, static_cast(source->length())); + int64_t rows_seen = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr batch, + ReadNextBatch(reader.get())); + if (batch == nullptr) { + break; + } + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_GT(batch->length(), 0); + ASSERT_LE(batch->length(), batch_size); + ASSERT_LE(rows_seen + batch->length(), source->length()); + ASSERT_TRUE(batch->type()->Equals(arrow::struct_(schema->fields()))); + ASSERT_TRUE(batch->Equals(source->Slice(rows_seen, batch->length()))) + << batch->ToString(); + for (int64_t i = 0; i < batch->length(); ++i) { + ASSERT_OK_AND_ASSIGN( + uint64_t physical_row, + reader->GetPreviousBatchFileRowId(static_cast(i))); + ASSERT_EQ(physical_row, static_cast(rows_seen + i)); + } + rows_seen += batch->length(); + } + ASSERT_EQ(rows_seen, source->length()); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), + "last Vortex batch was EOF"); + } + } + } +} + // Regression for the FIXED_SIZE_LIST branch of NormalizeViewArray: without recursion into the // elements, a fixed_size_list column comes back as fixed_size_list (Vortex // hardcodes Utf8 -> Utf8View), mismatching the normalized read schema and silently corrupting the @@ -494,7 +931,7 @@ TEST_F(VortexFileFormatTest, RejectNullVectorElements) { .ValueOrDie(); ASSERT_NOK_WITH_MSG(WriteFile(PathUtil::JoinPath(directory_->Str(), "invalid-vector.vortex"), schema, array, /*batch_size=*/1), - "VECTOR cannot contain null elements"); + "VECTOR cannot contain null elements, found one at row 0 position 1"); } TEST_F(VortexFileFormatTest, WriteThenReadVectorsWithNullAncestors) { @@ -539,7 +976,7 @@ TEST_F(VortexFileFormatTest, WriteThenReadVectorsWithNullAncestors) { input = arrow::StructArray::Make({arrow::MakeArray(visible_parent)}, schema->fields()) .ValueOrDie(); ASSERT_NOK_WITH_MSG(WriteFile(path + ".invalid", schema, input, /*batch_size=*/2), - "VECTOR cannot contain null elements"); + "VECTOR cannot contain null elements, found one at row 0 position 1"); } } @@ -696,32 +1133,9 @@ TEST_F(VortexFileFormatTest, ProjectedReadRejectsIncompatibleLeafType) { ASSERT_OK(WriteFile(path, schema, written, /*batch_size=*/2)); std::shared_ptr widened = arrow::schema({arrow::field("id", arrow::int64())}); - Result> rejected = - ReadFile(path, widened, /*batch_size=*/2, /*expected_row_count=*/3); - ASSERT_FALSE(rejected.ok()); - ASSERT_TRUE(rejected.status().IsInvalid()) << rejected.status().ToString(); -} - -// A read field the file does not carry is null-filled instead of rejected: that is the -// schema-evolution answer the rest of the read path gives, so every file yields the read schema. -TEST_F(VortexFileFormatTest, ProjectedReadNullFillsFieldMissingFromFile) { - std::string path = PathUtil::JoinPath(directory_->Str(), "missing-field.vortex"); - arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false)}; - std::shared_ptr schema = arrow::schema(fields); - std::shared_ptr written = - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([[1],[2],[3]])") - .ValueOrDie(); - ASSERT_OK(WriteFile(path, schema, written, /*batch_size=*/2)); - - arrow::FieldVector projected_fields = {fields[0], arrow::field("extra", arrow::utf8())}; - std::shared_ptr projected = arrow::schema(projected_fields); - std::shared_ptr expected = - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(projected_fields), - R"([[1,null],[2,null],[3,null]])") - .ValueOrDie(); - ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, - ReadFile(path, projected, /*batch_size=*/2, /*expected_row_count=*/3)); - ASSERT_TRUE(actual->Equals(expected)) << actual->ToString() << "\nvs\n" << expected->ToString(); + ASSERT_NOK_WITH_MSG( + ReadFile(path, widened, /*batch_size=*/2, /*expected_row_count=*/3), + "AlignArrayToReadType unsupported leaf type change: data int32 vs read int64"); } // Regression guard for ordered scans: 200 sequential rows span many Vortex chunks, and a small read @@ -997,4 +1411,1134 @@ TEST_F(VortexFileFormatTest, WriteSurfacesInjectedIOError) { ASSERT_TRUE(run_complete); } +// Selection pushdown must preserve physical row ids across output batches. +TEST_F(VortexFileFormatTest, SelectionPushdownReturnsSelectedRowsAndPhysicalRowIds) { + constexpr int64_t kRows = 10; + arrow::Int32Builder id_builder; + arrow::StringBuilder name_builder; + for (int64_t i = 0; i < kRows; ++i) { + ASSERT_TRUE(id_builder.Append(static_cast(i * 10)).ok()); + ASSERT_TRUE(name_builder.Append(fmt::format("row_{}", i)).ok()); + } + std::shared_ptr id_array; + std::shared_ptr name_array; + ASSERT_TRUE(id_builder.Finish(&id_array).ok()); + ASSERT_TRUE(name_builder.Finish(&name_array).ok()); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false), + arrow::field("name", arrow::utf8())}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr data = + arrow::StructArray::Make({id_array, name_array}, fields).ValueOrDie(); + + std::string path = PathUtil::JoinPath(directory_->Str(), "selection.vortex"); + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/4)); + + // Select physical rows {1, 3, 5, 7}; their id values are {10, 30, 50, 70}. + RoaringBitmap32 selection = RoaringBitmap32::From({1, 3, 5, 7}); + + // batch_size=2 spreads the 4 selected rows over two batches, exercising the cross-batch row id. + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader_builder, + format_->CreateReaderBuilder(/*batch_size=*/2)); + reader_builder->WithMemoryPool(pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system_->Open(path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, reader_builder->Build(input)); + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&ffi_schema, /*predicate=*/nullptr, selection)); + + std::vector got_ids; + std::vector got_row_ids; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + std::shared_ptr arr = + arrow::ImportArray(batch.first.get(), batch.second.get()).ValueOrDie(); + std::shared_ptr id_col = checked_pointer_cast( + checked_pointer_cast(arr)->GetFieldByName("id")); + for (int64_t i = 0; i < arr->length(); ++i) { + got_ids.push_back(id_col->Value(i)); + ASSERT_OK_AND_ASSIGN(uint64_t row_id, + reader->GetPreviousBatchFileRowId(static_cast(i))); + got_row_ids.push_back(row_id); + } + } + ASSERT_EQ(got_ids, (std::vector{10, 30, 50, 70})); + ASSERT_EQ(got_row_ids, (std::vector{1, 3, 5, 7})); +} + +TEST(VortexSelectionStateTest, LargeDenseSelectionsPreserveBitmapAndPhysicalRanks) { + constexpr int32_t kRows = 100000000; + for (const int32_t deleted_row : {-1, 0, kRows / 2, kRows - 1}) { + SCOPED_TRACE(deleted_row); + RoaringBitmap32 bitmap; + bitmap.AddRange(0, kRows); + if (deleted_row >= 0) { + bitmap.RemoveRange(deleted_row, deleted_row + 1); + } + const RoaringBitmap32 original = bitmap; + ASSERT_OK_AND_ASSIGN(VortexFileBatchReader::SelectionState state, + VortexFileBatchReader::SelectionState::Create(bitmap, kRows)); + ASSERT_TRUE(bitmap == original); + const uint64_t expected_count = kRows - (deleted_row >= 0 ? 1 : 0); + ASSERT_EQ(state.SelectedRowCount(), expected_count); + ASSERT_EQ(state.IndexCount(), deleted_row >= 0 ? 1 : 0); + for (const uint64_t rank : {uint64_t{0}, expected_count / 2, expected_count - 1}) { + ASSERT_OK_AND_ASSIGN(uint64_t physical_row, state.GetPhysicalRowId(rank)); + ASSERT_EQ(physical_row, + rank + (deleted_row >= 0 && rank >= static_cast(deleted_row))); + } + } +} + +TEST(VortexSelectionStateTest, DenseSelectionSupportsRowsBeyondSignedBitmapRange) { + constexpr int32_t kMaxRow = std::numeric_limits::max(); + for (const bool selects_max_row : {false, true}) { + SCOPED_TRACE(selects_max_row); + RoaringBitmap32 bitmap; + bitmap.AddRange(0, kMaxRow); + if (selects_max_row) { + bitmap.RemoveRange(kMaxRow - 1, kMaxRow); + bitmap.Add(kMaxRow); + ASSERT_NOK_WITH_MSG( + VortexFileBatchReader::SelectionState::Create(bitmap, kMaxRow), + fmt::format("Vortex selected row {} is out of range {}", kMaxRow, kMaxRow)); + } + const RoaringBitmap32 original = bitmap; + const uint64_t total_rows = static_cast(kMaxRow) + 2; + ASSERT_OK_AND_ASSIGN(VortexFileBatchReader::SelectionState state, + VortexFileBatchReader::SelectionState::Create(bitmap, total_rows)); + ASSERT_TRUE(bitmap == original); + ASSERT_EQ(state.SelectedRowCount(), static_cast(kMaxRow)); + const auto excluded = static_cast(selects_max_row ? kMaxRow - 1 : kMaxRow); + ASSERT_EQ(state.indices_, (std::vector{excluded, total_rows - 1})); + ASSERT_OK_AND_ASSIGN(uint64_t physical_row, state.GetPhysicalRowId(kMaxRow - 1)); + ASSERT_EQ(physical_row, static_cast(selects_max_row ? kMaxRow : kMaxRow - 1)); + ASSERT_NOK_WITH_MSG( + state.GetPhysicalRowId(kMaxRow), + fmt::format("emitted row offset {} exceeds selection size {}", kMaxRow, kMaxRow)); + } +} + +TEST(VortexSelectionStateTest, SelectionBoundsAreValidatedBeforeFastPaths) { + constexpr uint64_t kInvalidRow = std::numeric_limits::max(); + ASSERT_NOK_WITH_MSG( + VortexFileBatchReader::SelectionState::Create(RoaringBitmap32::From({0, 1, 2, 4}), 4), + "Vortex selected row 4 is out of range 4"); + ASSERT_NOK_WITH_MSG( + VortexFileBatchReader::SelectionState::Create(RoaringBitmap32::From({-1, 0, 1, 2}), 4), + fmt::format("Vortex selected row {} is out of range 4", kInvalidRow)); + constexpr int32_t kMaxRow = std::numeric_limits::max(); + const RoaringBitmap32 bitmap = RoaringBitmap32::From({0, kMaxRow}); + ASSERT_NOK_WITH_MSG(VortexFileBatchReader::SelectionState::Create(bitmap, kMaxRow), + fmt::format("Vortex selected row {} is out of range {}", kMaxRow, kMaxRow)); + ASSERT_OK_AND_ASSIGN( + VortexFileBatchReader::SelectionState state, + VortexFileBatchReader::SelectionState::Create(bitmap, static_cast(kMaxRow) + 1)); + ASSERT_OK_AND_ASSIGN(uint64_t physical_row, state.GetPhysicalRowId(1)); + ASSERT_EQ(physical_row, static_cast(kMaxRow)); + const uint64_t total_rows = static_cast(kMaxRow) + 2; + ASSERT_NOK_WITH_MSG( + VortexFileBatchReader::SelectionState::Create(RoaringBitmap32::From({0, -1}), total_rows), + fmt::format("Vortex selected row {} is out of range {}", kInvalidRow, total_rows)); +} + +TEST_F(VortexFileFormatTest, DenseSelectionKeepsOnlyExcludedIndicesAndPreservesNullValues) { + constexpr int32_t kRows = 4096; + constexpr int32_t kDeletedRow = 2048; + arrow::Int32Builder id_builder; + arrow::Int32Builder value_builder; + for (int32_t row = 0; row < kRows; ++row) { + ASSERT_TRUE(id_builder.Append(row).ok()); + if (row % 17 == 0) { + ASSERT_TRUE(value_builder.AppendNull().ok()); + } else { + ASSERT_TRUE(value_builder.Append(row * 10).ok()); + } + } + std::shared_ptr ids; + std::shared_ptr values; + ASSERT_TRUE(id_builder.Finish(&ids).ok()); + ASSERT_TRUE(value_builder.Finish(&values).ok()); + const auto schema = arrow::schema( + {arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::int32())}); + const auto data = arrow::StructArray::Make({ids, values}, schema->fields()).ValueOrDie(); + const std::string path = PathUtil::JoinPath(directory_->Str(), "dense-selection.vortex"); + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/1024)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + CreateReader(path, /*batch_size=*/127)); + RoaringBitmap32 selection; + selection.AddRange(0, kRows); + selection.RemoveRange(kDeletedRow, kDeletedRow + 1); + ASSERT_OK(ConfigureReader(reader.get(), schema, nullptr, selection)); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), kRows - 1, 1)); + uint64_t emitted = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadNextBatch(reader.get())); + if (array == nullptr) { + break; + } + ASSERT_TRUE(array->ValidateFull().ok()); + const auto& batch = checked_cast(*array); + const auto& batch_ids = checked_cast(*batch.field(0)); + const auto& batch_values = checked_cast(*batch.field(1)); + // Reverse requests also check that physical-id lookup does not depend on call order. + for (int64_t i = array->length(); i > 0; --i) { + const uint64_t output_row = emitted + static_cast(i - 1); + const uint64_t expected_row = output_row + (output_row >= kDeletedRow ? 1 : 0); + ASSERT_OK_AND_ASSIGN(uint64_t physical_row, + reader->GetPreviousBatchFileRowId(static_cast(i - 1))); + ASSERT_EQ(physical_row, expected_row); + ASSERT_EQ(batch_ids.Value(i - 1), static_cast(expected_row)); + ASSERT_EQ(batch_values.IsNull(i - 1), expected_row % 17 == 0); + if (!batch_values.IsNull(i - 1)) { + ASSERT_EQ(batch_values.Value(i - 1), static_cast(expected_row * 10)); + } + } + emitted += static_cast(array->length()); + } + ASSERT_EQ(emitted, uint64_t{kRows - 1}); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "last Vortex batch was EOF"); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), kRows - 1, 1)); +} + +TEST_F(VortexFileFormatTest, DenseSelectionMapsConsecutiveExcludedRowsAndSparseSelections) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + const std::vector> selections = { + {1, 2, 3, 4, 5}, {0, 1, 2, 3, 4}, {0, 1, 4, 5}, {2, 3, 4, 5}, + {0, 1, 2, 3}, {0, 2, 3, 5}, {0, 5}}; + for (const auto& selected_rows : selections) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), nullptr, + RoaringBitmap32::From(selected_rows))); + ASSERT_NO_FATAL_FAILURE( + AssertSelectionState(reader.get(), selected_rows.size(), + std::min(selected_rows.size(), size_t{6} - selected_rows.size()))); + std::vector actual; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadNextBatch(reader.get())); + if (array == nullptr) { + break; + } + const size_t offset = actual.size(); + actual.resize(offset + static_cast(array->length())); + for (int64_t i = array->length(); i > 0; --i) { + ASSERT_OK_AND_ASSIGN( + uint64_t row, reader->GetPreviousBatchFileRowId(static_cast(i - 1))); + actual[offset + static_cast(i - 1)] = row; + } + } + ASSERT_EQ(actual, (std::vector(selected_rows.begin(), selected_rows.end()))); + } +} + +TEST_F(VortexFileFormatTest, FullEmptyAndDenseSelectionsCanBeReconfiguredWithoutExpandedMapping) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + RoaringBitmap32 full; + full.AddRange(0, 6); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), nullptr, full)); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), 6, 0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, ReadNextBatch(reader.get())); + ASSERT_NE(first, nullptr); + ASSERT_EQ(first->length(), 2); + ASSERT_OK_AND_ASSIGN(uint64_t row, reader->GetPreviousBatchFileRowId(1)); + ASSERT_EQ(row, uint64_t{1}); + + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), nullptr, RoaringBitmap32())); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), 0, 0)); + ASSERT_OK_AND_ASSIGN(first, ReadNextBatch(reader.get())); + ASSERT_EQ(first, nullptr); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "no Vortex batch has been read yet"); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), nullptr, + RoaringBitmap32::From({1, 2, 3, 4, 5}))); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), 5, 1)); + ASSERT_OK_AND_ASSIGN(first, ReadNextBatch(reader.get())); + ASSERT_OK_AND_ASSIGN(row, reader->GetPreviousBatchFileRowId(0)); + ASSERT_EQ(row, uint64_t{1}); + ASSERT_OK_AND_ASSIGN(row, reader->GetPreviousBatchFileRowId(1)); + ASSERT_EQ(row, uint64_t{2}); + ASSERT_OK_AND_ASSIGN(first, ReadNextBatch(reader.get())); + ASSERT_OK_AND_ASSIGN(row, reader->GetPreviousBatchFileRowId(0)); + ASSERT_EQ(row, uint64_t{3}); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema())); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), 6, 0)); + ASSERT_OK_AND_ASSIGN(first, ReadNextBatch(reader.get())); + ASSERT_OK_AND_ASSIGN(row, reader->GetPreviousBatchFileRowId(0)); + ASSERT_EQ(row, uint64_t{0}); +} + +TEST_F(VortexFileFormatTest, NativePredicateErrorsPropagateFromLazyScanCreation) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + const Literal invalid_utf8(FieldType::STRING, "\xff", 1); + const std::shared_ptr invalid = + PredicateBuilder::Equal(1, "name", FieldType::STRING, invalid_utf8); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), invalid)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), + "create Vortex predicate scalar: Other error: invalid utf-8: " + "invalid utf-8 sequence of 1 bytes from index 0"); + + const std::shared_ptr valid = + PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(int32_t{1})); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), valid)); + ASSERT_OK_AND_ASSIGN(std::vector row_ids, CollectNativeRowIds(std::move(reader))); + ASSERT_EQ(row_ids, (std::vector{1})); +} + +TEST_F(VortexFileFormatTest, NativeTemporalLimitsFallBackAndPreserveResidualResults) { + const auto schema = + arrow::schema({arrow::field("date", arrow::date32()), + arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"))}); + const auto data = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(schema->fields()), "[[0,0],[1,1000],[null,null],[-1,-1000]]"); + ASSERT_TRUE(data.ok()) << data.status().ToString(); + const std::string path = PathUtil::JoinPath(directory_->Str(), "temporal-fallback.vortex"); + ASSERT_OK(WriteFile(path, schema, data.ValueOrDie(), /*batch_size=*/2)); + + const Literal max_date(FieldType::DATE, std::numeric_limits::max()); + // The timestamp fits in Paimon's calendar range but exceeds Vortex's native range. + const Literal end_of_year_9999(Timestamp::FromEpochMillis(253402300799999, 999000)); + const std::vector, std::vector>> cases = { + {PredicateBuilder::LessThan(0, "date", FieldType::DATE, max_date), {0, 1, 3}}, + {PredicateBuilder::NotIn(1, "ts", FieldType::TIMESTAMP, + {Literal(Timestamp::FromEpochMillis(0)), end_of_year_9999}), + {1, 3}}}; + for (const auto& [predicate, expected] : cases) { + SCOPED_TRACE(predicate->GetFunction().ToString()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_OK(ConfigureReader(reader.get(), schema, predicate)); + ASSERT_NO_FATAL_FAILURE(AssertNativeAndResidualRows( + std::move(reader), predicate, data.ValueOrDie(), {0, 1, 2, 3}, expected)); + } +} + +TEST_F(VortexFileFormatTest, IoCoalesceOptionsValidateLimitsAtReaderBuild) { + const std::string path = PathUtil::JoinPath(directory_->Str(), "io-coalesce-limits.vortex"); + const auto schema = arrow::schema({arrow::field("id", arrow::int32())}); + const auto data = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), "[[1]]") + .ValueOrDie(); + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2)); + + struct LimitCase { + std::map options; + const char* error; + }; + const std::string max_limit = std::to_string(std::numeric_limits::max()); + const std::string above_limit = + std::to_string(static_cast(std::numeric_limits::max()) + 1); + // Single overrides are checked against the other option's default. + const LimitCase cases[] = { + {{{kVortexReadIoCoalesceMaxGap, "16777216"}}, nullptr}, + {{{kVortexReadIoCoalesceMaxSize, "1048576"}}, nullptr}, + {{{kVortexReadIoCoalesceMaxGap, max_limit}, {kVortexReadIoCoalesceMaxSize, max_limit}}, + nullptr}, + {{{kVortexReadIoCoalesceMaxSize, "0"}}, "coalesced max_size must be positive"}, + {{{kVortexReadIoCoalesceMaxGap, "17"}, {kVortexReadIoCoalesceMaxSize, "16"}}, + "coalesced max_gap must not exceed max_size"}, + {{{kVortexReadIoCoalesceMaxGap, "16777217"}}, "coalesced max_gap must not exceed max_size"}, + {{{kVortexReadIoCoalesceMaxSize, "1048575"}}, "coalesced max_gap must not exceed max_size"}, + {{{kVortexReadIoCoalesceMaxGap, above_limit}, {kVortexReadIoCoalesceMaxSize, above_limit}}, + "coalesced max_size must not exceed INT64_MAX"}, + }; + for (const auto& test_case : cases) { + SCOPED_TRACE(::testing::PrintToString(test_case.options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr format, + FileFormatFactory::Get("vortex", test_case.options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr builder, + format->CreateReaderBuilder(2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system_->Open(path)); + Result> result = builder->Build(input); + if (test_case.error == nullptr) { + ASSERT_OK(result.status()); + ASSERT_NE(result.value(), nullptr); + } else { + ASSERT_NOK_WITH_MSG( + result, + fmt::format("open Vortex callback data source: Other error: {}", test_case.error)); + } + } +} + +TEST_F(VortexFileFormatTest, IoCoalesceOptionsChangePhysicalReadRanges) { + // Both payload start offsets must fit within max-gap regardless of which read is polled first. + constexpr int32_t kRows = 1536; + constexpr int32_t kPayloadBytes = 512; + constexpr int64_t kMaxGap = 2LL << 20; + constexpr int64_t kSmallMaxSize = 2LL << 20; + constexpr int64_t kLargeMaxSize = 4LL << 20; + constexpr int64_t kLargeRead = 128LL << 10; + arrow::Int32Builder id_builder; + arrow::BinaryBuilder left_builder; + arrow::BinaryBuilder omitted_builder; + arrow::BinaryBuilder right_builder; + uint64_t state = 0x9e3779b97f4a7c15ULL; + for (int32_t row = 0; row < kRows; ++row) { + ASSERT_TRUE(id_builder.Append(row).ok()); + for (arrow::BinaryBuilder* builder : {&left_builder, &omitted_builder, &right_builder}) { + std::string bytes(kPayloadBytes, '\0'); + for (char& byte : bytes) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + byte = static_cast(state & 0xff); + } + ASSERT_TRUE(builder->Append(bytes).ok()); + } + } + std::shared_ptr ids; + std::shared_ptr left; + std::shared_ptr omitted; + std::shared_ptr right; + ASSERT_TRUE(id_builder.Finish(&ids).ok()); + ASSERT_TRUE(left_builder.Finish(&left).ok()); + ASSERT_TRUE(omitted_builder.Finish(&omitted).ok()); + ASSERT_TRUE(right_builder.Finish(&right).ok()); + const auto schema = arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("left", arrow::binary(), false), + arrow::field("omitted", arrow::binary(), false), + arrow::field("right", arrow::binary(), false)}); + const auto data = + arrow::StructArray::Make({ids, left, omitted, right}, schema->fields()).ValueOrDie(); + const std::string path = PathUtil::JoinPath(directory_->Str(), "io-coalesce-ranges.vortex"); + // Distinct high-entropy columns leave a real unprojected payload between the two read columns. + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/kRows)); + const auto projection = arrow::schema({schema->field(1), schema->field(3)}); + const auto predicate = PredicateBuilder::GreaterOrEqual( + 0, "id", FieldType::INT, Literal(static_cast(kRows / 2))); + const auto& expected_left = checked_cast(*left); + const auto& expected_right = checked_cast(*right); + const std::vector> option_sets = { + {{kVortexReadIoCoalesceMaxGap, "0"}, + {kVortexReadIoCoalesceMaxSize, std::to_string(kLargeMaxSize)}}, + {{kVortexReadIoCoalesceMaxGap, std::to_string(kMaxGap)}, + {kVortexReadIoCoalesceMaxSize, std::to_string(kLargeMaxSize)}}, + {{kVortexReadIoCoalesceMaxGap, std::to_string(kMaxGap)}, + {kVortexReadIoCoalesceMaxSize, std::to_string(kSmallMaxSize)}}, + }; + std::vector>> physical_reads; + for (size_t i = 0; i < option_sets.size(); ++i) { + SCOPED_TRACE(i); + ASSERT_OK_AND_ASSIGN(std::shared_ptr format, + FileFormatFactory::Get("vortex", option_sets[i])); + ASSERT_OK_AND_ASSIGN(std::unique_ptr builder, + format->CreateReaderBuilder(/*batch_size=*/64)); + builder->WithMemoryPool(pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system_->Open(path)); + auto tracking = std::make_shared(std::move(input)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, builder->Build(tracking)); + ASSERT_OK(ConfigureReader(reader.get(), projection, predicate)); + // Opening the file reads its footer outside the segment coalescer under test. + tracking->ClearReadAtRanges(); + int64_t expected_row = kRows / 2; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadNextBatch(reader.get())); + if (array == nullptr) { + break; + } + ASSERT_TRUE(array->ValidateFull().ok()); + ASSERT_TRUE(array->type()->Equals(arrow::struct_(projection->fields()))); + const auto& batch = checked_cast(*array); + const auto& actual_left = checked_cast(*batch.field(0)); + const auto& actual_right = checked_cast(*batch.field(1)); + for (int64_t row = 0; row < array->length(); ++row) { + ASSERT_LT(expected_row, kRows); + ASSERT_OK_AND_ASSIGN(uint64_t physical_row, + reader->GetPreviousBatchFileRowId(static_cast(row))); + ASSERT_EQ(physical_row, static_cast(expected_row)); + ASSERT_EQ(actual_left.GetString(row), expected_left.GetString(expected_row)); + ASSERT_EQ(actual_right.GetString(row), expected_right.GetString(expected_row)); + ++expected_row; + } + } + ASSERT_EQ(expected_row, kRows); + reader.reset(); + physical_reads.push_back(tracking->GetReadAtRanges()); + } + + auto baseline = physical_reads[0]; + std::sort(baseline.begin(), baseline.end()); + const auto covers = [](const std::pair& merged, + const std::pair& first, + const std::pair& second) { + return merged.first <= first.first && + merged.first + merged.second >= second.first + second.second; + }; + bool observed_limited_merge = false; + for (size_t i = 0; i < baseline.size(); ++i) { + if (baseline[i].second <= kLargeRead) { + continue; + } + for (size_t j = i + 1; j < baseline.size(); ++j) { + const int64_t hole = baseline[j].first - (baseline[i].first + baseline[i].second); + const int64_t span = baseline[j].first + baseline[j].second - baseline[i].first; + if (baseline[j].second <= kLargeRead || hole <= kLargeRead || hole > kMaxGap || + span <= kSmallMaxSize + (64LL << 10)) { + continue; + } + const bool merged = std::any_of( + physical_reads[1].begin(), physical_reads[1].end(), + [&](const auto& range) { return covers(range, baseline[i], baseline[j]); }); + const bool limited = std::any_of( + physical_reads[2].begin(), physical_reads[2].end(), + [&](const auto& range) { return covers(range, baseline[i], baseline[j]); }); + observed_limited_merge |= merged && !limited; + } + } + ASSERT_TRUE(observed_limited_merge) + << "Expected a large read to bridge two baseline payload reads only with the larger span: " + << ::testing::PrintToString(physical_reads); +} + +TEST_F(VortexFileFormatTest, MembershipMaximumIsReaderLocalAndAppliesWithProjection) { + const std::string path = PathUtil::JoinPath(directory_->Str(), "membership-limits.vortex"); + const arrow::FieldVector fields = {arrow::field("id", arrow::int32()), + arrow::field("payload", arrow::utf8())}; + const auto schema = arrow::schema(fields); + const auto data = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(fields), R"([[null,"null-id"],[0,"zero"],[100,"hundred"],[1,"one"]])"); + ASSERT_TRUE(data.ok()) << data.status().ToString(); + ASSERT_OK(WriteFile(path, schema, data.ValueOrDie(), /*batch_size=*/4)); + + struct MembershipCase { + int32_t size; + const char* maximum; + bool pushed; + }; + const std::vector cases = { + {32, "", true}, {33, "", false}, {33, "33", true}, {33, "0", false}}; + const auto projection = arrow::schema({fields[1]}); + const std::vector all_rows = {0, 1, 2, 3}; + std::vector> readers; + // Create all readers before scanning to verify that later overrides do not affect earlier + // readers. The filter column is not part of the projected schema. + for (const MembershipCase& test_case : cases) { + std::vector literals; + for (int32_t value = 0; value < test_case.size; ++value) { + literals.emplace_back(value); + } + const auto predicate = PredicateBuilder::In(0, "id", FieldType::INT, literals); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + CreatePredicateReader(path, projection, predicate, test_case.maximum)); + readers.push_back(std::move(reader)); + } + const std::vector matches = {1, 3}; + for (size_t i = 0; i < cases.size(); ++i) { + SCOPED_TRACE(fmt::format("size={}, maximum={}", cases[i].size, cases[i].maximum)); + ASSERT_OK_AND_ASSIGN(std::vector rows, + CollectNativeRowIds(std::move(readers[i]))); + ASSERT_EQ(rows, cases[i].pushed ? matches : all_rows); + } + + const auto equality = PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(int32_t{0})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + CreatePredicateReader(path, projection, equality, /*max_values=*/"0")); + ASSERT_OK_AND_ASSIGN(std::vector rows, CollectNativeRowIds(std::move(reader))); + ASSERT_EQ(rows, (std::vector{1})); +} + +TEST_F(VortexFileFormatTest, FilterPushdownOnDateColumn) { + constexpr int64_t kRows = 10; + arrow::Date32Builder date_builder; + for (int64_t i = 0; i < kRows; ++i) { + ASSERT_TRUE(date_builder.Append(static_cast(18000 + i)).ok()); + } + std::shared_ptr date_array; + ASSERT_TRUE(date_builder.Finish(&date_array).ok()); + arrow::FieldVector fields = {arrow::field("d", arrow::date32(), false)}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr data = + arrow::StructArray::Make({date_array}, fields).ValueOrDie(); + + std::string path = PathUtil::JoinPath(directory_->Str(), "filter_date.vortex"); + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/4)); + + // DATE literals use epoch days; d > 18005 keeps physical rows 6..9. + std::shared_ptr predicate = PredicateBuilder::GreaterThan( + /*field_index=*/0, "d", FieldType::DATE, Literal(FieldType::DATE, 18005)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader_builder, + format_->CreateReaderBuilder(/*batch_size=*/3)); + reader_builder->WithMemoryPool(pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system_->Open(path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, reader_builder->Build(input)); + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&ffi_schema, predicate, /*selection_bitmap=*/std::nullopt)); + + std::vector got_days; + std::vector got_row_ids; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + std::shared_ptr arr = + arrow::ImportArray(batch.first.get(), batch.second.get()).ValueOrDie(); + std::shared_ptr date_column = checked_pointer_cast( + checked_pointer_cast(arr)->GetFieldByName("d")); + for (int64_t i = 0; i < arr->length(); ++i) { + got_days.push_back(date_column->Value(i)); + ASSERT_OK_AND_ASSIGN(uint64_t row_id, + reader->GetPreviousBatchFileRowId(static_cast(i))); + got_row_ids.push_back(row_id); + } + } + ASSERT_EQ(got_days, (std::vector{18006, 18007, 18008, 18009})); + ASSERT_EQ(got_row_ids, (std::vector{6, 7, 8, 9})); +} + +TEST_F(VortexFileFormatTest, ReaderWithoutReadSchemaReturnsFullDataAndChecksRowIdBounds) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "no Vortex batch has been read yet"); + ASSERT_OK_AND_ASSIGN(uint64_t total_rows, reader->GetNumberOfRows()); + ASSERT_EQ(total_rows, uint64_t{6}); + uint64_t rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadNextBatch(reader.get())); + if (array == nullptr) { + break; + } + ASSERT_TRUE(array->ValidateFull().ok()); + ASSERT_TRUE(array->type()->Equals(arrow::struct_(ReaderTestSchema()->fields()))); + ASSERT_OK_AND_ASSIGN(uint64_t first_row, reader->GetPreviousBatchFileRowId(0)); + ASSERT_EQ(first_row, rows); + const uint64_t max_row_id = std::numeric_limits::max(); + for (uint64_t row_id : {static_cast(array->length()), max_row_id}) { + ASSERT_NOK_WITH_MSG( + reader->GetPreviousBatchFileRowId(row_id), + fmt::format("batch row id {} is out of range {}", row_id, array->length())); + } + rows += static_cast(array->length()); + } + ASSERT_EQ(rows, total_rows); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "last Vortex batch was EOF"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr repeated_eof, ReadNextBatch(reader.get())); + ASSERT_EQ(repeated_eof, nullptr); +} + +TEST_F(VortexFileFormatTest, CombinedSelectionAndFilterPreservePhysicalRowsAfterProjection) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + const auto metadata = arrow::key_value_metadata({"projection-tag"}, {"retained"}); + const auto projection = + arrow::schema({arrow::field("name", arrow::utf8(), /*nullable=*/false, metadata)}); + const auto predicate = + PredicateBuilder::GreaterOrEqual(0, "id", FieldType::INT, Literal(static_cast(2))); + ASSERT_OK( + ConfigureReader(reader.get(), projection, predicate, RoaringBitmap32::From({0, 2, 3, 5}))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr file_schema, reader->GetFileSchema()); + const auto imported_file_schema = arrow::ImportSchema(file_schema.get()).ValueOrDie(); + ASSERT_TRUE(imported_file_schema->Equals(*ReaderTestSchema())); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), 4, 2)); + std::vector physical_rows; + std::vector> batches; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadNextBatch(reader.get())); + if (array == nullptr) { + break; + } + ASSERT_TRUE(array->ValidateFull().ok()); + ASSERT_TRUE(array->type()->Equals(arrow::struct_(projection->fields()), + /*check_metadata=*/true)); + for (int64_t i = 0; i < array->length(); ++i) { + ASSERT_OK_AND_ASSIGN(uint64_t physical_row, + reader->GetPreviousBatchFileRowId(static_cast(i))); + physical_rows.push_back(physical_row); + } + batches.push_back(std::move(array)); + } + ASSERT_EQ(physical_rows, (std::vector{2, 3, 5})); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), 4, 2)); + auto actual = arrow::Concatenate(batches, arrow_pool_.get()).ValueOrDie(); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(projection->fields()), + R"([["two"],["three"],["five"]])") + .ValueOrDie(); + ASSERT_TRUE(actual->Equals(expected)); +} + +TEST_F(VortexFileFormatTest, BoundedStringBatchesPreserveProjectionRowIdsAndBufferOwnership) { + constexpr int32_t kRows = 40; + constexpr int32_t kBatchSize = 4; + const std::string repeated_text(4096, 't'); + std::string repeated_binary(4096, '\0'); + repeated_binary.front() = '\xff'; + repeated_binary.back() = '\x7f'; + arrow::Int32Builder id_builder; + arrow::StringBuilder text_builder; + arrow::BinaryBuilder binary_builder; + RoaringBitmap32 selection; + std::vector expected_rows; + for (int32_t row = 0; row < kRows; ++row) { + ASSERT_TRUE(id_builder.Append(row).ok()); + if (row % 4 == 2) { + ASSERT_TRUE(text_builder.AppendNull().ok()); + } else { + ASSERT_TRUE(text_builder.Append(row % 4 == 3 ? "" : repeated_text).ok()); + } + if (row % 5 == 2) { + ASSERT_TRUE(binary_builder.AppendNull().ok()); + } else { + ASSERT_TRUE(binary_builder.Append(row % 5 == 3 ? "" : repeated_binary).ok()); + } + if (row % 3 != 1) { + selection.Add(row); + if (row >= 5) { + expected_rows.push_back(row); + } + } + } + std::shared_ptr ids; + std::shared_ptr texts; + std::shared_ptr binaries; + ASSERT_TRUE(id_builder.Finish(&ids).ok()); + ASSERT_TRUE(text_builder.Finish(&texts).ok()); + ASSERT_TRUE(binary_builder.Finish(&binaries).ok()); + const auto schema = arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("text", arrow::utf8()), + arrow::field("payload", arrow::binary())}); + const auto data = + arrow::StructArray::Make({ids, texts, binaries}, schema->fields()).ValueOrDie(); + const std::string path = PathUtil::JoinPath(directory_->Str(), "bounded-strings.vortex"); + // A single write batch contains repeated values much larger than an inline string view. + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/kRows)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path, kBatchSize)); + const auto projection = arrow::schema({schema->field(2), schema->field(1)}); + const auto predicate = + PredicateBuilder::GreaterOrEqual(0, "id", FieldType::INT, Literal(static_cast(5))); + ASSERT_OK(ConfigureReader(reader.get(), projection, predicate, selection)); + std::vector retained; + size_t rows_seen = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + ASSERT_OK(paimon::test::ReadResultCollector::CheckBatchOffset(batch)); + ASSERT_GT(batch.first->length, 0); + ASSERT_LE(batch.first->length, kBatchSize); + for (int64_t i = 0; i < batch.first->length; ++i) { + ASSERT_LT(rows_seen, expected_rows.size()); + ASSERT_OK_AND_ASSIGN(uint64_t physical_row, + reader->GetPreviousBatchFileRowId(static_cast(i))); + ASSERT_EQ(physical_row, expected_rows[rows_seen++]); + } + retained.push_back(std::move(batch)); + } + ASSERT_EQ(rows_seen, expected_rows.size()); + ASSERT_GT(retained.size(), 1); + // Keep the raw Arrow exports across a new scan, closing, and destroying the reader. + ASSERT_OK(ConfigureReader(reader.get(), schema)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr restarted, ReadNextBatch(reader.get())); + ASSERT_NE(restarted, nullptr); + reader->Close(); + reader.reset(); + ASSERT_TRUE(restarted->ValidateFull().ok()); + ASSERT_TRUE(restarted->Equals(data->Slice(0, restarted->length()))); + + const auto& expected_texts = checked_cast(*texts); + const auto& expected_binaries = checked_cast(*binaries); + rows_seen = 0; + for (auto& batch : retained) { + auto imported = arrow::ImportArray(batch.first.get(), batch.second.get()).ValueOrDie(); + ASSERT_TRUE(imported->ValidateFull().ok()); + ASSERT_TRUE(imported->type()->Equals(arrow::struct_(projection->fields()))); + const auto& values = checked_cast(*imported); + const auto& actual_binaries = checked_cast(*values.field(0)); + const auto& actual_texts = checked_cast(*values.field(1)); + for (int64_t i = 0; i < imported->length(); ++i) { + const auto physical_row = static_cast(expected_rows[rows_seen++]); + ASSERT_EQ(actual_texts.IsNull(i), expected_texts.IsNull(physical_row)); + ASSERT_EQ(actual_binaries.IsNull(i), expected_binaries.IsNull(physical_row)); + if (!actual_texts.IsNull(i)) { + ASSERT_EQ(actual_texts.GetString(i), expected_texts.GetString(physical_row)); + } + if (!actual_binaries.IsNull(i)) { + ASSERT_EQ(actual_binaries.GetString(i), expected_binaries.GetString(physical_row)); + } + } + } + ASSERT_EQ(rows_seen, expected_rows.size()); +} + +TEST_F(VortexFileFormatTest, ResetReadSchemaRestartsAndClearsSelectionAndFilter) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + const auto predicate = + PredicateBuilder::GreaterThan(0, "id", FieldType::INT, Literal(static_cast(2))); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), predicate, + RoaringBitmap32::From({1, 3, 5}))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, ReadNextBatch(reader.get())); + ASSERT_EQ(first->length(), 2); + ASSERT_OK_AND_ASSIGN(uint64_t first_row, reader->GetPreviousBatchFileRowId(0)); + ASSERT_EQ(first_row, uint64_t{3}); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema())); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "no Vortex batch has been read yet"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr restarted, ReadNextBatch(reader.get())); + ASSERT_EQ(restarted->length(), 2); + ASSERT_OK_AND_ASSIGN(first_row, reader->GetPreviousBatchFileRowId(0)); + ASSERT_EQ(first_row, uint64_t{0}); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), nullptr, RoaringBitmap32())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr empty, ReadNextBatch(reader.get())); + ASSERT_EQ(empty, nullptr); + ASSERT_OK( + ConfigureReader(reader.get(), ReaderTestSchema(), nullptr, RoaringBitmap32::From({0, 5}))); + ASSERT_OK_AND_ASSIGN(std::vector rows, CollectNativeRowIds(std::move(reader))); + ASSERT_EQ(rows, (std::vector{0, 5})); +} + +TEST_F(VortexFileFormatTest, EmptyProjectionPreservesRowCountsAndPhysicalIds) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + const auto projection = arrow::schema({}); + for (const bool filtered : {false, true}) { + SCOPED_TRACE(filtered); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + const std::shared_ptr predicate = + filtered ? PredicateBuilder::GreaterThan(0, "id", FieldType::INT, + Literal(static_cast(2))) + : nullptr; + const std::optional selection = + filtered ? std::optional(RoaringBitmap32::From({0, 1, 3, 5})) + : std::nullopt; + ASSERT_OK(ConfigureReader(reader.get(), projection, predicate, selection)); + std::vector rows; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadNextBatch(reader.get())); + if (array == nullptr) { + break; + } + ASSERT_TRUE(array->ValidateFull().ok()); + ASSERT_TRUE(array->type()->Equals(arrow::struct_(projection->fields()))); + const auto* native_reader = checked_cast(reader.get()); + // Inspect the native export before C++ alignment can discard payload columns. + ASSERT_EQ(native_reader->current_stream_schema_->num_fields(), filtered ? 1 : 0); + ASSERT_EQ(native_reader->current_batch_->type()->num_fields(), filtered ? 1 : 0); + if (filtered) { + ASSERT_EQ(native_reader->current_stream_schema_->field(0)->name(), "_row_idx"); + } + for (int64_t i = 0; i < array->length(); ++i) { + ASSERT_OK_AND_ASSIGN(uint64_t row, + reader->GetPreviousBatchFileRowId(static_cast(i))); + // File readers may return candidates; retain the residual predicate. + if (!filtered || row > 2) { + rows.push_back(row); + } + } + } + ASSERT_EQ(rows, filtered ? (std::vector{3, 5}) + : (std::vector{0, 1, 2, 3, 4, 5})); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "last Vortex batch was EOF"); + } +} + +TEST_F(VortexFileFormatTest, EmptyProjectionAvoidsPayloadReads) { + constexpr int32_t kRows = 64; + arrow::Int32Builder id_builder; + arrow::StringBuilder payload_builder; + uint32_t seed = 13; + std::string payload(16384, '\0'); + for (int32_t row = 0; row < kRows; ++row) { + ASSERT_TRUE(id_builder.Append(row).ok()); + for (char& byte : payload) { + seed = seed * 1664525U + 1013904223U; + byte = static_cast('!' + ((seed >> 24) % 90)); + } + ASSERT_TRUE(payload_builder.Append(payload).ok()); + } + std::shared_ptr ids; + std::shared_ptr payloads; + ASSERT_TRUE(id_builder.Finish(&ids).ok()); + ASSERT_TRUE(payload_builder.Finish(&payloads).ok()); + const auto schema = arrow::schema( + {arrow::field("id", arrow::int32(), false), arrow::field("payload", arrow::utf8(), false)}); + const auto data = arrow::StructArray::Make({ids, payloads}, schema->fields()).ValueOrDie(); + const std::string path = PathUtil::JoinPath(directory_->Str(), "empty-projection-io.vortex"); + ASSERT_OK(WriteFile(path, schema, data, kRows)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr format, + FileFormatFactory::Get("vortex", {{"file.format", "vortex"}, + {kVortexReadIoCoalesceMaxGap, "0"}, + {kVortexReadIoCoalesceMaxSize, "1"}})); + for (const bool filtered : {false, true}) { + SCOPED_TRACE(filtered); + std::vector bytes_read; + for (const bool empty_projection : {false, true}) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system_->Open(path)); + ASSERT_OK_AND_ASSIGN(int64_t file_bytes, input->Length()); + // Keep payload outside the native footer read-ahead cache for the I/O comparison. + ASSERT_GT(file_bytes, int64_t{256 * 1024}); + auto tracked_input = std::make_shared(input); + ASSERT_OK_AND_ASSIGN(std::unique_ptr builder, + format->CreateReaderBuilder(/*batch_size=*/16)); + builder->WithMemoryPool(pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + builder->Build(tracked_input)); + const std::shared_ptr predicate = + filtered ? PredicateBuilder::GreaterThan(0, "id", FieldType::INT, + Literal(static_cast(31))) + : nullptr; + ASSERT_OK(ConfigureReader(reader.get(), empty_projection ? arrow::schema({}) : schema, + predicate)); + tracked_input->ClearReadAtRanges(); + uint64_t rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr batch, + ReadNextBatch(reader.get())); + if (batch == nullptr) { + break; + } + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_EQ(batch->type()->num_fields(), empty_projection ? 0 : 2); + rows += static_cast(batch->length()); + } + ASSERT_EQ(rows, filtered ? kRows / 2 : kRows); + reader->Close(); + uint64_t bytes = 0; + for (const auto& range : tracked_input->GetReadAtRanges()) { + bytes += static_cast(range.second); + } + bytes_read.push_back(bytes); + } + ASSERT_GT(bytes_read[0], uint64_t{0}); + // Metadata and predicate columns may still be read; the large payload must be omitted. + ASSERT_LT(bytes_read[1], bytes_read[0] / 4); + } +} + +TEST_F(VortexFileFormatTest, NestedProjectionValidatesEveryRequestedField) { + const std::string path = PathUtil::JoinPath(directory_->Str(), "nested-schema-contract.vortex"); + const auto item_type = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + const auto schema = + arrow::schema({arrow::field("r", item_type), arrow::field("l", arrow::list(item_type))}); + const std::shared_ptr data = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(schema->fields()), + R"([[{"a":10,"b":"x"},[{"a":1,"b":"one"},{"a":2,"b":null}]], + [{"a":20,"b":"y"},[{"a":4,"b":"four"}]]])") + .ValueOrDie(); + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/1)); + const auto missing = arrow::struct_({arrow::field("missing", arrow::int32())}); + const auto pruned = arrow::struct_({arrow::field("b", arrow::utf8())}); + const auto invalid = + arrow::schema({arrow::field("r", pruned), arrow::field("l", arrow::list(missing))}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_NOK_WITH_MSG(ConfigureReader(reader.get(), invalid), + "Field 'missing' not found or duplicate in src schema"); + + // The list wrapper name is not a physical STRUCT field and may differ from the file. + const auto projection = + arrow::schema({arrow::field("l", arrow::list(arrow::field("element", pruned))), + arrow::field("r", pruned)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadFile(path, projection, /*batch_size=*/1, /*expected_row_count=*/2)); + const std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(projection->fields()), + R"([[[{"b":"one"},{"b":null}],{"b":"x"}],[[{"b":"four"}],{"b":"y"}]])") + .ValueOrDie(); + ASSERT_TRUE(actual->Equals(expected)) << actual->ToString(); +} + +TEST_F(VortexFileFormatTest, InvalidResetPreservesFilterSelectionAndPhysicalRows) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + const auto schema = ReaderTestSchema(); + const auto& id = schema->field(0); + struct ResetCase { + std::shared_ptr projection; + RoaringBitmap32 selection; + const char* error; + }; + const ResetCase cases[] = { + {arrow::schema({id, arrow::field("extra", arrow::utf8())}), RoaringBitmap32::From({0}), + "Field 'extra' not found or duplicate in src schema"}, + {schema, RoaringBitmap32::From({1, 6}), "Vortex selected row 6 is out of range 6"}, + }; + const auto predicate = + PredicateBuilder::GreaterThan(0, "id", FieldType::INT, Literal(int32_t{0})); + const RoaringBitmap32 selection = RoaringBitmap32::From({0, 1, 3, 4, 5}); + for (const ResetCase& test_case : cases) { + SCOPED_TRACE(test_case.error); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_OK(ConfigureReader(reader.get(), schema, predicate, selection)); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), 5, 1)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, ReadNextBatch(reader.get())); + ASSERT_NE(first, nullptr); + ASSERT_EQ(first->length(), 2); + + ASSERT_NOK_WITH_MSG( + ConfigureReader(reader.get(), test_case.projection, nullptr, test_case.selection), + test_case.error); + ASSERT_NO_FATAL_FAILURE(AssertSelectionState(reader.get(), 5, 1)); + ASSERT_OK_AND_ASSIGN(uint64_t row, reader->GetPreviousBatchFileRowId(0)); + ASSERT_EQ(row, uint64_t{1}); + ASSERT_OK_AND_ASSIGN(row, reader->GetPreviousBatchFileRowId(1)); + ASSERT_EQ(row, uint64_t{3}); + ASSERT_OK_AND_ASSIGN(std::vector remaining, + CollectNativeRowIds(std::move(reader))); + ASSERT_EQ(remaining, (std::vector{4, 5})); + } +} + +TEST_F(VortexFileFormatTest, FilterDoesNotOverwriteAFileColumnNamedRowIdx) { + const auto schema = arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("_row_idx", arrow::int64(), false)}); + auto data = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), + R"([[0,100],[1,101],[2,102],[3,103]])") + .ValueOrDie(); + const std::string path = PathUtil::JoinPath(directory_->Str(), "row-idx-column.vortex"); + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2)); + const auto predicate = + PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(static_cast(3))); + const auto filter = std::dynamic_pointer_cast(predicate); + ASSERT_NE(filter, nullptr); + for (const auto& read_schema : {schema, arrow::schema({schema->field(0)})}) { + SCOPED_TRACE(read_schema->ToString()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + ASSERT_OK( + ConfigureReader(reader.get(), read_schema, predicate, RoaringBitmap32::From({1, 3}))); + std::vector scanned_rows; + std::vector matched_rows; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, ReadNextBatch(reader.get())); + if (array == nullptr) { + break; + } + ASSERT_TRUE(array->ValidateFull().ok()); + ASSERT_TRUE(array->type()->Equals(arrow::struct_(read_schema->fields()))); + const auto& values = checked_cast(*array); + const auto& ids = checked_cast(*values.field(0)); + ASSERT_OK_AND_ASSIGN(std::vector mask, filter->Test(*array, arrow_pool_.get())); + for (int64_t i = 0; i < array->length(); ++i) { + ASSERT_OK_AND_ASSIGN(uint64_t physical_row, + reader->GetPreviousBatchFileRowId(static_cast(i))); + scanned_rows.push_back(physical_row); + ASSERT_EQ(ids.Value(i), static_cast(physical_row)); + if (read_schema->num_fields() == 2) { + const auto& original_column = + checked_cast(*values.field(1)); + ASSERT_EQ(original_column.Value(i), 100 + static_cast(physical_row)); + } + if (mask[i] != 0) { + matched_rows.push_back(physical_row); + } + } + } + // The stored _row_idx prevents native filtering even when it is not projected. + ASSERT_EQ(scanned_rows, (std::vector{1, 3})); + ASSERT_EQ(matched_rows, (std::vector{3})); + } +} + +TEST_F(VortexFileFormatTest, CloseReleasesSelectionAndRowPositionWhileReaderRemainsAlive) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + const auto predicate = + PredicateBuilder::GreaterThan(0, "id", FieldType::INT, Literal(static_cast(0))); + ASSERT_OK(ConfigureReader(reader.get(), ReaderTestSchema(), predicate, + RoaringBitmap32::From({1, 3, 5}))); + auto* native_reader = checked_cast(reader.get()); + ASSERT_GT(native_reader->selection_.indices_.capacity(), size_t{0}); + std::weak_ptr row_ids; + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr batch, ReadNextBatch(reader.get())); + ASSERT_NE(batch, nullptr); + ASSERT_NE(native_reader->row_position_.previous_batch_row_idx_, nullptr); + row_ids = native_reader->row_position_.previous_batch_row_idx_; + } + ASSERT_FALSE(row_ids.expired()); + reader->Close(); + ASSERT_NE(reader, nullptr); + // The index vector uses the standard allocator, so Arrow pool counters cannot check it. + ASSERT_EQ(native_reader->selection_.indices_.capacity(), size_t{0}); + ASSERT_TRUE(row_ids.expired()); + reader->Close(); + ASSERT_NOK_WITH_MSG(ConfigureReader(reader.get(), ReaderTestSchema()), + "Vortex reader is closed"); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "Vortex reader is closed"); +} + +TEST_F(VortexFileFormatTest, SchemaStreamErrorsPreserveDetailsAndReleaseStream) { + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + for (const bool has_native_message : {false, true}) { + for (const bool has_callback_error : {false, true}) { + SCOPED_TRACE(has_native_message); + SCOPED_TRACE(has_callback_error); + struct SchemaFailure { + std::string message = "native schema export sentinel"; + bool has_message = false; + bool error_read_before_release = false; + int32_t releases = 0; + } failure; + failure.has_message = has_native_message; + const std::string native_message = failure.message; + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + auto* native_reader = checked_cast(reader.get()); + ::ArrowArrayStream stream{}; + stream.private_data = &failure; + stream.get_schema = [](::ArrowArrayStream*, ::ArrowSchema*) -> int32_t { return EIO; }; + stream.get_last_error = [](::ArrowArrayStream* current) -> const char* { + auto* state = static_cast(current->private_data); + state->error_read_before_release = state->releases == 0; + return state->has_message ? state->message.c_str() : nullptr; + }; + stream.release = [](::ArrowArrayStream* current) { + auto* state = static_cast(current->private_data); + ++state->releases; + // Invalidate the stream-owned message without introducing a dangling pointer. + std::fill(state->message.begin(), state->message.end(), '#'); + current->release = nullptr; + }; + native_reader->current_stream_ = stream; + native_reader->stream_active_ = true; + if (has_callback_error) { + native_reader->input_context_->SetCallbackStatus( + Status::IOError("filesystem schema callback sentinel")); + } + const Status status = native_reader->ImportCurrentStreamSchema(); + ASSERT_EQ(failure.releases, 1); + ASSERT_FALSE(native_reader->stream_active_); + ASSERT_EQ(native_reader->current_stream_schema_, nullptr); + if (has_callback_error) { + ASSERT_NOK_WITH_MSG( + status, "read Vortex stream schema: filesystem schema callback sentinel"); + ASSERT_EQ(status.message().find(native_message), std::string::npos); + } else if (has_native_message) { + ASSERT_TRUE(failure.error_read_before_release); + ASSERT_NOK_WITH_MSG(status, + "Vortex Arrow stream get_schema failed: " + native_message); + } else { + ASSERT_NOK_WITH_MSG(status, "Vortex Arrow stream get_schema failed: unknown"); + } + reader->Close(); + ASSERT_EQ(failure.releases, 1); + } + } +} + +TEST_F(VortexFileFormatTest, EmptyFileAndZeroMatchFilterReturnEof) { + const auto schema = ReaderTestSchema(); + auto empty = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), "[]") + .ValueOrDie(); + const std::string empty_path = PathUtil::JoinPath(directory_->Str(), "empty.vortex"); + ASSERT_OK(WriteFile(empty_path, schema, empty, /*batch_size=*/2)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_reader, CreateReader(empty_path)); + ASSERT_OK_AND_ASSIGN(uint64_t count, empty_reader->GetNumberOfRows()); + ASSERT_EQ(count, uint64_t{0}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr batch, ReadNextBatch(empty_reader.get())); + ASSERT_EQ(batch, nullptr); + ASSERT_NOK_WITH_MSG(empty_reader->GetPreviousBatchFileRowId(0), + "no Vortex batch has been read yet"); + ASSERT_OK_AND_ASSIGN(std::string path, WriteReaderTestFile()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, CreateReader(path)); + const auto predicate = + PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(static_cast(100))); + ASSERT_OK(ConfigureReader(reader.get(), schema, predicate)); + ASSERT_OK_AND_ASSIGN(batch, ReadNextBatch(reader.get())); + ASSERT_EQ(batch, nullptr); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "no Vortex batch has been read yet"); +} + } // namespace paimon::vortex::test diff --git a/src/paimon/format/vortex/vortex_format_writer.cpp b/src/paimon/format/vortex/vortex_format_writer.cpp index ffa9be256..72de8999e 100644 --- a/src/paimon/format/vortex/vortex_format_writer.cpp +++ b/src/paimon/format/vortex/vortex_format_writer.cpp @@ -44,6 +44,42 @@ namespace { constexpr double COMPRESSION_RATIO = 0.25; +// Match Java's UTC LTZ storage without changing the epoch values or timestamp units. +std::shared_ptr NormalizeTimestampTimezones( + const std::shared_ptr& type) { + if (type->id() == arrow::Type::TIMESTAMP) { + const auto& timestamp = checked_cast(*type); + return timestamp.timezone().empty() || timestamp.timezone() == "UTC" + ? type + : arrow::timestamp(timestamp.unit(), "UTC"); + } + if (type->id() != arrow::Type::STRUCT && type->id() != arrow::Type::LIST && + type->id() != arrow::Type::FIXED_SIZE_LIST) { + return type; + } + arrow::FieldVector fields = type->fields(); + bool changed = false; + for (auto& field : fields) { + std::shared_ptr normalized = NormalizeTimestampTimezones(field->type()); + if (normalized != field->type()) { + field = field->WithType(normalized); + changed = true; + } + } + if (!changed) { + return type; + } + switch (type->id()) { + case arrow::Type::STRUCT: + return arrow::struct_(fields); + case arrow::Type::LIST: + return arrow::list(fields[0]); + default: + return arrow::fixed_size_list( + fields[0], checked_cast(*type).list_size()); + } +} + Status ValidateVisibleVectorElements(const std::shared_ptr& array, arrow::MemoryPool* pool) { if (!VectorUtils::ContainsVectorType(array->type())) { @@ -129,15 +165,20 @@ Result> VortexFormatWriter::Create( return Status::Invalid("Vortex writer requires non-null output, schema and arrow pool"); } PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVortexArrowSchema(schema)); - VxSessionPtr session(vx_session_new(), vx_session_free); + arrow::FieldVector fields = schema->fields(); + for (auto& field : fields) { + field = field->WithType(NormalizeTimestampTimezones(field->type())); + } + const auto write_schema = arrow::schema(std::move(fields), schema->metadata()); + VxSessionPtr session(vx_session_new()); if (session == nullptr) { return Status::IOError("failed to create Vortex session"); } ::ArrowSchema ffi_schema = {}; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema, &ffi_schema)); vx_error* error = nullptr; // vx_dtype_from_arrow_schema consumes ffi_schema on both success and failure. - VxDtypePtr dtype(vx_dtype_from_arrow_schema(&ffi_schema, &error), vx_dtype_free); + VxDtypePtr dtype(vx_dtype_from_arrow_schema(&ffi_schema, &error)); if (dtype == nullptr) { return VortexFfiError("convert Arrow schema to Vortex dtype", error); } @@ -150,7 +191,7 @@ Result> VortexFormatWriter::Create( output_context->GetCallbackStatus()); } return std::unique_ptr(new VortexFormatWriter( - output, schema, std::move(session), std::move(output_context), sink, arrow_pool)); + output, write_schema, std::move(session), std::move(output_context), sink, arrow_pool)); } VortexFormatWriter::~VortexFormatWriter() { @@ -193,8 +234,7 @@ Status VortexFormatWriter::AddBatch(::ArrowArray* batch) { PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, &ffi_schema)); vx_error* error = nullptr; // vx_array_from_arrow consumes both `vortex_batch` and `ffi_schema` on success and on failure. - VxArrayPtr array(vx_array_from_arrow(&rebased_batch, &ffi_schema, /*nullable=*/false, &error), - vx_array_free); + VxArrayPtr array(vx_array_from_arrow(&rebased_batch, &ffi_schema, /*nullable=*/false, &error)); if (array == nullptr) { return VortexFfiError("convert Arrow batch to Vortex array", error); } diff --git a/src/paimon/format/vortex/vortex_io_callbacks.h b/src/paimon/format/vortex/vortex_io_callbacks.h index 3a095d409..fdeed239c 100644 --- a/src/paimon/format/vortex/vortex_io_callbacks.h +++ b/src/paimon/format/vortex/vortex_io_callbacks.h @@ -52,7 +52,7 @@ class VortexInputContext { /// reader has dropped its own reference. /// /// @param context The context to forward callbacks to; a shared reference is retained. - /// @return The callback struct to hand to `vx_data_source_new_callback`. + /// @return The callback struct for `vx_data_source_new_callback_with_coalesce`. static vx_input_callbacks MakeCallbacks(const std::shared_ptr& context); /// `vx_read_at_fn`: fills `length` bytes at `offset` from the paimon stream. diff --git a/src/paimon/format/vortex/vortex_predicate_converter.cpp b/src/paimon/format/vortex/vortex_predicate_converter.cpp new file mode 100644 index 000000000..8c6099036 --- /dev/null +++ b/src/paimon/format/vortex/vortex_predicate_converter.cpp @@ -0,0 +1,499 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/format/vortex/vortex_predicate_converter.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/type.h" +#include "arrow/util/decimal.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/core/casting/decimal_to_decimal_cast_executor.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/function.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate.h" + +namespace paimon::vortex { +namespace { + +Status CheckNativeResult(const std::string& operation, const void* handle, + vx_error* error = nullptr) { + PAIMON_RETURN_NOT_OK(VortexFfiError(operation, error)); + if (handle == nullptr) { + return Status::Invalid(operation, ": Vortex returned a null handle"); + } + return Status::OK(); +} + +// vx_expression_not returns an owned handle typed const; reclaim it as an owned non-const pointer. +Result MakeNotExpr(const vx_expression* child) { + VxExpressionPtr expression(const_cast(vx_expression_not(child))); + PAIMON_RETURN_NOT_OK(CheckNativeResult("create Vortex NOT expression", expression.get())); + return expression; +} + +// Reject precision loss and overflow before constructing a native timestamp scalar. +bool GetTimestampValue(const Literal& literal, const std::shared_ptr& arrow_type, + int64_t* result, vx_time_unit* result_unit) { + if (arrow_type == nullptr || arrow_type->id() != arrow::Type::TIMESTAMP) { + return false; + } + const auto timestamp_type = checked_pointer_cast(arrow_type); + if (!timestamp_type->timezone().empty() && timestamp_type->timezone() != "UTC") { + return false; + } + const auto timestamp = literal.GetValue(); + int64_t value = timestamp.GetMillisecond(); + const int32_t nanos = timestamp.GetNanoOfMillisecond(); + vx_time_unit unit = VX_TIME_UNIT_MILLISECOND; + int64_t factor = 1; + int64_t fraction = 0; + switch (timestamp_type->unit()) { + case arrow::TimeUnit::SECOND: + if (value % 1000 != 0 || nanos != 0) { + return false; + } + unit = VX_TIME_UNIT_SECOND; + break; + case arrow::TimeUnit::MILLI: + if (nanos != 0) { + return false; + } + break; + case arrow::TimeUnit::MICRO: + if (nanos % 1000 != 0) { + return false; + } + factor = 1000; + fraction = nanos / 1000; + unit = VX_TIME_UNIT_MICROSECOND; + break; + case arrow::TimeUnit::NANO: + factor = 1000000; + fraction = nanos; + unit = VX_TIME_UNIT_NANOSECOND; + break; + default: + return false; + } + if (value < std::numeric_limits::min() / factor || + value > std::numeric_limits::max() / factor) { + return false; + } + value *= factor; + if (value > std::numeric_limits::max() - fraction) { + return false; + } + *result = DateTimeUtils::TimestampToInteger( + timestamp, DateTimeUtils::GetTimeTypeFromArrowType(timestamp_type)); + *result_unit = unit; + return true; +} + +// Normalize constants only; shared CAST may round, so accept it only if the value is unchanged. +Result> NormalizeDecimalLiteral( + const Literal& literal, const std::shared_ptr& arrow_type) { + const auto decimal = literal.GetValue(); + const auto& target_type = checked_cast(*arrow_type); + // Keep literals within Paimon's decimal range and the shared scale table. + if (decimal.Precision() < Decimal::MIN_PRECISION || + decimal.Precision() > Decimal::MAX_PRECISION || decimal.Scale() < 0 || + decimal.Scale() > decimal.Precision() || target_type.precision() < Decimal::MIN_PRECISION || + target_type.precision() > Decimal::MAX_PRECISION || target_type.scale() < 0 || + target_type.scale() > target_type.precision()) { + return std::optional(); + } + if (!arrow::Decimal128(decimal.HighBits(), decimal.LowBits()) + .FitsInPrecision(decimal.Precision())) { + return std::optional(); + } + if (decimal.Precision() == target_type.precision() && decimal.Scale() == target_type.scale()) { + return std::make_optional(decimal); + } + const DecimalToDecimalCastExecutor cast_executor; + PAIMON_ASSIGN_OR_RAISE(Literal converted, cast_executor.Cast(literal, arrow_type)); + if (converted.IsNull()) { + return std::optional(); + } + const auto normalized = converted.GetValue(); + if (decimal.CompareTo(normalized) != 0) { + return std::optional(); + } + return std::make_optional(normalized); +} + +Result MakeLiteralScalar(const Literal& literal, + const std::shared_ptr& arrow_type) { + VxScalarPtr scalar; + if (literal.IsNull()) { + return scalar; + } + vx_error* error = nullptr; + switch (literal.GetType()) { + case FieldType::BOOLEAN: + scalar.reset(vx_scalar_new_bool(literal.GetValue(), false)); + break; + case FieldType::TINYINT: + scalar.reset(vx_scalar_new_i8(literal.GetValue(), false)); + break; + case FieldType::SMALLINT: + scalar.reset(vx_scalar_new_i16(literal.GetValue(), false)); + break; + case FieldType::INT: + scalar.reset(vx_scalar_new_i32(literal.GetValue(), false)); + break; + // Vortex stores a date column as a date[days] extension and rejects comparing it against a + // plain int scalar, so build a date-typed scalar. + case FieldType::DATE: + scalar.reset(vx_scalar_new_date(literal.GetValue(), false, &error)); + if (scalar == nullptr && error == nullptr) { + // A valid Paimon constant can exceed the native civil-date range. + return scalar; + } + break; + case FieldType::BIGINT: + scalar.reset(vx_scalar_new_i64(literal.GetValue(), false)); + break; + case FieldType::FLOAT: + case FieldType::DOUBLE: + // Native totalOrder distinguishes NaN signs and payloads; Paimon treats all NaNs alike. + return scalar; + case FieldType::STRING: { + const auto value = literal.GetValue(); + scalar.reset(vx_scalar_new_utf8(value.data(), value.size(), false, &error)); + break; + } + case FieldType::DECIMAL: { + PAIMON_ASSIGN_OR_RAISE(std::optional normalized, + NormalizeDecimalLiteral(literal, arrow_type)); + if (!normalized.has_value()) { + return scalar; + } + // Cast validated the full source value before selecting a narrower representation. + const Decimal& decimal = normalized.value(); + const auto precision = static_cast(decimal.Precision()); + const auto scale = static_cast(decimal.Scale()); + if (decimal.IsCompact()) { + scalar.reset(vx_scalar_new_decimal_i64(decimal.ToUnscaledLong(), precision, scale, + false, &error)); + } else { + // Little-endian 16-byte two's complement, low word first. + uint8_t bytes16[16]; + const uint64_t low = decimal.LowBits(); + const uint64_t high = decimal.HighBits(); + std::memcpy(bytes16, &low, sizeof(low)); + std::memcpy(bytes16 + sizeof(low), &high, sizeof(high)); + scalar.reset( + vx_scalar_new_decimal_i128_le(bytes16, precision, scale, false, &error)); + } + break; + } + case FieldType::TIMESTAMP: { + int64_t value = 0; + vx_time_unit unit = VX_TIME_UNIT_MILLISECOND; + if (!GetTimestampValue(literal, arrow_type, &value, &unit)) { + return scalar; + } + const auto& timestamp_type = checked_cast(*arrow_type); + const char* zone = timestamp_type.timezone().empty() ? nullptr : "UTC"; + scalar.reset(vx_scalar_new_timestamp(value, unit, zone, false, &error)); + if (scalar == nullptr && error == nullptr) { + // Preserve residual comparison when native temporal limits are narrower. + return scalar; + } + break; + } + default: + return scalar; + } + PAIMON_RETURN_NOT_OK(CheckNativeResult("create Vortex predicate scalar", scalar.get(), error)); + return scalar; +} + +Result MakeLiteralExpr(const Literal& literal, + const std::shared_ptr& arrow_type) { + PAIMON_ASSIGN_OR_RAISE(VxScalarPtr scalar, MakeLiteralScalar(literal, arrow_type)); + if (scalar == nullptr) { + return VxExpressionPtr(); + } + vx_error* error = nullptr; + VxExpressionPtr expression(vx_expression_literal(scalar.get(), &error)); + PAIMON_RETURN_NOT_OK( + CheckNativeResult("create Vortex literal expression", expression.get(), error)); + return expression; +} + +// Literal types are validated by the caller. Check the actual file type once per leaf; +// scalar construction still checks value safety against its parameters. +Result CheckFieldTypePushdownEligibility(const std::shared_ptr& leaf, + const std::shared_ptr& arrow_type) { + if (arrow_type == nullptr) { + return false; + } + const FieldType field_type = leaf->GetFieldType(); + const Result file_type = FieldTypeUtils::ConvertToFieldType(arrow_type->id()); + if (!file_type.ok()) { + // Arrow types outside the Paimon type mapping cannot be pushed. + return false; + } + if (file_type.value() != field_type) { + return Status::Invalid("Predicate type mismatch for field '", leaf->FieldName(), + "': expected ", FieldTypeUtils::FieldTypeToString(file_type.value()), + " from file type ", arrow_type->ToString(), ", actual ", + FieldTypeUtils::FieldTypeToString(field_type)); + } + switch (field_type) { + case FieldType::BOOLEAN: + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::BIGINT: + case FieldType::STRING: + case FieldType::DATE: + case FieldType::DECIMAL: + case FieldType::TIMESTAMP: + return true; + default: + // NaN and signed-zero equivalence needs separate validation; keep the residual path. + return false; + } +} + +Result ConvertInList(const std::vector& literals, + const std::shared_ptr& arrow_type, + const vx_expression* column) { + std::vector scalars; + std::vector raw; + scalars.reserve(literals.size()); + raw.reserve(literals.size()); + for (const Literal& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(VxScalarPtr scalar, MakeLiteralScalar(literal, arrow_type)); + if (scalar == nullptr) { + return VxExpressionPtr(); + } + raw.push_back(scalar.get()); + scalars.push_back(std::move(scalar)); + } + vx_error* error = nullptr; + VxScalarPtr list( + vx_scalar_new_list(vx_scalar_dtype(raw[0]), raw.data(), raw.size(), false, &error)); + PAIMON_RETURN_NOT_OK(CheckNativeResult("create Vortex list scalar", list.get(), error)); + error = nullptr; + VxExpressionPtr literal_expr(vx_expression_literal(list.get(), &error)); + PAIMON_RETURN_NOT_OK( + CheckNativeResult("create Vortex literal expression", literal_expr.get(), error)); + VxExpressionPtr expression(vx_expression_list_contains(literal_expr.get(), column)); + PAIMON_RETURN_NOT_OK( + CheckNativeResult("create Vortex list membership expression", expression.get())); + return expression; +} + +Result ConvertMembership(const std::shared_ptr& leaf, + const std::shared_ptr& arrow_type, + const vx_expression* column, bool negate, + const VortexPredicateOptions& options) { + const std::vector& literals = leaf->Literals(); + // The query engine owns deduplication and integer-range rewrites. Apply value conversions + // and create native scalars only within the original-list limit. + if (literals.empty() || literals.size() > options.max_in_values) { + return VxExpressionPtr(); + } + PAIMON_ASSIGN_OR_RAISE(bool type_eligible, CheckFieldTypePushdownEligibility(leaf, arrow_type)); + if (!type_eligible) { + return VxExpressionPtr(); + } + PAIMON_ASSIGN_OR_RAISE(VxExpressionPtr expression, ConvertInList(literals, arrow_type, column)); + if (expression == nullptr || !negate) { + return expression; + } + PAIMON_ASSIGN_OR_RAISE(expression, MakeNotExpr(expression.get())); + VxExpressionPtr is_null(vx_expression_is_null(column)); + PAIMON_RETURN_NOT_OK(CheckNativeResult("create Vortex IS NULL expression", is_null.get())); + PAIMON_ASSIGN_OR_RAISE(VxExpressionPtr is_not_null, MakeNotExpr(is_null.get())); + // Neither IN nor NOT IN may retain null rows. Validation also rejects null/empty lists. + const vx_expression* children[] = {is_not_null.get(), expression.get()}; + VxExpressionPtr combined(vx_expression_and(children, 2)); + PAIMON_RETURN_NOT_OK(CheckNativeResult("create Vortex AND expression", combined.get())); + return combined; +} + +bool ComparisonOperator(Function::Type type, vx_binary_operator* op) { + switch (type) { + case Function::Type::EQUAL: + *op = VX_OPERATOR_EQ; + return true; + case Function::Type::NOT_EQUAL: + *op = VX_OPERATOR_NOT_EQ; + return true; + case Function::Type::GREATER_THAN: + *op = VX_OPERATOR_GT; + return true; + case Function::Type::GREATER_OR_EQUAL: + *op = VX_OPERATOR_GTE; + return true; + case Function::Type::LESS_THAN: + *op = VX_OPERATOR_LT; + return true; + case Function::Type::LESS_OR_EQUAL: + *op = VX_OPERATOR_LTE; + return true; + default: + return false; + } +} + +Result ConvertLeaf(const std::shared_ptr& leaf, + const std::shared_ptr& file_schema, + const vx_expression* root, + const VortexPredicateOptions& options) { + const Function::Type type = leaf->GetFunction().GetType(); + vx_binary_operator op = VX_OPERATOR_EQ; + const bool is_comparison = ComparisonOperator(type, &op); + if (is_comparison && leaf->Literals().size() != 1) { + return Status::Invalid("Comparison predicate for field '", leaf->FieldName(), + "' requires exactly one literal"); + } + const int32_t field_index = + file_schema != nullptr ? file_schema->GetFieldIndex(leaf->FieldName()) : -1; + if (field_index < 0) { + // Missing fields are null-filled by the reader, but cannot bind to a native file scan. + return VxExpressionPtr(); + } + const std::shared_ptr& arrow_type = file_schema->field(field_index)->type(); + VxExpressionPtr column(vx_expression_get_item(leaf->FieldName().c_str(), root)); + PAIMON_RETURN_NOT_OK(CheckNativeResult("create Vortex field expression", column.get())); + if (type == Function::Type::IS_NULL || type == Function::Type::IS_NOT_NULL) { + VxExpressionPtr is_null(vx_expression_is_null(column.get())); + PAIMON_RETURN_NOT_OK(CheckNativeResult("create Vortex IS NULL expression", is_null.get())); + if (type == Function::Type::IS_NULL) { + return is_null; + } + return MakeNotExpr(is_null.get()); + } + if (type == Function::Type::IN) { + return ConvertMembership(leaf, arrow_type, column.get(), false, options); + } + if (type == Function::Type::NOT_IN) { + return ConvertMembership(leaf, arrow_type, column.get(), true, options); + } + if (!is_comparison) { + return VxExpressionPtr(); + } + PAIMON_ASSIGN_OR_RAISE(bool type_eligible, CheckFieldTypePushdownEligibility(leaf, arrow_type)); + if (!type_eligible) { + return VxExpressionPtr(); + } + PAIMON_ASSIGN_OR_RAISE(VxExpressionPtr literal_expr, + MakeLiteralExpr(leaf->Literals()[0], arrow_type)); + if (literal_expr == nullptr) { + return VxExpressionPtr(); + } + VxExpressionPtr expression(vx_expression_binary(op, column.get(), literal_expr.get())); + PAIMON_RETURN_NOT_OK( + CheckNativeResult("create Vortex comparison expression", expression.get())); + return expression; +} + +Result ConvertImpl(const std::shared_ptr& predicate, + const std::shared_ptr& file_schema, + const vx_expression* root, + const VortexPredicateOptions& options) { + if (predicate == nullptr) { + return VxExpressionPtr(); + } + const Function::Type type = predicate->GetFunction().GetType(); + if (type == Function::Type::AND || type == Function::Type::OR) { + const auto compound = std::dynamic_pointer_cast(predicate); + if (compound == nullptr) { + return Status::Invalid("AND/OR predicate must be a CompoundPredicate"); + } + if (compound->Children().empty()) { + return Status::Invalid("AND/OR predicate must have at least one child"); + } + bool all_children_pushed = true; + std::vector children; + children.reserve(compound->Children().size()); + for (const std::shared_ptr& child : compound->Children()) { + if (child == nullptr) { + return Status::Invalid("AND/OR predicate must not contain a null child"); + } + PAIMON_ASSIGN_OR_RAISE(VxExpressionPtr converted, + ConvertImpl(child, file_schema, root, options)); + if (converted != nullptr) { + children.push_back(std::move(converted)); + } else { + all_children_pushed = false; + } + // Visit all branches so an unsupported OR child cannot hide a later error. + } + // AND can retain supported children; dropping any OR branch could exclude matching rows. + if (children.empty() || (type == Function::Type::OR && !all_children_pushed)) { + return VxExpressionPtr(); + } + std::vector raw; + raw.reserve(children.size()); + for (const VxExpressionPtr& child : children) { + raw.push_back(child.get()); + } + VxExpressionPtr combined(type == Function::Type::AND + ? vx_expression_and(raw.data(), raw.size()) + : vx_expression_or(raw.data(), raw.size())); + PAIMON_RETURN_NOT_OK( + CheckNativeResult("create Vortex compound expression", combined.get())); + return combined; + } + const auto leaf = std::dynamic_pointer_cast(predicate); + if (leaf == nullptr) { + vx_binary_operator op = VX_OPERATOR_EQ; + if (ComparisonOperator(type, &op) || type == Function::Type::IS_NULL || + type == Function::Type::IS_NOT_NULL || type == Function::Type::IN || + type == Function::Type::NOT_IN) { + return Status::Invalid("Leaf predicate must be a LeafPredicate"); + } + return VxExpressionPtr(); + } + return ConvertLeaf(leaf, file_schema, root, options); +} + +} // namespace + +Result VortexPredicateConverter::Convert( + const std::shared_ptr& predicate, const std::shared_ptr& file_schema, + const VortexPredicateOptions& options) { + if (predicate == nullptr) { + return VxExpressionPtr(); + } + VxExpressionPtr root(vx_expression_root()); + PAIMON_RETURN_NOT_OK(CheckNativeResult("create Vortex root expression", root.get())); + return ConvertImpl(predicate, file_schema, root.get(), options); +} + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_predicate_converter.h b/src/paimon/format/vortex/vortex_predicate_converter.h new file mode 100644 index 000000000..ca3c97ffe --- /dev/null +++ b/src/paimon/format/vortex/vortex_predicate_converter.h @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/format/vortex/vortex_defs.h" +#include "paimon/format/vortex/vortex_ffi_util.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { +class Predicate; +} // namespace paimon + +namespace paimon::vortex { + +/// Converts Paimon predicates into Vortex scan filters, leaving unsupported conditions to the +/// caller's residual filter. +class VortexPredicateConverter { + public: + VortexPredicateConverter() = delete; + ~VortexPredicateConverter() = delete; + + /// @param predicate The predicate to convert, with literal types already checked by the + /// caller (for example, through PredicateValidator). + /// @param file_schema Stored column types, including timestamp units and zones. + /// @param options Maximum input list length for IN and NOT IN pushdown. + /// @return An owned expression, nullptr if no condition can be pushed, or an error if + /// the predicate is malformed, its field types do not match the file, or native expression + /// construction fails. + static Result Convert(const std::shared_ptr& predicate, + const std::shared_ptr& file_schema, + const VortexPredicateOptions& options); +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_predicate_converter_test.cpp b/src/paimon/format/vortex/vortex_predicate_converter_test.cpp new file mode 100644 index 000000000..fefd98a4a --- /dev/null +++ b/src/paimon/format/vortex/vortex_predicate_converter_test.cpp @@ -0,0 +1,875 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/format/vortex/vortex_predicate_converter.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/format/vortex/vortex_defs.h" +#include "paimon/format/vortex/vortex_io_callbacks.h" +#include "paimon/fs/file_system.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::vortex::test { +namespace { + +constexpr char kInvalidUtf8Error[] = + "create Vortex predicate scalar: Other error: invalid utf-8: " + "invalid utf-8 sequence of 1 bytes from index 0"; + +std::shared_ptr TestSchema() { + return arrow::schema({ + arrow::field("i", arrow::int32()), + arrow::field("s", arrow::utf8()), + arrow::field("l", arrow::int64()), + arrow::field("b", arrow::boolean()), + arrow::field("dt", arrow::date32()), + arrow::field("dec_compact", arrow::decimal128(10, 2)), + arrow::field("dec_wide", arrow::decimal128(19, 2)), + }); +} + +bool Pushed(const std::shared_ptr& predicate, + const std::shared_ptr& schema, + const VortexPredicateOptions& options = {}) { + const Result result = + VortexPredicateConverter::Convert(predicate, schema, options); + EXPECT_OK(result.status()); + return result.ok() && result.value() != nullptr; +} + +struct NativeScanCase { + const char* name; + std::shared_ptr type; + FieldType field_type; + const char* values_json; + std::vector literals; +}; + +std::string NativeErrorMessage(const vx_error* error) { + if (error == nullptr) { + return ""; + } + const vx_string* message = vx_error_get_message(error); + return message == nullptr ? "unknown Vortex error" + : std::string(vx_string_ptr(message), vx_string_len(message)); +} + +// Check rows returned by the native scan, then optionally check the Paimon residual filter. +void AssertNativeRows( + const NativeScanCase& test_case, const std::shared_ptr& predicate, + const std::vector& expected_rows, + const std::optional>& expected_residual_rows = std::nullopt, + const VortexPredicateOptions& options = {}, bool expect_native_filter = true, + bool project_rowid_only = false, const vx_expression* native_filter_override = nullptr) { + SCOPED_TRACE(test_case.name); + auto values_result = + arrow::ipc::internal::json::ArrayFromJSON(test_case.type, test_case.values_json); + ASSERT_TRUE(values_result.ok()) << values_result.status().ToString(); + auto ids_result = arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), "[0,1,2,3]"); + ASSERT_TRUE(ids_result.ok()) << ids_result.status().ToString(); + const arrow::FieldVector fields = {arrow::field("rowid", arrow::int64(), false), + arrow::field("value", test_case.type)}; + const std::shared_ptr schema = arrow::schema(fields); + auto array_result = + arrow::StructArray::Make({ids_result.ValueOrDie(), values_result.ValueOrDie()}, fields); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + + std::unique_ptr directory = + paimon::test::UniqueTestDirectory::Create(); + ASSERT_NE(directory, nullptr); + const std::string path = PathUtil::JoinPath(directory->Str(), "values.vortex"); + vx_error* error = nullptr; + ScopeGuard release_error([&error]() { + if (error != nullptr) { + vx_error_free(error); + } + }); + VxSessionPtr session(vx_session_new()); + ASSERT_NE(session, nullptr); + ::ArrowArray exported_array{}; + ::ArrowSchema exported_schema{}; + ScopeGuard release_export([&exported_array, &exported_schema]() { + if (exported_array.release != nullptr) { + exported_array.release(&exported_array); + } + if (exported_schema.release != nullptr) { + exported_schema.release(&exported_schema); + } + }); + ASSERT_TRUE( + arrow::ExportArray(*array_result.ValueOrDie(), &exported_array, &exported_schema).ok()); + // Import consumes both Arrow handles; the guard also covers an incomplete failed import. + VxArrayPtr array(vx_array_from_arrow(&exported_array, &exported_schema, false, &error)); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + ASSERT_NE(array, nullptr); + LocalFileSystem file_system; + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr output, + file_system.Create(path, /*overwrite=*/false)); + auto output_context = std::make_shared(output); + vx_callback_sink* sink = nullptr; + bool output_closed = false; + ScopeGuard release_sink([&sink, &output_context, &output, &output_closed]() { + if (!output_closed) { + output_context->MarkAborted(); + if (sink != nullptr) { + vx_callback_sink_abort(std::exchange(sink, nullptr)); + } + output_context->WaitForIdleCallbacks(); + [[maybe_unused]] Status close_status = output->Close(); + } + }); + // Use the production callback I/O path, keeping Rust's asynchronous filesystem adapter + // out of this predicate test. The array owns the borrowed dtype until the sink closes. + const vx_dtype* dtype = vx_array_dtype(array.get()); + ASSERT_NE(dtype, nullptr); + sink = vx_callback_sink_open( + session.get(), VortexOutputContext::MakeCallbacks(output_context), dtype, &error); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + ASSERT_NE(sink, nullptr); + vx_callback_sink_push(sink, array.get(), &error); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + // Closing consumes the sink on both success and failure. + vx_callback_sink_close(std::exchange(sink, nullptr), &error); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + ASSERT_OK(output_context->GetCallbackStatus()); + output_closed = true; + ASSERT_OK(output->Close()); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, file_system.Open(path)); + ASSERT_OK_AND_ASSIGN(int64_t length, input->Length()); + auto input_context = std::make_shared(input); + VxDataSourcePtr source(vx_data_source_new_callback_with_coalesce( + session.get(), VortexInputContext::MakeCallbacks(input_context), + static_cast(length), kDefaultVortexReadIoCoalesceMaxGap, + kDefaultVortexReadIoCoalesceMaxSize, &error)); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + ASSERT_NE(source, nullptr); + ASSERT_OK_AND_ASSIGN(VxExpressionPtr filter, + VortexPredicateConverter::Convert(predicate, schema, options)); + ASSERT_EQ(filter != nullptr, expect_native_filter); + vx_scan_options scan_options{}; + scan_options.filter = native_filter_override == nullptr ? filter.get() : native_filter_override; + VxExpressionPtr root; + VxExpressionPtr projection; + if (project_rowid_only) { + root = VxExpressionPtr(vx_expression_root()); + ASSERT_NE(root, nullptr); + const char* names[] = {"rowid"}; + projection = VxExpressionPtr(vx_expression_select(names, 1, root.get())); + ASSERT_NE(projection, nullptr); + scan_options.projection = projection.get(); + } + VxScanPtr scan(vx_data_source_scan(source.get(), &scan_options, nullptr, &error)); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + ASSERT_NE(scan, nullptr); + std::vector row_ids; + std::vector residual_row_ids; + while (true) { + VxPartitionPtr partition(vx_scan_next_partition(scan.get(), &error)); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + if (partition == nullptr) { + break; + } + ::ArrowArrayStream stream{}; + ScopeGuard release_stream([&stream]() { + if (stream.release != nullptr) { + stream.release(&stream); + } + }); + // The scan consumes the partition even when it fails. + const int32_t result = + vx_partition_scan_arrow(session.get(), partition.release(), &stream, &error); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + ASSERT_EQ(result, 0); + auto reader_result = arrow::ImportRecordBatchReader(&stream); + ASSERT_TRUE(reader_result.ok()) << reader_result.status().ToString(); + const std::shared_ptr reader = reader_result.ValueOrDie(); + while (true) { + std::shared_ptr batch; + const arrow::Status status = reader->ReadNext(&batch); + ASSERT_TRUE(status.ok()) << status.ToString(); + if (batch == nullptr) { + break; + } + ASSERT_EQ(batch->num_columns(), project_rowid_only ? 1 : 2); + const std::shared_ptr ids = batch->GetColumnByName("rowid"); + ASSERT_NE(ids, nullptr); + ASSERT_EQ(ids->type_id(), arrow::Type::INT64); + const auto& typed_ids = checked_cast(*ids); + std::vector mask; + if (expected_residual_rows) { + ASSERT_FALSE(project_rowid_only); + const std::shared_ptr residual = + std::dynamic_pointer_cast(predicate); + ASSERT_NE(residual, nullptr); + const auto rows = + arrow::StructArray::Make(batch->columns(), batch->schema()->fields()); + ASSERT_TRUE(rows.ok()) << rows.status().ToString(); + ASSERT_OK_AND_ASSIGN( + mask, residual->Test(*rows.ValueOrDie(), arrow::default_memory_pool())); + ASSERT_EQ(mask.size(), static_cast(batch->num_rows())); + } + for (int64_t row = 0; row < batch->num_rows(); ++row) { + ASSERT_FALSE(typed_ids.IsNull(row)); + const int64_t row_id = typed_ids.Value(row); + row_ids.push_back(row_id); + if (expected_residual_rows) { + ASSERT_GE(row_id, 0); + ASSERT_LT(row_id, values_result.ValueOrDie()->length()); + const auto actual = batch->column(1)->GetScalar(row); + const auto expected = values_result.ValueOrDie()->GetScalar(row_id); + ASSERT_TRUE(actual.ok()); + ASSERT_TRUE(expected.ok()); + ASSERT_TRUE(actual.ValueOrDie()->Equals(*expected.ValueOrDie())); + if (mask[row] != 0) { + residual_row_ids.push_back(row_id); + } + } + } + } + } + std::sort(row_ids.begin(), row_ids.end()); + ASSERT_EQ(row_ids, expected_rows); + if (expected_residual_rows) { + std::sort(residual_row_ids.begin(), residual_row_ids.end()); + ASSERT_EQ(residual_row_ids, *expected_residual_rows); + } +} + +// Compare an unsafe direct native expression with the converter's conservative fallback. +void AssertNativeComparisonRows(const NativeScanCase& test_case, + const std::shared_ptr& predicate, + const vx_scalar* scalar, vx_binary_operator op, + const std::vector& expected_rows) { + VxExpressionPtr root(vx_expression_root()); + ASSERT_NE(root, nullptr); + VxExpressionPtr column(vx_expression_get_item("value", root.get())); + ASSERT_NE(column, nullptr); + vx_error* error = nullptr; + ScopeGuard release_error([&error]() { + if (error != nullptr) { + vx_error_free(error); + } + }); + VxExpressionPtr literal(vx_expression_literal(scalar, &error)); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + ASSERT_NE(literal, nullptr); + VxExpressionPtr filter(vx_expression_binary(op, column.get(), literal.get())); + ASSERT_NE(filter, nullptr); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, predicate, expected_rows, std::nullopt, {}, + /*expect_native_filter=*/false, + /*project_rowid_only=*/false, filter.get())); +} + +template +NativeScanCase IntegerRangeBoundaryCase(const char* name, + const std::shared_ptr& type, + FieldType field_type, const char* values_json) { + std::vector literals; + // Two disjoint four-value ranges at the extrema. Keep them unsorted and repeat endpoints. + for (int32_t offset = 0; offset < 4; ++offset) { + literals.emplace_back(static_cast(std::numeric_limits::max() - offset)); + literals.emplace_back(static_cast(std::numeric_limits::min() + offset)); + } + literals.emplace_back(std::numeric_limits::max()); + literals.emplace_back(std::numeric_limits::min()); + return {name, type, field_type, values_json, std::move(literals)}; +} +} // namespace + +TEST(VortexPredicateConverterTest, ComparisonsOnSupportedTypesArePushed) { + const std::shared_ptr schema = TestSchema(); + ASSERT_TRUE( + Pushed(PredicateBuilder::Equal(0, "i", FieldType::INT, Literal(static_cast(42))), + schema)); + ASSERT_TRUE( + Pushed(PredicateBuilder::NotEqual(0, "i", FieldType::INT, Literal(static_cast(1))), + schema)); + ASSERT_TRUE(Pushed( + PredicateBuilder::GreaterThan(0, "l", FieldType::BIGINT, Literal(static_cast(7))), + schema)); + ASSERT_TRUE(Pushed( + PredicateBuilder::GreaterOrEqual(0, "i", FieldType::INT, Literal(static_cast(1))), + schema)); + ASSERT_TRUE( + Pushed(PredicateBuilder::LessThan(0, "i", FieldType::INT, Literal(static_cast(1))), + schema)); + ASSERT_TRUE(Pushed(PredicateBuilder::Equal(0, "b", FieldType::BOOLEAN, Literal(true)), schema)); + ASSERT_TRUE(Pushed( + PredicateBuilder::Equal(0, "s", FieldType::STRING, Literal(FieldType::STRING, "abc", 3)), + schema)); + ASSERT_TRUE( + Pushed(PredicateBuilder::Equal(0, "dt", FieldType::DATE, Literal(FieldType::DATE, 19000)), + schema)); + // Compact decimal (precision <= 18) and wide decimal (precision > 18) both push. + ASSERT_TRUE(Pushed(PredicateBuilder::Equal(0, "dec_compact", FieldType::DECIMAL, + Literal(Decimal::FromUnscaledLong(12345, 10, 2))), + schema)); + ASSERT_TRUE(Pushed(PredicateBuilder::Equal(0, "dec_wide", FieldType::DECIMAL, + Literal(Decimal::FromUnscaledLong(12345, 19, 2))), + schema)); +} + +TEST(VortexPredicateConverterTest, IsNullAndIsNotNullArePushed) { + const std::shared_ptr schema = TestSchema(); + ASSERT_TRUE(Pushed(PredicateBuilder::IsNull(0, "i", FieldType::INT), schema)); + ASSERT_TRUE(Pushed(PredicateBuilder::IsNotNull(0, "i", FieldType::INT), schema)); + const auto floating_schema = + arrow::schema({arrow::field("f", arrow::float32()), arrow::field("d", arrow::float64())}); + ASSERT_TRUE(Pushed(PredicateBuilder::IsNull(0, "f", FieldType::FLOAT), floating_schema)); + ASSERT_TRUE(Pushed(PredicateBuilder::IsNotNull(1, "d", FieldType::DOUBLE), floating_schema)); +} + +TEST(VortexPredicateConverterTest, UnknownOrAmbiguousFileColumnsFallBack) { + const auto equal = PredicateBuilder::Equal(0, "value", FieldType::INT, Literal(int32_t{1})); + const auto is_null = PredicateBuilder::IsNull(0, "value", FieldType::INT); + const auto is_not_null = PredicateBuilder::IsNotNull(0, "value", FieldType::INT); + ASSERT_FALSE(Pushed(equal, nullptr)); + ASSERT_FALSE(Pushed(is_null, arrow::schema({arrow::field("other", arrow::int32())}))); + ASSERT_FALSE(Pushed(is_not_null, arrow::schema({arrow::field("value", arrow::int32()), + arrow::field("value", arrow::int32())}))); +} + +TEST(VortexPredicateConverterTest, LosslessDecimalComparisonsNormalizeToFileType) { + using ComparisonBuilder = decltype(&PredicateBuilder::Equal); + struct DecimalCase { + NativeScanCase native; + ComparisonBuilder build; + std::vector expected_rows; + }; + constexpr char values[] = R"(["-1.20",null,"0.00","1.20"])"; + const DecimalCase cases[] = { + {{"precision_only", + arrow::decimal128(10, 2), + FieldType::DECIMAL, + values, + {Literal(Decimal(19, 2, 120))}}, + PredicateBuilder::Equal, + {3}}, + {{"scale_up_compact_to_wide", + arrow::decimal128(19, 2), + FieldType::DECIMAL, + values, + {Literal(Decimal(18, 1, 12))}}, + PredicateBuilder::LessOrEqual, + {0, 2, 3}}, + }; + for (const DecimalCase& test_case : cases) { + const auto predicate = + test_case.build(1, "value", FieldType::DECIMAL, test_case.native.literals[0]); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case.native, predicate, + test_case.expected_rows, test_case.expected_rows)); + } +} + +TEST(VortexPredicateConverterTest, LosslessDecimalMembershipNormalizesEachLiteral) { + const NativeScanCase test_case = { + "mixed_precisions", + arrow::decimal128(10, 2), + FieldType::DECIMAL, + R"(["-1.20",null,"7.89","4.56"])", + {Literal(Decimal(19, 2, -120)), Literal(Decimal(20, 2, 456))}}; + const auto in = PredicateBuilder::In(1, "value", FieldType::DECIMAL, test_case.literals); + const auto not_in = PredicateBuilder::NotIn(1, "value", FieldType::DECIMAL, test_case.literals); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, in, {0, 3}, {{0, 3}})); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, not_in, {2}, {{2}})); +} + +TEST(VortexPredicateConverterTest, UnrepresentableDecimalsPreserveNativeCandidatesAndResidualRows) { + const Literal unrepresentable(Decimal(10, 3, 1234)); + const NativeScanCase test_case = {"lossy_decimal", + arrow::decimal128(10, 2), + FieldType::DECIMAL, + R"(["-1.23",null,"1.23","1.24"])", + {}}; + const std::vector all_rows = {0, 1, 2, 3}; + + // Rounding 1.234 to 1.23 would exclude row 2 from both native filters. + const auto less_than = + PredicateBuilder::LessThan(1, "value", FieldType::DECIMAL, unrepresentable); + const auto not_in = PredicateBuilder::NotIn(1, "value", FieldType::DECIMAL, + {Literal(Decimal(10, 2, -123)), unrepresentable}); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, less_than, all_rows, {{0, 2}}, {}, + /*expect_native_filter=*/false)); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, not_in, all_rows, {{2, 3}}, {}, + /*expect_native_filter=*/false)); +} + +TEST(VortexPredicateConverterTest, DecimalNormalizationRejectsInvalidLiteralMetadata) { + const auto schema = arrow::schema({arrow::field("value", arrow::decimal128(10, 2))}); + // The value fits the column's precision, but the literal's precision is outside Paimon's range. + const auto predicate = + PredicateBuilder::Equal(0, "value", FieldType::DECIMAL, Literal(Decimal(39, 2, 123))); + ASSERT_FALSE(Pushed(predicate, schema)); +} + +TEST(VortexPredicateConverterTest, MismatchedFileTypesReturnInvalid) { + const auto schema = arrow::schema({arrow::field("value", arrow::int32())}); + const auto predicate = + PredicateBuilder::Equal(0, "value", FieldType::BIGINT, Literal(int64_t{1})); + ASSERT_NOK_WITH_MSG(VortexPredicateConverter::Convert(predicate, schema, {}), + "Predicate type mismatch for field 'value': expected INT from file type " + "int32, actual BIGINT"); +} + +TEST(VortexPredicateConverterTest, TimestampComparisonsRequireLosslessRepresentableValues) { + const auto schema = arrow::schema({ + arrow::field("seconds", arrow::timestamp(arrow::TimeUnit::SECOND)), + arrow::field("millis", arrow::timestamp(arrow::TimeUnit::MILLI)), + arrow::field("local", arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai")), + }); + const std::pair cases[] = { + {"seconds", Literal(Timestamp::FromEpochMillis(1))}, + {"seconds", Literal(Timestamp::FromEpochMillis(1000, 1))}, + {"millis", Literal(Timestamp::FromEpochMillis(0, 1))}, + {"local", Literal(Timestamp::FromEpochMillis(1000))}, + }; + for (const auto& [field, literal] : cases) { + SCOPED_TRACE(field); + ASSERT_FALSE( + Pushed(PredicateBuilder::Equal(0, field, FieldType::TIMESTAMP, literal), schema)); + } +} + +TEST(VortexPredicateConverterTest, LosslessTimestampComparisonsFilterNativeFiles) { + const std::vector cases = { + {"seconds", + arrow::timestamp(arrow::TimeUnit::SECOND), + FieldType::TIMESTAMP, + "[-1,null,7,2]", + {Literal(Timestamp::FromEpochMillis(2000))}}, + {"millis", + arrow::timestamp(arrow::TimeUnit::MILLI), + FieldType::TIMESTAMP, + "[-1,null,7,2]", + {Literal(Timestamp::FromEpochMillis(2))}}, + {"micros", + arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + FieldType::TIMESTAMP, + "[-1,null,7,2]", + {Literal(Timestamp::FromEpochMillis(0, 2000))}}, + {"nanos", + arrow::timestamp(arrow::TimeUnit::NANO), + FieldType::TIMESTAMP, + "[-1,null,7,2]", + {Literal(Timestamp::FromEpochMillis(0, 2))}}, + }; + for (const NativeScanCase& test_case : cases) { + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + test_case, + PredicateBuilder::Equal(1, "value", FieldType::TIMESTAMP, test_case.literals[0]), {3})); + } +} + +TEST(VortexPredicateConverterTest, TimestampComparisonsPreserveBoundaryDecisionsAndNativeValues) { + const std::vector cases = { + {"micros_calendar_limits", + arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + FieldType::TIMESTAMP, + "[-1,null,7,2]", + {Literal(Timestamp::FromEpochMillis(253402207200999, 999000)), + Literal(Timestamp::FromEpochMillis(-62167219200000)), + Literal(Timestamp::FromEpochMillis(-1, 999000)), + Literal(Timestamp::FromEpochMillis(9223372036854775, 808000)), + Literal(Timestamp::FromEpochMillis(-9223372036854776, 192000))}}, + {"nanos_boundaries", + arrow::timestamp(arrow::TimeUnit::NANO), + FieldType::TIMESTAMP, + "[-1,null,7,2]", + {Literal(Timestamp::FromEpochMillis(9223372036854, 775807)), + Literal(Timestamp::FromEpochMillis(-9223372036854)), + Literal(Timestamp::FromEpochMillis(-1, 999999)), + Literal(Timestamp::FromEpochMillis(9223372036854, 775808)), + Literal(Timestamp::FromEpochMillis(-9223372036855, 224192))}}, + }; + for (const NativeScanCase& test_case : cases) { + SCOPED_TRACE(test_case.name); + const std::shared_ptr schema = + arrow::schema({arrow::field("value", test_case.type)}); + // Accepted expressions must also fit the native calendar range. + // The microsecond values cover year 0000 through Jiff's year 9999 limit. + ASSERT_TRUE( + Pushed(PredicateBuilder::Equal(0, "value", FieldType::TIMESTAMP, test_case.literals[0]), + schema)); + ASSERT_TRUE(Pushed( + PredicateBuilder::LessOrEqual(0, "value", FieldType::TIMESTAMP, test_case.literals[1]), + schema)); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + test_case, + PredicateBuilder::Equal(1, "value", FieldType::TIMESTAMP, test_case.literals[2]), {0})); + // One unit beyond INT64_MAX must fall back rather than overflow. + ASSERT_FALSE( + Pushed(PredicateBuilder::Equal(0, "value", FieldType::TIMESTAMP, test_case.literals[3]), + schema)); + // Although the final value is INT64_MIN, the intermediate millisecond product + // overflows. Preserve the existing fallback before calling the common utility. + ASSERT_FALSE( + Pushed(PredicateBuilder::Equal(0, "value", FieldType::TIMESTAMP, test_case.literals[4]), + schema)); + } +} + +TEST(VortexPredicateConverterTest, TruncatedTimestampComparisonFallsBackWithoutLosingRows) { + const NativeScanCase test_case = {"micros_fraction", + arrow::timestamp(arrow::TimeUnit::MICRO), + FieldType::TIMESTAMP, + "[0,null,1,-1]", + {}}; + const Literal one_ns(Timestamp::FromEpochMillis(0, 1)); + const std::shared_ptr less_than = + PredicateBuilder::LessThan(1, "value", FieldType::TIMESTAMP, one_ns); + ASSERT_OK_AND_ASSIGN(int32_t comparison, + Literal(Timestamp::FromEpochMillis(0)).CompareTo(one_ns)); + ASSERT_LT(comparison, 0); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, less_than, {0, 1, 2, 3}, std::nullopt, {}, + /*expect_native_filter=*/false)); + + vx_error* error = nullptr; + ScopeGuard release_error([&error]() { + if (error != nullptr) { + vx_error_free(error); + } + }); + VxScalarPtr truncated( + vx_scalar_new_timestamp(0, VX_TIME_UNIT_MICROSECOND, nullptr, false, &error)); + ASSERT_EQ(error, nullptr) << NativeErrorMessage(error); + ASSERT_NE(truncated, nullptr); + // Truncating 1ns to 0us loses row 0, which the exact comparison above must retain. + ASSERT_NO_FATAL_FAILURE( + AssertNativeComparisonRows(test_case, less_than, truncated.get(), VX_OPERATOR_LT, {3})); +} + +TEST(VortexPredicateConverterTest, NativeFloatingPointOrderRequiresConservativeFallback) { + const auto float_schema = arrow::schema({arrow::field("f", arrow::float32())}); + ASSERT_FALSE( + Pushed(PredicateBuilder::Equal(0, "f", FieldType::FLOAT, Literal(1.5F)), float_schema)); + + const NativeScanCase test_case = { + "floating_order", arrow::float64(), FieldType::DOUBLE, "[NaN,null,-0.0,0.0]", {}}; + const double nan = std::numeric_limits::quiet_NaN(); + ASSERT_OK_AND_ASSIGN(int32_t nan_comparison, Literal(nan).CompareTo(Literal(-nan))); + ASSERT_EQ(nan_comparison, 0); + ASSERT_OK_AND_ASSIGN(int32_t zero_comparison, Literal(-0.0).CompareTo(Literal(0.0))); + ASSERT_LT(zero_comparison, 0); + const std::shared_ptr equal_nan = + PredicateBuilder::Equal(1, "value", FieldType::DOUBLE, Literal(-nan)); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, equal_nan, {0, 1, 2, 3}, std::nullopt, {}, + /*expect_native_filter=*/false)); + VxScalarPtr negative_nan(vx_scalar_new_f64(-nan, false)); + ASSERT_NE(negative_nan, nullptr); + // Native totalOrder distinguishes NaN signs, while Paimon must retain row 0. + ASSERT_NO_FATAL_FAILURE( + AssertNativeComparisonRows(test_case, equal_nan, negative_nan.get(), VX_OPERATOR_EQ, {})); + + const std::shared_ptr equal_negative_zero = + PredicateBuilder::Equal(1, "value", FieldType::DOUBLE, Literal(-0.0)); + VxScalarPtr negative_zero(vx_scalar_new_f64(-0.0, false)); + ASSERT_NE(negative_zero, nullptr); + // Signed-zero ordering agrees in this case; it does not establish compatible NaN semantics. + ASSERT_NO_FATAL_FAILURE(AssertNativeComparisonRows(test_case, equal_negative_zero, + negative_zero.get(), VX_OPERATOR_EQ, {2})); +} + +TEST(VortexPredicateConverterTest, MembershipScalarsFilterNativeFilesAndExcludeNullRows) { + const Decimal::int128_t wide_value = static_cast(999999999999999999LL) * 10; + const std::vector cases = { + {"boolean", + arrow::boolean(), + FieldType::BOOLEAN, + "[true,null,false,true]", + {Literal(true)}}, + {"string", + arrow::utf8(), + FieldType::STRING, + R"(["",null,"miss","last"])", + {Literal(FieldType::STRING, "", 0), Literal(FieldType::STRING, "last", 4)}}, + {"date", + arrow::date32(), + FieldType::DATE, + "[-1,null,7,19000]", + {Literal(FieldType::DATE, -1), Literal(FieldType::DATE, 19000)}}, + {"wide_decimal", + arrow::decimal128(19, 2), + FieldType::DECIMAL, + R"(["-99999999999999999.90",null,"7.00","99999999999999999.90"])", + {Literal(Decimal(19, 2, -wide_value)), Literal(Decimal(19, 2, wide_value))}}, + {"timestamp_utc", + arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + FieldType::TIMESTAMP, + "[-1,null,7000,2001]", + {Literal(Timestamp::FromEpochMillis(-1, 999000)), + Literal(Timestamp::FromEpochMillis(2, 1000))}}, + }; + for (const NativeScanCase& test_case : cases) { + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + test_case, PredicateBuilder::In(1, "value", test_case.field_type, test_case.literals), + {0, 3})); + } + + // NOT IN excludes nulls even when all non-null values are in the list. + const NativeScanCase all_booleans = {"all_booleans", + arrow::boolean(), + FieldType::BOOLEAN, + "[true,null,false,true]", + {Literal(false), Literal(true)}}; + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + all_booleans, + PredicateBuilder::NotIn(1, "value", FieldType::BOOLEAN, all_booleans.literals), {})); +} + +TEST(VortexPredicateConverterTest, MembershipMaximumPreservesNativeAndResidualRows) { + const NativeScanCase test_case = { + "membership_list_sizes", arrow::int32(), FieldType::INT, "[0,null,2048,1]", {}}; + const std::vector expected_rows = {2}; + for (int32_t size : {32, 33}) { + SCOPED_TRACE(size); + std::vector literals; + for (int32_t value = 0; value < size; ++value) { + literals.emplace_back(value); + } + const auto predicate = PredicateBuilder::NotIn(1, "value", FieldType::INT, literals); + if (size == 32) { + ASSERT_NO_FATAL_FAILURE( + AssertNativeRows(test_case, predicate, expected_rows, expected_rows)); + } else { + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, predicate, {0, 1, 2, 3}, + expected_rows, {}, + /*expect_native_filter=*/false)); + ASSERT_NO_FATAL_FAILURE( + AssertNativeRows(test_case, predicate, expected_rows, expected_rows, {33})); + } + } + + const auto schema = TestSchema(); + const auto singleton = PredicateBuilder::NotIn(0, "i", FieldType::INT, {Literal(int32_t{1})}); + ASSERT_TRUE(Pushed(singleton, schema)); + ASSERT_FALSE(Pushed(singleton, schema, {0})); +} + +TEST(VortexPredicateConverterTest, IntegerListsAtTypeLimitsFilterNativeFiles) { + const std::vector cases = { + IntegerRangeBoundaryCase("tinyint_range_limits", arrow::int8(), FieldType::TINYINT, + "[-128,null,0,127]"), + IntegerRangeBoundaryCase("smallint_range_limits", arrow::int16(), + FieldType::SMALLINT, "[-32768,null,0,32767]"), + IntegerRangeBoundaryCase("int_range_limits", arrow::int32(), FieldType::INT, + "[-2147483648,null,0,2147483647]"), + IntegerRangeBoundaryCase("bigint_range_limits", arrow::int64(), FieldType::BIGINT, + "[-9223372036854775808,null,0,9223372036854775807]"), + }; + for (const NativeScanCase& test_case : cases) { + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + test_case, PredicateBuilder::In(1, "value", test_case.field_type, test_case.literals), + {0, 3})); + } +} + +TEST(VortexPredicateConverterTest, IntegerListsComposeWithNativePredicatesAndProjection) { + NativeScanCase test_case = { + "integer_range_compound", arrow::int32(), FieldType::INT, "[-4,null,8,4]", {}}; + for (int32_t value = -4; value <= 4; ++value) { + test_case.literals.emplace_back(value); + } + const std::shared_ptr in = + PredicateBuilder::In(1, "value", FieldType::INT, test_case.literals); + const std::shared_ptr not_in = + PredicateBuilder::NotIn(1, "value", FieldType::INT, test_case.literals); + const std::shared_ptr first_three = + PredicateBuilder::LessThan(0, "rowid", FieldType::BIGINT, Literal(int64_t{3})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in_and, + PredicateBuilder::And({in, first_three})); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, in_and, {0}, std::nullopt, {}, + /*expect_native_filter=*/true, + /*project_rowid_only=*/true)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr not_in_or_null, + PredicateBuilder::Or({not_in, PredicateBuilder::IsNull(1, "value", FieldType::INT)})); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, not_in_or_null, {1, 2}, std::nullopt, {}, + /*expect_native_filter=*/true, + /*project_rowid_only=*/true)); +} + +TEST(VortexPredicateConverterTest, NotInDuplicatesMissesAndAllExcludedFilterNativeFiles) { + const NativeScanCase test_case = { + "not_in_edges", arrow::int32(), FieldType::INT, "[-1,null,7,2]", {}}; + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + test_case, + PredicateBuilder::NotIn(1, "value", FieldType::INT, + {Literal(int32_t{-1}), Literal(int32_t{-1}), Literal(int32_t{2})}), + {2})); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + test_case, PredicateBuilder::NotIn(1, "value", FieldType::INT, {Literal(int32_t{99})}), + {0, 2, 3})); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + test_case, + PredicateBuilder::NotIn(1, "value", FieldType::INT, + {Literal(int32_t{-1}), Literal(int32_t{7}), Literal(int32_t{2})}), + {})); + const NativeScanCase all_null = { + "not_in_all_null", arrow::int32(), FieldType::INT, "[null,null,null,null]", {}}; + ASSERT_NO_FATAL_FAILURE(AssertNativeRows( + all_null, PredicateBuilder::NotIn(1, "value", FieldType::INT, {Literal(int32_t{1})}), {})); +} + +TEST(VortexPredicateConverterTest, MembershipUnsupportedInputsFallBack) { + const auto schema = TestSchema(); + ASSERT_FALSE(Pushed(PredicateBuilder::In(0, "i", FieldType::INT, {}), schema)); + ASSERT_FALSE(Pushed( + PredicateBuilder::In(0, "value", FieldType::BINARY, {Literal(FieldType::BINARY, "x", 1)}), + arrow::schema({arrow::field("value", arrow::binary())}))); + ASSERT_FALSE(Pushed(PredicateBuilder::In(0, "value", FieldType::FLOAT, {Literal(1.5F)}), + arrow::schema({arrow::field("value", arrow::float32())}))); +} + +TEST(VortexPredicateConverterTest, NestedCompoundsPreserveNativeCandidates) { + const NativeScanCase test_case = { + "nested-fallback", arrow::utf8(), FieldType::STRING, R"(["aa",null,"ab","ba"])", {}}; + const Literal ab(FieldType::STRING, "ab", 2); + const std::shared_ptr supported = + PredicateBuilder::GreaterOrEqual(1, "value", FieldType::STRING, ab); + const std::shared_ptr equal = + PredicateBuilder::Equal(1, "value", FieldType::STRING, Literal(FieldType::STRING, "aa", 2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr unsupported, + PredicateBuilder::StartsWith(1, "value", FieldType::STRING, + Literal(FieldType::STRING, "a", 1))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr partial_and, + PredicateBuilder::And({supported, unsupported})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_or, + PredicateBuilder::Or({unsupported, supported})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr nested_and, + PredicateBuilder::And({fallback_or, equal})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr nested_or, + PredicateBuilder::Or({partial_and, equal})); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, partial_and, {2, 3}, {{2}})); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, fallback_or, {0, 1, 2, 3}, {{0, 2, 3}}, {}, + /*expect_native_filter=*/false)); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, nested_and, {0}, {{0}})); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, nested_or, {0, 2, 3}, {{0, 2}})); +} + +TEST(VortexPredicateConverterTest, NativeMembershipErrorPropagatesThroughOr) { + const auto schema = TestSchema(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr unsupported, + PredicateBuilder::StartsWith(1, "s", FieldType::STRING, + Literal(FieldType::STRING, "a", 1))); + const auto in = PredicateBuilder::In( + 1, "s", FieldType::STRING, + {Literal(FieldType::STRING, "valid", 5), Literal(FieldType::STRING, "\xff", 1)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr disjunction, + PredicateBuilder::Or({unsupported, in})); + ASSERT_NOK_WITH_MSG(VortexPredicateConverter::Convert(disjunction, schema, {}), + kInvalidUtf8Error); +} + +TEST(VortexPredicateConverterTest, NativeTemporalScalarBoundsFallBackWithoutErrors) { + const int64_t maximum = std::numeric_limits::max(); + struct TemporalCase { + const char* name; + std::shared_ptr type; + FieldType field_type; + Literal literal; + }; + const std::vector cases = { + {"maximum-micros", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), FieldType::TIMESTAMP, + Literal(Timestamp::FromEpochMillis(maximum / 1000, + static_cast((maximum % 1000) * 1000)))}, + {"minimum-micros", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), FieldType::TIMESTAMP, + Literal(Timestamp::FromEpochMillis(std::numeric_limits::min() / 1000))}, + {"native-epoch-seconds", arrow::timestamp(arrow::TimeUnit::SECOND, "UTC"), + FieldType::TIMESTAMP, Literal(Timestamp::FromEpochMillis(253402207201000))}, + {"native-epoch-millis", arrow::timestamp(arrow::TimeUnit::MILLI), FieldType::TIMESTAMP, + Literal(Timestamp::FromEpochMillis(253402207201000))}, + {"minimum-date", arrow::date32(), FieldType::DATE, + Literal(FieldType::DATE, std::numeric_limits::min())}}; + for (const TemporalCase& test_case : cases) { + SCOPED_TRACE(test_case.name); + const std::shared_ptr schema = + arrow::schema({arrow::field("value", test_case.type)}); + ASSERT_FALSE(Pushed( + PredicateBuilder::Equal(0, "value", test_case.field_type, test_case.literal), schema)); + } +} + +TEST(VortexPredicateConverterTest, UnsafeTimestampInListPreservesNativeAndResidualRows) { + const NativeScanCase test_case = { + "timestamp-precision-fallback", + arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + FieldType::TIMESTAMP, + "[0,null,1,2]", + {Literal(Timestamp::FromEpochMillis(0, 1000)), Literal(Timestamp::FromEpochMillis(0, 1))}}; + const auto predicate = + PredicateBuilder::In(1, "value", FieldType::TIMESTAMP, test_case.literals); + ASSERT_NO_FATAL_FAILURE(AssertNativeRows(test_case, predicate, {0, 1, 2, 3}, {{2}}, {}, + /*expect_native_filter=*/false)); +} + +TEST(VortexPredicateConverterTest, MembershipMaximumAppliesIndependentlyToEachLeaf) { + const std::shared_ptr schema = TestSchema(); + std::vector integers; + std::vector longs; + for (int32_t value = 0; value < 32; ++value) { + integers.emplace_back(value); + longs.emplace_back(static_cast(value)); + } + const auto in = PredicateBuilder::In(0, "i", FieldType::INT, integers); + const auto not_in = PredicateBuilder::NotIn(2, "l", FieldType::BIGINT, longs); + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_predicate, + PredicateBuilder::And({in, not_in})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_predicate, + PredicateBuilder::Or({in, not_in})); + ASSERT_TRUE(Pushed(and_predicate, schema)); + ASSERT_TRUE(Pushed(or_predicate, schema)); + ASSERT_FALSE(Pushed(and_predicate, schema, {31})); + ASSERT_FALSE(Pushed(or_predicate, schema, {31})); +} + +TEST(VortexPredicateConverterTest, DuplicateValuesCountTowardMembershipMaximum) { + const auto schema = TestSchema(); + const std::vector literals(33, Literal(FieldType::STRING, "same", 4)); + const auto predicate = PredicateBuilder::NotIn(1, "s", FieldType::STRING, literals); + ASSERT_FALSE(Pushed(predicate, schema)); + ASSERT_TRUE(Pushed(predicate, schema, {33})); +} + +} // namespace paimon::vortex::test diff --git a/src/paimon/format/vortex/vortex_reader_builder.h b/src/paimon/format/vortex/vortex_reader_builder.h index 42d5cfd17..fb35a53e4 100644 --- a/src/paimon/format/vortex/vortex_reader_builder.h +++ b/src/paimon/format/vortex/vortex_reader_builder.h @@ -23,6 +23,7 @@ #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/format/reader_builder.h" +#include "paimon/format/vortex/vortex_defs.h" #include "paimon/format/vortex/vortex_file_batch_reader.h" #include "paimon/memory/memory_pool.h" @@ -30,8 +31,14 @@ namespace paimon::vortex { class VortexReaderBuilder : public ReaderBuilder { public: - explicit VortexReaderBuilder(int32_t batch_size) - : batch_size_(batch_size), pool_(GetDefaultPool()), arrow_pool_(GetArrowPool(pool_)) {} + explicit VortexReaderBuilder(int32_t batch_size, + const VortexPredicateOptions& predicate_options, + const VortexIoCoalesceOptions& coalesce_options) + : batch_size_(batch_size), + predicate_options_(predicate_options), + coalesce_options_(coalesce_options), + pool_(GetDefaultPool()), + arrow_pool_(GetArrowPool(pool_)) {} ReaderBuilder* WithMemoryPool(const std::shared_ptr& pool) override { pool_ = pool; @@ -48,11 +55,14 @@ class VortexReaderBuilder : public ReaderBuilder { if (pool_ == nullptr) { return Status::Invalid("Vortex reader memory pool is nullptr"); } - return VortexFileBatchReader::Create(input, batch_size_, arrow_pool_); + return VortexFileBatchReader::Create(input, batch_size_, predicate_options_, + coalesce_options_, arrow_pool_); } private: int32_t batch_size_; + VortexPredicateOptions predicate_options_; + VortexIoCoalesceOptions coalesce_options_; std::shared_ptr pool_; std::shared_ptr arrow_pool_; }; diff --git a/src/paimon/format/vortex/vortex_stats_extractor.cpp b/src/paimon/format/vortex/vortex_stats_extractor.cpp index c589e7c29..e4b8ed3df 100644 --- a/src/paimon/format/vortex/vortex_stats_extractor.cpp +++ b/src/paimon/format/vortex/vortex_stats_extractor.cpp @@ -28,6 +28,7 @@ #include "paimon/common/utils/math.h" #include "paimon/defs.h" #include "paimon/format/column_stats.h" +#include "paimon/format/vortex/vortex_defs.h" #include "paimon/format/vortex/vortex_ffi.h" #include "paimon/format/vortex/vortex_ffi_util.h" #include "paimon/format/vortex/vortex_io_callbacks.h" @@ -53,15 +54,14 @@ VortexStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& fil auto length = static_cast(signed_length); auto input_context = std::make_shared(input); - VxSessionPtr session(vx_session_new(), vx_session_free); + VxSessionPtr session(vx_session_new()); if (session == nullptr) { return Status::IOError("failed to create Vortex session"); } vx_error* error = nullptr; - VxDataSourcePtr data_source( - vx_data_source_new_callback(session.get(), VortexInputContext::MakeCallbacks(input_context), - length, &error), - vx_data_source_free); + VxDataSourcePtr data_source(vx_data_source_new_callback_with_coalesce( + session.get(), VortexInputContext::MakeCallbacks(input_context), length, + kDefaultVortexReadIoCoalesceMaxGap, kDefaultVortexReadIoCoalesceMaxSize, &error)); if (data_source == nullptr) { return VortexCallbackError("open Vortex file for stats", error, input_context->GetCallbackStatus()); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 17807e254..a19dc374a 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -40,6 +40,7 @@ #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" @@ -3798,45 +3799,122 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { #endif #ifdef PAIMON_ENABLE_VORTEX -TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibility) { - // Fixture written by Paimon Java 2.0.0 with file.format=vortex; see - // test/test_data/vortex/append_java_compat.db/append_java_compat/README.md. This verifies that - // paimon-cpp reads back a Vortex table produced by the Java implementation. - arrow::FieldVector fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("id", arrow::int32()), - arrow::field("name", arrow::utf8()), - }; - std::shared_ptr data_type = arrow::struct_(fields); - std::shared_ptr expected_array = - arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ -[0, 1, "one"], -[0, 2, null], -[0, 3, "three"], -[0, 4, "four"], -[0, 5, "five"] -])") - .ValueOrDie(); - auto expected = std::make_shared(expected_array); +namespace { + +Result> ReadTableRows( + const std::string& table_path, const std::vector>& splits, + const std::vector& projection, const std::shared_ptr& predicate, + bool exact_filter, bool late_materializing) { + ReadContextBuilder builder(table_path); + builder.SetReadFieldNames(projection) + .SetPredicate(predicate) + .EnablePredicateFilter(exact_filter) + .EnableLateMaterializing(late_materializing) + .AddOption(Options::READ_BATCH_SIZE, "2"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(splits)); + return ReadResultCollector::CollectResult(std::move(reader)); +} - const std::string table_name = "append_java_compat"; - const std::string table_path = GetDataDir() + "/vortex/" + table_name + ".db/" + table_name; - ScanContextBuilder scan_context_builder(table_path); - ReadContextBuilder read_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, - table_read->CreateReader(plan->Splits())); +} // namespace + +TEST_F(ScanAndReadInteTest, TestVortexPredicatesWithLateMaterialization) { + auto dir = UniqueTestDirectory::Create("local"); + ASSERT_NE(dir, nullptr); + const arrow::FieldVector fields = {arrow::field("id", arrow::int32()), + arrow::field("tag", arrow::utf8()), + arrow::field("payload", arrow::utf8())}; + const std::map options = {{Options::FILE_FORMAT, "vortex"}, + {Options::FILE_SYSTEM, "local"}, + {Options::BUCKET, "-1"}, + {Options::WRITE_ONLY, "true"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), arrow::schema(fields), {}, {}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([ + [0,"skip","zero"],[1,null,"one"],[2,"keep","two"],[3,"drop","three"], + [4,"keep",null],[5,"keep","five"],[6,"skip","six"],[7,"keep","seven"] + ])", + {}, 0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt)); + const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), std::nullopt, false)); + const auto result_type = arrow::struct_( + {arrow::field("_VALUE_KIND", arrow::int8()), fields[2], fields[0], fields[1]}); + auto greater = PredicateBuilder::GreaterThan(1, "id", FieldType::INT, Literal(int32_t{1})); + auto tag = + PredicateBuilder::Equal(2, "tag", FieldType::STRING, Literal(FieldType::STRING, "keep", 4)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({greater, tag})); + std::shared_ptr expected; + ASSERT_OK(ToPaimonStatus(arrow::ipc::internal::json::ChunkedArrayFromJSON( + result_type, + {R"([[0,"two",2,"keep"],[0,null,4,"keep"],[0,"five",5,"keep"],[0,"seven",7,"keep"]])"}, + &expected))); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, - ReadResultCollector::CollectResult(std::move(batch_reader))); - ASSERT_TRUE(expected->Equals(actual)) - << "actual: " << (actual == nullptr ? "null" : actual->ToString()) - << "\nexpected: " << expected->ToString(); + ReadTableRows(table_path, splits, {"payload", "id", "tag"}, predicate, + /*exact_filter=*/false, /*late_materializing=*/true)); + ASSERT_TRUE(expected->Equals(actual)); +} + +TEST_F(ScanAndReadInteTest, TestVortexFallbackWithAndWithoutPayload) { + auto dir = UniqueTestDirectory::Create("local"); + ASSERT_NE(dir, nullptr); + const arrow::FieldVector fields = {arrow::field("value", arrow::int32()), + arrow::field("payload", arrow::utf8())}; + const std::map options = {{Options::FILE_FORMAT, "vortex"}, + {Options::FILE_SYSTEM, "local"}, + {Options::BUCKET, "-1"}, + {Options::WRITE_ONLY, "true"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), arrow::schema(fields), {}, {}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[-1,"minus"],[0,"zero"],[1,"one"],[null,"null"]])", {}, 0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt)); + const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), std::nullopt, false)); + + // This list exceeds the default 32-value native pushdown limit. + std::vector values = {Literal(int32_t{1})}; + for (int32_t value = 100; value < 132; ++value) { + values.emplace_back(value); + } + const auto predicate = PredicateBuilder::NotIn(0, "value", FieldType::INT, values); + const auto value_kind = arrow::field("_VALUE_KIND", arrow::int8()); + const auto probe_type = arrow::struct_({value_kind, fields[0]}); + + // A probe-only projection bypasses the two-phase reader, even with late materialization on. + ASSERT_OK_AND_ASSIGN(std::shared_ptr candidates, + ReadTableRows(table_path, splits, {"value"}, predicate, + /*exact_filter=*/false, /*late_materializing=*/true)); + std::shared_ptr expected_candidates; + ASSERT_OK(ToPaimonStatus(arrow::ipc::internal::json::ChunkedArrayFromJSON( + probe_type, {R"([[0,-1],[0,0],[0,1],[0,null]])"}, &expected_candidates))); + ASSERT_TRUE(expected_candidates->Equals(candidates)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadTableRows(table_path, splits, {"value"}, predicate, + /*exact_filter=*/true, /*late_materializing=*/true)); + std::shared_ptr expected_filtered; + ASSERT_OK(ToPaimonStatus(arrow::ipc::internal::json::ChunkedArrayFromJSON( + probe_type, {R"([[0,-1],[0,0]])"}, &expected_filtered))); + ASSERT_TRUE(expected_filtered->Equals(actual)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr with_payload, + ReadTableRows(table_path, splits, {"value", "payload"}, predicate, + /*exact_filter=*/false, /*late_materializing=*/true)); + const auto payload_type = arrow::struct_({value_kind, fields[0], fields[1]}); + std::shared_ptr expected_with_payload; + ASSERT_OK(ToPaimonStatus(arrow::ipc::internal::json::ChunkedArrayFromJSON( + payload_type, {R"([[0,-1,"minus"],[0,0,"zero"]])"}, &expected_with_payload))); + ASSERT_TRUE(expected_with_payload->Equals(with_payload)); } // The Java fixture stores TIMESTAMP_LTZ in UTC while paimon reads it in the reader's own zone, so @@ -3875,9 +3953,8 @@ TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibilityLtzAcrossTimezones) { shanghai = column; } } - arrow::Result> viewed = shanghai->View(utc->type()); - ASSERT_TRUE(viewed.ok()) << viewed.status().ToString(); - ASSERT_TRUE((*viewed)->Equals(*utc)) + std::shared_ptr viewed = shanghai->View(utc->type()).ValueOrDie(); + ASSERT_TRUE(viewed->Equals(*utc)) << "shanghai: " << shanghai->ToString() << "\nutc: " << utc->ToString(); } diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/README.md b/test/test_data/vortex/append_java_compat.db/append_java_compat/README.md deleted file mode 100644 index f9ad78bc8..000000000 --- a/test/test_data/vortex/append_java_compat.db/append_java_compat/README.md +++ /dev/null @@ -1,22 +0,0 @@ -Table: append_java_compat -Writer: Paimon Java 2.0.0 (paimon-vortex-format + paimon-vortex-jni 2.0.0) - -Schema: -id INT NOT NULL -name STRING - -Options: -bucket = -1 -file.format = vortex - -Data is a single APPEND commit (snapshot 1) with five rows. The second row has a -NULL name: -(1, "one") -(2, NULL) -(3, "three") -(4, "four") -(5, "five") - -Generated by a standalone Java program using the Paimon batch write API -(BatchWriteBuilder / BatchTableWrite / BatchTableCommit) on JDK 17 with ---add-opens=java.base/java.nio=ALL-UNNAMED (required by Arrow Java memory). diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/bucket-0/data-430e1867-52a2-4264-ae8e-a2d9fe20bf1e-0.vortex b/test/test_data/vortex/append_java_compat.db/append_java_compat/bucket-0/data-430e1867-52a2-4264-ae8e-a2d9fe20bf1e-0.vortex deleted file mode 100644 index 4b062f482..000000000 Binary files a/test/test_data/vortex/append_java_compat.db/append_java_compat/bucket-0/data-430e1867-52a2-4264-ae8e-a2d9fe20bf1e-0.vortex and /dev/null differ diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-ac2dfb39-417f-42a4-887a-7618b40580a5-0 b/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-ac2dfb39-417f-42a4-887a-7618b40580a5-0 deleted file mode 100644 index 886c17378..000000000 Binary files a/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-ac2dfb39-417f-42a4-887a-7618b40580a5-0 and /dev/null differ diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-0 b/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-0 deleted file mode 100644 index f091992a2..000000000 Binary files a/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-0 and /dev/null differ diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-1 b/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-1 deleted file mode 100644 index 2a79d6a17..000000000 Binary files a/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-1 and /dev/null differ diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/schema/schema-0 b/test/test_data/vortex/append_java_compat.db/append_java_compat/schema/schema-0 deleted file mode 100644 index 75918adbf..000000000 --- a/test/test_data/vortex/append_java_compat.db/append_java_compat/schema/schema-0 +++ /dev/null @@ -1,21 +0,0 @@ -{ - "version" : 3, - "id" : 0, - "fields" : [ { - "id" : 0, - "name" : "id", - "type" : "INT NOT NULL" - }, { - "id" : 1, - "name" : "name", - "type" : "STRING" - } ], - "highestFieldId" : 1, - "partitionKeys" : [ ], - "primaryKeys" : [ ], - "options" : { - "bucket" : "-1", - "file.format" : "vortex" - }, - "timeMillis" : 1789455840340 -} \ No newline at end of file diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/EARLIEST b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/EARLIEST deleted file mode 100644 index 56a6051ca..000000000 --- a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/EARLIEST +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/LATEST b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/LATEST deleted file mode 100644 index 56a6051ca..000000000 --- a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/LATEST +++ /dev/null @@ -1 +0,0 @@ -1 \ No newline at end of file diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/snapshot-1 b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/snapshot-1 deleted file mode 100644 index ef7d4ea12..000000000 --- a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/snapshot-1 +++ /dev/null @@ -1,17 +0,0 @@ -{ - "version" : 3, - "uuid" : "70b40861-aaad-4ae5-8380-7e88af4f2100", - "id" : 1, - "schemaId" : 0, - "baseManifestList" : "manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-0", - "baseManifestListSize" : 1006, - "deltaManifestList" : "manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-1", - "deltaManifestListSize" : 1110, - "commitUser" : "420305a8-591b-49b8-93e8-fe76caea2cef", - "commitIdentifier" : 9223372036854775807, - "commitKind" : "APPEND", - "timeMillis" : 1789456011021, - "totalRecordCount" : 5, - "deltaRecordCount" : 5, - "nextRowId" : 0 -} \ No newline at end of file