From 91a569c54b76cbea14969785f88a7a9253d4ee09 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Thu, 12 Nov 2026 03:51:53 -0800 Subject: [PATCH 1/2] feat(full-text): search unindexed rows in full and detail modes Use the snapshot's next row ID in full mode and the selected data files' row ranges in detail mode to find gaps in single-field full-text coverage. Plan raw splits only when a selected partition has a full-text index for the column. Keep fast mode index-only. Assign disjoint row ranges to stored and raw splits and pin both to the planning snapshot and schema. Index live, non-null raw text in memory with Tantivy or Lucene++. Apply row filters before ranking and merge stored and raw hits for the final top-k. When scalar search mode is full or detail, evaluate the row filter against data files for uncovered ranges, or for all searched rows if no scalar index can evaluate the predicate. Refs: #406 --- LICENSE | 20 + cmake_modules/ThirdpartyToolchain.cmake | 60 +- cmake_modules/full_text.diff | 237 ++++ cmake_modules/lucene.diff | 45 + docs/source/user_guide/global_index.rst | 137 ++- include/paimon/defs.h | 23 +- include/paimon/table/source/full_text_read.h | 34 +- include/paimon/table/source/full_text_scan.h | 38 +- .../table/source/full_text_search_builder.h | 26 +- .../table/source/full_text_search_split.h | 14 + src/paimon/CMakeLists.txt | 3 + src/paimon/common/defs.cpp | 1 + src/paimon/common/utils/range.cpp | 17 +- src/paimon/common/utils/range_test.cpp | 8 + src/paimon/core/core_options.cpp | 9 +- src/paimon/core/core_options.h | 5 +- src/paimon/core/core_options_test.cpp | 3 +- .../data_evolution_global_index_coverage.cpp | 83 ++ .../data_evolution_global_index_coverage.h | 62 + ...a_evolution_global_index_coverage_test.cpp | 161 +++ .../global_index_live_row_filter.cpp | 26 + .../global_index_live_row_filter.h | 11 + .../core/table/source/abstract_table_scan.h | 9 + .../source/data_evolution_full_text_read.cpp | 461 ++++++-- .../source/data_evolution_full_text_read.h | 34 +- .../data_evolution_full_text_read_test.cpp | 319 +++++- .../source/data_evolution_full_text_scan.cpp | 159 ++- .../source/data_evolution_full_text_scan.h | 29 +- .../data_evolution_full_text_scan_test.cpp | 54 +- .../table/source/filtered_row_id_reader.cpp | 252 ++-- .../table/source/filtered_row_id_reader.h | 37 +- .../table/source/full_text_search_builder.cpp | 178 ++- .../table/source/full_text_search_split.cpp | 246 +++- .../source/full_text_search_split_test.cpp | 275 ++++- .../index_full_text_search_split_impl.h | 30 +- .../table/source/raw_full_text_read_impl.cpp | 380 ++++++ .../table/source/raw_full_text_read_impl.h | 91 ++ .../source/raw_full_text_search_split_impl.h | 91 ++ .../full_text_global_index_writer.cpp | 27 +- .../full_text/full_text_global_index_writer.h | 3 +- src/paimon/global_index/lucene/CMakeLists.txt | 1 + .../lucene/lucene_global_index_writer.cpp | 78 +- .../lucene/lucene_global_index_writer.h | 13 +- test/inte/global_index_test.cpp | 1020 ++++++++++++++++- 44 files changed, 3977 insertions(+), 833 deletions(-) create mode 100644 cmake_modules/full_text.diff create mode 100644 cmake_modules/lucene.diff create mode 100644 src/paimon/core/global_index/data_evolution_global_index_coverage.cpp create mode 100644 src/paimon/core/global_index/data_evolution_global_index_coverage.h create mode 100644 src/paimon/core/global_index/data_evolution_global_index_coverage_test.cpp create mode 100644 src/paimon/core/table/source/raw_full_text_read_impl.cpp create mode 100644 src/paimon/core/table/source/raw_full_text_read_impl.h create mode 100644 src/paimon/core/table/source/raw_full_text_search_split_impl.h diff --git a/LICENSE b/LICENSE index e986c187b..98523fe93 100644 --- a/LICENSE +++ b/LICENSE @@ -414,6 +414,26 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product includes code from Apache Paimon Full Text 0.1.0. + +* Upstream source patch in cmake_modules/full_text.diff + +Copyright: 2026 The Apache Software Foundation. +Home page: https://github.com/apache/paimon-full-text +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Lucene++ 3.0.9. + +* Upstream source patch in cmake_modules/lucene.diff + +Copyright (c) 2009-2014 Alan Wright. All rights reserved. +Home page: https://github.com/luceneplusplus/LucenePlusPlus +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This product includes code from Vortex 0.75.0. * Callback I/O sink utilities in crates/vortex_callback_io/callback_io.rs diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 4b78d0e74..c29988aef 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -887,10 +887,14 @@ macro(build_lucene) "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}") set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a") + set(LUCENE_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/lucene.diff") + paimon_patch_command(lucene_ep PATCH_COMMAND) externalproject_add(lucene_ep ${EP_COMMON_OPTIONS} URL ${LUCENE_SOURCE_URL} URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}" + LOG_PATCH ON + PATCH_COMMAND ${PATCH_COMMAND} CMAKE_ARGS ${LUCENE_CMAKE_ARGS} BUILD_BYPRODUCTS ${LUCENE_LIB} DEPENDS zlib @@ -903,6 +907,21 @@ macro(build_lucene) boost_chrono boost_atomic) + paimon_register_patched_project(lucene_ep + VERSION + "${PAIMON_LUCENE_BUILD_VERSION}" + PATCHES + "${LUCENE_PATCH_FILE}" + INSTALL_DIR + "${LUCENE_PREFIX}" + VERSION_VARIABLES + PAIMON_LUCENE_BUILD_VERSION + PAIMON_LUCENE_BUILD_SHA256_CHECKSUM + PAIMON_BOOST_BUILD_VERSION + PAIMON_BOOST_BUILD_SHA256_CHECKSUM + PAIMON_ZLIB_BUILD_VERSION + PAIMON_ZLIB_BUILD_SHA256_CHECKSUM) + set(LUCENE_INCLUDE_DIR "${LUCENE_PREFIX}/include") # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${LUCENE_INCLUDE_DIR}") @@ -912,6 +931,12 @@ macro(build_lucene) PROPERTIES IMPORTED_LOCATION "${LUCENE_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${LUCENE_INCLUDE_DIR}") + if(APPLE) + # Boost 1.66 headers still use the function base classes removed in C++17. + target_compile_definitions(lucene + INTERFACE _LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION + ) + endif() target_link_libraries(lucene INTERFACE zlib @@ -1094,6 +1119,16 @@ macro(build_boost) file(WRITE ${BOOST_USER_CONFIG} "using ${BOOST_TOOLSET} : : \"${CMAKE_CXX_COMPILER}\" ;\n") + set(BOOST_CXX_FLAGS "${EP_CXX_FLAGS}") + if(APPLE) + string(APPEND BOOST_CXX_FLAGS + " -D_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION") + endif() + if(APPLE AND CMAKE_OSX_SYSROOT) + # b2 does not inherit the SDK flags CMake adds to native targets. + string(APPEND BOOST_CXX_FLAGS " -isysroot '${CMAKE_OSX_SYSROOT}'") + endif() + externalproject_add(boost_ep URL ${BOOST_SOURCE_URL} URL_HASH "SHA256=${PAIMON_BOOST_BUILD_SHA256_CHECKSUM}" @@ -1109,8 +1144,8 @@ macro(build_boost) --user-config=${BOOST_USER_CONFIG} toolset=${BOOST_TOOLSET} cxxstd=${CMAKE_CXX_STANDARD} - cxxflags=${EP_CXX_FLAGS} linkflags=${EP_CXX_FLAGS} - install + cxxflags=${BOOST_CXX_FLAGS} + linkflags=${BOOST_CXX_FLAGS} install INSTALL_COMMAND "" DEPENDS zlib BUILD_BYPRODUCTS ${BOOST_BYPRODUCTS} @@ -1460,10 +1495,8 @@ macro(build_full_text) set(FULL_TEXT_DYNAMIC_LIB "${FULL_TEXT_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" ) - set(FULL_TEXT_CARGO_TARGET_DIR - "${CMAKE_CURRENT_BINARY_DIR}/full_text_ep-prefix/src/full_text_ep-build") set(FULL_TEXT_CARGO_DYNAMIC_LIB - "${FULL_TEXT_CARGO_TARGET_DIR}/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" + "/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" ) file(MAKE_DIRECTORY "${FULL_TEXT_INCLUDE_DIR}") @@ -1472,13 +1505,17 @@ macro(build_full_text) # Build with the lock file shipped in the source release: a reader rejects index files whose # recorded Tantivy version differs from the linked one, so the Tantivy version must match the # one used by Java Paimon. + set(FULL_TEXT_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/full_text.diff") + paimon_patch_command(full_text_ep PATCH_COMMAND) externalproject_add(full_text_ep URL ${FULL_TEXT_SOURCE_URL} URL_HASH "SHA256=${PAIMON_FULL_TEXT_BUILD_SHA256_CHECKSUM}" ${THIRDPARTY_LOG_OPTIONS} + LOG_PATCH ON + PATCH_COMMAND ${PATCH_COMMAND} CONFIGURE_COMMAND "" BUILD_COMMAND ${CMAKE_COMMAND} -E env - CARGO_TARGET_DIR=${FULL_TEXT_CARGO_TARGET_DIR} + "CARGO_TARGET_DIR=" ${PAIMON_CARGO_EXECUTABLE} build --release --locked --manifest-path /ffi/Cargo.toml COMMAND ${CMAKE_COMMAND} -E copy_if_different @@ -1487,6 +1524,17 @@ macro(build_full_text) /include ${FULL_TEXT_INCLUDE_DIR} BUILD_BYPRODUCTS "${FULL_TEXT_DYNAMIC_LIB}") + paimon_register_patched_project(full_text_ep + VERSION + "${PAIMON_FULL_TEXT_BUILD_VERSION}" + PATCHES + "${FULL_TEXT_PATCH_FILE}" + INSTALL_DIR + "${FULL_TEXT_PREFIX}" + VERSION_VARIABLES + PAIMON_FULL_TEXT_BUILD_VERSION + PAIMON_FULL_TEXT_BUILD_SHA256_CHECKSUM) + add_library(paimon_ftindex_ffi SHARED IMPORTED GLOBAL) set_target_properties(paimon_ftindex_ffi PROPERTIES IMPORTED_LOCATION "${FULL_TEXT_DYNAMIC_LIB}" diff --git a/cmake_modules/full_text.diff b/cmake_modules/full_text.diff new file mode 100644 index 000000000..dceb482ee --- /dev/null +++ b/cmake_modules/full_text.diff @@ -0,0 +1,237 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Paimon Full Text 0.1.0 uses a temporary directory for its Tantivy writer. +# Provide a RAM-backed writer for unindexed rows with the same archive and reader. + +diff --git a/core/src/index.rs b/core/src/index.rs +--- a/core/src/index.rs ++++ b/core/src/index.rs +@@ -20,7 +20,9 @@ + use crate::error::{FtIndexError, Result}; + use crate::io::{FullTextReadMetrics, ReadMetrics, ReadRequest, SeekRead, SeekWrite}; + use crate::query::{BooleanOccur, MatchOperator, QuerySpec}; +-use crate::storage::{read_header, write_envelope_from_paths, ArchiveFileEntry, IndexHeader}; ++use crate::storage::{ ++ read_header, write_envelope_from_paths, write_envelope_header, ArchiveFileEntry, IndexHeader, ++}; + use crate::tokenizer::{TokenizerConfig, TokenizerKind}; + use levenshtein_automata::{Distance, LevenshteinAutomatonBuilder, DFA, SINK_STATE}; + use roaring::RoaringTreemap; +@@ -30,6 +32,7 @@ + use std::path::{Path, PathBuf}; + use std::sync::{Arc, Mutex}; + use tantivy::collector::{FilterCollector, TopDocs}; ++use tantivy::directory::{Directory, RamDirectory}; + use tantivy::query::{ + BooleanQuery, BoostQuery, ConstScorer, EmptyQuery, EnableScoring, Explanation, Occur, Query, + QueryParser, Scorer, TermQuery, Weight, +@@ -64,7 +67,12 @@ + + struct FullTextIndexWriterState { + index_writer: IndexWriter, +- temp_dir: TempDir, ++ directory: WriterDirectory, ++} ++ ++enum WriterDirectory { ++ Disk(TempDir), ++ Memory(RamDirectory), + } + + #[derive(Clone, Debug, PartialEq, Eq)] +@@ -77,8 +85,23 @@ + pub fn new(config: FullTextIndexConfig) -> Result { + config.validate()?; + let temp_dir = TempDir::new()?; +- let schema = build_schema(&config); +- let mut index = Index::create_in_dir(temp_dir.path(), schema.clone())?; ++ let index = Index::create_in_dir(temp_dir.path(), build_schema(&config))?; ++ Self::with_index(config, index, WriterDirectory::Disk(temp_dir)) ++ } ++ ++ pub fn new_in_memory(config: FullTextIndexConfig) -> Result { ++ config.validate()?; ++ let directory = RamDirectory::create(); ++ let index = Index::create(directory.clone(), build_schema(&config), Default::default())?; ++ Self::with_index(config, index, WriterDirectory::Memory(directory)) ++ } ++ ++ fn with_index( ++ config: FullTextIndexConfig, ++ mut index: Index, ++ directory: WriterDirectory, ++ ) -> Result { ++ let schema = index.schema(); + register_tokenizer(&mut index, &config.tokenizer)?; + let row_id_field = schema + .get_field(&config.row_id_field) +@@ -89,7 +112,7 @@ + config, + state: Some(FullTextIndexWriterState { + index_writer, +- temp_dir, ++ directory, + }), + row_id_field, + text_fields, +@@ -157,11 +180,23 @@ + })?; + let FullTextIndexWriterState { + mut index_writer, +- temp_dir, ++ directory, + } = state; + index_writer.commit()?; ++ let index = index_writer.index().clone(); + index_writer.wait_merging_threads()?; + ++ let metadata = FullTextIndexMetadata { ++ config: self.config.clone(), ++ document_count: self.document_count, ++ tantivy_version: tantivy::version().to_string(), ++ }; ++ let temp_dir = match directory { ++ WriterDirectory::Disk(temp_dir) => temp_dir, ++ WriterDirectory::Memory(ram) => { ++ return write_ram_archive(output, metadata, index, ram); ++ } ++ }; + let files = collect_index_files(temp_dir.path())?; + let mut offset = 0u64; + let mut entries = Vec::with_capacity(files.len()); +@@ -177,11 +212,7 @@ + } + + let header = IndexHeader { +- metadata: FullTextIndexMetadata { +- config: self.config.clone(), +- document_count: self.document_count, +- tantivy_version: tantivy::version().to_string(), +- }, ++ metadata, + files: entries, + }; + let paths = files.iter().map(|file| &file.path).collect::>(); +@@ -485,6 +516,43 @@ + "document field '{field}' is not configured for this index" + ))) + } ++} ++ ++/// Write the same archive as the on-disk writer, including Tantivy's segment footers. ++fn write_ram_archive( ++ output: &mut W, ++ metadata: FullTextIndexMetadata, ++ index: Index, ++ directory: RamDirectory, ++) -> Result<()> { ++ let mut paths: Vec<_> = index.directory().list_managed_files().into_iter().collect(); ++ paths.push(PathBuf::from(".managed.json")); ++ paths.sort(); ++ let mut files = Vec::with_capacity(paths.len()); ++ let mut contents = Vec::with_capacity(paths.len()); ++ let mut offset = 0u64; ++ for path in paths { ++ let bytes = directory ++ .open_read(&path) ++ .map_err(tantivy::TantivyError::from)? ++ .read_bytes()?; ++ let length = bytes.len() as u64; ++ files.push(ArchiveFileEntry { ++ name: path.to_string_lossy().into_owned(), ++ offset, ++ length, ++ }); ++ offset = offset.checked_add(length).ok_or_else(|| { ++ FtIndexError::InvalidStorage("full-text archive size overflow".to_string()) ++ })?; ++ contents.push(bytes); ++ } ++ write_envelope_header(output, &IndexHeader { metadata, files })?; ++ for bytes in contents { ++ output.write_all(bytes.as_slice())?; ++ } ++ output.flush()?; ++ Ok(()) + } + + struct IndexFile { +diff --git a/core/src/storage.rs b/core/src/storage.rs +--- a/core/src/storage.rs ++++ b/core/src/storage.rs +@@ -87,7 +87,10 @@ + Ok(()) + } + +-fn write_envelope_header(output: &mut W, header: &IndexHeader) -> Result<()> { ++pub(crate) fn write_envelope_header( ++ output: &mut W, ++ header: &IndexHeader, ++) -> Result<()> { + let header_json = serde_json::to_vec(header)?; + if header_json.len() > MAX_HEADER_BYTES { + return Err(FtIndexError::InvalidStorage(format!( +diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs +--- a/ffi/src/lib.rs ++++ b/ffi/src/lib.rs +@@ -220,10 +220,28 @@ + values: *const *const c_char, + len: usize, + ) -> *mut PaimonFtindexWriterHandle { ++ open_writer(keys, values, len, FullTextIndexWriter::new) ++} ++ ++#[no_mangle] ++pub unsafe extern "C" fn paimon_ftindex_writer_open_in_memory( ++ keys: *const *const c_char, ++ values: *const *const c_char, ++ len: usize, ++) -> *mut PaimonFtindexWriterHandle { ++ open_writer(keys, values, len, FullTextIndexWriter::new_in_memory) ++} ++ ++unsafe fn open_writer( ++ keys: *const *const c_char, ++ values: *const *const c_char, ++ len: usize, ++ create: fn(FullTextIndexConfig) -> paimon_ftindex_core::Result, ++) -> *mut PaimonFtindexWriterHandle { + ffi_ptr(|| { + let options = options_from_raw(keys, values, len)?; + let config = FullTextIndexConfig::from_options(&options).map_err(|e| e.to_string())?; +- let writer = FullTextIndexWriter::new(config).map_err(|e| e.to_string())?; ++ let writer = create(config).map_err(|e| e.to_string())?; + Ok(Box::into_raw(Box::new(PaimonFtindexWriterHandle { + inner: writer, + }))) +diff --git a/include/paimon_ftindex.h b/include/paimon_ftindex.h +--- a/include/paimon_ftindex.h ++++ b/include/paimon_ftindex.h +@@ -59,6 +59,12 @@ + const char **values, + size_t len); + ++/* Builds the index segments in memory for a temporary search index. */ ++PaimonFtindexWriterHandle *paimon_ftindex_writer_open_in_memory( ++ const char **keys, ++ const char **values, ++ size_t len); ++ + int paimon_ftindex_writer_add_document( + PaimonFtindexWriterHandle *writer, + int64_t row_id, diff --git a/cmake_modules/lucene.diff b/cmake_modules/lucene.diff new file mode 100644 index 000000000..32ba1570f --- /dev/null +++ b/cmake_modules/lucene.diff @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Lucene++ 3.0.9 uses pointer member access on reference parameters in these templates. +# Apple Clang rejects the definitions while parsing Lucene.h via LuceneInc.h, including +# when building the production library's precompiled headers, without any swap() calls. + +diff --git a/include/lucene++/Collection.h b/include/lucene++/Collection.h +--- a/include/lucene++/Collection.h ++++ b/include/lucene++/Collection.h +@@ -167,7 +167,7 @@ + } + + void swap(this_type& other) { +- container.swap(other->container); ++ container.swap(other.container); + } + + TYPE& operator[] (int32_t pos) { +diff --git a/include/lucene++/Set.h b/include/lucene++/Set.h +--- a/include/lucene++/Set.h ++++ b/include/lucene++/Set.h +@@ -108,7 +108,7 @@ + } + + void swap(this_type& other) { +- setContainer.swap(other->setContainer); ++ setContainer.swap(other.setContainer); + } + + operator bool() const { diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 5fcc3b1e2..38fd8be11 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -140,7 +140,7 @@ a column name containing only such whitespace selects the indexed field. - **Option**: ``lucene-fts.write.tmp.directory`` - - **Description**: Specifies the temporary directory used during Lucene index writing. No default value; must be explicitly set. + - **Description**: Required when building a persistent Lucene index. Full-text searches build their temporary indexes in memory. - **Environment Variable**: ``PAIMON_JIEBA_DICT_DIR`` @@ -298,12 +298,8 @@ Table-Level Full-Text Search full-text indexes of one column of a data-evolution table, as Java Paimon's ``FullTextSearchBuilder`` does, so a caller does not create index readers or choose index types and shard ranges. It searches the index files whose indexer supports full-text search -(``GlobalIndexer::SupportsFullTextSearch()``): ``full-text`` and ``lucene-fts``. Only the rows that -an index shard covers are searched, as in the default ``fast`` ``full-text-index.search-mode`` of -Java Paimon, so rows written since the index was last built are not found until it is built for -them. A search whose ``full-text-index.search-mode``, or else ``global-index.search-mode``, is -``full`` or ``detail``, which also search the rows that no index covers, fails with -``NotImplemented``. +(``GlobalIndexer::SupportsFullTextSearch()``): ``full-text`` and ``lucene-fts``. By default it +searches only indexed rows. To include rows appended since the last build, see `Search Modes`_. .. code-block:: cpp @@ -330,11 +326,10 @@ them. A search whose ``full-text-index.search-mode``, or else ``global-index.sea PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(data_plan->Splits())); -The result holds the ``limit`` highest-scoring rows among the candidates of all index shards, and -a read of the rows returns their scores in the ``_INDEX_SCORE`` field. Each shard returns at most -``limit`` candidates, chosen by its index, so ties are broken by ascending row id among the -candidates only: which of the rows tied at the ``limit``-th score of a shard become candidates is up -to its index. +The search keeps the ``limit`` highest-scoring candidates across the stored shards and any +temporary index built for unindexed rows. Read ``_INDEX_SCORE`` to retrieve their scores. +Each index returns up to ``limit`` candidates and decides which rows to return when scores tie +at its cutoff. The merge breaks ties among the returned candidates by ascending row id. - ``SetPartitionFilter()`` restricts the search to some partitions, as in ``ScanContextBuilder``. - ``WithFilter()`` restricts the search to the rows matching a predicate; see @@ -345,8 +340,13 @@ to its index. options may be set, and it must agree with an explicit ``scan.mode``: ``from-snapshot`` needs ``scan.snapshot-id`` or ``scan.tag-name``, ``from-snapshot-full`` needs ``scan.snapshot-id``, ``from-timestamp`` needs ``scan.timestamp`` or ``scan.timestamp-millis``, and ``latest-full`` and - ``latest`` take none of them. The search still uses the latest schema of the table, not the - schema of that snapshot. + ``latest`` take none of them. For an explicit snapshot, tag or timestamp, the search resolves + text and filter columns in the selected snapshot's schema. BLOB field declarations also + follow that schema unless explicitly overridden. Each call to ``CreatePlan()`` resolves the + selected tag or timestamp again. Splits from earlier plans still refer to their original + snapshot. A current search uses the latest table schema. +- Tables with ``query-auth.enabled=true`` require catalog row filters and column masks. Full-text + search rejects these tables, including for historical queries. - A shard whose indexed field is the column or that carries it as an extra field is searched. When shards overlap, each row is ranked by one of them: an index whose indexed field is the column comes first, then the order of the index type and fields, then the start and end of the @@ -360,6 +360,53 @@ to its index. - A column with a primary-key full-text index (``pk-full-text.index.columns``) cannot be searched yet. +Search Modes +~~~~~~~~~~~~ + +``full-text-index.search-mode``, or else ``global-index.search-mode``, selects the rows that a +search covers, as in Java Paimon: + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Mode + - Rows searched + * - ``fast`` (default) + - Only the rows that an index shard covers. + * - ``full`` + - Also the rows below the next row id of the snapshot that no single-field full-text index + covers. Planning reads no data manifest, and the read skips the rows outside the searched + partitions. + * - ``detail`` + - Also the rows of the data files in the searched partitions that no single-field full-text + index covers. Planning reads the data manifests to find them. + +If the selected partitions have no full-text index for the column, the search returns no rows. +Otherwise, ``full`` and ``detail`` add one ``RawFullTextSearchSplit`` for the gaps in coverage when +the snapshot has a positive next row id. +As in Java Paimon, this coverage counts only single-field full-text index definitions. Multi-field +indexes can serve the searched column as their primary or extra field, but do not reduce the raw +ranges. Coverage includes every single-field definition, even if shard selection chooses another +index to rank its rows. +The worker combines the raw ranges and reads the text column, ``_ROW_ID``, and any filter columns +from the planned snapshot. If the ranges contain live, non-null text, the worker builds one +in-memory index with the first selected shard's backend and searches it with the same query and +limit. Both ``full-text`` and ``lucene-fts`` keep the temporary index segments and archive in memory. +A worker can search these rows without a writable local temporary directory. +Set analyzer options such as ``full-text.tokenizer`` in the table or search options to +match the existing index. + +The temporary index scores against its own corpus of live, non-null text. Planning excludes raw +ranges from every stored split's search ranges and omits stored splits with no remaining search +rows. It preserves physical shard ranges for row id translation. A row in a raw range therefore +appears only if its current text matches, even when workers read the splits independently. +The final top-k combines hits from the temporary index and the remaining indexed rows. + +.. code-block:: cpp + + search_builder.AddOption("full-text-index.search-mode", "full"); + Row Filters ~~~~~~~~~~~ @@ -378,20 +425,21 @@ implies: ``(p = 1 AND a > 0) OR (p = 2 AND a > 0)`` searches only partitions ``p paimon::FieldType::INT, paimon::Literal(7)); search_builder.WithFilter(filter); -The scan keeps the scalar global index files that index a field of the filter, and the read -resolves the filter with them before ranking: - -- If the indexes decide the whole filter, the rows they return are searched. -- If they can only narrow the filter down to candidates, because a conjunct has no index or the - filter has a ``Contains``, ``EndsWith`` or ``Like`` predicate, the candidates are excluded and a - warning is logged. The search then never returns a non-matching row, but may return fewer rows - than ``limit``. With ``global-index.filter.refine-from-data=true``, the read instead verifies the - candidates by reading their filter columns and ``_ROW_ID`` in the planned snapshot, on the caller - and possibly for every candidate. -- Rows that no scalar index covers are excluded. If no index can evaluate the filter at all, no row - is searched and a warning is logged. This is the default ``fast`` ``scalar-index.search-mode``, - which falls back on ``global-index.search-mode``. A row filter in ``full`` or ``detail`` mode, - which decide such rows by reading their filter columns, fails with ``NotImplemented``. +On a full-text index shard, ``scalar-index.search-mode`` controls how the reader finds rows that +satisfy the filter before ranking. It falls back to ``global-index.search-mode``, then to ``fast``. + +- In ``fast`` mode, only rows identified by a scalar index can qualify. If no index can evaluate + the filter, the reader skips the shard's rows and logs a warning. +- In ``full`` and ``detail``, the reader checks the filter directly against the data wherever a + filter column lacks index coverage. If no index can evaluate the filter, the reader checks the + covered rows against the data as well. See `Search Modes`_ for how each mode finds gaps. +- In any mode, an index may return candidates that still need verification, for example for a + ``Contains``, ``EndsWith`` or ``Like`` predicate. By default the reader excludes those candidates + and logs a warning. Set ``global-index.filter.refine-from-data=true`` to check them against the + planned snapshot. This option does not affect rows read directly to fill gaps. + +A raw split always evaluates the filter on its data. All its live, non-null text contributes to +the temporary index's scores, even when the filter excludes a row from the results. Only single-field indexes that do not support full-text search evaluate a filter. A filter with a ``PredicateBuilder::FullTextMatch`` predicate fails with ``NotImplemented``. @@ -399,11 +447,11 @@ Only single-field indexes that do not support full-text search evaluate a filter Pinning the Data Read ~~~~~~~~~~~~~~~~~~~~~ -The row ids of the result refer to the snapshot the search was planned from. If the table may -change before the rows are read, plan with ``NewFullTextScan()`` and pin the data scan to the -snapshot of the plan, setting ``scan.mode`` too so that a ``scan.mode`` of the table cannot override -it. For a search of ``scan.tag-name``, pin the data scan with the same tag instead, as the tag may -outlive the snapshot. +The row ids in a search result belong to the planning snapshot. Use ``NewFullTextScan()`` to get +its ID and set both ``scan.mode`` and ``scan.snapshot-id`` on the data scan. If a tag retains an +expired snapshot, set ``scan.mode`` to ``from-snapshot`` and select the tag with ``scan.tag-name``. +A tag can move between the search and the data scan. Check that both plans have the same +``SnapshotId()`` before reading the rows. .. code-block:: cpp @@ -426,13 +474,22 @@ outlive the snapshot. ``ExecuteLocal()`` plans and searches in one process. To plan in one process and search in others, plan with ``NewFullTextScan()``, ship the splits with ``FullTextSearchSplit::Serialize()`` and ``FullTextSearchSplit::Deserialize()``, and search them with the ``NewFullTextRead()`` of a builder -for the same table, column, query and filter. The serialized form is versioned and specific to -Paimon C++: Java Paimon serializes its splits with Java serialization, so splits cannot be exchanged -with it. Each split records the snapshot it was planned from, and the read ranks the rows live in -that snapshot. The read rejects a split whose index files do not index the column of the latest -schema, such as after the column was replaced. A read returns the top rows of the splits it is -given, so keep the ``limit`` highest-scoring rows of all reads if the splits are searched by several -of them. +for the same table, column, query, limit, partitions, row filter and search options. The serialized +form is versioned and specific to Paimon C++. Each split records its snapshot. A worker configured +with an explicit snapshot ID can use a tag that retains that snapshot after its file expires. +Stored index splits from version 3 and all raw splits carry the planning schema ID. A worker +rejects these splits if the searched column name now refers to a different field ID, even when +both fields belong to the same index. Raw split version 1 records the text field ID explicitly. +Workers use the planning schema for index fields, row filters and BLOB storage declarations. +For example, if an integer filter column is replaced by a BLOB view with the same name, the +worker still reads the original integer column. Explicit BLOB field options take precedence. +When all supplied splits lack a schema ID, workers use their own schema. +Stored splits carry scalar index files only for their assigned search ranges. A worker also +reads deletion vectors only for data files that intersect those ranges. +When several workers search the splits, merge their results and keep the ``limit`` highest-scoring +rows. The planner assigns disjoint search ranges to stored and raw splits before serialization. +If you supply overlapping splits to one read, it excludes the raw ranges from the stored search +before ranking. .. code-block:: cpp diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 9b96d7220..a39a2e128 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -637,20 +637,21 @@ struct PAIMON_EXPORT Options { /// and checkpoints use independent round robin. If not set, the global index /// files will be stored under the index directory. static const char GLOBAL_INDEX_EXTERNAL_PATH[]; - /// "global-index.search-mode" - Fallback search mode for global index queries, used when the - /// search mode of the index family is not set: "fast", "full" or "detail". No default value. - /// Paimon C++ reads it only for full-text search so far. + /// "global-index.search-mode" - Fallback for full-text, scalar and vector index search modes. + /// Accepts "fast", "full" or "detail". Families default to "fast" when neither option is set. static const char GLOBAL_INDEX_SEARCH_MODE[]; - /// "full-text-index.search-mode" - Search mode for full-text index queries: "fast" searches - /// only the rows that an index covers, while "full" and "detail" also search the rows that no - /// index covers. Default value is "global-index.search-mode", or else "fast". Paimon C++ - /// supports only "fast" so far, and a full-text search in another mode fails. + /// "full-text-index.search-mode" - "fast" searches indexed rows. "full" finds gaps below the + /// snapshot's next row id. "detail" finds gaps in the selected partitions' data files. Both + /// search the gaps if a selected partition has a full-text index for the column. + /// Defaults to "global-index.search-mode", then "fast". static const char FULL_TEXT_INDEX_SEARCH_MODE[]; - /// "scalar-index.search-mode" - Search mode for scalar index queries: "fast" evaluates only the - /// rows that an index covers, while "full" and "detail" also evaluate the rows that no index - /// covers. Default value is "global-index.search-mode", or else "fast". Paimon C++ supports - /// only "fast" so far, and a full-text search with a row filter in another mode fails. + /// "scalar-index.search-mode" - Controls row filtering on full-text index shards. "fast" + /// uses scalar index candidates. "full" and "detail" also read the filter columns for gaps + /// in scalar index coverage. Defaults to "global-index.search-mode", then "fast". static const char SCALAR_INDEX_SEARCH_MODE[]; + /// "vector-index.search-mode" - Reserved for table-level vector search. Accepts "fast", + /// "full" or "detail". Defaults to "global-index.search-mode", then "fast". + static const char VECTOR_INDEX_SEARCH_MODE[]; /// "global-index.filter.refine-from-data" - Whether a vector, hybrid or full-text search may /// read the filter columns of candidate rows to verify a row filter that the scalar global /// index can only answer with a superset, such as contains, ends-with or like on a BTree index diff --git a/include/paimon/table/source/full_text_read.h b/include/paimon/table/source/full_text_read.h index 1d4a6f718..ff71652dc 100644 --- a/include/paimon/table/source/full_text_read.h +++ b/include/paimon/table/source/full_text_read.h @@ -40,28 +40,22 @@ class PAIMON_EXPORT FullTextRead { return Read(plan.Splits()); } - /// Searches `splits` and returns the `limit` highest-scoring rows among the candidates that - /// they return. The result holds global row ids and iterates them in ascending order; pass it - /// to `ScanContextBuilder::SetGlobalIndexResult()` to read the rows, whose scores the read - /// returns in the `_INDEX_SCORE` field. + /// Returns the top `limit` hits across `splits`, in ascending global row id order. Pass the + /// result to `ScanContextBuilder::SetGlobalIndexResult()` to read the rows with `_INDEX_SCORE`. /// - /// The splits are searched in parallel, and each one returns at most `limit` candidates, chosen - /// by its index. Ties among the candidates are broken by ascending row id, but which of the - /// rows tied at the `limit`-th score of a split become candidates is up to its index. A split - /// ranks only its search row ranges, and with deletion vectors enabled only the rows that are - /// live in the snapshot the splits were planned from; BM25 statistics still come from its whole - /// index shard. + /// Row filters apply before ranking. Each stored shard scores against its whole corpus. + /// The temporary index scores against all live, non-null text in the raw ranges. + /// Each backend resolves ties at its cutoff. The merge breaks ties by ascending row id. /// - /// With a row filter (`FullTextSearchBuilder::WithFilter()`), a split ranks only the rows that - /// satisfy it, as resolved before ranking by the scalar indexes that the splits carry. Rows - /// that no scalar index can evaluate are not ranked. Candidates that the indexes cannot decide - /// exactly, as for `Contains`, `EndsWith` and `Like`, are excluded too, so the result may hold - /// fewer than `limit` rows, unless `global-index.filter.refine-from-data` verifies them against - /// the data of the planned snapshot on this thread. - /// @param splits Splits planned by a `FullTextScan` of the same column and filter, all from one - /// snapshot. - /// @return The rows found, or an error if a split is invalid, its index files do not index the - /// column of the latest schema (for example, after the column was replaced), or an + /// Planned splits can be read independently. When raw and stored splits overlap, stored + /// shards exclude the raw ranges before selecting their top hits. + /// + /// On stored shards, `full` and `detail` scalar modes read filter columns wherever scalar + /// indexes lack coverage or cannot answer the predicate. + /// Inexact candidates rank only after verification with `global-index.filter.refine-from-data`. + /// Workers bind row-filter fields to the planning schema when the splits carry its ID. + /// @param splits Splits planned for the same snapshot, schema, column and filters. + /// @return The rows found, or an error if a split is invalid, its column was replaced, or an /// index or data file cannot be read. virtual Result> Read( const std::vector>& splits) = 0; diff --git a/include/paimon/table/source/full_text_scan.h b/include/paimon/table/source/full_text_scan.h index fc78111e1..2e32901da 100644 --- a/include/paimon/table/source/full_text_scan.h +++ b/include/paimon/table/source/full_text_scan.h @@ -60,28 +60,24 @@ class PAIMON_EXPORT FullTextScan { virtual ~FullTextScan() = default; - /// Plans the search on the snapshot that the options select, as Java - /// `TimeTravelUtil#tryTravelOrLatest` does: snapshot `scan.snapshot-id`, the snapshot of tag - /// `scan.tag-name`, the latest snapshot at or before `scan.timestamp` or - /// `scan.timestamp-millis`, or else the latest snapshot. As in Java, at most one of these - /// options may be set, and it must agree with an explicit `scan.mode`: `from-snapshot` needs - /// `scan.snapshot-id` or `scan.tag-name`, `from-snapshot-full` needs `scan.snapshot-id`, - /// `from-timestamp` needs `scan.timestamp` or `scan.timestamp-millis`, and `latest-full` and - /// `latest` take none of them. + /// Plans the search on the latest snapshot or the one selected by `scan.snapshot-id`, + /// `scan.tag-name`, `scan.timestamp` or `scan.timestamp-millis`. A timestamp selects the + /// latest snapshot at or before that time. Set at most one selector, consistent with + /// `scan.mode`. Each plan resolves its tag or timestamp anew. /// - /// Each index shard of the searched column becomes one split. The scan considers only the - /// index files that are in the partitions of the partition filter, that index the column as - /// their indexed field or as one of their extra fields, and whose indexer supports full-text - /// search (`GlobalIndexer::SupportsFullTextSearch()`). The plan fails if Paimon C++ is built - /// without the backend of a `full-text` or `lucene-fts` index it finds. When the row ranges of - /// index shards overlap, each row is ranked by one shard only: an index whose indexed field is - /// the column comes before one that carries it as an extra field, and further ties are broken - /// by index type and fields, then by the start and end of the shard. With a row filter, each - /// split also carries the files of single-field scalar indexes that intersect its shard and - /// index a field of the filter. - /// @return The plan, or an error if more than one time-travel option is set, the option does - /// not agree with `scan.mode`, the selected snapshot or tag does not exist, or reading - /// the index manifests fails; `NotImplemented` for an index whose backend is not built. + /// Each full-text shard assigned rows in the selected partitions becomes one split. For + /// overlapping shards, the planner prefers an index on the searched column over one that + /// carries it as an extra field. Ties are broken by index type, field IDs and shard range. + /// A stored split carries single-field scalar indexes for its row filter only where those + /// indexes intersect its assigned search ranges. + /// + /// When a selected partition has a full-text index for the column, `full` and `detail` assign + /// gaps in single-field index coverage to a `RawFullTextSearchSplit`. `full` checks all row + /// ids below the snapshot's next row id. `detail` checks data files in the selected partitions. + /// Raw and stored splits rank disjoint rows and can be read independently. + /// @return The plan, or an error for conflicting scan options, a missing snapshot or tag, or + /// unreadable manifests. A missing `full-text` or `lucene-fts` backend returns + /// `NotImplemented`. virtual Result> CreatePlan() = 0; }; diff --git a/include/paimon/table/source/full_text_search_builder.h b/include/paimon/table/source/full_text_search_builder.h index 2d07d9829..df80bbfca 100644 --- a/include/paimon/table/source/full_text_search_builder.h +++ b/include/paimon/table/source/full_text_search_builder.h @@ -50,13 +50,14 @@ class Executor; /// /// `ExecuteLocal()` plans and searches in this process. To run them in different processes, plan /// with `NewFullTextScan()`, serialize the splits with `FullTextSearchSplit::Serialize()`, and -/// search them with the `NewFullTextRead()` of a builder for the same table, column, query and -/// filter. +/// search them with `NewFullTextRead()` on each worker using the same table, column, query, +/// limit, partition and row filters, and search options. +/// Full-text search returns `NotImplemented` on tables that require catalog query authorization. /// /// The row ids of the result refer to the snapshot the search was planned from, which -/// `FullTextScan::Plan::SnapshotId()` returns. If the table may change in between, pin the data -/// scan to it with `scan.mode` = `from-snapshot` and `scan.snapshot-id`, or with the same -/// `scan.tag-name` for a search of a tag, as the tag may outlive the snapshot. +/// `FullTextScan::Plan::SnapshotId()` returns. Pin the data scan with `scan.mode` = `from-snapshot` +/// and `scan.snapshot-id`. If a tag retains an expired snapshot, use `scan.tag-name` and check +/// that the data plan's snapshot ID still matches the search plan's. class PAIMON_EXPORT FullTextSearchBuilder { public: /// @param table_path The root path of the table. @@ -111,17 +112,16 @@ class PAIMON_EXPORT FullTextSearchBuilder { FullTextSearchBuilder& WithFileSystem(const std::shared_ptr& file_system); /// Creates a scan that plans the search on the current settings. The query must be set, and - /// its column must exist in the latest schema of the table. + /// its column must exist in the selected snapshot's schema for time travel, or in the latest + /// table schema for a current search. /// @return The scan, or an error if the settings are invalid or the table cannot be loaded; - /// `NotImplemented` for a primary-key full-text index (`pk-full-text.index.columns`), a - /// search mode other than `fast` (`full-text-index.search-mode`, or else - /// `global-index.search-mode`), a row filter whose scalar search mode is not `fast` - /// (`scalar-index.search-mode`, or else `global-index.search-mode`), or a filter with a - /// full-text match. + /// `NotImplemented` for a primary-key full-text index (`pk-full-text.index.columns`) or + /// a filter with a full-text match. Result> NewFullTextScan() const; - /// Creates a read that searches the splits of a scan on the current settings. In addition to - /// the requirements of `NewFullTextScan()`, the limit must be positive. + /// Creates a worker for the configured query and positive limit. The worker validates the + /// row filter against the planning schema when it receives the splits in + /// `FullTextRead::Read()`. /// @return The read, or an error if the settings are invalid or the table cannot be loaded. Result> NewFullTextRead() const; diff --git a/include/paimon/table/source/full_text_search_split.h b/include/paimon/table/source/full_text_search_split.h index 839c12e6a..0918f28da 100644 --- a/include/paimon/table/source/full_text_search_split.h +++ b/include/paimon/table/source/full_text_search_split.h @@ -81,4 +81,18 @@ class PAIMON_EXPORT IndexFullTextSearchSplit : public FullTextSearchSplit { virtual const std::vector& SearchRowRanges() const = 0; }; +/// Ranges of unindexed rows that a full or detail search reads from the planned snapshot. +class PAIMON_EXPORT RawFullTextSearchSplit : public FullTextSearchSplit { + public: + /// @return The planning snapshot ID. + virtual int64_t SnapshotId() const = 0; + + /// @return The name of the searched table column. + virtual const std::string& ColumnName() const = 0; + + /// @return Sorted, disjoint and non-adjacent global row id ranges. Only live rows in the + /// selected partitions can match. + virtual const std::vector& RowRanges() const = 0; +}; + } // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 7b8d58aeb..e0954f778 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -283,6 +283,7 @@ set(PAIMON_CORE_SRCS core/deletionvectors/bitmap64_deletion_vector.cpp core/deletionvectors/deletion_vectors_index_file.cpp core/deletionvectors/deletion_vector_index_file_writer.cpp + core/global_index/data_evolution_global_index_coverage.cpp core/global_index/global_index_evaluator_impl.cpp core/global_index/global_index_live_row_filter.cpp core/global_index/global_index_scan.cpp @@ -512,6 +513,7 @@ set(PAIMON_CORE_SRCS core/table/source/filtered_row_id_reader.cpp core/table/source/full_text_search_builder.cpp core/table/source/full_text_search_split.cpp + core/table/source/raw_full_text_read_impl.cpp core/table/source/primary_key_sorted_index_scan.cpp core/table/source/primary_key_sorted_index_result.cpp core/table/source/primary_key_index_batch_scan.cpp @@ -917,6 +919,7 @@ if(PAIMON_BUILD_TESTS) core/io/rolling_blob_file_writer_test.cpp core/io/rolling_vector_file_writer_test.cpp core/global_index/global_index_file_manager_test.cpp + core/global_index/data_evolution_global_index_coverage_test.cpp core/global_index/global_index_evaluator_impl_test.cpp core/global_index/indexed_split_test.cpp core/global_index/live_row_global_index_reader_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 9fef4fe1c..703d13bda 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -168,6 +168,7 @@ const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; const char Options::GLOBAL_INDEX_SEARCH_MODE[] = "global-index.search-mode"; const char Options::FULL_TEXT_INDEX_SEARCH_MODE[] = "full-text-index.search-mode"; const char Options::SCALAR_INDEX_SEARCH_MODE[] = "scalar-index.search-mode"; +const char Options::VECTOR_INDEX_SEARCH_MODE[] = "vector-index.search-mode"; const char Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA[] = "global-index.filter.refine-from-data"; const char Options::PK_BTREE_INDEX_COLUMNS[] = "pk-btree.index.columns"; const char Options::PK_BITMAP_INDEX_COLUMNS[] = "pk-bitmap.index.columns"; diff --git a/src/paimon/common/utils/range.cpp b/src/paimon/common/utils/range.cpp index 753b80106..986ef5dc5 100644 --- a/src/paimon/common/utils/range.cpp +++ b/src/paimon/common/utils/range.cpp @@ -45,12 +45,9 @@ std::vector Range::SortAndMergeOverlap(const std::vector& ranges, for (size_t i = 1; i < sorted_ranges.size(); ++i) { Range next = sorted_ranges[i]; - // Check if current and next overlap (not just adjacent) - if (current.to + (adjacent ? 1 : 0) >= next.from) { - // Merge: extend current range + if (current.to >= next.from || (adjacent && current.to + 1 == next.from)) { current = Range(current.from, std::max(current.to, next.to)); } else { - // No overlap: add current to result and move to next results.push_back(current); current = next; } @@ -127,16 +124,12 @@ std::vector Range::Exclude(const std::vector& ranges) const { if (current < intersect.value().from) { result.emplace_back(current, intersect.value().from - 1); } - // Move current position past the intersection - current = intersect.value().to + 1; - if (current > to) { - break; + if (intersect->to == to) { + return result; } + current = intersect->to + 1; } - // Add the remaining part after all exclusions (if any) - if (current <= to) { - result.emplace_back(current, to); - } + result.emplace_back(current, to); return result; } diff --git a/src/paimon/common/utils/range_test.cpp b/src/paimon/common/utils/range_test.cpp index 65ee22eeb..69654b853 100644 --- a/src/paimon/common/utils/range_test.cpp +++ b/src/paimon/common/utils/range_test.cpp @@ -19,6 +19,8 @@ #include "paimon/utils/range.h" +#include + #include "gtest/gtest.h" namespace paimon::test { @@ -97,6 +99,9 @@ TEST(RangeTest, TestCompare) { } TEST(RangeTest, TestSortAndMergeOverlap) { + const int64_t last = std::numeric_limits::max(); + ASSERT_EQ(Range::SortAndMergeOverlap({Range(last - 2, last), Range(last, last)}, true), + std::vector({Range(last - 2, last)})); { // test simple std::vector ranges = {Range(0, 10), Range(5, 15)}; @@ -258,6 +263,9 @@ TEST(RangeTest, TestAnd) { } TEST(RangeTest, TestExclude) { + const int64_t last = std::numeric_limits::max(); + ASSERT_EQ(Range(last - 3, last).Exclude({Range(last - 2, last)}), + std::vector({Range(last - 3, last - 3)})); { // test basic // [0, 10000] exclude [1000,2000],[3000,4000],[5000,6000] diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 3a87f6616..2e92d7c98 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -454,6 +454,7 @@ struct CoreOptions::Impl { std::optional global_index_thread_num; GlobalIndexSearchMode full_text_index_search_mode = GlobalIndexSearchMode::FAST; GlobalIndexSearchMode scalar_index_search_mode = GlobalIndexSearchMode::FAST; + GlobalIndexSearchMode vector_index_search_mode = GlobalIndexSearchMode::FAST; bool global_index_filter_refine_from_data = false; bool realtime_enabled = false; @@ -942,12 +943,12 @@ struct CoreOptions::Impl { // index, no default value PAIMON_RETURN_NOT_OK( parser.Parse(Options::GLOBAL_INDEX_THREAD_NUM, &global_index_thread_num)); - // Parse full-text-index.search-mode, falling back to global-index.search-mode, default fast PAIMON_RETURN_NOT_OK(parser.ParseIndexSearchMode(Options::FULL_TEXT_INDEX_SEARCH_MODE, &full_text_index_search_mode)); - // Parse scalar-index.search-mode, falling back to global-index.search-mode, default fast PAIMON_RETURN_NOT_OK(parser.ParseIndexSearchMode(Options::SCALAR_INDEX_SEARCH_MODE, &scalar_index_search_mode)); + PAIMON_RETURN_NOT_OK(parser.ParseIndexSearchMode(Options::VECTOR_INDEX_SEARCH_MODE, + &vector_index_search_mode)); // Parse global-index.filter.refine-from-data, default false PAIMON_RETURN_NOT_OK(parser.Parse(Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, &global_index_filter_refine_from_data)); @@ -1976,6 +1977,10 @@ GlobalIndexSearchMode CoreOptions::GetScalarIndexSearchMode() const { return impl_->scalar_index_search_mode; } +GlobalIndexSearchMode CoreOptions::GetVectorIndexSearchMode() const { + return impl_->vector_index_search_mode; +} + bool CoreOptions::GlobalIndexFilterRefineFromData() const { return impl_->global_index_filter_refine_from_data; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index c478b2c1c..e513d5d93 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -307,12 +307,9 @@ class PAIMON_EXPORT CoreOptions { BucketFunctionType GetBucketFunctionType() const; std::optional GetGlobalIndexThreadNum() const; - /// Returns `full-text-index.search-mode`, or else `global-index.search-mode`, or else - /// `GlobalIndexSearchMode::FAST`, as Java `CoreOptions#fullTextIndexSearchMode` does. GlobalIndexSearchMode GetFullTextIndexSearchMode() const; - /// Returns `scalar-index.search-mode`, or else `global-index.search-mode`, or else - /// `GlobalIndexSearchMode::FAST`, as Java `CoreOptions#scalarIndexSearchMode` does. GlobalIndexSearchMode GetScalarIndexSearchMode() const; + GlobalIndexSearchMode GetVectorIndexSearchMode() const; bool GlobalIndexFilterRefineFromData() const; const std::vector& GetBlobFields() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index c3b78a696..0a6709471 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -1164,7 +1164,8 @@ TEST(CoreOptionsTest, TestParseIndexSearchMode) { }; const std::vector families = { {Options::FULL_TEXT_INDEX_SEARCH_MODE, &CoreOptions::GetFullTextIndexSearchMode}, - {Options::SCALAR_INDEX_SEARCH_MODE, &CoreOptions::GetScalarIndexSearchMode}}; + {Options::SCALAR_INDEX_SEARCH_MODE, &CoreOptions::GetScalarIndexSearchMode}, + {Options::VECTOR_INDEX_SEARCH_MODE, &CoreOptions::GetVectorIndexSearchMode}}; for (const IndexFamily& family : families) { SCOPED_TRACE(family.key); auto search_mode = [&family](const std::map& options) diff --git a/src/paimon/core/global_index/data_evolution_global_index_coverage.cpp b/src/paimon/core/global_index/data_evolution_global_index_coverage.cpp new file mode 100644 index 000000000..50a6226f6 --- /dev/null +++ b/src/paimon/core/global_index/data_evolution_global_index_coverage.cpp @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/global_index/data_evolution_global_index_coverage.h" + +#include +#include +#include + +namespace paimon { + +std::vector DataEvolutionGlobalIndexCoverage::IndexedRanges( + const std::vector>& index_files, + const std::set& field_ids) { + std::map> coverage_by_field; + for (const auto& file : index_files) { + const std::optional& meta = file->GetGlobalIndexMeta(); + if (!meta || (meta->extra_field_ids && !meta->extra_field_ids->empty())) { + continue; + } + coverage_by_field[meta->index_field_id].emplace_back(meta->row_range_start, + meta->row_range_end); + } + + std::vector indexed; + for (int32_t field_id : field_ids) { + auto iter = coverage_by_field.find(field_id); + if (iter == coverage_by_field.end()) { + return {}; + } + std::vector field_ranges = Range::SortAndMergeOverlap(iter->second, true); + indexed = indexed.empty() ? std::move(field_ranges) : Range::And(indexed, field_ranges); + if (indexed.empty()) { + break; + } + } + return indexed; +} + +Result> DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode mode, std::optional next_row_id, + const std::vector& indexed_ranges, const DataRangesSupplier& data_ranges) { + if (mode == GlobalIndexSearchMode::FAST || !next_row_id || next_row_id.value() <= 0) { + return std::vector(); + } + std::vector all_data_ranges; + if (mode == GlobalIndexSearchMode::DETAIL) { + PAIMON_ASSIGN_OR_RAISE(all_data_ranges, data_ranges()); + } else { + all_data_ranges.emplace_back(0, next_row_id.value() - 1); + } + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex covered, RowRangeIndex::Create(indexed_ranges)); + return ExcludeRanges(Range::SortAndMergeOverlap(all_data_ranges, /*adjacent=*/true), covered); +} + +std::vector DataEvolutionGlobalIndexCoverage::ExcludeRanges(const std::vector& ranges, + const RowRangeIndex& covered) { + std::vector uncovered; + for (const Range& range : ranges) { + std::vector remaining = + range.Exclude(covered.IntersectedRanges(range.from, range.to)); + uncovered.insert(uncovered.end(), remaining.begin(), remaining.end()); + } + return uncovered; +} + +} // namespace paimon diff --git a/src/paimon/core/global_index/data_evolution_global_index_coverage.h b/src/paimon/core/global_index/data_evolution_global_index_coverage.h new file mode 100644 index 000000000..e31c60378 --- /dev/null +++ b/src/paimon/core/global_index/data_evolution_global_index_coverage.h @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/options/global_index_search_mode.h" +#include "paimon/result.h" +#include "paimon/utils/range.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon { +class DataEvolutionGlobalIndexCoverage { + public: + using DataRangesSupplier = std::function>()>; + + DataEvolutionGlobalIndexCoverage() = delete; + ~DataEvolutionGlobalIndexCoverage() = delete; + + /// Returns the ranges where every requested field has a single-field index. + /// An empty field set yields no ranges. + static std::vector IndexedRanges( + const std::vector>& index_files, + const std::set& field_ids); + + /// Subtracts covered rows from sorted, disjoint ranges. + static std::vector ExcludeRanges(const std::vector& ranges, + const RowRangeIndex& covered); + + /// Returns sorted, disjoint gaps below the snapshot's next row id in FULL mode or within + /// the supplied data file ranges in DETAIL mode. Only DETAIL loads the data ranges. + /// FAST and snapshots without a positive next row id yield no gaps. + static Result> UnindexedRanges(GlobalIndexSearchMode mode, + std::optional next_row_id, + const std::vector& indexed_ranges, + const DataRangesSupplier& data_ranges); +}; + +} // namespace paimon diff --git a/src/paimon/core/global_index/data_evolution_global_index_coverage_test.cpp b/src/paimon/core/global_index/data_evolution_global_index_coverage_test.cpp new file mode 100644 index 000000000..fe3e85d22 --- /dev/null +++ b/src/paimon/core/global_index/data_evolution_global_index_coverage_test.cpp @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/global_index/data_evolution_global_index_coverage.h" + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/range.h" + +namespace paimon::test { +namespace { +std::shared_ptr NewIndexFile( + const std::string& file_name, const Range& range, int32_t index_field_id, + const std::optional>& extra_field_ids = std::nullopt) { + GlobalIndexMeta meta(range.from, range.to, index_field_id, extra_field_ids, nullptr); + return std::make_shared("btree", file_name, 128, range.Count(), std::nullopt, + std::nullopt, meta); +} + +DataEvolutionGlobalIndexCoverage::DataRangesSupplier Ranges(const std::vector& ranges) { + return [ranges]() -> Result> { return ranges; }; +} + +DataEvolutionGlobalIndexCoverage::DataRangesSupplier Unused() { + return []() -> Result> { + return Status::Invalid("data ranges are not needed"); + }; +} +} // namespace + +TEST(DataEvolutionGlobalIndexCoverageTest, TestIndexedRangesOfOneField) { + std::vector> files = { + NewIndexFile("a", Range(10, 19), 1), + NewIndexFile("b", Range(0, 9), 1), + NewIndexFile("overlap", Range(5, 14), 1), + NewIndexFile("c", Range(30, 39), 1), + NewIndexFile("other", Range(20, 29), 2), + std::make_shared("deletion-vector", "dv", 128, 10, std::nullopt, + std::nullopt)}; + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {1}), + std::vector({Range(0, 19), Range(30, 39)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {3}), std::vector()); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {}), std::vector()); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestIndexedRangesIntersectFields) { + std::vector> files = {NewIndexFile("disjoint", Range(50, 59), 0), + NewIndexFile("a", Range(0, 49), 1), + NewIndexFile("b", Range(20, 79), 2)}; + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {1, 2}), + std::vector({Range(20, 49)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {2}), + std::vector({Range(20, 79)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {1, 4}), std::vector()); + ASSERT_TRUE(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {0, 1, 2}).empty()); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestIndexedRangesIgnoreMultiFieldIndexes) { + const std::vector> multi_field_files = { + NewIndexFile("multi", Range(0, 9), 1, std::vector({2}))}; + for (const std::set& field_ids : + {std::set{1}, std::set{2}, std::set{1, 2}}) { + SCOPED_TRACE(::testing::PrintToString(field_ids)); + ASSERT_TRUE( + DataEvolutionGlobalIndexCoverage::IndexedRanges(multi_field_files, field_ids).empty()); + } + + std::vector> mixed_files = multi_field_files; + mixed_files.push_back(NewIndexFile("primary", Range(0, 5), 1)); + mixed_files.push_back(NewIndexFile("extra", Range(3, 9), 2, std::vector())); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(mixed_files, {1}), + std::vector({Range(0, 5)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(mixed_files, {2}), + std::vector({Range(3, 9)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(mixed_files, {1, 2}), + std::vector({Range(3, 5)})); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestFullExcludesIndexedRangesFromNextRowId) { + ASSERT_OK_AND_ASSIGN( + std::vector ranges, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::FULL, 100, + {Range(50, 59), Range(0, 9), Range(10, 19), Range(95, 120)}, Unused())); + ASSERT_EQ(ranges, std::vector({Range(20, 49), Range(60, 94)})); + + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::FULL, 100, {Range(0, 99)}, Unused())); + ASSERT_TRUE(ranges.empty()); + + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::FULL, 100, {}, Unused())); + ASSERT_EQ(ranges, std::vector({Range(0, 99)})); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestDetailExcludesIndexedRangesFromDataFiles) { + ASSERT_OK_AND_ASSIGN(std::vector ranges, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {Range(0, 14)}, + Ranges({Range(60, 69), Range(0, 9), Range(10, 19), Range(65, 79)}))); + ASSERT_EQ(ranges, std::vector({Range(15, 19), Range(60, 79)})); + + ASSERT_OK_AND_ASSIGN( + ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {}, Ranges({Range(2, 3), Range(7, 9)}))); + ASSERT_EQ(ranges, std::vector({Range(2, 3), Range(7, 9)})); + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {}, Ranges({}))); + ASSERT_TRUE(ranges.empty()); + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {Range(0, 99)}, + Ranges({Range(0, 9), Range(60, 79)}))); + ASSERT_TRUE(ranges.empty()); + + ASSERT_NOK_WITH_MSG(DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {}, Unused()), + "data ranges are not needed"); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestSkipUnindexedRanges) { + ASSERT_OK_AND_ASSIGN(std::vector ranges, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::FAST, 100, {}, Unused())); + ASSERT_TRUE(ranges.empty()); + for (const std::optional& next_row_id : + {std::optional(), std::optional(0), std::optional(-1)}) { + for (GlobalIndexSearchMode mode : + {GlobalIndexSearchMode::FULL, GlobalIndexSearchMode::DETAIL}) { + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + mode, next_row_id, {}, Unused())); + ASSERT_TRUE(ranges.empty()); + } + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/global_index/global_index_live_row_filter.cpp b/src/paimon/core/global_index/global_index_live_row_filter.cpp index 95c22d721..a34186415 100644 --- a/src/paimon/core/global_index/global_index_live_row_filter.cpp +++ b/src/paimon/core/global_index/global_index_live_row_filter.cpp @@ -217,4 +217,30 @@ Result> GlobalIndexLiveRowFilter::LiveRows( return std::optional(std::move(live_rows)); } +Result> GlobalIndexLiveRowFilter::DataRowRanges( + const std::string& root_path, const std::shared_ptr& table_schema, + const Snapshot& snapshot, const std::shared_ptr& partition_filter, + const CoreOptions& options, const std::shared_ptr& executor, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr snapshot_reader, + CreateSnapshotReader(root_path, table_schema, partition_filter, options, executor, pool)); + snapshot_reader->WithMode(ScanMode::ALL)->WithSnapshot(snapshot); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, snapshot_reader->Read()); + std::vector ranges; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("cannot cast split to DataSplitImpl when reading row ranges"); + } + for (const auto& file : data_split->DataFiles()) { + if (file->first_row_id && file->row_count > 0) { + PAIMON_ASSIGN_OR_RAISE(Range row_id_range, file->NonNullRowIdRange()); + ranges.push_back(row_id_range); + } + } + } + return ranges; +} + } // namespace paimon diff --git a/src/paimon/core/global_index/global_index_live_row_filter.h b/src/paimon/core/global_index/global_index_live_row_filter.h index 13cd15869..6523361db 100644 --- a/src/paimon/core/global_index/global_index_live_row_filter.h +++ b/src/paimon/core/global_index/global_index_live_row_filter.h @@ -22,6 +22,7 @@ #include #include #include +#include #include "paimon/common/predicate/predicate_filter.h" #include "paimon/core/core_options.h" @@ -30,6 +31,7 @@ #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" +#include "paimon/utils/range.h" #include "paimon/utils/roaring_bitmap64.h" #include "paimon/utils/row_range_index.h" @@ -62,6 +64,15 @@ class GlobalIndexLiveRowFilter { const Snapshot& snapshot, const std::shared_ptr& partition_filter, const std::optional& row_range_index, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool); + + /// Returns the global row id ranges of the data files of `snapshot` in the partitions accepted + /// by `partition_filter`. Each non-empty file with a first row id contributes its range, + /// including rows marked by deletion vectors. + static Result> DataRowRanges( + const std::string& root_path, const std::shared_ptr& table_schema, + const Snapshot& snapshot, const std::shared_ptr& partition_filter, + const CoreOptions& options, const std::shared_ptr& executor, + const std::shared_ptr& pool); }; } // namespace paimon diff --git a/src/paimon/core/table/source/abstract_table_scan.h b/src/paimon/core/table/source/abstract_table_scan.h index f8dd15aac..6da732e9c 100644 --- a/src/paimon/core/table/source/abstract_table_scan.h +++ b/src/paimon/core/table/source/abstract_table_scan.h @@ -43,6 +43,15 @@ class AbstractTableScan : public TableScan { return snapshot_reader_->GetMetrics(); } + /// Uses the supplied snapshot even after its snapshot file expires. + Result> PlanSnapshotRanges(const Snapshot& snapshot, + const RowRangeIndex& row_ranges) { + return snapshot_reader_->WithMode(ScanMode::ALL) + ->WithSnapshot(snapshot) + ->WithRowRangeIndex(row_ranges) + ->Read(); + } + protected: Result> CreateStartingScanner(bool is_streaming) const { const auto& snapshot_manager = snapshot_reader_->GetSnapshotManager(); diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.cpp b/src/paimon/core/table/source/data_evolution_full_text_read.cpp index 465b2e37f..57c6e7ff1 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_read.cpp @@ -19,9 +19,13 @@ #include "paimon/core/table/source/data_evolution_full_text_read.h" +#include #include #include +#include #include +#include +#include #include #include #include @@ -34,18 +38,22 @@ #include "paimon/common/global_index/global_index_utils.h" #include "paimon/common/global_index/offset_global_index_reader.h" #include "paimon/common/global_index/scored_global_index_result_utils.h" +#include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/global_index/data_evolution_global_index_coverage.h" #include "paimon/core/global_index/global_index_evaluator_impl.h" #include "paimon/core/global_index/global_index_live_row_filter.h" #include "paimon/core/global_index/global_index_scan_impl.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_evolution_full_text_scan.h" #include "paimon/core/table/source/filtered_row_id_reader.h" -#include "paimon/core/tag/tag.h" +#include "paimon/core/table/source/raw_full_text_read_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" #include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/core/utils/tag_manager.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" #include "paimon/global_index/global_index_io_meta.h" @@ -65,45 +73,7 @@ Logger* FullTextReadLogger() { /// live rows: at most 65536 run containers, cheaper than copying the live rows of a large table for /// every split. A wider span trims the live rows instead. constexpr int64_t kMaxExpandedSearchRows = static_cast(1) << 32; - -/// Loads snapshot `snapshot_id`, or the copy that a tag keeps of it once the snapshot has expired, -/// since a plan of `scan.tag-name` may come from a tag that outlives its snapshot. -Result LoadPlannedSnapshot(const CoreOptions& options, const std::string& root_path, - int64_t snapshot_id) { - std::string branch = BranchManager::NormalizeBranch(options.GetBranch()); - SnapshotManager snapshot_manager(options.GetFileSystem(), root_path, branch); - Result snapshot = snapshot_manager.LoadSnapshot(snapshot_id); - if (snapshot.ok()) { - return snapshot; - } - // The snapshot may expire between planning and loading, or even while it is loaded, so a - // failed load falls back to the tags unless the snapshot is still there. - PAIMON_ASSIGN_OR_RAISE(bool exists, snapshot_manager.SnapshotExists(snapshot_id)); - if (exists) { - return snapshot.status(); - } - TagManager tag_manager(options.GetFileSystem(), root_path, branch); - PAIMON_ASSIGN_OR_RAISE(std::vector tag_names, tag_manager.ListTagNames()); - for (const std::string& tag_name : tag_names) { - // A tag deleted after it was listed, even while it is read, is skipped, as it no longer - // keeps the snapshot; any other failure to read a tag is an error. - Result> tag = tag_manager.Get(tag_name); - if (!tag.ok()) { - PAIMON_ASSIGN_OR_RAISE(bool tag_exists, - options.GetFileSystem()->Exists(tag_manager.TagPath(tag_name))); - if (tag_exists) { - return tag.status(); - } - continue; - } - if (tag.value() && tag.value()->Id() == snapshot_id) { - return tag.value()->TrimToSnapshot(); - } - } - return Status::NotExist(fmt::format( - "snapshot {} that the full-text search splits were planned from no longer exists", - snapshot_id)); -} +constexpr size_t kMaxReportedRanges = 3; Result ToBitmap(const std::shared_ptr& result) { if (auto bitmap_result = std::dynamic_pointer_cast(result)) { @@ -119,24 +89,76 @@ Result ToBitmap(const std::shared_ptr& resul return bitmap; } -/// Describes `split` for the errors of its search. std::string Describe(const IndexFullTextSearchSplitImpl& split) { std::vector file_names; file_names.reserve(split.FullTextIndexFiles().size()); for (const auto& file : split.FullTextIndexFiles()) { file_names.push_back(file->FileName()); } + const auto& ranges = split.SearchRowRanges(); + std::vector sample(ranges.begin(), + ranges.begin() + std::min(ranges.size(), kMaxReportedRanges)); return fmt::format( "while searching the split of column {} in snapshot {} over rows [{}, {}] with {} index " - "files [{}]", + "files [{}], schema {}, {} search ranges, first {}", split.ColumnName(), split.SnapshotId(), split.RowRangeStart(), split.RowRangeEnd(), - split.FullTextIndexFiles().front()->IndexType(), fmt::join(file_names, ", ")); + split.FullTextIndexFiles().front()->IndexType(), fmt::join(file_names, ", "), + split.SchemaId() ? fmt::format("{}", *split.SchemaId()) : "unknown", ranges.size(), + StringUtils::VectorToString(sample)); +} + +Result> ScalarIndexCoverage( + const std::vector>& files, const std::set& field_ids, + const CoreOptions& options) { + // A worker may have fewer index backends than the process that planned its splits. + std::map readable_types; + std::vector> readable_files; + for (const auto& file : files) { + const auto& meta = file->GetGlobalIndexMeta().value(); + if (meta.extra_field_ids && !meta.extra_field_ids->empty()) { + continue; + } + auto [type, inserted] = readable_types.try_emplace(file->IndexType(), false); + if (inserted) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(type->first, options.ToMap())); + type->second = indexer && !indexer->SupportsFullTextSearch(); + } + if (type->second) { + readable_files.push_back(file); + } + } + return DataEvolutionGlobalIndexCoverage::IndexedRanges(readable_files, field_ids); +} + +Status ExcludeRawRanges(const RowRangeIndex& raw_rows, + std::vector>* index_splits) { + for (auto& split : *index_splits) { + if (!raw_rows.Intersects(split->RowRangeStart(), split->RowRangeEnd())) { + continue; + } + std::vector remaining = + DataEvolutionGlobalIndexCoverage::ExcludeRanges(split->SearchRowRanges(), raw_rows); + if (remaining.empty()) { + split.reset(); + } else if (remaining != split->SearchRowRanges()) { + PAIMON_ASSIGN_OR_RAISE( + split, IndexFullTextSearchSplitImpl::Create( + split->SnapshotId(), split->ColumnName(), split->RowRangeStart(), + split->RowRangeEnd(), split->FullTextIndexFiles(), std::move(remaining), + split->ScalarIndexFiles(), split->SchemaId())); + } + } + index_splits->erase(std::remove(index_splits->begin(), index_splits->end(), nullptr), + index_splits->end()); + return Status::OK(); } } // namespace DataEvolutionFullTextRead::DataEvolutionFullTextRead( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& index_path_factory, + const CoreOptions& options, const std::map& option_overrides, + const std::shared_ptr& index_path_factory, const std::shared_ptr& index_file_manager, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, const std::string& query, @@ -145,6 +167,7 @@ DataEvolutionFullTextRead::DataEvolutionFullTextRead( : root_path_(root_path), table_schema_(table_schema), options_(options), + option_overrides_(option_overrides), index_path_factory_(index_path_factory), index_file_manager_(index_file_manager), partition_filter_(partition_filter), @@ -157,7 +180,8 @@ DataEvolutionFullTextRead::DataEvolutionFullTextRead( Result> DataEvolutionFullTextRead::Create( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& path_factory, + const CoreOptions& options, const std::map& option_overrides, + const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, const std::string& query, int32_t limit, const std::shared_ptr& executor, @@ -172,8 +196,39 @@ Result> DataEvolutionFullTextRead::Cr auto index_file_manager = std::make_shared( options.GetFileSystem(), index_path_factory, /*checkpoint_path_factory=*/nullptr); return std::unique_ptr(new DataEvolutionFullTextRead( - root_path, table_schema, options, index_path_factory, index_file_manager, partition_filter, - filter, text_field, query, limit, final_executor, pool)); + root_path, table_schema, options, option_overrides, index_path_factory, index_file_manager, + partition_filter, filter, text_field, query, limit, final_executor, pool)); +} + +Status DataEvolutionFullTextRead::BindSchemaOptions( + const TableSchema& schema, const std::map& option_overrides, + CoreOptions* options) { + PAIMON_RETURN_NOT_OK( + SchemaValidation::RejectQueryAuth(schema, option_overrides, "Full-text search")); + auto merged = options->ToMap(); + auto restore = [&](std::initializer_list keys) { + if (std::any_of(keys.begin(), keys.end(), + [&](const char* key) { return option_overrides.count(key) > 0; })) { + return; + } + for (const char* key : keys) { + merged.erase(key); + auto entry = schema.Options().find(key); + if (entry != schema.Options().end()) { + merged.insert(*entry); + } + } + }; + restore({Options::BLOB_FIELD}); + restore({Options::BLOB_DESCRIPTOR_FIELD, Options::FALLBACK_BLOB_DESCRIPTOR_FIELD}); + restore({Options::BLOB_VIEW_FIELD}); + if (merged != options->ToMap()) { + PAIMON_ASSIGN_OR_RAISE(CoreOptions bound, + CoreOptions::FromMap(merged, options->GetFileSystem())); + bound.WithCache(options->GetCache()); + *options = bound; + } + return Status::OK(); } std::optional DataEvolutionFullTextRead::IncludeRowIds( @@ -208,23 +263,25 @@ std::optional DataEvolutionFullTextRead::IncludeRowIds( } Result> DataEvolutionFullTextRead::LiveRows( + const std::vector>& index_splits, int64_t snapshot_id) const { - // Without deletion vectors or row tracking every row is live, so the planned snapshot, which - // may be gone, is not loaded. - if (!options_.DeletionVectorsEnabled() || !options_.RowTrackingEnabled()) { + if (index_splits.empty() || !options_.DeletionVectorsEnabled() || + !options_.RowTrackingEnabled()) { return std::optional(); } PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, - LoadPlannedSnapshot(options_, root_path_, snapshot_id)); + FilteredRowIdReader::LoadSnapshot(options_, root_path_, snapshot_id)); + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex search_rows, + RowRangeIndex::Create(CollectSearchRanges(index_splits))); return GlobalIndexLiveRowFilter::LiveRows(root_path_, table_schema_, snapshot, - partition_filter_, /*row_range_index=*/std::nullopt, - options_, executor_, pool_); + partition_filter_, std::move(search_rows), options_, + executor_, pool_); } Result> DataEvolutionFullTextRead::MatchedRows( const std::vector>& index_splits, int64_t snapshot_id, const std::optional& live_rows) const { - if (!filter_) { + if (!filter_ || index_splits.empty() || (live_rows && live_rows->IsEmpty())) { return std::optional(); } // Keyed by file name: a scalar index file may intersect more than one split. @@ -239,60 +296,131 @@ Result> DataEvolutionFullTextRead::MatchedRows( for (const auto& [file_name, file] : files_by_name) { scalar_index_files.push_back(file); } - std::unique_ptr index_scan = GlobalIndexScanImpl::CreateForScalarFilters( - table_schema_, options_, index_path_factory_, scalar_index_files, executor_, pool_); - PAIMON_ASSIGN_OR_RAISE(std::optional evaluation, - index_scan->ScanWithCoverage(filter_)); - if (!evaluation) { - PAIMON_LOG_WARN(FullTextReadLogger(), - "Full-text search on table %s in snapshot %ld has a row filter %s that no " - "scalar global index can evaluate; indexed rows are excluded from the " - "result because %s is fast. Build a scalar index on the filtered columns.", - root_path_.c_str(), snapshot_id, filter_->ToString().c_str(), - Options::SCALAR_INDEX_SEARCH_MODE); - return std::optional(RoaringBitmap64()); + + const GlobalIndexSearchMode search_mode = options_.GetScalarIndexSearchMode(); + std::vector search_ranges; + std::vector unindexed_ranges; + bool use_scalar_indexes = !scalar_index_files.empty(); + if (search_mode != GlobalIndexSearchMode::FAST) { + search_ranges = Range::SortAndMergeOverlap(CollectSearchRanges(index_splits), true); + PAIMON_ASSIGN_OR_RAISE(std::set filter_field_ids, + GlobalIndexEvaluatorImpl::CollectFieldIds(*table_schema_, filter_)); + PAIMON_ASSIGN_OR_RAISE(std::vector indexed_ranges, + ScalarIndexCoverage(scalar_index_files, filter_field_ids, options_)); + const std::vector covered = Range::And(search_ranges, indexed_ranges); + use_scalar_indexes = + std::any_of(covered.begin(), covered.end(), [&live_rows](const Range& range) { + if (!live_rows) { + return true; + } + auto first = live_rows->EqualOrLarger(range.from); + return first != live_rows->End() && *first <= range.to; + }); + if (use_scalar_indexes && covered != search_ranges) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, FilteredRowIdReader::LoadSnapshot( + options_, root_path_, snapshot_id)); + auto data_ranges = [&]() { + return GlobalIndexLiveRowFilter::DataRowRanges(root_path_, table_schema_, snapshot, + partition_filter_, options_, + executor_, pool_); + }; + PAIMON_ASSIGN_OR_RAISE(std::vector gaps, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + search_mode, snapshot.NextRowId(), covered, data_ranges)); + unindexed_ranges = Range::And(search_ranges, gaps); + } } + std::optional evaluation; + if (use_scalar_indexes) { + std::unique_ptr index_scan = + GlobalIndexScanImpl::CreateForScalarFilters( + table_schema_, options_, index_path_factory_, scalar_index_files, executor_, pool_); + PAIMON_ASSIGN_OR_RAISE(evaluation, index_scan->ScanWithCoverage(filter_)); + } + FilteredRowIdReader row_id_reader(root_path_, table_schema_, options_, partition_filter_, + filter_, snapshot_id, executor_, pool_); + auto read_matches = [&](const std::vector& ranges) -> Result { + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex rows, RowRangeIndex::Create(ranges)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 matches, row_id_reader.MatchingRowIds(rows)); + if (live_rows) { + matches &= *live_rows; + } + return matches; + }; + if (!evaluation) { + if (search_mode == GlobalIndexSearchMode::FAST) { + PAIMON_LOG_WARN( + FullTextReadLogger(), + "Full-text search on table %s in snapshot %ld excluded indexed rows because no " + "scalar global index can evaluate the row filter %s in fast mode. Build a scalar " + "index on the filtered columns or set %s to full or detail.", + root_path_.c_str(), snapshot_id, filter_->ToString().c_str(), + Options::SCALAR_INDEX_SEARCH_MODE); + return std::optional(RoaringBitmap64()); + } + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 matches, read_matches(search_ranges)); + return std::optional(std::move(matches)); + } PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 index_rows, ToBitmap(evaluation->result)); if (live_rows) { index_rows &= live_rows.value(); } + for (const Range& range : unindexed_ranges) { + if (range.to == RoaringBitmap64::MAX_VALUE) { + index_rows.RemoveRange(range.from, range.to); + index_rows -= RoaringBitmap64::From({range.to}); + } else { + index_rows.RemoveRange(range.from, range.to + 1); + } + } PAIMON_ASSIGN_OR_RAISE( bool exact, FilteredRowIdReader::IsExact(*table_schema_, filter_, evaluation->contributing_field_ids)); - if (exact) { - return std::optional(std::move(index_rows)); - } - - // Only the candidates that a split ranks are worth verifying. - RoaringBitmap64 candidates; - for (const auto& split : index_splits) { - std::optional split_rows = IncludeRowIds(*split, &index_rows); - if (split_rows) { - candidates |= split_rows.value(); + if (!exact) { + RoaringBitmap64 candidates; + for (const auto& split : index_splits) { + std::optional split_rows = IncludeRowIds(*split, &index_rows); + if (split_rows) { + candidates |= split_rows.value(); + } else { + candidates |= + GlobalIndexUtils::RowIdsOf(Range(split->RowRangeStart(), split->RowRangeEnd())); + } + } + if (candidates.IsEmpty()) { + index_rows = std::move(candidates); + } else if (!options_.GlobalIndexFilterRefineFromData()) { + PAIMON_LOG_WARN(FullTextReadLogger(), + "The scalar global index can only answer the row filter %s on table " + "%s in snapshot %ld with candidates, and %s is false, so those %ld " + "rows are excluded from the search; the result may hold fewer rows " + "than requested. Set the option to true to verify the candidates " + "against the data, or build an index that answers the predicate " + "exactly.", + filter_->ToString().c_str(), root_path_.c_str(), snapshot_id, + Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, + candidates.Cardinality()); + index_rows = RoaringBitmap64(); } else { - candidates |= - GlobalIndexUtils::RowIdsOf(Range(split->RowRangeStart(), split->RowRangeEnd())); + PAIMON_ASSIGN_OR_RAISE(index_rows, row_id_reader.MatchingRowIds(std::move(candidates))); } } - if (candidates.IsEmpty()) { - return std::optional(std::move(candidates)); + if (!unindexed_ranges.empty()) { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 unindexed_matches, read_matches(unindexed_ranges)); + index_rows |= unindexed_matches; } - if (!options_.GlobalIndexFilterRefineFromData()) { - PAIMON_LOG_WARN(FullTextReadLogger(), - "The scalar global index can only answer the row filter %s on table %s " - "in snapshot %ld with candidates, and %s is false, so those %ld rows " - "are excluded from the search; the result may hold fewer rows than " - "requested. Set the option to true to verify the candidates against the " - "data, or build an index that answers the predicate exactly.", - filter_->ToString().c_str(), root_path_.c_str(), snapshot_id, - Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, candidates.Cardinality()); - return std::optional(RoaringBitmap64()); + return std::optional(std::move(index_rows)); +} + +std::vector DataEvolutionFullTextRead::CollectSearchRanges( + const std::vector>& index_splits) { + std::vector ranges; + for (const auto& split : index_splits) { + const std::vector& search_ranges = split->SearchRowRanges(); + ranges.insert(ranges.end(), search_ranges.begin(), search_ranges.end()); } - FilteredRowIdReader row_id_reader(root_path_, table_schema_, options_, partition_filter_, - filter_, snapshot_id, executor_, pool_); - PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 matched_rows, row_id_reader.MatchingRowIds(candidates)); - return std::optional(std::move(matched_rows)); + return ranges; } Result> DataEvolutionFullTextRead::SearchSplit( @@ -355,17 +483,62 @@ Result> DataEvolutionFullTextRead::Read const std::vector>& splits) { std::vector> index_splits; index_splits.reserve(splits.size()); + std::string raw_index_type; + std::vector raw_ranges; std::optional snapshot_id; + std::optional schema_id; + auto check_split = [&](const std::string& column_name, int64_t split_snapshot_id, + std::optional split_schema_id) -> Status { + if (column_name != text_field_.Name()) { + return Status::Invalid(fmt::format( + "a split of column {} cannot be read by a full-text search of column {}", + column_name, text_field_.Name())); + } + if (snapshot_id && snapshot_id.value() != split_snapshot_id) { + return Status::Invalid(fmt::format( + "full-text search splits planned from snapshots {} and {} cannot be read together", + snapshot_id.value(), split_snapshot_id)); + } + snapshot_id = split_snapshot_id; + if (split_schema_id) { + if (schema_id && schema_id != split_schema_id) { + return Status::Invalid( + fmt::format("full-text search splits planned with schemas {} and {} cannot be " + "read together", + *schema_id, *split_schema_id)); + } + schema_id = split_schema_id; + } + return Status::OK(); + }; for (const auto& split : splits) { + if (auto raw_split = std::dynamic_pointer_cast(split)) { + PAIMON_RETURN_NOT_OK(check_split(raw_split->ColumnName(), raw_split->SnapshotId(), + raw_split->SchemaId())); + if (raw_split->FieldId() != text_field_.Id()) { + return Status::Invalid(fmt::format( + "a raw split of column {} was planned for field id {}, but the column now has " + "field id {}", + text_field_.Name(), raw_split->FieldId(), text_field_.Id())); + } + if (!raw_index_type.empty() && raw_index_type != raw_split->IndexType()) { + return Status::Invalid( + fmt::format("cannot combine raw full-text splits with " + "index types {} and {}", + raw_index_type, raw_split->IndexType())); + } + raw_index_type = raw_split->IndexType(); + const auto& ranges = raw_split->RowRanges(); + raw_ranges.insert(raw_ranges.end(), ranges.begin(), ranges.end()); + continue; + } auto index_split = std::dynamic_pointer_cast(split); if (!index_split) { - return Status::Invalid("invalid full-text search split, cannot cast to an index split"); - } - if (index_split->ColumnName() != text_field_.Name()) { - return Status::Invalid(fmt::format( - "a split of column {} cannot be read by a full-text search of column {}", - index_split->ColumnName(), text_field_.Name())); + return Status::Invalid( + "invalid full-text search split, cannot cast to an index or raw split"); } + PAIMON_RETURN_NOT_OK(check_split(index_split->ColumnName(), index_split->SnapshotId(), + index_split->SchemaId())); // A column of the same name may have replaced the planned one, so match field ids too. const GlobalIndexMeta& meta = index_split->FullTextIndexFiles().front()->GetGlobalIndexMeta().value(); @@ -375,27 +548,51 @@ Result> DataEvolutionFullTextRead::Read "replaced after the split was planned", text_field_.Name(), text_field_.Id())); } - if (snapshot_id && snapshot_id.value() != index_split->SnapshotId()) { - return Status::Invalid(fmt::format( - "full-text search splits planned from snapshots {} and {} cannot be read together", - snapshot_id.value(), index_split->SnapshotId())); - } - snapshot_id = index_split->SnapshotId(); index_splits.push_back(std::move(index_split)); } - if (index_splits.empty()) { + if (!snapshot_id) { return std::shared_ptr( std::make_shared(RoaringBitmap64(), std::vector())); } - // Live rows are read on this thread before the searches fan out, so a read of manifests on - // `executor_` never waits for it from one of its own threads. They are pinned to the snapshot - // the splits were planned from (apache/paimon#9031). - PAIMON_ASSIGN_OR_RAISE(std::optional live_rows, LiveRows(snapshot_id.value())); - // Likewise for the row filter, which reads indexes and possibly data. + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex raw_rows, RowRangeIndex::Create(raw_ranges)); + if (!raw_ranges.empty()) { + PAIMON_RETURN_NOT_OK(ExcludeRawRanges(raw_rows, &index_splits)); + } + if (schema_id && *schema_id != table_schema_->Id()) { + SchemaManager schemas(options_.GetFileSystem(), root_path_, + BranchManager::NormalizeBranch(options_.GetBranch())); + DataEvolutionFullTextRead planned_read(*this); + PAIMON_ASSIGN_OR_RAISE(planned_read.table_schema_, schemas.ReadSchema(*schema_id)); + PAIMON_RETURN_NOT_OK(BindSchemaOptions(*planned_read.table_schema_, option_overrides_, + &planned_read.options_)); + PAIMON_ASSIGN_OR_RAISE(planned_read.text_field_, + planned_read.table_schema_->GetField(text_field_.Name())); + if (planned_read.text_field_.Id() != text_field_.Id()) { + return Status::Invalid(fmt::format( + "a split of column {} in schema {} was planned for field id {}, but the column " + "now has field id {}", + text_field_.Name(), *schema_id, planned_read.text_field_.Id(), text_field_.Id())); + } + return planned_read.SearchSplits(index_splits, *snapshot_id, raw_rows, raw_index_type); + } + return SearchSplits(index_splits, *snapshot_id, raw_rows, raw_index_type); +} + +Result> DataEvolutionFullTextRead::SearchSplits( + const std::vector>& index_splits, + int64_t snapshot_id, const RowRangeIndex& raw_rows, const std::string& raw_index_type) const { + if (filter_) { + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()), filter_, + /*validate_field_idx=*/false)); + } + // Keep data reads on the caller: they can submit work to the same executor as the shards. + PAIMON_ASSIGN_OR_RAISE(std::optional live_rows, + LiveRows(index_splits, snapshot_id)); PAIMON_ASSIGN_OR_RAISE(std::optional matched_rows, - MatchedRows(index_splits, snapshot_id.value(), live_rows)); + MatchedRows(index_splits, snapshot_id, live_rows)); const std::optional& eligible_rows = matched_rows ? matched_rows : live_rows; using SearchResult = Result>; std::vector> searches; @@ -416,20 +613,50 @@ Result> DataEvolutionFullTextRead::Read }); } std::vector results; + std::vector> futures; + // Drain tasks that borrow this reader if submission, the raw scan or a shard throws. + ScopeGuard drain([&futures] { + for (auto& future : futures) { + if (future.valid()) { + future.wait(); + } + } + }); if (searches.size() <= 1) { for (const auto& search : searches) { results.push_back(search()); } } else { - std::vector> futures; futures.reserve(searches.size()); for (const auto& search : searches) { futures.push_back(Via(executor_.get(), search)); } + } + std::optional raw_result; + if (!raw_rows.Ranges().empty()) { + RawFullTextReadImpl raw_read(root_path_, table_schema_, options_, partition_filter_, + filter_, text_field_, query_, limit_, executor_, pool_); + raw_result = raw_read.Search(snapshot_id, raw_index_type, raw_rows); + } + if (!futures.empty()) { results = CollectAll(futures); } std::vector> scored_results; - scored_results.reserve(results.size()); + scored_results.reserve(results.size() + 1); + if (raw_result) { + if (!raw_result->ok()) { + const Status& status = raw_result->status(); + const auto& ranges = raw_rows.Ranges(); + std::vector sample(ranges.begin(), + ranges.begin() + std::min(ranges.size(), kMaxReportedRanges)); + return status.WithMessage(fmt::format( + "{}; while searching column {} in snapshot {} with a temporary {} index " + "over {} unindexed row ranges, first {}, schema {}, field id {}", + status.message(), text_field_.Name(), snapshot_id, raw_index_type, ranges.size(), + StringUtils::VectorToString(sample), table_schema_->Id(), text_field_.Id())); + } + scored_results.push_back(std::move(*raw_result).value()); + } for (auto& result : results) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scored_result, std::move(result)); diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.h b/src/paimon/core/table/source/data_evolution_full_text_read.h index 62e1a99b4..91f4c6ab2 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read.h +++ b/src/paimon/core/table/source/data_evolution_full_text_read.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -37,7 +38,9 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate.h" #include "paimon/table/source/full_text_read.h" +#include "paimon/utils/range.h" #include "paimon/utils/roaring_bitmap64.h" +#include "paimon/utils/row_range_index.h" namespace paimon { /// `FullTextRead` of a data-evolution table, aligned with Java `DataEvolutionFullTextRead`. @@ -46,6 +49,7 @@ class DataEvolutionFullTextRead : public FullTextRead { /// @param root_path Root directory of the table. /// @param table_schema Schema of the table. /// @param options Options of the table, merged with those of the search. + /// @param option_overrides Options explicitly set on the search builder. /// @param path_factory Path factory of the table. /// @param partition_filter Predicate over the partition fields used to read live rows and /// candidate rows, or null for all partitions. @@ -58,12 +62,18 @@ class DataEvolutionFullTextRead : public FullTextRead { /// @param pool Memory pool for reading indexes, manifests and candidate rows. static Result> Create( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& path_factory, + const CoreOptions& options, const std::map& option_overrides, + const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, const std::string& query, int32_t limit, const std::shared_ptr& executor, const std::shared_ptr& pool); + /// Checks query authorization and binds BLOB field options to the selected schema. + static Status BindSchemaOptions(const TableSchema& schema, + const std::map& option_overrides, + CoreOptions* options); + using FullTextRead::Read; Result> Read( @@ -82,6 +92,7 @@ class DataEvolutionFullTextRead : public FullTextRead { DataEvolutionFullTextRead(const std::string& root_path, const std::shared_ptr& table_schema, const CoreOptions& options, + const std::map& option_overrides, const std::shared_ptr& index_path_factory, const std::shared_ptr& index_file_manager, const std::shared_ptr& partition_filter, @@ -90,19 +101,23 @@ class DataEvolutionFullTextRead : public FullTextRead { const std::shared_ptr& executor, const std::shared_ptr& pool); - /// Returns the rows live in snapshot `snapshot_id`, or nullopt if every row counts as live. - Result> LiveRows(int64_t snapshot_id) const; + Result> SearchSplits( + const std::vector>& index_splits, + int64_t snapshot_id, const RowRangeIndex& raw_rows, + const std::string& raw_index_type) const; - /// Resolves the row filter into the rows of `index_splits` that satisfy it, as Java - /// `DataEvolutionFullTextRead#matchedRows` does in the `fast` `scalar-index.search-mode`, the - /// only one supported so far: rows that no scalar index can evaluate are excluded. Candidates - /// that the indexes cannot decide exactly are verified against the data with - /// `global-index.filter.refine-from-data`, and excluded without it. - /// @return The matched rows, all of them live, or nullopt if there is no row filter. + Result> LiveRows( + const std::vector>& index_splits, + int64_t snapshot_id) const; + + /// Returns nullopt when the live-row filter alone determines which indexed rows can rank. Result> MatchedRows( const std::vector>& index_splits, int64_t snapshot_id, const std::optional& live_rows) const; + static std::vector CollectSearchRanges( + const std::vector>& index_splits); + /// Searches the index shard of `split` among `include_row_ids`, or all of its rows if not set, /// and returns at most `limit_` rows with global row ids. Result> SearchSplit( @@ -112,6 +127,7 @@ class DataEvolutionFullTextRead : public FullTextRead { std::string root_path_; std::shared_ptr table_schema_; CoreOptions options_; + std::map option_overrides_; std::shared_ptr index_path_factory_; std::shared_ptr index_file_manager_; std::shared_ptr partition_filter_; diff --git a/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp b/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp index 9d8507cd6..aafc9a3a6 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp @@ -20,11 +20,15 @@ #include "paimon/core/table/source/data_evolution_full_text_read.h" #include +#include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -43,7 +47,12 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/table/source/raw_full_text_read_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" +#include "paimon/executor.h" #include "paimon/factories/factory.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" @@ -54,6 +63,8 @@ #include "paimon/predicate/predicate_builder.h" #include "paimon/table/source/full_text_search_builder.h" #include "paimon/testing/mock/mock_global_index_reader.h" +#include "paimon/testing/utils/snapshot_test_helper.h" +#include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" #include "paimon/utils/range.h" @@ -64,6 +75,53 @@ constexpr char kFakeScalarIndexType[] = "fake-scalar"; constexpr char kOtherFakeScalarIndexType[] = "other-fake-scalar"; constexpr char kQuery[] = R"({"match":{"query":"paimon"}})"; +class RejectingSearchExecutor : public Executor { + public: + explicit RejectingSearchExecutor(uint32_t rejected_submission) + : rejected_submission_(rejected_submission) {} + + void Add(std::function task) override { + std::lock_guard lock(mutex_); + if (++submissions_ == rejected_submission_) { + rejected_.set_value(); + throw std::bad_alloc(); + } + tasks_.push_back(std::move(task)); + } + + void ShutdownNow() override { + std::lock_guard lock(mutex_); + tasks_.clear(); + } + + uint32_t GetThreadNum() const override { + return 2; + } + + bool WaitUntilRejected() { + return rejected_.get_future().wait_for(std::chrono::seconds(5)) == + std::future_status::ready; + } + + void RunAcceptedTasks() { + std::vector> tasks; + { + std::lock_guard lock(mutex_); + tasks.swap(tasks_); + } + for (const auto& task : tasks) { + task(); + } + } + + private: + uint32_t rejected_submission_; + uint32_t submissions_ = 0; + std::mutex mutex_; + std::promise rejected_; + std::vector> tasks_; +}; + struct RecordedSearch { std::string file_name; std::vector schema_fields; @@ -299,6 +357,50 @@ ScoredRows ToRows(const ScoredGlobalIndexResult& result) { } } // namespace +TEST(InMemoryGlobalIndexFilesTest, TestTemporaryFileLifetimeAndIsolation) { + const std::string contents("a\0b", 3); + std::shared_ptr pool = GetMemoryPool(); + std::unique_ptr input; + { + InMemoryGlobalIndexFiles files(pool); + ASSERT_OK_AND_ASSIGN(std::string path, files.NewPath("full-text")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, files.NewOutputStream(path)); + ASSERT_NOK(output->Write(contents.data(), -1)); + ASSERT_NOK(output->Write(nullptr, 1)); + ASSERT_OK_AND_ASSIGN(int64_t written, output->Write(contents.data(), contents.size())); + ASSERT_EQ(written, 3); + ASSERT_NOK(files.GetInputStream(path)); + ASSERT_OK(output->Close()); + ASSERT_OK(output->Close()); + ASSERT_NOK(output->Write(contents.data(), contents.size())); + ASSERT_OK_AND_ASSIGN(int64_t position, output->GetPos()); + ASSERT_EQ(position, written); + ASSERT_OK_AND_ASSIGN(int64_t size, files.GetFileSize(path)); + ASSERT_EQ(size, written); + ASSERT_OK_AND_ASSIGN(input, files.GetInputStream(path)); + + InMemoryGlobalIndexFiles other_search(pool); + ASSERT_OK_AND_ASSIGN(std::string other_path, other_search.NewPath("full-text")); + ASSERT_NE(path, other_path); + ASSERT_NOK(other_search.GetInputStream(path)); + ASSERT_OK_AND_ASSIGN(output, other_search.NewOutputStream(other_path)); + ASSERT_OK_AND_ASSIGN(written, output->Write(nullptr, 0)); + ASSERT_EQ(written, 0); + ASSERT_OK(output->Close()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_file, + other_search.GetInputStream(other_path)); + ASSERT_OK_AND_ASSIGN(size, empty_file->Length()); + ASSERT_EQ(size, 0); + } + ASSERT_GT(pool->CurrentUsage(), 0u); + std::string bytes(contents.size(), '\0'); + ASSERT_OK_AND_ASSIGN(int64_t read, input->Read(bytes.data(), bytes.size(), 0)); + ASSERT_EQ(read, 3); + ASSERT_EQ(bytes, contents); + input.reset(); + ASSERT_EQ(pool->CurrentUsage(), 0u); +} + class DataEvolutionFullTextReadTest : public ::testing::Test { protected: void SetUp() override { @@ -335,7 +437,8 @@ class DataEvolutionFullTextReadTest : public ::testing::Test { index_type, file_name, /*file_size=*/128, /*row_count=*/range.Count(), /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, meta); return IndexFullTextSearchSplitImpl::Create(snapshot_id, "content", range.from, range.to, - {file}, search_row_ranges, scalar_index_files) + {file}, search_row_ranges, scalar_index_files, + TableSchema::FIRST_SCHEMA_ID) .value(); } @@ -411,6 +514,56 @@ TEST_F(DataEvolutionFullTextReadTest, TestMergesShardResultsIntoGlobalTopK) { ASSERT_EQ(ToRows(*result).size(), 10u); } +TEST_F(DataEvolutionFullTextReadTest, TestSubmissionFailureDrainsShardTasks) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(table_path_, {}, false)); + auto row_type = + arrow::struct_({arrow::field("id", arrow::int32()), arrow::field("content", arrow::utf8()), + arrow::field("title", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(row_type, + R"([[0, "paimon", "title"], [1, "paimon", "title"], + [2, "paimon", "title"], [3, "paimon", "title"]])", + {}, 0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt)); + + constexpr uint32_t kSecondShard = 2; + constexpr uint32_t kFirstRawManifest = 3; + for (uint32_t rejected_submission : {kSecondShard, kFirstRawManifest}) { + SCOPED_TRACE(rejected_submission); + FakeIndexState::Instance().Reset(); + std::vector> splits = { + NewSplit("a.index", Range(0, 0), {0.1f}, {Range(0, 0)}), + NewSplit("b.index", Range(1, 1), {0.2f}, {Range(1, 1)})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr raw_split, + RawFullTextSearchSplitImpl::Create(1, "content", kFakeIndexType, {Range(2, 3)}, 1, 0)); + splits.push_back(raw_split); + auto executor = std::make_shared(rejected_submission); + auto builder = NewBuilder(3); + builder->WithExecutor(executor); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, builder->NewFullTextRead()); + auto reading = std::async(std::launch::async, [&read, &splits] { + try { + (void)read->Read(splits); + return false; + } catch (const std::bad_alloc&) { + return true; + } + }); + bool rejected = executor->WaitUntilRejected(); + bool returned_early = + reading.wait_for(std::chrono::milliseconds(200)) == std::future_status::ready; + executor->RunAcceptedTasks(); + bool threw = reading.get(); + ASSERT_TRUE(rejected); + ASSERT_FALSE(returned_early) << "Read exited while shard tasks were still pending"; + ASSERT_TRUE(threw); + ASSERT_EQ(FakeIndexState::Instance().Searches().size(), rejected_submission - 1); + } +} + TEST_F(DataEvolutionFullTextReadTest, TestRanksOnlySearchRowRanges) { std::vector> splits = { NewSplit("a.index", Range(0, 4), {0.1f, 0.9f, 0.5f, 0.3f, 0.7f}, @@ -597,6 +750,14 @@ TEST_F(DataEvolutionFullTextReadTest, TestReadsEachSplitWithItsOwnIndexFields) { NewSplit("dedicated.index", Range(100, 102), {0.1f, 0.5f, 0.2f}, {Range(100, 102)}), NewSplit("multi.index", Range(200, 202), {0.3f, 0.4f, 0.6f}, {Range(200, 202)}, /*index_field_id=*/2, std::vector({1}))}; + SchemaManager schemas(dir_->GetFileSystem(), table_path_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr planned_schema, + schemas.ReadSchema(TableSchema::FIRST_SCHEMA_ID)); + auto fields = planned_schema->Fields(); + fields[2] = DataField(2, arrow::field("renamed_title", arrow::utf8())); + ASSERT_OK(TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_, fields, + planned_schema->HighestFieldId(), + planned_schema->Options())); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(splits)); ASSERT_EQ(ToRows(*result), ScoredRows({{101, 0.5f}, {201, 0.4f}, {202, 0.6f}})); @@ -621,7 +782,24 @@ TEST_F(DataEvolutionFullTextReadTest, TestRejectsInvalidSplits) { /*index_field_id=*/1, std::nullopt, /*snapshot_id=*/2); ASSERT_NOK_WITH_MSG(read->Read({split, other_snapshot}), "planned from snapshots 1 and 2 cannot be read together"); - ASSERT_NOK_WITH_MSG(read->Read({nullptr}), "cannot cast to an index split"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr raw_split, + RawFullTextSearchSplitImpl::Create(1, "content", kFakeIndexType, {Range(2, 3)}, 1, 0)); + ASSERT_NOK_WITH_MSG(read->Read({raw_split, other_snapshot}), + "planned from snapshots 1 and 2 cannot be read together"); + ASSERT_NOK_WITH_MSG(read->Read({other_snapshot, raw_split}), + "planned from snapshots 2 and 1 cannot be read together"); + ASSERT_OK_AND_ASSIGN( + auto second_schema_split, + RawFullTextSearchSplitImpl::Create(1, "content", kFakeIndexType, {Range(4, 5)}, 1, 1)); + ASSERT_NOK_WITH_MSG(read->Read({raw_split, second_schema_split}), + "planned with schemas 0 and 1 cannot be read together"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr other_backend, + RawFullTextSearchSplitImpl::Create(1, "content", "lucene-fts", {Range(4, 5)}, 1, 0)); + ASSERT_NOK_WITH_MSG(read->Read({raw_split, other_backend}), + "cannot combine raw full-text splits with index types"); + ASSERT_NOK_WITH_MSG(read->Read({nullptr}), "cannot cast to an index or raw split"); FullTextSearchBuilder title_builder(table_path_); title_builder.SetQuery("title", kQuery).SetLimit(3); @@ -629,14 +807,34 @@ TEST_F(DataEvolutionFullTextReadTest, TestRejectsInvalidSplits) { ASSERT_NOK_WITH_MSG(title_read->Read({split}), "a split of column content cannot be read by a full-text search of " "column title"); + ASSERT_NOK_WITH_MSG(title_read->Read({raw_split}), + "a split of column content cannot be read by a full-text search of " + "column title"); - auto unknown_type = NewSplit("c.index", Range(0, 1), {0.1f, 0.2f}, {Range(0, 1)}, + auto unknown_type = NewSplit("c.index", Range(0, 15), std::vector(16, 0.1f), + {Range(0, 1), Range(4, 5), Range(8, 9), Range(12, 13)}, /*index_field_id=*/1, std::nullopt, /*snapshot_id=*/1, /*index_type=*/"no-such-index"); - ASSERT_NOK_WITH_MSG(read->Read({unknown_type}), + auto failed = read->Read({unknown_type}); + ASSERT_NOK_WITH_MSG(failed, "no global indexer is registered for index type no-such-index; while " - "searching the split of column content in snapshot 1 over rows [0, 1] " + "searching the split of column content in snapshot 1 over rows [0, 15] " "with no-such-index index files [c.index]"); + ASSERT_NE( + failed.status().message().find("schema 0, 4 search ranges, first [[0, 1], [4, 5], [8, 9]]"), + std::string::npos); + ASSERT_EQ(failed.status().message().find("[12, 13]"), std::string::npos); + ASSERT_OK_AND_ASSIGN(raw_split, + RawFullTextSearchSplitImpl::Create( + 1, "content", "no-such-index", + {Range(2, 3), Range(6, 7), Range(10, 11), Range(14, 15)}, 1, 0)); + failed = read->Read({raw_split}); + ASSERT_FALSE(failed.ok()); + ASSERT_NE(failed.status().message().find( + "while searching column content in snapshot 1 with a temporary no-such-index " + "index over 4 unindexed row ranges, first [[2, 3], [6, 7], [10, 11]]"), + std::string::npos); + ASSERT_EQ(failed.status().message().find("[14, 15]"), std::string::npos); } TEST_F(DataEvolutionFullTextReadTest, TestRejectsSplitOfReplacedColumn) { @@ -647,29 +845,39 @@ TEST_F(DataEvolutionFullTextReadTest, TestRejectsSplitOfReplacedColumn) { ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); ASSERT_OK(read->Read(splits)); - SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); - ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, - schema_manager.Latest()); - ASSERT_TRUE(latest_schema); - const std::shared_ptr& old_schema = latest_schema.value(); - auto replaced_content = arrow::schema( - {arrow::field("id", arrow::int32()), arrow::field("old_content", arrow::utf8()), - arrow::field("title", arrow::utf8()), arrow::field("content", arrow::utf8())}); ASSERT_OK_AND_ASSIGN( - std::unique_ptr new_schema, - TableSchema::Create(old_schema->Id() + 1, replaced_content, old_schema->PartitionKeys(), - old_schema->PrimaryKeys(), old_schema->Options())); - ASSERT_OK_AND_ASSIGN(std::string new_schema_json, new_schema->ToJsonString()); - ASSERT_OK(dir_->GetFileSystem()->AtomicStore( - PathUtil::JoinPath(schema_manager.SchemaDirectory(), - "schema-" + std::to_string(new_schema->Id())), - new_schema_json)); + std::shared_ptr raw_split, + RawFullTextSearchSplitImpl::Create(1, "content", kFakeIndexType, {Range(4, 5)}, 1, 0)); + ASSERT_OK_AND_ASSIGN(std::string bytes, + FullTextSearchSplit::Serialize(raw_split, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr received_raw_split, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), GetDefaultPool())); + + auto fs = dir_->GetFileSystem(); + SchemaManager schemas(fs, table_path_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr planned_schema, + schemas.ReadSchema(TableSchema::FIRST_SCHEMA_ID)); + auto fields = planned_schema->Fields(); + const auto content = fields[1].ArrowField(); + fields[1] = DataField(1, arrow::field("old_content", arrow::utf8())); + fields.emplace_back(3, content); + ASSERT_OK(TestHelper::WriteNextSchema(fs, table_path_, fields, 3, planned_schema->Options())); ASSERT_OK_AND_ASSIGN(read, NewBuilder(3)->NewFullTextRead()); for (const auto& split : splits) { ASSERT_NOK_WITH_MSG(read->Read({split}), "a split of column content does not index its field id 3"); } + ASSERT_NOK_WITH_MSG(read->Read({received_raw_split}), + "a raw split of column content was planned for field id 1"); + + fields.pop_back(); + fields[2] = DataField(2, content); + ASSERT_OK(TestHelper::WriteNextSchema(fs, table_path_, fields, 3, planned_schema->Options())); + ASSERT_OK_AND_ASSIGN(read, NewBuilder(3)->NewFullTextRead()); + ASSERT_NOK_WITH_MSG(read->Read({splits.back()}), + "planned for field id 1, but the column now has field id 2"); } TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { @@ -691,8 +899,15 @@ TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { std::unique_ptr full_mode = NewBuilder(3); full_mode->AddOption(Options::GLOBAL_INDEX_SEARCH_MODE, "full"); - ASSERT_NOK_WITH_MSG(full_mode->NewFullTextScan(), "full and detail are not supported yet"); - ASSERT_NOK_WITH_MSG(full_mode->NewFullTextRead(), "full and detail are not supported yet"); + ASSERT_OK(full_mode->NewFullTextScan()); + ASSERT_OK(full_mode->NewFullTextRead()); + full_mode->AddOption(Options::QUERY_AUTH_ENABLED, "true"); + ASSERT_NOK_WITH_MSG(full_mode->NewFullTextScan(), "Full-text search is not supported"); + ASSERT_NOK_WITH_MSG(full_mode->NewFullTextRead(), "Full-text search is not supported"); + std::unique_ptr invalid_mode = NewBuilder(3); + invalid_mode->AddOption(Options::FULL_TEXT_INDEX_SEARCH_MODE, "invalid"); + ASSERT_NOK_WITH_MSG(invalid_mode->NewFullTextScan(), + "invalid full-text-index.search-mode: invalid"); std::unique_ptr no_thread = NewBuilder(3); no_thread->AddOption(Options::GLOBAL_INDEX_THREAD_NUM, "0"); @@ -710,7 +925,9 @@ TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { missing_field->WithFilter( PredicateBuilder::Equal(/*field_index=*/0, "missing", FieldType::INT, Literal(1))); ASSERT_NOK_WITH_MSG(missing_field->NewFullTextScan(), "field missing does not exist in schema"); - ASSERT_NOK_WITH_MSG(missing_field->NewFullTextRead(), "field missing does not exist in schema"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr worker, missing_field->NewFullTextRead()); + auto split = NewSplit("a.index", Range(0, 0), {1.0f}, {Range(0, 0)}); + ASSERT_NOK_WITH_MSG(worker->Read({split}), "field missing does not exist in schema"); std::unique_ptr row_id_field = NewBuilder(3); row_id_field->WithFilter(PredicateBuilder::Equal( @@ -734,10 +951,8 @@ TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { scalar_full_mode->AddOption(Options::SCALAR_INDEX_SEARCH_MODE, "full"); ASSERT_OK(scalar_full_mode->NewFullTextScan()); scalar_full_mode->WithFilter(IdEquals(1)); - ASSERT_NOK_WITH_MSG(scalar_full_mode->NewFullTextScan(), - "full and detail are not supported yet"); - ASSERT_NOK_WITH_MSG(scalar_full_mode->NewFullTextRead(), - "full and detail are not supported yet"); + ASSERT_OK(scalar_full_mode->NewFullTextScan()); + ASSERT_OK(scalar_full_mode->NewFullTextRead()); std::unique_ptr null_filter = NewBuilder(3); null_filter->WithFilter(nullptr); @@ -775,7 +990,7 @@ TEST_F(DataEvolutionFullTextReadTest, TestResolvesPlanSnapshot) { NewBuilder(3)->ExecuteLocal()); ASSERT_TRUE(ToRows(*result).empty()); - std::vector> time_travel_errors = { + std::vector> unavailable_snapshots = { {"", Options::SCAN_SNAPSHOT_ID, "snapshot-1"}, {"", Options::SCAN_TAG_NAME, "Tag '1' doesn't exist."}, {"", Options::SCAN_TIMESTAMP, @@ -785,10 +1000,8 @@ TEST_F(DataEvolutionFullTextReadTest, TestResolvesPlanSnapshot) { {"from-snapshot", Options::SCAN_TAG_NAME, "Tag '1' doesn't exist."}, {"from-snapshot-full", Options::SCAN_SNAPSHOT_ID, "snapshot-1"}, {"from-timestamp", Options::SCAN_TIMESTAMP_MILLIS, - "There is currently no snapshot earlier than or equal to timestamp [1]"}, - {"latest-full", Options::SCAN_SNAPSHOT_ID, - "scan.snapshot-id must be null when you use latest-full for scan.mode"}}; - for (const auto& [mode, key, error] : time_travel_errors) { + "There is currently no snapshot earlier than or equal to timestamp [1]"}}; + for (const auto& [mode, key, error] : unavailable_snapshots) { SCOPED_TRACE(mode + " " + key); std::unique_ptr builder = NewBuilder(3); if (!mode.empty()) { @@ -797,7 +1010,47 @@ TEST_F(DataEvolutionFullTextReadTest, TestResolvesPlanSnapshot) { builder->AddOption(key, key == Options::SCAN_TIMESTAMP ? "2023-06-01 00:00:00" : "1"); ASSERT_OK_AND_ASSIGN(scan, builder->NewFullTextScan()); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), error); + ASSERT_OK(builder->NewFullTextRead()); } + + auto builder = NewBuilder(3); + builder->AddOption(Options::SCAN_MODE, "latest-full").AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + const std::string error = + "scan.snapshot-id must be null when you use latest-full for scan.mode"; + ASSERT_OK_AND_ASSIGN(scan, builder->NewFullTextScan()); + ASSERT_NOK_WITH_MSG(scan->CreatePlan(), error); + ASSERT_NOK_WITH_MSG(builder->NewFullTextRead(), error); +} + +TEST_F(DataEvolutionFullTextReadTest, TestReplansTagAndTimestamp) { + const auto fs = dir_->GetFileSystem(); + SnapshotManager snapshots(fs, table_path_); + const auto tag_path = TagManager(fs, table_path_).TagPath("current"); + auto publish = [&](int64_t id) { + ASSERT_OK_AND_ASSIGN(std::string json, BuildTestSnapshot(id).ToJsonString()); + ASSERT_OK(fs->WriteFile(snapshots.SnapshotPath(id), json, false)); + ASSERT_OK(fs->WriteFile(tag_path, json, true)); + }; + ASSERT_NO_FATAL_FAILURE(publish(1)); + auto new_scan = [&](const char* key, const std::string& value) { + return NewBuilder(3)->AddOption(key, value).NewFullTextScan(); + }; + ASSERT_OK_AND_ASSIGN(auto tag_scan, new_scan(Options::SCAN_TAG_NAME, "current")); + ASSERT_OK_AND_ASSIGN(auto timestamp_scan, + new_scan(Options::SCAN_TIMESTAMP_MILLIS, + std::to_string(BuildTestSnapshot(1).TimeMillis()))); + ASSERT_OK_AND_ASSIGN(auto fixed_scan, new_scan(Options::SCAN_SNAPSHOT_ID, "1")); + auto check_plans = [&](int64_t latest) { + ASSERT_OK_AND_ASSIGN(auto tag_plan, tag_scan->CreatePlan()); + EXPECT_EQ(tag_plan->SnapshotId(), latest); + ASSERT_OK_AND_ASSIGN(auto timestamp_plan, timestamp_scan->CreatePlan()); + EXPECT_EQ(timestamp_plan->SnapshotId(), latest); + }; + ASSERT_NO_FATAL_FAILURE(check_plans(1)); + ASSERT_NO_FATAL_FAILURE(publish(2)); + ASSERT_NO_FATAL_FAILURE(check_plans(2)); + ASSERT_OK_AND_ASSIGN(auto fixed_plan, fixed_scan->CreatePlan()); + ASSERT_EQ(fixed_plan->SnapshotId(), 1); } } // namespace paimon::test diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan.cpp b/src/paimon/core/table/source/data_evolution_full_text_scan.cpp index 983f58a29..0a0ed6723 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_scan.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -34,23 +33,24 @@ #include "fmt/format.h" #include "fmt/ranges.h" +#include "paimon/core/global_index/data_evolution_global_index_coverage.h" #include "paimon/core/global_index/global_index_evaluator_impl.h" +#include "paimon/core/global_index/global_index_live_row_filter.h" +#include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/manifest/index_manifest_entry.h" #include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/schema/schema_validation.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" -#include "paimon/core/tag/tag.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/core/utils/tag_manager.h" -#include "paimon/defs.h" #include "paimon/global_index/full_text/full_text_defs.h" #include "paimon/global_index/global_indexer.h" #include "paimon/global_index/global_indexer_factory.h" #include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/utils/row_range_index.h" namespace paimon { namespace { @@ -65,63 +65,6 @@ struct IndexRangeCandidate { std::vector assigned; }; -/// Resolves the snapshot to search as Java `TimeTravelUtil#tryTravelOrLatest` does: the one that -/// the time-travel option selects, or else the latest one. -/// -/// TODO: take the column, the partition keys and the options from the schema of the selected -/// snapshot, as Java time travel does. They still come from the latest schema, so a column dropped -/// and added again with the same name since that snapshot finds no index files. -Result> ResolveSnapshot(const CoreOptions& options, - const SnapshotManager& snapshot_manager, - const TagManager& tag_manager) { - // Java checks the options against `scan.mode` when it loads the table with them, which leaves - // at most one time-travel option set. - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateStartupMode(options)); - // Java `TimeTravelUtil#SCAN_KEYS`, without `scan.watermark`, which Paimon C++ does not define. - const char* const time_travel_keys[] = {Options::SCAN_SNAPSHOT_ID, Options::SCAN_TAG_NAME, - Options::SCAN_TIMESTAMP, - Options::SCAN_TIMESTAMP_MILLIS}; - const std::map& raw_options = options.ToMap(); - auto key_iter = - std::find_if(std::begin(time_travel_keys), std::end(time_travel_keys), - [&raw_options](const char* key) { return raw_options.count(key) > 0; }); - if (key_iter == std::end(time_travel_keys)) { - return snapshot_manager.LatestSnapshot(); - } - const std::string key = *key_iter; - if (key == Options::SCAN_SNAPSHOT_ID) { - const std::optional& snapshot_id = options.GetScanSnapshotId(); - if (!snapshot_id) { - return Status::Invalid(fmt::format("invalid {}", key)); - } - PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, - snapshot_manager.LoadSnapshot(snapshot_id.value())); - return std::optional(std::move(snapshot)); - } - if (key == Options::SCAN_TAG_NAME) { - std::optional tag_name = options.GetScanTagName(); - if (!tag_name) { - return Status::Invalid(fmt::format("invalid {}", key)); - } - PAIMON_ASSIGN_OR_RAISE(Tag tag, tag_manager.GetOrThrow(tag_name.value())); - PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, tag.TrimToSnapshot()); - return std::optional(std::move(snapshot)); - } - // `CoreOptions` parses both `scan.timestamp` and `scan.timestamp-millis` into milliseconds. - std::optional timestamp_millis = options.GetScanTimestampMillis(); - if (!timestamp_millis) { - return Status::Invalid(fmt::format("invalid {}", key)); - } - PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, - snapshot_manager.EarlierOrEqualTimeMillis(timestamp_millis.value())); - if (!snapshot) { - return Status::Invalid( - fmt::format("There is currently no snapshot earlier than or equal to timestamp [{}]", - timestamp_millis.value())); - } - return snapshot; -} - /// Returns whether `left` is chosen before `right` for a row that both cover, as Java /// `DataEvolutionFullTextScan#compareCandidates` does. struct CandidateRanksBefore { @@ -137,17 +80,20 @@ struct CandidateRanksBefore { DataEvolutionFullTextScan::DataEvolutionFullTextScan( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& path_factory, + const std::optional& time_travel_snapshot, const CoreOptions& options, + const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, - const std::shared_ptr& pool) + const std::shared_ptr& executor, const std::shared_ptr& pool) : root_path_(root_path), table_schema_(table_schema), + time_travel_snapshot_(time_travel_snapshot), options_(options), path_factory_(path_factory), partition_filter_(partition_filter), filter_(filter), text_field_(text_field), + executor_(executor), pool_(pool) {} bool DataEvolutionFullTextScan::IndexesField(const GlobalIndexMeta& meta, int32_t field_id) { @@ -167,20 +113,6 @@ bool DataEvolutionFullTextScan::IsScalarPreFilterFile(const GlobalIndexMeta& met filter_field_ids.count(meta.index_field_id) > 0; } -Result>> DataEvolutionFullTextScan::IntersectingFiles( - const std::vector>& index_files, const Range& row_range) { - std::vector> files; - for (const auto& file : index_files) { - // Checked before a `Range` is built from it, as a reversed range breaks its invariant. - PAIMON_RETURN_NOT_OK(IndexFullTextSearchSplitImpl::CheckScalarIndexFile(file)); - const GlobalIndexMeta& meta = file->GetGlobalIndexMeta().value(); - if (Range::HasIntersection(Range(meta.row_range_start, meta.row_range_end), row_range)) { - files.push_back(file); - } - } - return files; -} - std::string DataEvolutionFullTextScan::IndexIdentity(const IndexFileMeta& file) { const std::optional& meta = file.GetGlobalIndexMeta(); assert(meta); @@ -189,7 +121,7 @@ std::string DataEvolutionFullTextScan::IndexIdentity(const IndexFileMeta& file) fmt::join(extra_field_ids, ", ")); } -std::vector +Result> DataEvolutionFullTextScan::ChooseIndexRanges( const std::vector>& index_files, int32_t field_id) { // (physical range, identity) -> candidate @@ -199,6 +131,7 @@ DataEvolutionFullTextScan::ChooseIndexRanges( if (!meta || !IndexesField(meta.value(), field_id)) { continue; } + PAIMON_RETURN_NOT_OK(IndexFullTextSearchSplitImpl::CheckIndexFile(file, "full-text")); Range file_range(meta->row_range_start, meta->row_range_end); std::string identity = IndexIdentity(*file); auto iter = candidates @@ -282,12 +215,41 @@ Result DataEvolutionFullTextScan::ServesFullTextSearch(const std::string& return false; } +Result> DataEvolutionFullTextScan::PlanRawSplit( + const Snapshot& snapshot, + const std::vector>& full_text_index_files, + const std::vector& selections) const { + GlobalIndexSearchMode search_mode = options_.GetFullTextIndexSearchMode(); + if (search_mode == GlobalIndexSearchMode::FAST || selections.empty()) { + return std::shared_ptr(); + } + // Coverage counts every single-field definition, including unselected shards. + std::vector indexed_ranges = + DataEvolutionGlobalIndexCoverage::IndexedRanges(full_text_index_files, {text_field_.Id()}); + auto data_ranges = [&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, + GlobalIndexScanImpl::GetOrCreateExecutor(executor_, options_)); + return GlobalIndexLiveRowFilter::DataRowRanges( + root_path_, table_schema_, snapshot, partition_filter_, options_, executor, pool_); + }; + PAIMON_ASSIGN_OR_RAISE(std::vector raw_ranges, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + search_mode, snapshot.NextRowId(), indexed_ranges, data_ranges)); + if (raw_ranges.empty()) { + return std::shared_ptr(); + } + return RawFullTextSearchSplitImpl::Create( + snapshot.Id(), text_field_.Name(), selections.front().files.front()->IndexType(), + std::move(raw_ranges), text_field_.Id(), table_schema_->Id()); +} + Result> DataEvolutionFullTextScan::CreatePlan() { - std::string branch = BranchManager::NormalizeBranch(options_.GetBranch()); - SnapshotManager snapshot_manager(options_.GetFileSystem(), root_path_, branch); - TagManager tag_manager(options_.GetFileSystem(), root_path_, branch); - PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, - ResolveSnapshot(options_, snapshot_manager, tag_manager)); + std::optional snapshot = time_travel_snapshot_; + if (!snapshot) { + SnapshotManager snapshots(options_.GetFileSystem(), root_path_, + BranchManager::NormalizeBranch(options_.GetBranch())); + PAIMON_ASSIGN_OR_RAISE(snapshot, snapshots.LatestSnapshot()); + } if (!snapshot) { return std::make_shared(std::nullopt, std::vector>()); @@ -344,24 +306,41 @@ Result> DataEvolutionFullTextScan::CreatePla full_text_index_files.push_back(entry.index_file); } } - bool supports_full_text = indexer != nullptr && indexer->SupportsFullTextSearch(); - if (IsScalarPreFilterFile(meta, filter_field_ids, supports_full_text)) { + if (indexer && + IsScalarPreFilterFile(meta, filter_field_ids, indexer->SupportsFullTextSearch())) { scalar_index_files.push_back(entry.index_file); } } std::vector> splits; - for (auto& selection : ChooseIndexRanges(full_text_index_files, field_id)) { - PAIMON_ASSIGN_OR_RAISE(std::vector> split_scalar_files, - IntersectingFiles(scalar_index_files, selection.file_range)); + PAIMON_ASSIGN_OR_RAISE(std::vector selections, + ChooseIndexRanges(full_text_index_files, field_id)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr raw_split, + PlanRawSplit(*snapshot, full_text_index_files, selections)); + std::optional raw_rows; + if (raw_split) { + PAIMON_ASSIGN_OR_RAISE(raw_rows, RowRangeIndex::Create(raw_split->RowRanges())); + } + for (auto& selection : selections) { + if (raw_rows) { + // Keep the physical shard range for translating index-local row ids. + selection.search_ranges = + DataEvolutionGlobalIndexCoverage::ExcludeRanges(selection.search_ranges, *raw_rows); + if (selection.search_ranges.empty()) { + continue; + } + } PAIMON_ASSIGN_OR_RAISE( std::shared_ptr split, IndexFullTextSearchSplitImpl::Create( snapshot->Id(), text_field_.Name(), selection.file_range.from, selection.file_range.to, std::move(selection.files), - std::move(selection.search_ranges), std::move(split_scalar_files))); + std::move(selection.search_ranges), scalar_index_files, table_schema_->Id())); splits.push_back(std::move(split)); } + if (raw_split) { + splits.push_back(std::move(raw_split)); + } return std::make_shared(snapshot->Id(), std::move(splits)); } diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan.h b/src/paimon/core/table/source/data_evolution_full_text_scan.h index b439582dd..443e9eb4a 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan.h +++ b/src/paimon/core/table/source/data_evolution_full_text_scan.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -31,7 +32,9 @@ #include "paimon/core/index/global_index_meta.h" #include "paimon/core/index/index_file_meta.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/executor.h" #include "paimon/global_index/global_indexer.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate.h" @@ -39,9 +42,8 @@ #include "paimon/utils/range.h" namespace paimon { -/// `FullTextScan` of a data-evolution table, aligned with Java `DataEvolutionFullTextScan`: one -/// split per index shard of the searched column, planned from the global index files of one -/// snapshot. +class RawFullTextSearchSplitImpl; + class DataEvolutionFullTextScan : public FullTextScan { public: /// The index shard chosen to rank some rows of the searched column. @@ -58,18 +60,24 @@ class DataEvolutionFullTextScan : public FullTextScan { /// @param root_path Root directory of the table. /// @param table_schema Schema of the table. + /// @param time_travel_snapshot Explicit snapshot used to bind the schema, or nullopt to plan + /// the latest snapshot. /// @param options Options of the table, merged with those of the search. /// @param path_factory Path factory of the table. /// @param partition_filter Predicate over the partition fields, or null for all partitions. /// @param filter Row filter beyond the partition filter, or null for all rows. /// @param text_field The searched column. + /// @param executor Executor to read manifests on in the `detail` search mode, or null + /// to create one with `global-index.thread-num` threads if needed. /// @param pool Memory pool for reading index manifests. DataEvolutionFullTextScan(const std::string& root_path, const std::shared_ptr& table_schema, + const std::optional& time_travel_snapshot, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, + const std::shared_ptr& executor, const std::shared_ptr& pool); Result> CreatePlan() override; @@ -88,12 +96,6 @@ class DataEvolutionFullTextScan : public FullTextScan { const std::set& filter_field_ids, bool supports_full_text); - /// Returns the files of `index_files` whose row range intersects `row_range`, or `Invalid` if - /// a split cannot carry one of them; see - /// `IndexFullTextSearchSplitImpl::CheckScalarIndexFile()`. - static Result>> IntersectingFiles( - const std::vector>& index_files, const Range& row_range); - /// Returns the identity of the index that `file` belongs to: its index type, indexed field id /// and extra field ids, as `type|index_field_id|[extra_field_ids]`. Files of one identity /// differ only by shard, while files of different identities are never read together. @@ -108,7 +110,7 @@ class DataEvolutionFullTextScan : public FullTextScan { /// @param index_files Global index files; those that do not index `field_id` are ignored. /// @param field_id The id of the searched field. /// @return The chosen shards, ordered by start, end and identity. - static std::vector ChooseIndexRanges( + static Result> ChooseIndexRanges( const std::vector>& index_files, int32_t field_id); /// Returns whether the index files of `index_type` serve a full-text search. A `full-text` or @@ -123,13 +125,20 @@ class DataEvolutionFullTextScan : public FullTextScan { const std::string& table_path, int64_t snapshot_id); private: + Result> PlanRawSplit( + const Snapshot& snapshot, + const std::vector>& full_text_index_files, + const std::vector& selections) const; + std::string root_path_; std::shared_ptr table_schema_; + std::optional time_travel_snapshot_; CoreOptions options_; std::shared_ptr path_factory_; std::shared_ptr partition_filter_; std::shared_ptr filter_; DataField text_field_; + std::shared_ptr executor_; std::shared_ptr pool_; }; diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp b/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp index 47e8ae789..73019b9bb 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp @@ -75,17 +75,19 @@ std::ostream& operator<<(std::ostream& os, const Selection& selection) { return os << "]}"; } -std::vector Choose(const std::vector>& files) { +void CheckSelections(const std::vector>& files, + const std::vector& expected) { + ASSERT_OK_AND_ASSIGN(auto chosen, + DataEvolutionFullTextScan::ChooseIndexRanges(files, kTextFieldId)); std::vector selections; - for (const auto& selection : - DataEvolutionFullTextScan::ChooseIndexRanges(files, kTextFieldId)) { + for (const auto& selection : chosen) { std::vector file_names; for (const auto& file : selection.files) { file_names.push_back(file->FileName()); } selections.push_back(Selection{selection.file_range, file_names, selection.search_ranges}); } - return selections; + ASSERT_EQ(selections, expected); } } // namespace @@ -120,8 +122,8 @@ TEST(DataEvolutionFullTextScanTest, TestDisjointShards) { {Range(100, 199), {"shard-1"}, {Range(100, 199)}}, {Range(300, 399), {"shard-3"}, {Range(300, 399)}}, {Range(400, max_row_id), {"shard-4"}, {Range(400, max_row_id)}}}; - ASSERT_EQ(Choose(files), expected); - ASSERT_TRUE(Choose({}).empty()); + CheckSelections(files, expected); + CheckSelections({}, {}); } TEST(DataEvolutionFullTextScanTest, TestOverlappingShardsOfOneIndex) { @@ -131,7 +133,7 @@ TEST(DataEvolutionFullTextScanTest, TestOverlappingShardsOfOneIndex) { NewIndexFile("1-3", "full-text", Range(1, 3), kTextFieldId)}; std::vector expected = {{Range(0, 2), {"0-2"}, {Range(0, 2)}}, {Range(1, 3), {"1-3"}, {Range(3, 3)}}}; - ASSERT_EQ(Choose(files), expected); + CheckSelections(files, expected); } TEST(DataEvolutionFullTextScanTest, TestDedicatedIndexBeforeExtraField) { @@ -141,31 +143,27 @@ TEST(DataEvolutionFullTextScanTest, TestDedicatedIndexBeforeExtraField) { NewIndexFile("dedicated", "test-full-text", Range(50, 149), kTextFieldId)}; std::vector expected = {{Range(0, 99), {"multi"}, {Range(0, 49)}}, {Range(50, 149), {"dedicated"}, {Range(50, 149)}}}; - ASSERT_EQ(Choose(files), expected); - - ASSERT_EQ(Choose({files[0]}), - std::vector({{Range(0, 99), {"multi"}, {Range(0, 99)}}})); + CheckSelections(files, expected); + CheckSelections({files[0]}, {{Range(0, 99), {"multi"}, {Range(0, 99)}}}); } TEST(DataEvolutionFullTextScanTest, TestIdentityOrderBreaksTies) { std::vector> same_range = { NewIndexFile("lucene", "lucene-fts", Range(0, 9), kTextFieldId), NewIndexFile("full-text", "full-text", Range(0, 9), kTextFieldId)}; - ASSERT_EQ(Choose(same_range), - std::vector({{Range(0, 9), {"full-text"}, {Range(0, 9)}}})); + CheckSelections(same_range, {{Range(0, 9), {"full-text"}, {Range(0, 9)}}}); std::vector> lucene_inside = { NewIndexFile("full-text", "full-text", Range(0, 99), kTextFieldId), NewIndexFile("lucene", "lucene-fts", Range(50, 59), kTextFieldId)}; - ASSERT_EQ(Choose(lucene_inside), - std::vector({{Range(0, 99), {"full-text"}, {Range(0, 99)}}})); + CheckSelections(lucene_inside, {{Range(0, 99), {"full-text"}, {Range(0, 99)}}}); std::vector> full_text_inside = { NewIndexFile("lucene", "lucene-fts", Range(0, 99), kTextFieldId), NewIndexFile("full-text", "full-text", Range(40, 59), kTextFieldId)}; std::vector expected = {{Range(0, 99), {"lucene"}, {Range(0, 39), Range(60, 99)}}, {Range(40, 59), {"full-text"}, {Range(40, 59)}}}; - ASSERT_EQ(Choose(full_text_inside), expected); + CheckSelections(full_text_inside, expected); } TEST(DataEvolutionFullTextScanTest, TestScalarPreFilterFiles) { @@ -191,21 +189,7 @@ TEST(DataEvolutionFullTextScanTest, TestScalarPreFilterFiles) { ASSERT_FALSE(is_pre_filter_file(1, std::vector({2}), /*supports_full_text=*/true)); } -TEST(DataEvolutionFullTextScanTest, TestIntersectingFiles) { - auto first = NewIndexFile("first.index", "btree", Range(0, 9), /*index_field_id=*/2); - auto second = NewIndexFile("second.index", "btree", Range(10, 19), /*index_field_id=*/2); - auto third = NewIndexFile("third.index", "btree", Range(20, 29), /*index_field_id=*/2); - std::vector> files = {first, second, third}; - ASSERT_OK_AND_ASSIGN(std::vector> intersecting, - DataEvolutionFullTextScan::IntersectingFiles(files, Range(9, 10))); - ASSERT_EQ(intersecting, std::vector>({first, second})); - ASSERT_OK_AND_ASSIGN(intersecting, - DataEvolutionFullTextScan::IntersectingFiles(files, Range(25, 40))); - ASSERT_EQ(intersecting, std::vector>({third})); - ASSERT_OK_AND_ASSIGN(intersecting, - DataEvolutionFullTextScan::IntersectingFiles(files, Range(30, 40))); - ASSERT_TRUE(intersecting.empty()); - +TEST(DataEvolutionFullTextScanTest, TestRejectsInvalidIndexRanges) { std::vector> invalid_ranges = { {-1, 9}, {9, 0}, {0, std::numeric_limits::max()}}; for (const auto& [start, end] : invalid_ranges) { @@ -214,11 +198,9 @@ TEST(DataEvolutionFullTextScanTest, TestIntersectingFiles) { auto invalid = std::make_shared( "btree", "invalid.index", /*file_size=*/128, /*row_count=*/10, /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, meta); - std::vector> with_invalid = {first, invalid}; - ASSERT_NOK_WITH_MSG(DataEvolutionFullTextScan::IntersectingFiles(with_invalid, Range(0, 9)), - fmt::format("scalar index file invalid.index covers an invalid row " - "range [{}, {}]", - start, end)); + const std::string error = fmt::format( + "index file invalid.index covers an invalid row range [{}, {}]", start, end); + ASSERT_NOK_WITH_MSG(DataEvolutionFullTextScan::ChooseIndexRanges({invalid}, 2), error); } } diff --git a/src/paimon/core/table/source/filtered_row_id_reader.cpp b/src/paimon/core/table/source/filtered_row_id_reader.cpp index 6c7d54f1a..78a674439 100644 --- a/src/paimon/core/table/source/filtered_row_id_reader.cpp +++ b/src/paimon/core/table/source/filtered_row_id_reader.cpp @@ -20,7 +20,6 @@ #include "paimon/core/table/source/filtered_row_id_reader.h" #include -#include #include #include #include @@ -36,14 +35,14 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/global_index/global_index_evaluator_impl.h" -#include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/source/abstract_table_scan.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" #include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" -#include "paimon/format/file_format.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/leaf_predicate.h" @@ -52,7 +51,6 @@ #include "paimon/reader/batch_reader.h" #include "paimon/scan_context.h" #include "paimon/table/source/plan.h" -#include "paimon/table/source/startup_mode.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" @@ -71,65 +69,52 @@ bool HasCandidateOnlyLeaf(const std::shared_ptr& predicate) { } return false; } +} // namespace -/// Returns `options` with the scan pinned to snapshot `snapshot_id`, as Java does. A tag may -/// outlive its snapshot, so once the snapshot has expired a search of a tag reads the tag instead, -/// which the caller must check still refers to the snapshot. -Result> PinToSnapshot(const std::string& root_path, - const CoreOptions& options, - int64_t snapshot_id) { - std::map pinned = options.ToMap(); - pinned[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); - pinned.erase(Options::SCAN_TIMESTAMP); - pinned.erase(Options::SCAN_TIMESTAMP_MILLIS); - if (pinned.count(Options::SCAN_TAG_NAME) > 0) { - SnapshotManager snapshot_manager(options.GetFileSystem(), root_path, - BranchManager::NormalizeBranch(options.GetBranch())); - PAIMON_ASSIGN_OR_RAISE(bool exists, snapshot_manager.SnapshotExists(snapshot_id)); - if (!exists) { - pinned.erase(Options::SCAN_SNAPSHOT_ID); - return pinned; +Result FilteredRowIdReader::LoadSnapshot(const CoreOptions& options, + const std::string& root_path, + int64_t snapshot_id) { + std::string branch = BranchManager::NormalizeBranch(options.GetBranch()); + SnapshotManager snapshot_manager(options.GetFileSystem(), root_path, branch); + Result snapshot = snapshot_manager.LoadSnapshot(snapshot_id); + if (snapshot.ok()) { + return snapshot; + } + PAIMON_ASSIGN_OR_RAISE(bool exists, snapshot_manager.SnapshotExists(snapshot_id)); + if (exists) { + return snapshot.status(); + } + TagManager tag_manager(options.GetFileSystem(), root_path, branch); + if (std::optional name = options.GetScanTagName()) { + PAIMON_ASSIGN_OR_RAISE(Tag tag, tag_manager.GetOrThrow(*name)); + if (tag.Id() != snapshot_id) { + return Status::Invalid(fmt::format( + "cannot read the rows of snapshot {}, as the data scan resolved another one " + "({} from tag {})", + snapshot_id, tag.Id(), *name)); } - pinned.erase(Options::SCAN_TAG_NAME); + return tag.TrimToSnapshot(); } - pinned[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); - return pinned; -} - -/// Creates a read of `read_context` with `table_schema` rather than the latest schema, as Java -/// `FilteredRowIdReader` reads with the table that the search loaded, so that a column renamed or -/// replaced since then cannot point the filter at another field. -Result> CreateReadWithSchema( - std::unique_ptr read_context, const std::shared_ptr& table_schema, - const std::map& options, const std::shared_ptr& pool, - const std::shared_ptr& executor) { - // Row ids come from row tracking, which a primary-key table does not support. - if (!table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented("cannot evaluate a row filter on a primary-key table"); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_names, tag_manager.ListTagNames()); + for (const std::string& name : tag_names) { + // A tag may disappear after listing. Propagate read errors for tags that still exist. + Result> tag = tag_manager.Get(name); + if (!tag.ok()) { + PAIMON_ASSIGN_OR_RAISE(bool tag_exists, + options.GetFileSystem()->Exists(tag_manager.TagPath(name))); + if (tag_exists) { + return tag.status(); + } + continue; + } + if (tag.value() && tag.value()->Id() == snapshot_id) { + return tag.value()->TrimToSnapshot(); + } } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, options)); - const CoreOptions& core_options = internal_context->GetCoreOptions(); - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, - core_options.GetExternalPathWeights()); - PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, - core_options.CreateGlobalIndexExternalPaths()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), - external_path_weights, pool)); - return std::unique_ptr( - std::make_unique(path_factory, internal_context, pool, executor)); + return Status::NotExist(fmt::format( + "snapshot {} that the full-text search splits were planned from no longer exists", + snapshot_id)); } -} // namespace FilteredRowIdReader::FilteredRowIdReader( const std::string& root_path, const std::shared_ptr& table_schema, @@ -157,38 +142,18 @@ Result FilteredRowIdReader::IsExact(const TableSchema& table_schema, return !HasCandidateOnlyLeaf(filter); } -Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap64& rows) const { - RoaringBitmap64 matching; - if (rows.IsEmpty()) { - return matching; - } - std::map pinned_options; - PAIMON_ASSIGN_OR_RAISE(pinned_options, PinToSnapshot(root_path_, options_, snapshot_id_)); +Result FilteredRowIdReader::MatchingRowIds(RoaringBitmap64 rows) const { + BitmapGlobalIndexResult result(std::move(rows)); + PAIMON_ASSIGN_OR_RAISE(std::vector ranges, result.ToRanges()); + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex row_ranges, RowRangeIndex::Create(ranges)); + return MatchingRowIds(row_ranges); +} - // Unlike the read, the scan may load the latest schema, as it selects files by partition and - // row id alone. - ScanContextBuilder scan_builder(root_path_); - scan_builder.SetOptions(pinned_options) - .SetPredicate(partition_filter_) - .SetGlobalIndexResult(std::make_shared(rows)) - .WithFileSystem(options_.GetFileSystem()) - .WithMemoryPool(pool_) - .WithExecutor(executor_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - // An empty plan of another snapshot would drop every candidate, so it is checked too. - if (plan->SnapshotId() != snapshot_id_) { - return Status::Invalid(fmt::format( - "cannot evaluate a row filter on snapshot {}, as the data scan resolved another one", - snapshot_id_)); - } - if (plan->Splits().empty()) { +Result FilteredRowIdReader::MatchingRowIds(const RowRangeIndex& row_ranges) const { + RoaringBitmap64 matching; + if (row_ranges.Ranges().empty()) { return matching; } - - // The read schema decides the field indexes that a predicate of the read must refer to. std::set filter_fields; PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(filter_, &filter_fields)); std::vector read_fields; @@ -197,24 +162,82 @@ Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap6 read_fields.push_back(field.Name()); } } - read_fields.push_back(SpecialFields::RowId().Name()); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_filter, - PredicateUtils::CreatePickedFieldFilter( - filter_, ObjectUtils::CreateIdentifierToIndexMap(read_fields))); + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, LoadSnapshot(options_, root_path_, snapshot_id_)); + PAIMON_RETURN_NOT_OK( + ReadRows(root_path_, table_schema_, options_, partition_filter_, snapshot, row_ranges, + read_fields, filter_, executor_, pool_, + [&row_ranges, &matching](const arrow::StructArray& /*batch*/, + const arrow::Int64Array& row_ids) -> Status { + for (int64_t i = 0; i < row_ids.length(); ++i) { + int64_t row_id = row_ids.Value(i); + if (row_ranges.Intersects(row_id, row_id)) { + matching.Add(row_id); + } + } + return Status::OK(); + })); + return matching; +} + +Status FilteredRowIdReader::ReadRows( + const std::string& root_path, const std::shared_ptr& table_schema, + const CoreOptions& options, const std::shared_ptr& partition_filter, + const Snapshot& snapshot, const RowRangeIndex& row_ranges, + const std::vector& read_fields, const std::shared_ptr& predicate, + const std::shared_ptr& executor, const std::shared_ptr& pool, + const BatchConsumer& consumer) { + const std::string branch = BranchManager::NormalizeBranch(options.GetBranch()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(options.GetFileSystem(), root_path, table_schema, + CatalogUtils::IdentifierFromPath(root_path, branch), /*uuid=*/"", + branch)); + ScanContextBuilder scan_builder(root_path); + scan_builder.SetOptions(options.ToMap()) + .SetPredicate(partition_filter) + .WithFileSystem(options.GetFileSystem()) + .WithCache(options.GetCache()) + .WithMemoryPool(pool) + .WithExecutor(executor); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + table->NewScan(std::move(scan_context))); + auto* data_scan = dynamic_cast(table_scan.get()); + if (!data_scan) { + return Status::Invalid("reading row id ranges requires a native table scan"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + data_scan->PlanSnapshotRanges(snapshot, row_ranges)); + if (plan->Splits().empty()) { + return Status::OK(); + } + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented("cannot read full-text row id ranges on a primary-key table"); + } + if (options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE(plan, DataEvolutionBatchScan::WrapToIndexedSplits( + plan, row_ranges, /*id_to_score=*/{})); + } - ReadContextBuilder read_builder(root_path_); - read_builder.SetOptions(pinned_options) - .SetReadFieldNames(read_fields) - .SetPredicate(read_filter) - .EnablePredicateFilter(true) + std::vector fields = read_fields; + fields.push_back(SpecialFields::RowId().Name()); + ReadContextBuilder read_builder(root_path); + read_builder.SetOptions(options.ToMap()) + .SetReadFieldNames(fields) .EnablePrefetch(true) - .WithFileSystem(options_.GetFileSystem()) - .WithMemoryPool(pool_) - .WithExecutor(executor_); + .WithFileSystem(options.GetFileSystem()) + .WithCache(options.GetCache()) + .WithMemoryPool(pool) + .WithExecutor(executor); + if (predicate) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_predicate, + PredicateUtils::CreatePickedFieldFilter( + predicate, ObjectUtils::CreateIdentifierToIndexMap(fields))); + read_builder.SetPredicate(read_predicate).EnablePredicateFilter(true); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - CreateReadWithSchema(std::move(read_context), table_schema_, - pinned_options, pool_, executor_)); + table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ScopeGuard guard([&reader]() { reader->Close(); }); @@ -227,24 +250,19 @@ Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap6 PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, arrow::ImportArray(c_array.get(), c_schema.get())); if (array->type_id() != arrow::Type::STRUCT) { - return Status::Invalid("cannot evaluate a row filter on a batch that is not a struct"); + return Status::Invalid("cannot read rows from a batch that is not a struct"); } + auto struct_array = checked_pointer_cast(array); std::shared_ptr row_id_array = - checked_pointer_cast(array)->GetFieldByName( - SpecialFields::RowId().Name()); + struct_array->GetFieldByName(SpecialFields::RowId().Name()); if (!row_id_array || row_id_array->type_id() != arrow::Type::INT64) { - return Status::Invalid(fmt::format("cannot evaluate a row filter without an int64 {}", - SpecialFields::RowId().Name())); - } - auto row_ids = checked_pointer_cast(row_id_array); - for (int64_t i = 0; i < row_ids->length(); ++i) { - int64_t row_id = row_ids->Value(i); - if (rows.Contains(row_id)) { - matching.Add(row_id); - } + return Status::Invalid( + fmt::format("cannot read rows without an int64 {}", SpecialFields::RowId().Name())); } + PAIMON_RETURN_NOT_OK( + consumer(*struct_array, *checked_pointer_cast(row_id_array))); } - return matching; + return Status::OK(); } } // namespace paimon diff --git a/src/paimon/core/table/source/filtered_row_id_reader.h b/src/paimon/core/table/source/filtered_row_id_reader.h index e70d5d26e..bd10f4d4f 100644 --- a/src/paimon/core/table/source/filtered_row_id_reader.h +++ b/src/paimon/core/table/source/filtered_row_id_reader.h @@ -20,24 +20,26 @@ #pragma once #include +#include #include #include #include +#include +#include "arrow/api.h" #include "paimon/common/predicate/predicate_filter.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/utils/roaring_bitmap64.h" +#include "paimon/utils/row_range_index.h" namespace paimon { -/// Evaluates a row filter on the data of the given row ids and returns those that satisfy it, -/// aligned with Java `FilteredRowIdReader`. Only the filter columns and `_ROW_ID` are read, so this -/// is the exact counterpart of a scalar global index lookup: a search can hand the result to its -/// index as include row ids without touching the index corpus. +/// Verifies a row filter against the data of the planned snapshot. class FilteredRowIdReader { public: /// @param root_path Root directory of the table. @@ -45,9 +47,7 @@ class FilteredRowIdReader { /// @param options Options of the table, merged with those of the search. /// @param partition_filter Predicate over the partition fields, or null for all partitions. /// @param filter The row filter; must not be null. - /// @param snapshot_id The snapshot to read. Once it has expired, a `scan.tag-name` in - /// `options` is read instead, as a tag may outlive its snapshot, and - /// must still refer to it. + /// @param snapshot_id The planned snapshot, which a tag may retain after expiry. /// @param executor Executor to read data on. /// @param pool Memory pool for reading data. FilteredRowIdReader(const std::string& root_path, @@ -67,7 +67,28 @@ class FilteredRowIdReader { const std::set& contributing_field_ids); /// Returns the subset of `rows` whose data satisfies the filter. - Result MatchingRowIds(const RoaringBitmap64& rows) const; + Result MatchingRowIds(RoaringBitmap64 rows) const; + Result MatchingRowIds(const RowRangeIndex& row_ranges) const; + + /// Loads the planned snapshot, including a copy retained by a tag on the same branch. + static Result LoadSnapshot(const CoreOptions& options, const std::string& root_path, + int64_t snapshot_id); + + using BatchConsumer = + std::function; + + /// Reads live rows of files intersecting `row_ranges` in the given snapshot. + /// The consumer must check row ids if a file reader returns rows outside the ranges. + /// `read_fields` excludes `_ROW_ID`, which is appended to each batch. + static Status ReadRows(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& partition_filter, + const Snapshot& snapshot, const RowRangeIndex& row_ranges, + const std::vector& read_fields, + const std::shared_ptr& predicate, + const std::shared_ptr& executor, + const std::shared_ptr& pool, const BatchConsumer& consumer); private: std::string root_path_; diff --git a/src/paimon/core/table/source/full_text_search_builder.cpp b/src/paimon/core/table/source/full_text_search_builder.cpp index 5f81c282f..98b51c3e9 100644 --- a/src/paimon/core/table/source/full_text_search_builder.cpp +++ b/src/paimon/core/table/source/full_text_search_builder.cpp @@ -33,11 +33,17 @@ #include "paimon/core/index/pk/primary_key_index_definitions.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_evolution_full_text_read.h" #include "paimon/core/table/source/data_evolution_full_text_scan.h" +#include "paimon/core/table/source/filtered_row_id_reader.h" +#include "paimon/core/tag/tag.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/format/file_format.h" @@ -52,19 +58,30 @@ namespace paimon { class FullTextSearchBuilder::Impl { public: - /// What a scan and a read of the search are created from. - struct ResolvedSearch { + class TimeTravelScan; + + struct LoadedTable { std::string root_path; - std::shared_ptr table_schema; + std::string branch; + std::shared_ptr schema; CoreOptions options; + + Result> TimeTravelSnapshot(bool for_worker) const; + }; + + struct ResolvedSearch { + LoadedTable table; + std::optional time_travel_snapshot; std::shared_ptr path_factory; std::shared_ptr partition_filter; std::shared_ptr filter; DataField text_field; }; - /// Loads the table and checks the settings shared by scans and reads. - Result Resolve() const; + Result LoadTable() const; + Result Bind(LoadedTable table, std::optional snapshot, + bool validate_data_filter) const; + Result ResolveForPlan() const; Status CheckLimit() const; @@ -85,6 +102,51 @@ class FullTextSearchBuilder::Impl { std::shared_ptr file_system_; }; +class FullTextSearchBuilder::Impl::TimeTravelScan : public FullTextScan { + public: + explicit TimeTravelScan(const Impl& builder) : builder_(builder) {} + + Result> CreatePlan() override { + PAIMON_ASSIGN_OR_RAISE(ResolvedSearch search, builder_.ResolveForPlan()); + return builder_.CreateScan(search)->CreatePlan(); + } + + private: + Impl builder_; +}; + +Result> FullTextSearchBuilder::Impl::LoadedTable::TimeTravelSnapshot( + bool for_worker) const { + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateStartupMode(options)); + if (std::optional tag_name = options.GetScanTagName()) { + TagManager tags(options.GetFileSystem(), root_path, branch); + PAIMON_ASSIGN_OR_RAISE(Tag tag, tags.GetOrThrow(*tag_name)); + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, tag.TrimToSnapshot()); + return std::optional(std::move(snapshot)); + } + SnapshotManager snapshots(options.GetFileSystem(), root_path, branch); + if (std::optional id = options.GetScanSnapshotId()) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, for_worker ? FilteredRowIdReader::LoadSnapshot( + options, root_path, *id) + : snapshots.LoadSnapshot(*id)); + return std::optional(std::move(snapshot)); + } + if (std::optional timestamp = options.GetScanTimestampMillis()) { + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, + snapshots.EarlierOrEqualTimeMillis(*timestamp)); + if (!snapshot && !for_worker) { + return Status::Invalid(fmt::format( + "There is currently no snapshot earlier than or equal to timestamp [{}]", + *timestamp)); + } + return snapshot; + } + if (options.ToMap().count(Options::SCAN_TIMESTAMP) > 0) { + return Status::Invalid(fmt::format("invalid {}", Options::SCAN_TIMESTAMP)); + } + return std::optional(); +} + namespace { /// Rejects a column with a primary-key full-text index, which Java searches with /// `PrimaryKeyFullTextScan` and `PrimaryKeyFullTextRead` instead. @@ -151,10 +213,10 @@ struct SearchFilters { std::shared_ptr data_filter; }; -/// Validates `filters` and splits their conjunction. Result ResolveFilters(const std::vector>& filters, const arrow::Schema& schema, - const std::vector& partition_keys) { + const std::vector& partition_keys, + bool validate_data_filter) { if (filters.empty()) { return SearchFilters(); } @@ -162,9 +224,6 @@ Result ResolveFilters(const std::vector partition_key_to_idx = ObjectUtils::CreateIdentifierToIndexMap(partition_keys); std::vector> partition_conjuncts; @@ -192,11 +251,16 @@ Result ResolveFilters(const std::vector FullTextSearchBuilder::Impl::Resolve() const { +Result FullTextSearchBuilder::Impl::LoadTable() const { if (!field_name_) { return Status::Invalid("Query must be set via SetQuery()"); } @@ -217,18 +281,26 @@ Result FullTextSearchBuilder::Impl: fmt::format("schema file not found in path {}, branch {}", root_path, branch)); } std::shared_ptr table_schema = latest_schema.value(); + PAIMON_RETURN_NOT_OK( + SchemaValidation::RejectQueryAuth(*table_schema, options_, "Full-text search")); std::map merged_options = table_schema->Options(); for (const auto& [key, value] : options_) { merged_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(merged_options, file_system_)); - if (options.GetFullTextIndexSearchMode() != GlobalIndexSearchMode::FAST) { - return Status::NotImplemented(fmt::format( - "full-text search only supports the fast {} (or {}), which searches only the rows " - "that an index covers; full and detail are not supported yet", - Options::FULL_TEXT_INDEX_SEARCH_MODE, Options::GLOBAL_INDEX_SEARCH_MODE)); - } + return LoadedTable{std::move(root_path), std::move(branch), std::move(table_schema), + std::move(options)}; +} +Result FullTextSearchBuilder::Impl::Bind( + LoadedTable table, std::optional snapshot, bool validate_data_filter) const { + auto& [root_path, branch, table_schema, options] = table; + if (snapshot && snapshot->SchemaId() != table_schema->Id()) { + SchemaManager schema_manager(options.GetFileSystem(), root_path, branch); + PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(snapshot->SchemaId())); + PAIMON_RETURN_NOT_OK( + DataEvolutionFullTextRead::BindSchemaOptions(*table_schema, options_, &options)); + } Result text_field = table_schema->GetField(field_name_.value()); if (!text_field.ok()) { return Status::Invalid( @@ -239,15 +311,8 @@ Result FullTextSearchBuilder::Impl: auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(SearchFilters search_filters, - ResolveFilters(filters_, *arrow_schema, table_schema->PartitionKeys())); - if (search_filters.data_filter && - options.GetScalarIndexSearchMode() != GlobalIndexSearchMode::FAST) { - return Status::NotImplemented(fmt::format( - "a full-text search with a row filter only supports the fast {} (or {}), which " - "evaluates only the rows that a scalar index covers; full and detail are not " - "supported yet", - Options::SCALAR_INDEX_SEARCH_MODE, Options::GLOBAL_INDEX_SEARCH_MODE)); - } + ResolveFilters(filters_, *arrow_schema, table_schema->PartitionKeys(), + validate_data_filter)); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, options.GetExternalPathWeights()); @@ -279,13 +344,17 @@ Result FullTextSearchBuilder::Impl: return Status::Invalid("invalid partition predicate, cannot cast to predicate filter"); } } - return ResolvedSearch{root_path, - table_schema, - std::move(options), - path_factory, - partition_filter, - search_filters.data_filter, - std::move(text_field).value()}; + return ResolvedSearch{ + std::move(table), std::move(snapshot), path_factory, + partition_filter, search_filters.data_filter, std::move(text_field).value()}; +} + +Result FullTextSearchBuilder::Impl::ResolveForPlan() + const { + PAIMON_ASSIGN_OR_RAISE(LoadedTable table, LoadTable()); + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, + table.TimeTravelSnapshot(/*for_worker=*/false)); + return Bind(std::move(table), std::move(snapshot), /*validate_data_filter=*/true); } Status FullTextSearchBuilder::Impl::CheckLimit() const { @@ -298,17 +367,19 @@ Status FullTextSearchBuilder::Impl::CheckLimit() const { std::unique_ptr FullTextSearchBuilder::Impl::CreateScan( const ResolvedSearch& search) const { return std::make_unique( - search.root_path, search.table_schema, search.options, search.path_factory, - search.partition_filter, search.filter, search.text_field, memory_pool_); + search.table.root_path, search.table.schema, search.time_travel_snapshot, + search.table.options, search.path_factory, search.partition_filter, search.filter, + search.text_field, executor_, memory_pool_); } Result> FullTextSearchBuilder::Impl::CreateRead( const ResolvedSearch& search) const { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - DataEvolutionFullTextRead::Create( - search.root_path, search.table_schema, search.options, - search.path_factory, search.partition_filter, search.filter, - search.text_field, query_, limit_.value(), executor_, memory_pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr read, + DataEvolutionFullTextRead::Create(search.table.root_path, search.table.schema, + search.table.options, options_, search.path_factory, + search.partition_filter, search.filter, search.text_field, + query_, limit_.value(), executor_, memory_pool_)); return std::unique_ptr(std::move(read)); } @@ -376,20 +447,39 @@ FullTextSearchBuilder& FullTextSearchBuilder::WithFileSystem( } Result> FullTextSearchBuilder::NewFullTextScan() const { - PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); - return impl_->CreateScan(search); + PAIMON_ASSIGN_OR_RAISE(Impl::LoadedTable table, impl_->LoadTable()); + Result> snapshot = table.TimeTravelSnapshot(/*for_worker=*/false); + if (snapshot.ok()) { + // Report schema and filter errors at construction even for selectors resolved again later. + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, + impl_->Bind(std::move(table), std::move(snapshot).value(), + /*validate_data_filter=*/true)); + if (!search.table.options.GetScanTagName() && + !search.table.options.GetScanTimestampMillis()) { + return impl_->CreateScan(search); + } + } + return std::make_unique(*impl_); } Result> FullTextSearchBuilder::NewFullTextRead() const { PAIMON_RETURN_NOT_OK(impl_->CheckLimit()); - PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); + PAIMON_ASSIGN_OR_RAISE(Impl::LoadedTable table, impl_->LoadTable()); + Result> snapshot = table.TimeTravelSnapshot(/*for_worker=*/true); + // A stored split can outlive its snapshot if the search needs no table data. + if (!snapshot.ok() && !snapshot.status().IsNotExist()) { + return snapshot.status(); + } + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, + impl_->Bind(std::move(table), std::move(snapshot).value_or(std::nullopt), + /*validate_data_filter=*/false)); return impl_->CreateRead(search); } Result> FullTextSearchBuilder::ExecuteLocal() const { PAIMON_RETURN_NOT_OK(impl_->CheckLimit()); // The scan and the read share one load of the table, so they see the same schema. - PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->ResolveForPlan()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, impl_->CreateRead(search)); std::unique_ptr scan = impl_->CreateScan(search); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); diff --git a/src/paimon/core/table/source/full_text_search_split.cpp b/src/paimon/core/table/source/full_text_search_split.cpp index 7f52cde4d..4cf6a97bd 100644 --- a/src/paimon/core/table/source/full_text_search_split.cpp +++ b/src/paimon/core/table/source/full_text_search_split.cpp @@ -19,6 +19,7 @@ #include "paimon/table/source/full_text_search_split.h" +#include #include #include #include @@ -37,8 +38,10 @@ #include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/index/index_file_meta_serializer.h" +#include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_evolution_full_text_scan.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" @@ -75,6 +78,68 @@ Status CheckEntryCount(const DataInputStream& in, int32_t count, int64_t min_ent return Status::OK(); } +Status CheckSerializableName(const std::string& value, const char* what) { + if (value.empty()) { + return Status::Invalid(fmt::format("full-text search split requires {}", what)); + } + if (value.size() > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("full-text search split requires {} of at most {} bytes, but got {} bytes", + what, std::numeric_limits::max(), value.size())); + } + return Status::OK(); +} + +Status CheckSplitMetadata(int64_t snapshot_id, const std::string& column_name, + std::optional schema_id) { + if (snapshot_id < Snapshot::FIRST_SNAPSHOT_ID) { + return Status::Invalid(fmt::format( + "full-text search split snapshot id must be positive, got {}", snapshot_id)); + } + if (schema_id && *schema_id < 0) { + return Status::Invalid("full-text search split requires a non-negative schema id"); + } + return CheckSerializableName(column_name, "a column name"); +} + +void WriteRowRanges(const std::vector& ranges, MemorySegmentOutputStream* out) { + out->WriteValue(static_cast(ranges.size())); + for (const Range& range : ranges) { + out->WriteValue(range.from); + out->WriteValue(range.to); + } +} + +Result> ReadRowRanges(DataInputStream* in, const std::string& entry) { + std::string entries = entry + "s"; + PAIMON_ASSIGN_OR_RAISE(int32_t range_count, in->ReadValue()); + PAIMON_RETURN_NOT_OK(CheckEntryCount(*in, range_count, kSearchRowRangeBytes, entries.c_str())); + std::vector ranges; + ranges.reserve(range_count); + for (int32_t i = 0; i < range_count; ++i) { + PAIMON_ASSIGN_OR_RAISE(int64_t from, in->ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int64_t to, in->ReadValue()); + if (from > to) { + return Status::Invalid( + fmt::format("invalid full-text search split, {} [{}, {}] ends before it starts", + entry, from, to)); + } + ranges.emplace_back(from, to); + } + return ranges; +} + +Status CheckFullyRead(const DataInputStream& in) { + PAIMON_ASSIGN_OR_RAISE(int64_t pos, in.GetPos()); + PAIMON_ASSIGN_OR_RAISE(int64_t stream_length, in.Length()); + if (pos != stream_length) { + return Status::Invalid( + fmt::format("invalid full-text search split, remaining {} bytes after deserializing", + stream_length - pos)); + } + return Status::OK(); +} + Status CheckBinaryRowLayout(const MemorySegment& segment, int64_t offset, int64_t size, const arrow::StructType& type); @@ -213,16 +278,16 @@ Result>> ReadIndexFiles( } } // namespace -Status IndexFullTextSearchSplitImpl::CheckScalarIndexFile( - const std::shared_ptr& file) { +Status IndexFullTextSearchSplitImpl::CheckIndexFile(const std::shared_ptr& file, + const char* kind) { if (!file || !file->GetGlobalIndexMeta()) { return Status::Invalid("full-text search split requires global index files"); } const GlobalIndexMeta& meta = file->GetGlobalIndexMeta().value(); if (!IsValidRowRange(meta.row_range_start, meta.row_range_end)) { - return Status::Invalid( - fmt::format("scalar index file {} covers an invalid row range [{}, {}]", - file->FileName(), meta.row_range_start, meta.row_range_end)); + return Status::Invalid(fmt::format("{} index file {} covers an invalid row range [{}, {}]", + kind, file->FileName(), meta.row_range_start, + meta.row_range_end)); } return Status::OK(); } @@ -231,16 +296,9 @@ Result> IndexFullTextSearchSplitIm int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, std::vector search_row_ranges, - std::vector> scalar_index_files) { - if (column_name.empty()) { - return Status::Invalid("full-text search split requires a column name"); - } - // `Serialize()` writes the column name with a 16-bit length. - if (column_name.size() > std::numeric_limits::max()) { - return Status::Invalid(fmt::format( - "full-text search split requires a column name of at most {} bytes, but got {} bytes", - std::numeric_limits::max(), column_name.size())); - } + const std::vector>& scalar_index_files, + std::optional schema_id) { + PAIMON_RETURN_NOT_OK(CheckSplitMetadata(snapshot_id, column_name, schema_id)); if (!IsValidRowRange(row_range_start, row_range_end)) { return Status::Invalid( fmt::format("invalid physical index range [{}, {}]", row_range_start, row_range_end)); @@ -292,12 +350,20 @@ Result> IndexFullTextSearchSplitIm previous.ToString(), range.ToString())); } } + std::vector> intersecting_scalar_files; for (const auto& file : scalar_index_files) { - PAIMON_RETURN_NOT_OK(CheckScalarIndexFile(file)); + PAIMON_RETURN_NOT_OK(CheckIndexFile(file, "scalar")); + const GlobalIndexMeta& meta = *file->GetGlobalIndexMeta(); + auto first = std::lower_bound( + search_row_ranges.begin(), search_row_ranges.end(), meta.row_range_start, + [](const Range& range, int64_t start) { return range.to < start; }); + if (first != search_row_ranges.end() && first->from <= meta.row_range_end) { + intersecting_scalar_files.push_back(file); + } } return std::shared_ptr(new IndexFullTextSearchSplitImpl( snapshot_id, column_name, row_range_start, row_range_end, std::move(full_text_index_files), - std::move(search_row_ranges), std::move(scalar_index_files))); + std::move(search_row_ranges), std::move(intersecting_scalar_files), schema_id)); } bool IndexFullTextSearchSplitImpl::operator==(const IndexFullTextSearchSplitImpl& other) const { @@ -306,7 +372,7 @@ bool IndexFullTextSearchSplitImpl::operator==(const IndexFullTextSearchSplitImpl } return snapshot_id_ == other.snapshot_id_ && column_name_ == other.column_name_ && row_range_start_ == other.row_range_start_ && row_range_end_ == other.row_range_end_ && - search_row_ranges_ == other.search_row_ranges_ && + schema_id_ == other.schema_id_ && search_row_ranges_ == other.search_row_ranges_ && ObjectUtils::Equal(full_text_index_files_, other.full_text_index_files_) && ObjectUtils::Equal(scalar_index_files_, other.scalar_index_files_); } @@ -314,35 +380,91 @@ bool IndexFullTextSearchSplitImpl::operator==(const IndexFullTextSearchSplitImpl std::string IndexFullTextSearchSplitImpl::ToString() const { return fmt::format( "IndexFullTextSearchSplit{{snapshotId={}, columnName={}, rowRangeStart={}, " - "rowRangeEnd={}, searchRowRanges={}, fullTextIndexFiles={}, scalarIndexFiles={}}}", + "rowRangeEnd={}, searchRowRanges={}, fullTextIndexFiles={}, scalarIndexFiles={}, " + "schemaId={}}}", snapshot_id_, column_name_, row_range_start_, row_range_end_, StringUtils::VectorToString(search_row_ranges_), StringUtils::VectorToString(full_text_index_files_), - StringUtils::VectorToString(scalar_index_files_)); + StringUtils::VectorToString(scalar_index_files_), + schema_id_ ? fmt::format("{}", *schema_id_) : "unknown"); +} + +Result> RawFullTextSearchSplitImpl::Create( + int64_t snapshot_id, const std::string& column_name, const std::string& index_type, + std::vector row_ranges, int32_t field_id, int64_t schema_id) { + PAIMON_RETURN_NOT_OK(CheckSplitMetadata(snapshot_id, column_name, schema_id)); + if (field_id < 0) { + return Status::Invalid("a raw full-text split requires a non-negative field id"); + } + PAIMON_RETURN_NOT_OK(CheckSerializableName(index_type, "an index type")); + if (row_ranges.empty()) { + return Status::Invalid("raw full-text search split requires at least one row range"); + } + for (size_t i = 0; i < row_ranges.size(); ++i) { + const Range& range = row_ranges[i]; + if (!IsValidRowRange(range.from, range.to)) { + return Status::Invalid( + fmt::format("invalid raw full-text search row range {}", range.ToString())); + } + if (i == 0) { + continue; + } + const Range& previous = row_ranges[i - 1]; + if (range.from <= previous.to || range.from - 1 == previous.to) { + return Status::Invalid( + fmt::format("row ranges {} and {} are not sorted, disjoint and non-adjacent", + previous.ToString(), range.ToString())); + } + } + return std::shared_ptr(new RawFullTextSearchSplitImpl( + snapshot_id, column_name, index_type, std::move(row_ranges), field_id, schema_id)); +} + +bool RawFullTextSearchSplitImpl::operator==(const RawFullTextSearchSplitImpl& other) const { + return snapshot_id_ == other.snapshot_id_ && column_name_ == other.column_name_ && + index_type_ == other.index_type_ && row_ranges_ == other.row_ranges_ && + field_id_ == other.field_id_ && schema_id_ == other.schema_id_; +} + +std::string RawFullTextSearchSplitImpl::ToString() const { + return fmt::format( + "RawFullTextSearchSplit{{snapshotId={}, columnName={}, indexType={}, rowRanges={}, " + "fieldId={}, schemaId={}}}", + snapshot_id_, column_name_, index_type_, StringUtils::VectorToString(row_ranges_), + field_id_, schema_id_); } Result FullTextSearchSplit::Serialize( const std::shared_ptr& split, const std::shared_ptr& pool) { - auto index_split = std::dynamic_pointer_cast(split); - if (!index_split) { - return Status::Invalid("invalid full-text search split, cannot cast to an index split"); - } MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); - out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); - out.WriteValue(IndexFullTextSearchSplitImpl::VERSION); - out.WriteValue(index_split->SnapshotId()); - out.WriteString(index_split->ColumnName()); - out.WriteValue(index_split->RowRangeStart()); - out.WriteValue(index_split->RowRangeEnd()); - const std::vector& search_row_ranges = index_split->SearchRowRanges(); - out.WriteValue(static_cast(search_row_ranges.size())); - for (const auto& range : search_row_ranges) { - out.WriteValue(range.from); - out.WriteValue(range.to); + if (auto raw_split = std::dynamic_pointer_cast(split)) { + out.WriteValue(RawFullTextSearchSplitImpl::MAGIC); + out.WriteValue(RawFullTextSearchSplitImpl::VERSION); + out.WriteValue(raw_split->SnapshotId()); + out.WriteString(raw_split->ColumnName()); + out.WriteValue(raw_split->FieldId()); + out.WriteString(raw_split->IndexType()); + WriteRowRanges(raw_split->RowRanges(), &out); + out.WriteValue(raw_split->SchemaId()); + } else if (auto index_split = std::dynamic_pointer_cast(split)) { + out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); + out.WriteValue(index_split->SchemaId() ? IndexFullTextSearchSplitImpl::VERSION + : 2); + out.WriteValue(index_split->SnapshotId()); + out.WriteString(index_split->ColumnName()); + out.WriteValue(index_split->RowRangeStart()); + out.WriteValue(index_split->RowRangeEnd()); + WriteRowRanges(index_split->SearchRowRanges(), &out); + IndexFileMetaSerializer serializer(pool); + PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->FullTextIndexFiles(), &out)); + PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->ScalarIndexFiles(), &out)); + if (index_split->SchemaId()) { + out.WriteValue(*index_split->SchemaId()); + } + } else { + return Status::Invalid( + "invalid full-text search split, cannot cast to an index or raw split"); } - IndexFileMetaSerializer serializer(pool); - PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->FullTextIndexFiles(), &out)); - PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->ScalarIndexFiles(), &out)); PAIMON_UNIQUE_PTR bytes = MemorySegmentUtils::CopyToBytes(out.Segments(), 0, out.CurrentSize(), pool.get()); return std::string(bytes->data(), bytes->size()); @@ -362,6 +484,25 @@ Result> FullTextSearchSplit::Deserialize( std::make_shared(buffer, static_cast(length)); DataInputStream in(input_stream); PAIMON_ASSIGN_OR_RAISE(int64_t magic, in.ReadValue()); + if (magic == RawFullTextSearchSplitImpl::MAGIC) { + PAIMON_ASSIGN_OR_RAISE(int32_t version, in.ReadValue()); + if (version != RawFullTextSearchSplitImpl::VERSION) { + return Status::Invalid( + fmt::format("unsupported RawFullTextSearchSplit version: {}", version)); + } + PAIMON_ASSIGN_OR_RAISE(int64_t snapshot_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(std::string column_name, in.ReadString()); + PAIMON_ASSIGN_OR_RAISE(int32_t field_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(std::string index_type, in.ReadString()); + PAIMON_ASSIGN_OR_RAISE(std::vector row_ranges, ReadRowRanges(&in, "row range")); + PAIMON_ASSIGN_OR_RAISE(int64_t schema_id, in.ReadValue()); + PAIMON_RETURN_NOT_OK(CheckFullyRead(in)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr split, + RawFullTextSearchSplitImpl::Create(snapshot_id, column_name, index_type, + std::move(row_ranges), field_id, schema_id)); + return std::shared_ptr(std::move(split)); + } if (magic != IndexFullTextSearchSplitImpl::MAGIC) { return Status::Invalid("invalid full-text search split, unknown magic number"); } @@ -374,39 +515,24 @@ Result> FullTextSearchSplit::Deserialize( PAIMON_ASSIGN_OR_RAISE(std::string column_name, in.ReadString()); PAIMON_ASSIGN_OR_RAISE(int64_t row_range_start, in.ReadValue()); PAIMON_ASSIGN_OR_RAISE(int64_t row_range_end, in.ReadValue()); - PAIMON_ASSIGN_OR_RAISE(int32_t range_count, in.ReadValue()); - PAIMON_RETURN_NOT_OK( - CheckEntryCount(in, range_count, kSearchRowRangeBytes, "search row ranges")); - std::vector search_row_ranges; - search_row_ranges.reserve(range_count); - for (int32_t i = 0; i < range_count; ++i) { - PAIMON_ASSIGN_OR_RAISE(int64_t from, in.ReadValue()); - PAIMON_ASSIGN_OR_RAISE(int64_t to, in.ReadValue()); - if (from > to) { - return Status::Invalid(fmt::format( - "invalid full-text search split, search row range [{}, {}] ends before it starts", - from, to)); - } - search_row_ranges.emplace_back(from, to); - } + PAIMON_ASSIGN_OR_RAISE(std::vector search_row_ranges, + ReadRowRanges(&in, "search row range")); PAIMON_ASSIGN_OR_RAISE(std::vector> full_text_index_files, ReadIndexFiles(&in, pool)); std::vector> scalar_index_files; if (version >= 2) { PAIMON_ASSIGN_OR_RAISE(scalar_index_files, ReadIndexFiles(&in, pool)); } - PAIMON_ASSIGN_OR_RAISE(int64_t pos, in.GetPos()); - PAIMON_ASSIGN_OR_RAISE(int64_t stream_length, in.Length()); - if (pos != stream_length) { - return Status::Invalid( - fmt::format("invalid full-text search split, remaining {} bytes after deserializing", - stream_length - pos)); + std::optional schema_id; + if (version >= 3) { + PAIMON_ASSIGN_OR_RAISE(schema_id, in.ReadValue()); } + PAIMON_RETURN_NOT_OK(CheckFullyRead(in)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr split, IndexFullTextSearchSplitImpl::Create( snapshot_id, column_name, row_range_start, row_range_end, std::move(full_text_index_files), std::move(search_row_ranges), - std::move(scalar_index_files))); + scalar_index_files, schema_id)); return std::shared_ptr(std::move(split)); } diff --git a/src/paimon/core/table/source/full_text_search_split_test.cpp b/src/paimon/core/table/source/full_text_search_split_test.cpp index f94673668..68e533dfb 100644 --- a/src/paimon/core/table/source/full_text_search_split_test.cpp +++ b/src/paimon/core/table/source/full_text_search_split_test.cpp @@ -38,6 +38,7 @@ #include "paimon/core/index/index_file_meta.h" #include "paimon/core/index/index_file_meta_serializer.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" @@ -79,14 +80,15 @@ std::string FromHex(const std::string& hex) { return bytes; } -std::string SplitBytes(const std::function& write_body, - const std::shared_ptr& pool) { +std::string Version2IndexSplitBytes( + const std::function& write_body, + const std::shared_ptr& pool) { const int64_t snapshot_id = 1; const int64_t row_range_start = 0; const int64_t row_range_end = 9; MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); - out.WriteValue(IndexFullTextSearchSplitImpl::VERSION); + out.WriteValue(2); out.WriteValue(snapshot_id); out.WriteString("f0"); out.WriteValue(row_range_start); @@ -98,6 +100,23 @@ std::string SplitBytes(const std::function& wr class FullTextSearchSplitTest : public ::testing::Test { protected: + void CheckInvalidSplitBytes(const std::string& bytes) const { + std::string trailing = bytes + std::string(1, '\0'); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(trailing.data(), trailing.size(), pool_), + "remaining 1 bytes after deserializing"); + for (size_t size = 0; size < bytes.size(); ++size) { + ASSERT_NOK(FullTextSearchSplit::Deserialize(bytes.data(), size, pool_)) << size; + } + for (const auto& snapshot_id : {FromHex("0000000000000000"), FromHex("FFFFFFFFFFFFFFFF")}) { + std::string invalid = bytes; + invalid.replace(sizeof(int64_t) + sizeof(int32_t), sizeof(int64_t), snapshot_id); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(invalid.data(), invalid.size(), pool_), + "snapshot id must be positive"); + } + } + std::shared_ptr pool_ = GetDefaultPool(); }; @@ -107,10 +126,11 @@ TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { /*extra_field_ids=*/std::vector({1, 3}), pool_), NewIndexFile("full-text-1.index", "full-text", Range(100, 199), /*index_field_id=*/2, /*extra_field_ids=*/std::vector({1, 3}), pool_)}; - ASSERT_OK_AND_ASSIGN(std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create( - /*snapshot_id=*/7, "content", /*row_range_start=*/100, - /*row_range_end=*/199, files, {Range(100, 119), Range(150, 199)})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create( + /*snapshot_id=*/7, "content", /*row_range_start=*/100, + /*row_range_end=*/199, files, {Range(100, 119), Range(150, 199)}, {}, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); const std::string index_files = FromHex( @@ -172,10 +192,18 @@ TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { ASSERT_TRUE(version_1_impl); ASSERT_EQ(*version_1_impl, *split) << version_1_impl->ToString() << " vs " << split->ToString(); + ASSERT_OK_AND_ASSIGN(split, + IndexFullTextSearchSplitImpl::Create(7, "content", 100, 199, files, + split->SearchRowRanges(), {}, 5)); + ASSERT_OK_AND_ASSIGN(bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_EQ(bytes, bytes_of_version("00000003") + FromHex("00000000" + "0000000000000005")); + std::string longest_column(std::numeric_limits::max(), 'c'); - ASSERT_OK_AND_ASSIGN(split, IndexFullTextSearchSplitImpl::Create( - /*snapshot_id=*/7, longest_column, /*row_range_start=*/100, - /*row_range_end=*/199, files, {Range(100, 199)})); + ASSERT_OK_AND_ASSIGN(split, + IndexFullTextSearchSplitImpl::Create( + /*snapshot_id=*/7, longest_column, /*row_range_start=*/100, + /*row_range_end=*/199, files, {Range(100, 199)}, {}, std::nullopt)); ASSERT_OK_AND_ASSIGN(bytes, FullTextSearchSplit::Serialize(split, pool_)); ASSERT_OK_AND_ASSIGN(result, FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); @@ -188,45 +216,47 @@ TEST_F(FullTextSearchSplitTest, TestSerializeScalarIndexFiles) { std::vector> files = {NewIndexFile( "full-text-0.index", "full-text", Range(0, 99), /*index_field_id=*/1, std::nullopt, pool_)}; std::vector> scalar_files = { - NewIndexFile("btree-0.index", "btree", Range(0, 49), /*index_field_id=*/2, std::nullopt, - pool_), - NewIndexFile("bitmap-0.index", "bitmap", Range(50, 149), /*index_field_id=*/3, std::nullopt, - pool_)}; - ASSERT_OK_AND_ASSIGN(std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 99, files, - {Range(0, 99)}, scalar_files)); - ASSERT_EQ(split->ScalarIndexFiles().size(), 2u); + NewIndexFile("btree-0.index", "btree", Range(0, 10), 2, std::nullopt, pool_), + NewIndexFile("gap.index", "btree", Range(20, 59), 2, std::nullopt, pool_), + NewIndexFile("bitmap-0.index", "bitmap", Range(69, 149), 3, std::nullopt, pool_), + NewIndexFile("outside.index", "bitmap", Range(100, 109), 3, std::nullopt, pool_)}; + const std::vector search_ranges = {Range(10, 19), Range(60, 69)}; + for (const auto& schema_id : {std::optional(), std::optional(5)}) { + SCOPED_TRACE(::testing::PrintToString(schema_id)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create( + 1, "f0", 0, 99, files, search_ranges, scalar_files, schema_id)); + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + auto impl = std::dynamic_pointer_cast(result); + ASSERT_TRUE(impl); + ASSERT_EQ(*impl, *split) << impl->ToString() << " vs " << split->ToString(); + ASSERT_EQ(impl->SchemaId(), schema_id); + ASSERT_EQ(impl->ScalarIndexFiles().size(), 2u); + ASSERT_EQ(impl->ScalarIndexFiles()[0]->FileName(), "btree-0.index"); + ASSERT_EQ(impl->ScalarIndexFiles()[1]->FileName(), "bitmap-0.index"); + ASSERT_NE(impl->ToString().find("scalarIndexFiles=["), std::string::npos); + ASSERT_NOK(FullTextSearchSplit::Deserialize(bytes.data(), bytes.size() - 1, pool_)); - ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); - auto impl = std::dynamic_pointer_cast(result); - ASSERT_TRUE(impl); - ASSERT_EQ(*impl, *split) << impl->ToString() << " vs " << split->ToString(); - ASSERT_EQ(impl->ScalarIndexFiles()[0]->FileName(), "btree-0.index"); - ASSERT_EQ(impl->ScalarIndexFiles()[1]->FileName(), "bitmap-0.index"); - ASSERT_NE(impl->ToString().find("scalarIndexFiles=["), std::string::npos); - - ASSERT_OK_AND_ASSIGN(std::shared_ptr no_scalar_files, - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 99, files, - {Range(0, 99)})); - ASSERT_FALSE(*no_scalar_files == *split); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr no_scalar_files, + IndexFullTextSearchSplitImpl::Create(1, "f0", 0, 99, files, search_ranges, + {scalar_files[1]}, schema_id)); + ASSERT_TRUE(no_scalar_files->ScalarIndexFiles().empty()); + ASSERT_FALSE(*no_scalar_files == *split); + } } TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsInvalidBytes) { std::vector> files = {NewIndexFile( "full-text-0.index", "full-text", Range(0, 9), /*index_field_id=*/1, std::nullopt, pool_)}; - ASSERT_OK_AND_ASSIGN( - std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, {Range(0, 9)})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, + {Range(0, 9)}, {}, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - std::string trailing = bytes + std::string(1, '\0'); - ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Deserialize(trailing.data(), trailing.size(), pool_), - "remaining 1 bytes after deserializing"); - for (size_t size = 0; size < bytes.size(); ++size) { - ASSERT_NOK(FullTextSearchSplit::Deserialize(bytes.data(), size, pool_)) << size; - } + ASSERT_NO_FATAL_FAILURE(CheckInvalidSplitBytes(bytes)); std::string too_many_scalar_files = bytes; too_many_scalar_files.replace(too_many_scalar_files.size() - sizeof(int32_t), sizeof(int32_t), FromHex("7FFFFFFF")); @@ -249,22 +279,22 @@ TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsInvalidBytes) { MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); out.WriteValue(IndexFullTextSearchSplitImpl::VERSION + 1); - std::string newer_version = ToBytes(&out, pool_); + std::string unsupported = ToBytes(&out, pool_); ASSERT_NOK_WITH_MSG( - FullTextSearchSplit::Deserialize(newer_version.data(), newer_version.size(), pool_), - "unsupported IndexFullTextSearchSplit version: 3"); + FullTextSearchSplit::Deserialize(unsupported.data(), unsupported.size(), pool_), + "unsupported IndexFullTextSearchSplit version: 4"); } ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Serialize(nullptr, pool_), - "cannot cast to an index split"); + "cannot cast to an index or raw split"); } TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptFileLayout) { std::vector> files = { NewIndexFile("full-text-0.index", "full-text", Range(0, 9), /*index_field_id=*/2, /*extra_field_ids=*/std::vector({1}), pool_)}; - ASSERT_OK_AND_ASSIGN( - std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, {Range(0, 9)})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, + {Range(0, 9)}, {}, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); ASSERT_OK(FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); @@ -316,7 +346,7 @@ TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptFileLayout) { TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptEntries) { auto deserialize = [this](const std::function& write_body) { - std::string bytes = SplitBytes(write_body, pool_); + std::string bytes = Version2IndexSplitBytes(write_body, pool_); return FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_); }; auto write_one_range = [](MemorySegmentOutputStream* out) { @@ -387,9 +417,12 @@ TEST_F(FullTextSearchSplitTest, TestCreateValidatesArguments) { auto create = [&](const std::vector>& index_files, const std::vector& search_row_ranges) { return IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 10, 19, index_files, - search_row_ranges); + search_row_ranges, {}, std::nullopt); }; ASSERT_OK(create(files, {Range(10, 11), Range(13, 19)})); + ASSERT_NOK_WITH_MSG( + IndexFullTextSearchSplitImpl::Create(1, "f0", 10, 19, files, {Range(10, 19)}, {}, -1), + "non-negative schema id"); ASSERT_NOK_WITH_MSG(create({}, {Range(10, 19)}), "at least one index file"); ASSERT_NOK_WITH_MSG(create(files, {}), "at least one search row range"); ASSERT_NOK_WITH_MSG(create(files, {Range(9, 19)}), "outside physical index range [10, 19]"); @@ -418,27 +451,145 @@ TEST_F(FullTextSearchSplitTest, TestCreateValidatesArguments) { "full-text", "full-text-3.index", /*file_size=*/128, /*row_count=*/10, /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt); ASSERT_NOK_WITH_MSG(create({no_global_meta}, {Range(10, 19)}), "global index files"); - ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 10, 19, files, - {Range(10, 19)}, {no_global_meta}), - "global index files"); ASSERT_NOK_WITH_MSG( - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "", 10, 19, files, {Range(10, 19)}), - "requires a column name"); + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 10, 19, files, + {Range(10, 19)}, {no_global_meta}, std::nullopt), + "global index files"); + ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "", 10, 19, files, + {Range(10, 19)}, {}, std::nullopt), + "requires a column name"); std::string too_long_column(std::numeric_limits::max() + 1, 'c'); - ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, too_long_column, 10, - 19, files, {Range(10, 19)}), - "column name of at most 65535 bytes, but got 65536 bytes"); + ASSERT_NOK_WITH_MSG( + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, too_long_column, 10, 19, files, + {Range(10, 19)}, {}, std::nullopt), + "column name of at most 65535 bytes, but got 65536 bytes"); std::vector> invalid_ranges = { {-1, 19}, {19, 10}, {0, std::numeric_limits::max()}}; for (const auto& [start, end] : invalid_ranges) { - ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", start, - end, files, {Range(10, 19)}), - "invalid physical index range"); + ASSERT_NOK_WITH_MSG( + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", start, end, files, + {Range(10, 19)}, {}, std::nullopt), + "invalid physical index range"); ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create( /*snapshot_id=*/1, "f0", 10, 19, files, {Range(10, 19)}, - {NewScalarIndexFile(start, end, pool_)}), + {NewScalarIndexFile(start, end, pool_)}, std::nullopt), "scalar index file btree-0.index covers an invalid row range"); } } +TEST_F(FullTextSearchSplitTest, TestRawSplitSerializeRoundTrip) { + const std::vector ranges = {Range(0, 4), Range(10, 19)}; + const std::string range_bytes = FromHex( + "00000002" + "0000000000000000" + "0000000000000004" + "000000000000000A" + "0000000000000013"); + const std::string bytes = "PFTRAWS1" + FromHex("0000000100000000000000070002") + "f0" + + FromHex("000000030009") + "full-text" + range_bytes + + FromHex("0000000000000005"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + auto split = std::dynamic_pointer_cast(result); + ASSERT_TRUE(split); + ASSERT_EQ(split->SnapshotId(), 7); + ASSERT_EQ(split->ColumnName(), "f0"); + ASSERT_EQ(split->IndexType(), "full-text"); + ASSERT_EQ(split->RowRanges(), ranges); + ASSERT_EQ(split->FieldId(), 3); + ASSERT_EQ(split->SchemaId(), 5); + ASSERT_OK_AND_ASSIGN(std::string serialized, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_EQ(serialized, bytes); + ASSERT_OK_AND_ASSIGN(std::shared_ptr created, + RawFullTextSearchSplitImpl::Create(7, "f0", "full-text", ranges, 3, 5)); + ASSERT_EQ(*created, *split); + ASSERT_OK_AND_ASSIGN(std::shared_ptr other_type, + RawFullTextSearchSplitImpl::Create(7, "f0", "lucene-fts", ranges, 3, 5)); + ASSERT_FALSE(*other_type == *split); +} + +TEST_F(FullTextSearchSplitTest, TestRawSplitCreateValidatesArguments) { + ASSERT_NOK_WITH_MSG( + RawFullTextSearchSplitImpl::Create(1, "f0", "full-text", {Range(0, 4)}, 0, -1), + "non-negative schema id"); + ASSERT_NOK_WITH_MSG( + RawFullTextSearchSplitImpl::Create(1, "f0", "full-text", {Range(0, 4)}, -1, 0), + "non-negative field id"); + auto create = [](const std::vector& row_ranges) { + return RawFullTextSearchSplitImpl::Create(1, "f0", "full-text", row_ranges, 0, 0); + }; + ASSERT_OK(create({Range(0, 0), Range(2, std::numeric_limits::max() - 1)})); + ASSERT_NOK_WITH_MSG(create({}), "at least one row range"); + ASSERT_NOK_WITH_MSG(create({Range(-1, 3)}), "invalid raw full-text search row range"); + ASSERT_NOK_WITH_MSG(create({Range(0, std::numeric_limits::max())}), + "invalid raw full-text search row range"); + ASSERT_NOK_WITH_MSG(create({Range(5, 9), Range(0, 3)}), + "not sorted, disjoint and non-adjacent"); + ASSERT_NOK_WITH_MSG(create({Range(0, 4), Range(4, 9)}), + "not sorted, disjoint and non-adjacent"); + ASSERT_NOK_WITH_MSG(create({Range(0, 4), Range(5, 9)}), + "not sorted, disjoint and non-adjacent"); + ASSERT_NOK_WITH_MSG(RawFullTextSearchSplitImpl::Create(1, "", "full-text", {Range(0, 4)}, 0, 0), + "requires a column name"); + ASSERT_NOK_WITH_MSG(RawFullTextSearchSplitImpl::Create(1, "f0", "", {Range(0, 4)}, 0, 0), + "requires an index type"); + std::string too_long(std::numeric_limits::max() + 1, 'c'); + ASSERT_NOK_WITH_MSG(RawFullTextSearchSplitImpl::Create(1, "f0", too_long, {Range(0, 4)}, 0, 0), + "index type of at most 65535 bytes, but got 65536 bytes"); +} + +TEST_F(FullTextSearchSplitTest, TestRawSplitDeserializeRejectsInvalidBytes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + RawFullTextSearchSplitImpl::Create(1, "f0", "full-text", + {Range(0, 4), Range(10, 19)}, 0, 5)); + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_NO_FATAL_FAILURE(CheckInvalidSplitBytes(bytes)); + auto write_header = [](MemorySegmentOutputStream* out, int32_t version) { + out->WriteValue(RawFullTextSearchSplitImpl::MAGIC); + out->WriteValue(version); + out->WriteValue(1); + out->WriteString("f0"); + out->WriteValue(0); + out->WriteString("full-text"); + }; + for (int32_t version : {0, 2, 3}) { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + write_header(&out, version); + std::string newer_version = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(newer_version.data(), newer_version.size(), pool_), + "unsupported RawFullTextSearchSplit version"); + } + { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + write_header(&out, RawFullTextSearchSplitImpl::VERSION); + out.WriteValue(std::numeric_limits::max()); + std::string too_many_ranges = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(too_many_ranges.data(), too_many_ranges.size(), pool_), + "2147483647 row ranges exceed the remaining 0 bytes"); + } + { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + write_header(&out, RawFullTextSearchSplitImpl::VERSION); + out.WriteValue(1); + out.WriteValue(9); + out.WriteValue(0); + std::string reversed_range = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(reversed_range.data(), reversed_range.size(), pool_), + "row range [9, 0] ends before it starts"); + } + { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + write_header(&out, RawFullTextSearchSplitImpl::VERSION); + out.WriteValue(0); + out.WriteValue(5); + std::string no_ranges = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(no_ranges.data(), no_ranges.size(), pool_), + "at least one row range"); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/table/source/index_full_text_search_split_impl.h b/src/paimon/core/table/source/index_full_text_search_split_impl.h index eaaac04aa..32bc586f0 100644 --- a/src/paimon/core/table/source/index_full_text_search_split_impl.h +++ b/src/paimon/core/table/source/index_full_text_search_split_impl.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -37,9 +38,8 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { public: /// "PFTSPLT1" in ASCII. static constexpr int64_t MAGIC = 0x50465453504C5431L; - /// Version 2 appends the scalar index files that pre-filter rows, as version 2 of Java - /// `IndexFullTextSearchSplit` does. - static constexpr int32_t VERSION = 2; + /// Version 2 adds scalar index files. Version 3 adds the planning schema ID. + static constexpr int32_t VERSION = 3; /// Creates a split. /// @param snapshot_id The snapshot the split is planned from. @@ -52,18 +52,19 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { /// global index meta of the shard range. /// @param search_row_ranges The sorted, disjoint and non-adjacent row id ranges to rank, at /// least one and all within the shard range. - /// @param scalar_index_files The global index files that intersect the shard range and can - /// pre-filter its rows, each accepted by `CheckScalarIndexFile()`. + /// @param scalar_index_files Scalar indexes for the row filter. The split keeps the files + /// that intersect its assigned search ranges. + /// @param schema_id The schema used when planning, absent from versions 1 and 2. /// @return The split, or `Invalid` if an argument breaks these rules. static Result> Create( int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, std::vector search_row_ranges, - std::vector> scalar_index_files = {}); + const std::vector>& scalar_index_files, + std::optional schema_id); - /// Checks that `file` has a global index meta whose row range is not negative, is ordered, and - /// has a row count that fits in int64, as a split requires of its scalar index files. - static Status CheckScalarIndexFile(const std::shared_ptr& file); + /// Checks that the global index range is ordered, non-negative and has a representable count. + static Status CheckIndexFile(const std::shared_ptr& file, const char* kind); int64_t SnapshotId() const override { return snapshot_id_; @@ -93,6 +94,10 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { return scalar_index_files_; } + std::optional SchemaId() const { + return schema_id_; + } + bool operator==(const IndexFullTextSearchSplitImpl& other) const; std::string ToString() const; @@ -102,14 +107,16 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, std::vector search_row_ranges, - std::vector> scalar_index_files) + std::vector> scalar_index_files, + std::optional schema_id) : snapshot_id_(snapshot_id), column_name_(column_name), row_range_start_(row_range_start), row_range_end_(row_range_end), full_text_index_files_(std::move(full_text_index_files)), search_row_ranges_(std::move(search_row_ranges)), - scalar_index_files_(std::move(scalar_index_files)) {} + scalar_index_files_(std::move(scalar_index_files)), + schema_id_(schema_id) {} int64_t snapshot_id_; std::string column_name_; @@ -118,6 +125,7 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { std::vector> full_text_index_files_; std::vector search_row_ranges_; std::vector> scalar_index_files_; + std::optional schema_id_; }; } // namespace paimon diff --git a/src/paimon/core/table/source/raw_full_text_read_impl.cpp b/src/paimon/core/table/source/raw_full_text_read_impl.cpp new file mode 100644 index 000000000..977ba521b --- /dev/null +++ b/src/paimon/core/table/source/raw_full_text_read_impl.cpp @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/raw_full_text_read_impl.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "arrow/compute/api_vector.h" +#include "arrow/io/memory.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/math.h" +#include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/uuid.h" +#include "paimon/core/table/source/filtered_row_id_reader.h" +#include "paimon/core/utils/nested_projection_utils.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/utils/roaring_bitmap64.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon { +struct InMemoryGlobalIndexFiles::Storage { + explicit Storage(const std::shared_ptr& pool) : arrow_pool(GetArrowPool(pool)) {} + + std::shared_ptr arrow_pool; + std::mutex mutex; + std::map> files; +}; + +class InMemoryGlobalIndexFiles::IndexOutputStream : public OutputStream { + public: + IndexOutputStream(const std::string& file_path, const std::shared_ptr& storage, + const std::shared_ptr& output) + : file_path_(file_path), storage_(storage), output_(output) {} + + Result Write(const char* buffer, int64_t size) override { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "write length")); + if (size > 0 && buffer == nullptr) { + return Status::Invalid("write buffer must not be null when size is positive"); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(output_->Write(buffer, size)); + return size; + } + + Status Flush() override { + return Status::OK(); + } + + Result GetPos() const override { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(int64_t position, output_->Tell()); + return position; + } + + Result GetUri() const override { + return file_path_; + } + + Status Close() override { + if (output_->closed()) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr bytes, output_->Finish()); + std::lock_guard lock(storage_->mutex); + storage_->files[file_path_] = std::move(bytes); + return Status::OK(); + } + + private: + std::string file_path_; + std::shared_ptr storage_; + std::shared_ptr output_; +}; + +namespace { +class SharedBytesInputStream : public ByteArrayInputStream { + public: + SharedBytesInputStream(const std::shared_ptr& pool, + const std::shared_ptr& bytes, + const std::string& file_path) + : ByteArrayInputStream(bytes->size() == 0 ? "" : bytes->data_as(), bytes->size()), + pool_(pool), + bytes_(bytes), + file_path_(file_path) {} + + Result GetUri() const override { + return file_path_; + } + + private: + // Arrow buffers free their memory through this pool, which must outlive bytes_. + std::shared_ptr pool_; + std::shared_ptr bytes_; + std::string file_path_; +}; + +std::shared_ptr EmptyResult() { + return std::make_shared(RoaringBitmap64(), std::vector()); +} +} // namespace + +InMemoryGlobalIndexFiles::InMemoryGlobalIndexFiles(const std::shared_ptr& pool) + : storage_(std::make_shared(pool)) {} + +Result InMemoryGlobalIndexFiles::NewPath(const std::string& prefix) const { + std::string uuid; + if (!UUID::Generate(&uuid)) { + return Status::IOError("failed to generate a UUID for a temporary full-text index file"); + } + return fmt::format("memory://global-index/{}-{}.index", prefix, uuid); +} + +Result> InMemoryGlobalIndexFiles::NewOutputStream( + const std::string& file_path) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr output, + arrow::io::BufferOutputStream::Create(0, storage_->arrow_pool.get())); + return std::unique_ptr( + std::make_unique(file_path, storage_, output)); +} + +Result InMemoryGlobalIndexFiles::GetFileSize(const std::string& file_path) const { + std::lock_guard lock(storage_->mutex); + auto iter = storage_->files.find(file_path); + if (iter == storage_->files.end()) { + return Status::NotExist(fmt::format("in-memory index file {} does not exist", file_path)); + } + return iter->second->size(); +} + +Result> InMemoryGlobalIndexFiles::GetInputStream( + const std::string& file_path) const { + std::shared_ptr bytes; + { + std::lock_guard lock(storage_->mutex); + auto iter = storage_->files.find(file_path); + if (iter == storage_->files.end()) { + return Status::NotExist( + fmt::format("in-memory index file {} does not exist", file_path)); + } + bytes = iter->second; + } + return std::unique_ptr( + std::make_unique(storage_->arrow_pool, bytes, file_path)); +} + +RawFullTextReadImpl::RawFullTextReadImpl(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, + const DataField& text_field, const std::string& query, + int32_t limit, const std::shared_ptr& executor, + const std::shared_ptr& pool) + : root_path_(root_path), + table_schema_(table_schema), + options_(options), + partition_filter_(partition_filter), + filter_(filter), + text_field_(text_field), + query_(query), + limit_(limit), + executor_(executor), + pool_(pool) {} + +Result> RawFullTextReadImpl::Search( + int64_t snapshot_id, const std::string& index_type, const RowRangeIndex& raw_rows) const { + if (raw_rows.Ranges().empty()) { + return EmptyResult(); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, options_.ToMap())); + if (!indexer || !indexer->SupportsFullTextSearch()) { + return Status::NotImplemented(fmt::format( + "cannot search unindexed rows with index type {}, which has no registered global " + "indexer that supports full-text search", + index_type)); + } + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, + FilteredRowIdReader::LoadSnapshot(options_, root_path_, snapshot_id)); + auto index_schema = DataField::ConvertDataFieldsToArrowSchema({text_field_}); + auto files = std::make_shared(pool_); + std::shared_ptr writer; + + std::vector read_fields = {text_field_.Name()}; + if (filter_) { + std::set filter_fields; + PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(filter_, &filter_fields)); + for (const DataField& field : table_schema_->Fields()) { + if (field.Name() != text_field_.Name() && filter_fields.count(field.Name()) > 0) { + read_fields.push_back(field.Name()); + } + } + } + + std::shared_ptr arrow_pool = GetArrowPool(pool_); + std::shared_ptr bound_filter; + const int64_t first_row_id = raw_rows.Ranges().front().from; + const int64_t last_relative_id = raw_rows.Ranges().back().to - first_row_id; + // Lucene requires contiguous document ids. Dense ids also keep the row count within int64_t. + const bool use_relative_ids = + index_type != lucene::kIdentifier && last_relative_id < std::numeric_limits::max(); + std::vector document_row_ids; + RoaringBitmap64 matching_documents; + auto consumer = [&](const arrow::StructArray& batch, + const arrow::Int64Array& row_ids) -> Status { + std::shared_ptr text_array = batch.GetFieldByName(text_field_.Name()); + if (!text_array) { + return Status::Invalid( + fmt::format("unindexed rows are missing text column {}", text_field_.Name())); + } + PAIMON_ASSIGN_OR_RAISE(text_array, NestedProjectionUtils::AlignArrayToReadType( + text_array, arrow::utf8(), arrow_pool.get())); + std::vector accepted; + if (filter_) { + if (!bound_filter) { + const auto& struct_type = checked_cast(*batch.type()); + std::map field_indexes = + ObjectUtils::CreateIdentifierToIndexMap( + struct_type.fields(), + [](const std::shared_ptr& field) { return field->name(); }); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr bound_predicate, + PredicateUtils::CreatePickedFieldFilter(filter_, field_indexes)); + bound_filter = std::dynamic_pointer_cast(bound_predicate); + if (!bound_filter) { + return Status::Invalid(fmt::format( + "cannot evaluate row filter {} on unindexed rows", filter_->ToString())); + } + } + PAIMON_ASSIGN_OR_RAISE(accepted, bound_filter->Test(batch, arrow_pool.get())); + } + + arrow::BooleanBuilder selected(arrow_pool.get()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(selected.Reserve(batch.length())); + std::vector document_ids; + document_ids.reserve(batch.length()); + for (int64_t i = 0; i < batch.length(); ++i) { + int64_t row_id = row_ids.Value(i); + bool include = text_array->IsValid(i) && raw_rows.Intersects(row_id, row_id); + selected.UnsafeAppend(include); + if (!include) { + continue; + } + int64_t document_id = row_id - first_row_id; + if (!use_relative_ids) { + document_id = static_cast(document_row_ids.size()); + document_row_ids.push_back(row_id); + } + if (filter_ && accepted[i]) { + matching_documents.Add(document_id); + } + document_ids.push_back(document_id); + } + if (document_ids.empty()) { + return Status::OK(); + } + if (!writer) { + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*index_schema, &c_schema)); + ScopeGuard guard([&c_schema]() { ArrowSchemaRelease(&c_schema); }); + PAIMON_ASSIGN_OR_RAISE( + writer, indexer->CreateWriter(text_field_.Name(), &c_schema, files, pool_)); + } + if (static_cast(document_ids.size()) != batch.length()) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr mask, + selected.Finish()); + arrow::compute::ExecContext context(arrow_pool.get()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Datum filtered, + arrow::compute::Filter(text_array, mask, arrow::compute::FilterOptions::Defaults(), + &context)); + text_array = filtered.make_array(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr document_batch, + arrow::StructArray::Make({text_array}, index_schema->fields())); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*document_batch, &c_array)); + ScopeGuard guard([&c_array]() { ArrowArrayRelease(&c_array); }); + return writer->AddBatch(&c_array, std::move(document_ids)); + }; + PAIMON_RETURN_NOT_OK(FilteredRowIdReader::ReadRows( + root_path_, table_schema_, options_, partition_filter_, snapshot, raw_rows, read_fields, + /*predicate=*/nullptr, executor_, pool_, consumer)); + if (!writer || (filter_ && matching_documents.IsEmpty())) { + return EmptyResult(); + } + + PAIMON_ASSIGN_OR_RAISE(std::vector index_metas, writer->Finish()); + if (index_metas.empty()) { + return EmptyResult(); + } + const int64_t reader_row_count = + use_relative_ids ? last_relative_id + 1 : static_cast(document_row_ids.size()); + std::shared_ptr reader; + { + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*index_schema, &c_schema)); + ScopeGuard guard([&c_schema]() { ArrowSchemaRelease(&c_schema); }); + PAIMON_ASSIGN_OR_RAISE( + reader, indexer->CreateReader(&c_schema, files, index_metas, reader_row_count, + std::nullopt, pool_)); + } + std::optional include_documents; + if (filter_) { + include_documents = std::move(matching_documents); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + reader->VisitFullTextSearch(std::make_shared( + text_field_.Name(), query_, limit_, std::move(include_documents)))); + if (!result) { + return Status::NotImplemented( + fmt::format("global index type {} does not answer full-text searches", index_type)); + } + + std::vector> hits; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + result->CreateScoredIterator()); + while (iterator->HasNext()) { + std::pair hit = iterator->NextWithScore(); + if (hit.first < 0 || hit.first >= reader_row_count) { + return Status::Invalid(fmt::format("temporary {} index returned row {} outside [0, {})", + index_type, hit.first, reader_row_count)); + } + hits.emplace_back(use_relative_ids ? first_row_id + hit.first : document_row_ids[hit.first], + hit.second); + } + if (!use_relative_ids) { + std::sort(hits.begin(), hits.end(), + [](const auto& left, const auto& right) { return left.first < right.first; }); + } + RoaringBitmap64 row_ids; + std::vector scores; + scores.reserve(hits.size()); + for (const auto& [row_id, score] : hits) { + row_ids.Add(row_id); + scores.push_back(score); + } + return std::shared_ptr( + std::make_shared(std::move(row_ids), std::move(scores))); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/raw_full_text_read_impl.h b/src/paimon/core/table/source/raw_full_text_read_impl.h new file mode 100644 index 000000000..47f8c9f64 --- /dev/null +++ b/src/paimon/core/table/source/raw_full_text_read_impl.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/executor.h" +#include "paimon/global_index/global_index_result.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/global_index/io/global_index_file_writer.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate.h" +#include "paimon/result.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon { +/// Files become visible to readers when their output streams close. +class InMemoryGlobalIndexFiles : public GlobalIndexFileWriter, public GlobalIndexFileReader { + public: + explicit InMemoryGlobalIndexFiles(const std::shared_ptr& pool); + + Result NewPath(const std::string& prefix) const override; + + Result> NewOutputStream( + const std::string& file_path) const override; + + Result GetFileSize(const std::string& file_path) const override; + + Result> GetInputStream( + const std::string& file_path) const override; + + private: + struct Storage; + class IndexOutputStream; + + std::shared_ptr storage_; +}; + +class RawFullTextReadImpl { + public: + RawFullTextReadImpl(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, const DataField& text_field, + const std::string& query, int32_t limit, + const std::shared_ptr& executor, + const std::shared_ptr& pool); + + Result> Search(int64_t snapshot_id, + const std::string& index_type, + const RowRangeIndex& raw_rows) const; + + private: + std::string root_path_; + std::shared_ptr table_schema_; + CoreOptions options_; + std::shared_ptr partition_filter_; + std::shared_ptr filter_; + DataField text_field_; + std::string query_; + int32_t limit_; + std::shared_ptr executor_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/raw_full_text_search_split_impl.h b/src/paimon/core/table/source/raw_full_text_search_split_impl.h new file mode 100644 index 000000000..2768311e1 --- /dev/null +++ b/src/paimon/core/table/source/raw_full_text_search_split_impl.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/table/source/full_text_search_split.h" +#include "paimon/utils/range.h" + +namespace paimon { +/// Carries the index type so a raw split can be read independently of its plan's index splits. +class RawFullTextSearchSplitImpl : public RawFullTextSearchSplit { + public: + /// "PFTRAWS1" in ASCII. + static constexpr int64_t MAGIC = 0x5046545241575331L; + static constexpr int32_t VERSION = 1; + + static Result> Create( + int64_t snapshot_id, const std::string& column_name, const std::string& index_type, + std::vector row_ranges, int32_t field_id, int64_t schema_id); + + int64_t SnapshotId() const override { + return snapshot_id_; + } + + const std::string& ColumnName() const override { + return column_name_; + } + + const std::vector& RowRanges() const override { + return row_ranges_; + } + + const std::string& IndexType() const { + return index_type_; + } + + int32_t FieldId() const { + return field_id_; + } + + int64_t SchemaId() const { + return schema_id_; + } + + bool operator==(const RawFullTextSearchSplitImpl& other) const; + + std::string ToString() const; + + private: + RawFullTextSearchSplitImpl(int64_t snapshot_id, const std::string& column_name, + const std::string& index_type, std::vector row_ranges, + int32_t field_id, int64_t schema_id) + : snapshot_id_(snapshot_id), + column_name_(column_name), + index_type_(index_type), + row_ranges_(std::move(row_ranges)), + field_id_(field_id), + schema_id_(schema_id) {} + + int64_t snapshot_id_; + std::string column_name_; + std::string index_type_; + std::vector row_ranges_; + int32_t field_id_; + int64_t schema_id_; +}; + +} // namespace paimon diff --git a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp index 0df69ac95..c1c49a04a 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/rapidjson_util.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/full_text/full_text_defs.h" #include "paimon/memory/bytes.h" @@ -105,23 +106,30 @@ Result> FullTextGlobalIndexWriter::Cr keys.push_back(key.c_str()); values.push_back(value.c_str()); } - FtindexWriterPtr writer(paimon_ftindex_writer_open(keys.data(), values.data(), keys.size())); + PAIMON_ASSIGN_OR_RAISE(std::string file_path, file_writer->NewPath(kFileNamePrefix)); + auto open_writer = StringUtils::StartsWith(file_path, "memory://") + ? paimon_ftindex_writer_open_in_memory + : paimon_ftindex_writer_open; + FtindexWriterPtr writer(open_writer(keys.data(), values.data(), keys.size())); if (!writer) { return LastFtindexError( fmt::format("open full-text index writer for field {}", field_name)); } - return std::shared_ptr(new FullTextGlobalIndexWriter( - field_name, arrow_type, std::move(writer), file_writer, options, pool)); + return std::shared_ptr( + new FullTextGlobalIndexWriter(field_name, arrow_type, std::move(writer), + std::move(file_path), file_writer, options, pool)); } FullTextGlobalIndexWriter::FullTextGlobalIndexWriter( const std::string& field_name, const std::shared_ptr& arrow_type, - FtindexWriterPtr writer, const std::shared_ptr& file_writer, + FtindexWriterPtr writer, std::string file_path, + const std::shared_ptr& file_writer, const std::map& options, const std::shared_ptr& pool) : pool_(pool ? pool : GetDefaultPool()), field_name_(field_name), arrow_type_(arrow_type), writer_(std::move(writer)), + file_path_(std::move(file_path)), file_writer_(file_writer), options_(options), logger_(Logger::GetLogger("FullTextGlobalIndexWriter")) {} @@ -194,15 +202,14 @@ Result> FullTextGlobalIndexWriter::Finish() { Result FullTextGlobalIndexWriter::WriteIndex( PaimonFtindexWriterHandle* writer) const { - PAIMON_ASSIGN_OR_RAISE(std::string file_path, file_writer_->NewPath(kFileNamePrefix)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr out, - file_writer_->NewOutputStream(file_path)); + file_writer_->NewOutputStream(file_path_)); // Closes the stream on every error path, keeping the original error. ScopeGuard close_guard([&]() { Status status = out->Close(); if (!status.ok()) { PAIMON_LOG_WARN(logger_, "failed to close full-text index file %s: %s", - file_path.c_str(), status.ToString().c_str()); + file_path_.c_str(), status.ToString().c_str()); } }); OutputContext ctx; @@ -210,17 +217,17 @@ Result FullTextGlobalIndexWriter::WriteIndex( PaimonFtindexOutputFile output{static_cast(&ctx), &WriteToOutputStream, &FlushOutputStream}; if (paimon_ftindex_writer_write_index(writer, output) != 0) { - return LastFtindexError(fmt::format("write full-text index {}", file_path), ctx.status); + return LastFtindexError(fmt::format("write full-text index {}", file_path_), ctx.status); } PAIMON_RETURN_NOT_OK(out->Flush()); close_guard.Release(); PAIMON_RETURN_NOT_OK(out->Close()); - PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_path)); + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_path_)); std::string options_json; PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto metadata = std::make_shared(options_json, pool_.get()); - return GlobalIndexIOMeta(file_path, file_size, row_count_, metadata); + return GlobalIndexIOMeta(file_path_, file_size, row_count_, metadata); } } // namespace paimon::full_text diff --git a/src/paimon/global_index/full_text/full_text_global_index_writer.h b/src/paimon/global_index/full_text/full_text_global_index_writer.h index d10da2b7a..f433c2127 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_writer.h +++ b/src/paimon/global_index/full_text/full_text_global_index_writer.h @@ -54,7 +54,7 @@ class FullTextGlobalIndexWriter : public GlobalIndexWriter { private: FullTextGlobalIndexWriter(const std::string& field_name, const std::shared_ptr& arrow_type, - FtindexWriterPtr writer, + FtindexWriterPtr writer, std::string file_path, const std::shared_ptr& file_writer, const std::map& options, const std::shared_ptr& pool); @@ -65,6 +65,7 @@ class FullTextGlobalIndexWriter : public GlobalIndexWriter { std::string field_name_; std::shared_ptr arrow_type_; FtindexWriterPtr writer_; + std::string file_path_; std::shared_ptr file_writer_; std::map options_; /// Number of rows received, including null values. diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index 36548daea..be91347ef 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -63,6 +63,7 @@ if(PAIMON_ENABLE_LUCENE) ${LUCENE_INCLUDE_DIR} STATIC_LINK_LIBS paimon_shared + lucene test_utils_static ${PAIMON_LUCENE_INDEX_STATIC_LINK_LIBS} ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 23e495979..933aed05d 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -18,8 +18,6 @@ #include "paimon/global_index/lucene/lucene_global_index_writer.h" -#include - #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "lucene++/FileUtils.h" @@ -29,6 +27,7 @@ #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/uuid.h" #include "paimon/global_index/lucene/jieba_analyzer.h" #include "paimon/global_index/lucene/lucene_defs.h" @@ -42,35 +41,39 @@ namespace paimon::lucene { } while (0) LuceneGlobalIndexWriter::LuceneWriteContext::LuceneWriteContext( - const std::string& _tmp_index_path, const Lucene::FSDirectoryPtr& _lucene_dir, - const Lucene::IndexWriterPtr& _index_writer, const Lucene::DocumentPtr& _doc, - const Lucene::FieldPtr& _field) - : tmp_index_path(_tmp_index_path), - lucene_dir(_lucene_dir), - index_writer(_index_writer), - doc(_doc), - field(_field) {} + const std::string& output_path, const std::string& tmp_index_path, + const Lucene::DirectoryPtr& lucene_dir, const Lucene::IndexWriterPtr& index_writer, + const Lucene::DocumentPtr& doc, const Lucene::FieldPtr& field) + : output_path(output_path), + tmp_index_path(tmp_index_path), + lucene_dir(lucene_dir), + index_writer(index_writer), + doc(doc), + field(field) {} Result> LuceneGlobalIndexWriter::Create( const std::string& field_name, const std::shared_ptr& arrow_type, const std::shared_ptr& file_writer, const std::map& options, const std::shared_ptr& pool) { try { - std::string uuid; - if (!UUID::Generate(&uuid)) { - return Status::Invalid("generate uuid for lucene tmp path failed."); + PAIMON_ASSIGN_OR_RAISE(std::string output_path, file_writer->NewPath(kIdentifier)); + std::string tmp_path; + Lucene::DirectoryPtr lucene_dir; + if (StringUtils::StartsWith(output_path, "memory://")) { + lucene_dir = Lucene::newLucene(); + } else { + PAIMON_ASSIGN_OR_RAISE(std::string tmp_dir, OptionsUtils::GetValueFromMap( + options, kLuceneWriteTmpDir)); + std::string uuid; + if (!UUID::Generate(&uuid)) { + return Status::Invalid("generate uuid for lucene tmp path failed."); + } + tmp_path = PathUtil::JoinPath(tmp_dir, "paimon-lucene-" + uuid); + // FSDirectory mutates the lock prefix, so each writer needs its own lock factory. + lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(tmp_path), + Lucene::newLucene()); } - // get local tmp path - PAIMON_ASSIGN_OR_RAISE(std::string tmp_dir, OptionsUtils::GetValueFromMap( - options, std::string(kLuceneWriteTmpDir))); - std::string tmp_path = PathUtil::JoinPath(tmp_dir, "paimon-lucene-" + uuid); - - // FSDirectory calls Directory::setLockFactory(), which mutates lockPrefix. Use a separate - // factory per directory to avoid racing on the NoLockFactory singleton across writers. - auto lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(tmp_path), - Lucene::newLucene()); // TODO(xinyu.lxy): support other tokenizer - // open lucene index writer PAIMON_ASSIGN_OR_RAISE( bool omit_term_freq_and_positions, OptionsUtils::GetValueFromMap(options, kLuceneWriteOmitTermFreqAndPositions, false)); @@ -96,8 +99,9 @@ Result> LuceneGlobalIndexWriter::Create field->setOmitTermFreqAndPositions(omit_term_freq_and_positions); doc->add(field); return std::shared_ptr(new LuceneGlobalIndexWriter( - field_name, arrow_type, LuceneWriteContext(tmp_path, lucene_dir, writer, doc, field), - file_writer, options, pool)); + field_name, arrow_type, + LuceneWriteContext(output_path, tmp_path, lucene_dir, writer, doc, field), file_writer, + options, pool)); } catch (const std::exception& e) { return Status::Invalid( fmt::format("create lucene global index writer failed, with {} error.", e.what())); @@ -119,6 +123,9 @@ LuceneGlobalIndexWriter::LuceneGlobalIndexWriter( options_(options) {} LuceneGlobalIndexWriter::~LuceneGlobalIndexWriter() { + if (write_context_.tmp_index_path.empty()) { + return; + } try { [[maybe_unused]] bool ec = Lucene::FileUtils::removeDirectory( LuceneUtils::StringToWstring(write_context_.tmp_index_path)); @@ -167,9 +174,8 @@ Status LuceneGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array, return Status::OK(); } -Result LuceneGlobalIndexWriter::FlushIndexToFinal() { +Status LuceneGlobalIndexWriter::FlushIndexToFinal() { try { - // flush index to tmp dir if (write_context_.index_writer->numDocs() != row_id_) { return Status::Invalid( fmt::format("lucene writer row count {} mismatch paimon inner row count {}", @@ -178,19 +184,15 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { write_context_.index_writer->optimize(); write_context_.index_writer->close(); - // list tmp dir - auto tmp_file_names = write_context_.lucene_dir->listAll(); - PAIMON_ASSIGN_OR_RAISE(std::string index_file_path, file_writer_->NewPath(kIdentifier)); - // prepare output from file_writer + auto file_names = write_context_.lucene_dir->listAll(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, - file_writer_->NewOutputStream(index_file_path)); + file_writer_->NewOutputStream(write_context_.output_path)); DataOutputStream data_output_stream(out); PAIMON_RETURN_NOT_OK(data_output_stream.WriteValue(kVersion)); PAIMON_RETURN_NOT_OK( - data_output_stream.WriteValue(static_cast(tmp_file_names.size()))); - // read all data from index files and write to target output + data_output_stream.WriteValue(static_cast(file_names.size()))); auto buffer = std::make_shared(kDefaultReadBufferSize, pool_.get()); - for (const auto& wfile_name : tmp_file_names) { + for (const auto& wfile_name : file_names) { auto file_name = LuceneUtils::WstringToString(wfile_name); PAIMON_RETURN_NOT_OK( data_output_stream.WriteValue(static_cast(file_name.size()))); @@ -220,7 +222,7 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); write_context_.lucene_dir->close(); - return index_file_path; + return Status::OK(); } catch (const std::exception& e) { return Status::Invalid( fmt::format("finish for lucene global index writer failed, with {} error.", e.what())); @@ -231,8 +233,8 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { } Result> LuceneGlobalIndexWriter::Finish() { - PAIMON_ASSIGN_OR_RAISE(std::string index_file_path, FlushIndexToFinal()); - // prepare global index meta + PAIMON_RETURN_NOT_OK(FlushIndexToFinal()); + const std::string& index_file_path = write_context_.output_path; PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_path)); std::string options_json; PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.h b/src/paimon/global_index/lucene/lucene_global_index_writer.h index e11be99cd..115191090 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.h +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.h @@ -28,16 +28,17 @@ namespace paimon::lucene { class LuceneGlobalIndexWriter : public GlobalIndexWriter { public: struct LuceneWriteContext { - LuceneWriteContext(const std::string& _tmp_index_path, - const Lucene::FSDirectoryPtr& _lucene_dir, - const Lucene::IndexWriterPtr& _index_writer, - const Lucene::DocumentPtr& _doc, const Lucene::FieldPtr& _field); + LuceneWriteContext(const std::string& output_path, const std::string& tmp_index_path, + const Lucene::DirectoryPtr& lucene_dir, + const Lucene::IndexWriterPtr& index_writer, + const Lucene::DocumentPtr& doc, const Lucene::FieldPtr& field); LuceneWriteContext(LuceneWriteContext&&) = default; LuceneWriteContext& operator=(LuceneWriteContext&&) = default; + std::string output_path; std::string tmp_index_path; - Lucene::FSDirectoryPtr lucene_dir; + Lucene::DirectoryPtr lucene_dir; Lucene::IndexWriterPtr index_writer; Lucene::DocumentPtr doc; Lucene::FieldPtr field; @@ -62,7 +63,7 @@ class LuceneGlobalIndexWriter : public GlobalIndexWriter { const std::map& options, const std::shared_ptr& pool); - Result FlushIndexToFinal(); + Status FlushIndexToFinal(); private: std::shared_ptr pool_; diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 25a9eceb3..045642760 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -18,17 +18,21 @@ #include #include +#include #include #include #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "arrow/type.h" #include "fmt/format.h" #include "fmt/ranges.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_factory.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_options.h" +#include "paimon/common/global_index/scored_global_index_result_utils.h" #include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/global_index/union_global_index_reader.h" #include "paimon/common/table/special_fields.h" @@ -36,6 +40,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/deletion_vector.h" #include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/schema/schema_manager.h" @@ -43,6 +48,8 @@ #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/filtered_row_id_reader.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" +#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" @@ -173,6 +180,58 @@ class NestedFieldsIndexFactory : public GlobalIndexerFactory { REGISTER_PAIMON_FACTORY(NestedFieldsIndexFactory); #endif +#ifdef PAIMON_ENABLE_FULL_TEXT +class ExtraFieldFullTextIndexer : public GlobalIndexer { + public: + Result>> GetExtraFieldNames() const override { + return std::optional>({"f1"}); + } + + Result> CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const override { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get("full-text", {})); + return indexer->CreateWriter(field_name, arrow_schema, file_writer, pool); + } + + Result> CreateReader( + ::ArrowSchema* c_arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, + const std::shared_ptr& pool) const override { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(c_arrow_schema)); + auto text_schema = arrow::schema({schema->field(0)}); + ::ArrowSchema c_text_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*text_schema, &c_text_schema)); + ScopeGuard guard([&c_text_schema]() { ArrowSchemaRelease(&c_text_schema); }); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get("full-text", {})); + return indexer->CreateReader(&c_text_schema, file_reader, files, total_row_count, + row_ranges, pool); + } + + bool SupportsFullTextSearch() const override { + return true; + } +}; + +class ExtraFieldFullTextIndexerFactory : public GlobalIndexerFactory { + public: + const char* Identifier() const override { + return "extra-field-full-text-test-global"; + } + + Result> Create( + const std::map& options) const override { + return std::make_unique(); + } +}; +REGISTER_PAIMON_FACTORY(ExtraFieldFullTextIndexerFactory); +#endif + } // namespace using ScoredRows = std::vector>; @@ -336,9 +395,10 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter std::unique_ptr NewFullTextSearchBuilder( const std::string& table_path, const std::string& query, int32_t limit, - const std::map& options = {}) const { + const std::map& options = {}, + const std::string& column = "f0") const { auto builder = std::make_unique(table_path); - builder->SetQuery("f0", query).SetLimit(limit).SetOptions(options).WithFileSystem(fs_); + builder->SetQuery(column, query).SetLimit(limit).SetOptions(options).WithFileSystem(fs_); return builder; } @@ -416,6 +476,61 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter return shards; } + Result> PlanRawRanges( + const std::string& table_path, const std::string& query, + const std::map& options, + const std::vector>& partition_filters = {}, + const std::shared_ptr& filter = nullptr) const { + std::unique_ptr builder = + NewFullTextSearchBuilder(table_path, query, 1, options); + builder->SetPartitionFilter(partition_filters).WithFilter(filter); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, builder->NewFullTextScan()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + std::vector raw_ranges; + int32_t raw_splits = 0; + for (const auto& split : plan->Splits()) { + auto raw_split = std::dynamic_pointer_cast(split); + if (!raw_split) { + continue; + } + if (++raw_splits > 1 || raw_split->ColumnName() != "f0" || + raw_split->SnapshotId() != plan->SnapshotId()) { + return Status::Invalid("unexpected raw full-text search split"); + } + raw_ranges = raw_split->RowRanges(); + } + return raw_ranges; + } + + Result> ReadSerializedSplits( + const std::string& table_path, const std::string& query, int32_t limit, + const std::map& options, + const std::shared_ptr& filter = nullptr, + const std::vector>& partitions = {}, + const std::string& column = "f0") const { + auto new_builder = [&]() { + auto builder = NewFullTextSearchBuilder(table_path, query, limit, options, column); + builder->SetPartitionFilter(partitions).WithFilter(filter); + return builder; + }; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + new_builder()->NewFullTextScan()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + std::vector> partial_results; + for (const auto& split : plan->Splits()) { + PAIMON_ASSIGN_OR_RAISE(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr received_split, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + new_builder()->NewFullTextRead()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partial_result, + read->Read({received_split})); + partial_results.push_back(std::move(partial_result)); + } + return ScoredGlobalIndexResultUtils::MergeTopK(partial_results, limit); + } + void CheckTableSearchMatchesReader(const std::string& table_path, const std::string& query, const std::shared_ptr& reader, int32_t max_limit) const { @@ -672,6 +787,11 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_FALSE(unreadable_snapshot_result.ok()); ASSERT_EQ(unreadable_snapshot_result.status().ToString().find("no longer exists"), std::string::npos); + auto worker_builder = NewFullTextSearchBuilder( + table_path, query, 100, + {{Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_before_deletion)}}); + ASSERT_NOK_WITH_MSG(worker_builder->NewFullTextRead(), + unreadable_snapshot_result.status().message()); ASSERT_OK(file_system->Delete(snapshot_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr index_result, @@ -719,13 +839,18 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, schema->field_names(), AppleRows(fields, 0, 5))); ASSERT_OK(Commit(table_path, commit_msgs1)); - ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, schema->field_names(), - AppleRows(fields, 5, 10))); - ASSERT_OK(Commit(table_path, commit_msgs2)); std::string query = R"({"match":{"query":"apple"}})"; - + for (const char* mode : {"fast", "full", "detail"}) { + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTable(table_path, query, 10, {}, + {{Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}})); + ASSERT_TRUE(rows.empty()); + } ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", index_type, index_options, Range(0, 4))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, schema->field_names(), + AppleRows(fields, 5, 10))); + ASSERT_OK(Commit(table_path, commit_msgs2)); ASSERT_OK_AND_ASSIGN( std::unique_ptr first_shard_scan, NewFullTextSearchBuilder(table_path, query, /*limit=*/10)->NewFullTextScan()); @@ -733,19 +858,105 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter first_shard_scan->CreatePlan()); ASSERT_TRUE(first_shard_plan->SnapshotId()); ASSERT_EQ(first_shard_plan->Splits().size(), 1u); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr raw_split, + RawFullTextSearchSplitImpl::Create(first_shard_plan->SnapshotId().value(), "f0", + index_type, {Range(0, 9)}, 0, 0)); + auto worker_options = index_options; + worker_options.erase("lucene-fts.write.tmp.directory"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr top_one_read, + NewFullTextSearchBuilder(table_path, query, 1, worker_options)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw_result, + top_one_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows raw_top_row, ToScoredRows(raw_result)); + ASSERT_EQ(raw_top_row.size(), 1u); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr top_ten_read, + NewFullTextSearchBuilder(table_path, query, 10, worker_options)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(raw_result, top_ten_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows all_raw_rows, ToScoredRows(raw_result)); + ASSERT_EQ(all_raw_rows.size(), 10u); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr raw_prefix, + RawFullTextSearchSplitImpl::Create(first_shard_plan->SnapshotId().value(), "f0", + index_type, {Range(0, 5)}, 0, 0)); + ASSERT_OK_AND_ASSIGN(raw_split, RawFullTextSearchSplitImpl::Create( + first_shard_plan->SnapshotId().value(), "f0", + index_type, {Range(3, int64_t{1} << 40)}, 0, 0)); + const std::vector> overlapping_splits = { + raw_split, first_shard_plan->Splits().front(), raw_prefix}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr overlapping_result, + top_ten_read->Read(overlapping_splits)); + ASSERT_OK_AND_ASSIGN(ScoredRows overlapping_rows, ToScoredRows(overlapping_result)); + ASSERT_EQ(overlapping_rows, all_raw_rows); + ASSERT_OK_AND_ASSIGN(overlapping_result, top_one_read->Read(overlapping_splits)); + ASSERT_OK_AND_ASSIGN(ScoredRows overlapping_top_row, ToScoredRows(overlapping_result)); + ASSERT_EQ(overlapping_top_row, raw_top_row); + ASSERT_OK_AND_ASSIGN(raw_result, top_ten_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows raw_suffix, ToScoredRows(raw_result)); ASSERT_OK_AND_ASSIGN(ScoredRows indexed_rows, SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, {{Options::FULL_TEXT_INDEX_SEARCH_MODE, "fast"}, {Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); - ASSERT_EQ(indexed_rows.size(), 5u); - for (const auto& [row_id, score] : indexed_rows) { - ASSERT_LT(row_id, 5) << score; - } - for (const char* mode : {"full", "detail"}) { - ASSERT_NOK_WITH_MSG( - SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, - {{Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}), - "full and detail are not supported yet"); + ASSERT_EQ(RowIdsOf(indexed_rows), std::vector({0, 1, 2, 3, 4})); + ScoredRows expected_overlap(indexed_rows.begin(), indexed_rows.begin() + 3); + expected_overlap.insert(expected_overlap.end(), raw_suffix.begin(), raw_suffix.end()); + ASSERT_OK_AND_ASSIGN(overlapping_result, + top_ten_read->Read({raw_split, first_shard_plan->Splits().front()})); + ASSERT_OK_AND_ASSIGN(overlapping_rows, ToScoredRows(overlapping_result)); + ASSERT_EQ(overlapping_rows, expected_overlap); + ASSERT_OK_AND_ASSIGN(overlapping_result, + top_one_read->Read({first_shard_plan->Splits().front(), raw_split})); + ASSERT_OK_AND_ASSIGN(overlapping_top_row, ToScoredRows(overlapping_result)); + ASSERT_EQ(overlapping_top_row.size(), 1u); + const auto& best = *std::max_element( + expected_overlap.begin(), expected_overlap.end(), + [](const auto& left, const auto& right) { return left.second < right.second; }); + ASSERT_EQ(overlapping_top_row.front().second, best.second); + ASSERT_NE(std::find(expected_overlap.begin(), expected_overlap.end(), + overlapping_top_row.front()), + expected_overlap.end()); + ASSERT_OK_AND_ASSIGN( + raw_split, RawFullTextSearchSplitImpl::Create( + first_shard_plan->SnapshotId().value(), "f0", index_type, + {Range(0, 4), Range(7, std::numeric_limits::max())}, 0, 0)); + ASSERT_OK_AND_ASSIGN(raw_result, top_ten_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows sparse_rows, ToScoredRows(raw_result)); + ASSERT_EQ(RowIdsOf(sparse_rows), std::vector({0, 1, 2, 3, 4, 7, 8, 9})); + auto sparse_builder = NewFullTextSearchBuilder(table_path, query, 1, worker_options); + sparse_builder->WithFilter( + PredicateBuilder::In(1, "f1", FieldType::INT, {Literal(5), Literal(7)})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr sparse_read, + sparse_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(raw_result, sparse_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows filtered_sparse_rows, ToScoredRows(raw_result)); + auto expected_sparse_row = std::find_if(sparse_rows.begin(), sparse_rows.end(), + [](const auto& row) { return row.first == 7; }); + ASSERT_NE(expected_sparse_row, sparse_rows.end()); + ASSERT_EQ(filtered_sparse_rows, ScoredRows({*expected_sparse_row})); + const std::vector> raw_modes = { + {{Options::FULL_TEXT_INDEX_SEARCH_MODE, "full"}}, + {{Options::FULL_TEXT_INDEX_SEARCH_MODE, "detail"}}, + {{Options::GLOBAL_INDEX_SEARCH_MODE, "full"}}}; + std::vector raw_mode_rows; + std::vector raw_mode_top_rows; + for (std::map search_options : raw_modes) { + search_options.insert(index_options.begin(), index_options.end()); + ASSERT_OK_AND_ASSIGN(std::vector raw_ranges, + PlanRawRanges(table_path, query, search_options)); + ASSERT_EQ(raw_ranges, std::vector({Range(5, 9)})); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTable(table_path, query, 10, {}, search_options)); + ASSERT_EQ(RowIdsOf(rows), std::vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + raw_mode_rows.push_back(std::move(rows)); + ASSERT_OK_AND_ASSIGN(ScoredRows top_rows, + SearchTable(table_path, query, 3, {}, search_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr received_result, + ReadSerializedSplits(table_path, query, 3, search_options)); + ASSERT_OK_AND_ASSIGN(ScoredRows received_rows, ToScoredRows(received_result)); + ASSERT_EQ(received_rows, top_rows); + raw_mode_top_rows.push_back(std::move(top_rows)); } ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", index_type, index_options, Range(5, 9))); @@ -753,6 +964,20 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK_AND_ASSIGN(PlannedShards shards, PlanShards(table_path, query)); ASSERT_EQ(shards, PlannedShards({{Range(0, 4), {Range(0, 4)}}, {Range(5, 9), {Range(5, 9)}}})); + ASSERT_OK_AND_ASSIGN(ScoredRows fully_indexed_rows, SearchTable(table_path, query, 10)); + for (const ScoredRows& rows : raw_mode_rows) { + ASSERT_EQ(rows, fully_indexed_rows); + } + ASSERT_OK_AND_ASSIGN(ScoredRows fully_indexed_top_rows, SearchTable(table_path, query, 3)); + for (const ScoredRows& rows : raw_mode_top_rows) { + ASSERT_EQ(rows, fully_indexed_top_rows); + } + for (std::map search_options : raw_modes) { + search_options.insert(index_options.begin(), index_options.end()); + ASSERT_OK_AND_ASSIGN(std::vector raw_ranges, + PlanRawRanges(table_path, query, search_options)); + ASSERT_TRUE(raw_ranges.empty()); + } ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, /*partitions=*/std::nullopt, /*options=*/{}, @@ -762,28 +987,11 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_EQ(readers.size(), 1u); CheckTableSearchMatchesReader(table_path, query, readers[0], /*max_limit=*/11); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr scan, - NewFullTextSearchBuilder(table_path, query, /*limit=*/3)->NewFullTextScan()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - std::vector> received_splits; - for (const auto& split : plan->Splits()) { - ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr received_split, - FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); - received_splits.push_back(std::move(received_split)); - } - ASSERT_OK_AND_ASSIGN( - std::unique_ptr read, - NewFullTextSearchBuilder(table_path, query, /*limit=*/3)->NewFullTextRead()); ASSERT_OK_AND_ASSIGN(std::shared_ptr top_result, - read->Read(received_splits)); + ReadSerializedSplits(table_path, query, 3, {})); ASSERT_OK_AND_ASSIGN(ScoredRows top_rows, ToScoredRows(top_result)); ASSERT_EQ(top_rows.size(), 3u); - ASSERT_OK_AND_ASSIGN(ScoredRows local_top_rows, - SearchTable(table_path, query, /*limit=*/3)); - ASSERT_EQ(top_rows, local_top_rows); + ASSERT_EQ(top_rows, fully_indexed_top_rows); std::string first_shard_snapshot_id = std::to_string(first_shard_plan->SnapshotId().value()); @@ -803,10 +1011,18 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter {{Options::SCAN_MODE, "latest-full"}, {Options::SCAN_SNAPSHOT_ID, first_shard_snapshot_id}}), "scan.snapshot-id must be null when you use latest-full for scan.mode"); + auto tag_builder = NewFullTextSearchBuilder(table_path, query, 10, + {{Options::SCAN_TAG_NAME, "first-shard"}}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr tag_scan, + tag_builder->NewFullTextScan()); + ASSERT_NOK_WITH_MSG(tag_scan->CreatePlan(), "Tag 'first-shard' doesn't exist."); ASSERT_OK(TagSnapshot(table_path, first_shard_plan->SnapshotId().value(), "first-shard")); - ASSERT_OK_AND_ASSIGN(ScoredRows tag_rows, - SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, - {{Options::SCAN_TAG_NAME, "first-shard"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr tag_plan, tag_scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr tag_read, + tag_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr tag_result, + tag_read->Read(*tag_plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows tag_rows, ToScoredRows(tag_result)); ASSERT_EQ(tag_rows, first_shard_rows); ASSERT_OK_AND_ASSIGN(ScoredRows latest_rows, SearchTable(table_path, query, /*limit=*/10)); ASSERT_EQ(latest_rows.size(), 10u); @@ -847,6 +1063,27 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK_AND_ASSIGN(ScoredRows rows_with_bitmap, SearchTable(table_path, query, /*limit=*/3)); ASSERT_EQ(rows_with_bitmap, top_rows); + + auto null_rows = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + "[[null, 10], [null, 11]]") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> null_messages, + WriteArray(table_path, schema->field_names(), null_rows)); + ASSERT_OK(Commit(table_path, null_messages)); + for (std::map search_options : raw_modes) { + SCOPED_TRACE(search_options.begin()->second); + search_options.insert(index_options.begin(), index_options.end()); + search_options.erase("lucene-fts.write.tmp.directory"); + if (index_type == "full-text") { + search_options["full-text.tokenizer"] = "unavailable"; + } + ASSERT_OK_AND_ASSIGN(std::vector raw_ranges, + PlanRawRanges(table_path, query, search_options)); + ASSERT_EQ(raw_ranges, std::vector({Range(10, 11)})); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTable(table_path, query, 10, {}, search_options)); + ASSERT_EQ(rows, fully_indexed_rows); + } } void CheckTableSearchWithFilter(const std::string& index_type, @@ -1025,21 +1262,13 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK_AND_ASSIGN(std::shared_ptr unfiltered_plan, unfiltered_scan->CreatePlan()); ASSERT_EQ(unfiltered_plan->Splits().size(), 1u); - index_split = + auto unfiltered_split = std::dynamic_pointer_cast(unfiltered_plan->Splits()[0]); - ASSERT_TRUE(index_split); - ASSERT_TRUE(index_split->ScalarIndexFiles().empty()); + ASSERT_TRUE(unfiltered_split); + ASSERT_TRUE(unfiltered_split->ScalarIndexFiles().empty()); - std::vector> splits; - for (const auto& split : plan->Splits()) { - ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr deserialized, - FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); - splits.push_back(std::move(deserialized)); - } - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, builder->NewFullTextRead()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(splits)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadSerializedSplits(table_path, query, 100, {}, f1_is_1)); ASSERT_OK_AND_ASSIGN(ScoredRows rows, ToScoredRows(result)); ASSERT_EQ(rows, rows_of({2, 3})); @@ -1055,6 +1284,67 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter search_options.insert(time_travel_option); check_filter(partly_indexed, {3}, search_options); } + + ASSERT_OK(WriteIndex(table_path, {}, "f3", "btree", {}, Range(0, 2))); + ASSERT_OK(WriteIndex(table_path, {}, "f3", "bitmap", {}, Range(3, 4))); + auto f3_at_least_2 = PredicateBuilder::GreaterOrEqual(3, "f3", FieldType::INT, Literal(2)); + check_filter(f3_at_least_2, {2, 3, 4}); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + std::map scalar_options = { + {Options::SCALAR_INDEX_SEARCH_MODE, mode}}; + check_filter(f3_at_least_2, {2, 3, 4, 5}, scalar_options); + scalar_options[BitmapGlobalIndexOptions::kBitmapIndexFallbackScanMaxSize] = "1B"; + check_filter(f3_at_least_2, {2, 3, 4, 5}, scalar_options); + } + auto partial_builder = NewFullTextSearchBuilder(table_path, query, 100); + partial_builder->WithFilter(f3_at_least_2); + ASSERT_OK_AND_ASSIGN(std::unique_ptr partial_scan, + partial_builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr partial_plan, + partial_scan->CreatePlan()); + auto* factories = FactoryCreator::GetInstance(); + factories->TEST_Unregister("bitmap-global"); + ScopeGuard restore_bitmap([factories]() { + factories->Register("bitmap-global", new BitmapGlobalIndexFactory()); + }); + check_filter(f3_at_least_2, {2}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr contains_b, + PredicateBuilder::Contains(2, "f2", FieldType::STRING, + Literal(FieldType::STRING, "b", 1))); + auto f3_at_least_1 = PredicateBuilder::GreaterOrEqual(3, "f3", FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr partial_contains, + PredicateBuilder::And({contains_b, f3_at_least_1})); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + partial_builder->AddOption(Options::SCALAR_INDEX_SEARCH_MODE, mode); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + partial_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(result, read->Read(*partial_plan)); + ASSERT_OK_AND_ASSIGN(rows, ToScoredRows(result)); + ASSERT_EQ(rows, rows_of({2, 3, 4, 5})); + std::map scalar_options = { + {Options::SCALAR_INDEX_SEARCH_MODE, mode}}; + check_filter(partly_indexed, {3}, scalar_options); + check_filter(undecidable_or, {2, 3}, scalar_options); + check_filter(contains, {}, scalar_options); + check_filter(partial_contains, {3, 4, 5}, scalar_options); + scalar_options.insert(refine.begin(), refine.end()); + check_filter(partial_contains, {1, 3, 4, 5}, scalar_options); + } + + const auto& f1_index = index_split->ScalarIndexFiles()[0]; + ASSERT_OK(dir_->GetFileSystem()->Delete( + PathUtil::JoinPath(FileStorePathFactory::IndexPath(table_path), f1_index->FileName()))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr f1_and_f4, + PredicateBuilder::And({f1_is_1, f4_is_pear})); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + const std::map scalar_options = { + {Options::SCALAR_INDEX_SEARCH_MODE, mode}}; + check_filter(f1_and_f4, {2, 3}, scalar_options); + ASSERT_NOK(SearchTableWithFilter(table_path, query, 100, f1_is_1, {}, scalar_options)); + } } void CheckTableSearchWithPartitionAndDataFilter( @@ -1150,6 +1440,434 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter PlannedShards({{Range(0, 1), {Range(0, 1)}}, {Range(2, 3), {Range(2, 3)}}})); } + void CheckRawSearchSkipsDeletedRows(const std::string& index_type, + const std::map& index_options) { + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; + CreateTable({}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, schema->field_names(), + AppleRows(fields, 0, 5))); + ASSERT_OK(Commit(table_path, commit_msgs1)); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, schema->field_names(), + AppleRows(fields, 5, 10))); + ASSERT_OK(Commit(table_path, commit_msgs2)); + ASSERT_OK(WriteIndex(table_path, {}, "f0", index_type, index_options, Range(0, 4))); + + std::string query = R"({"match":{"query":"apple"}})"; + std::map full_options = index_options; + full_options[Options::FULL_TEXT_INDEX_SEARCH_MODE] = "full"; + std::unique_ptr pinned_builder = + NewFullTextSearchBuilder(table_path, query, 100, full_options); + ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_scan, + pinned_builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned_plan, + pinned_scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_read, + pinned_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned_result, + pinned_read->Read(*pinned_plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows rows_before_deletion, ToScoredRows(pinned_result)); + ASSERT_EQ(RowIdsOf(rows_before_deletion), + std::vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + + ASSERT_OK_AND_ASSIGN(auto split, ScanData(table_path, {})); + ASSERT_OK_AND_ASSIGN(std::vector anchor_file_names, + DeletionVectorTestHelper::RetrieveAnchorFileNames(split->DataFiles())); + ASSERT_EQ(anchor_file_names.size(), 2u); + ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_deletes, + DeletionVectorTestHelper::CreateDeletionVectorCommitMessage( + dir_->GetFileSystem(), table_path, file_format_, commit_msgs1[0], + {{anchor_file_names[0], {0, 1}}}, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw_deletes, + DeletionVectorTestHelper::CreateDeletionVectorCommitMessage( + dir_->GetFileSystem(), table_path, file_format_, commit_msgs2[0], + {{anchor_file_names[1], {0, 2}}}, pool_)); + ASSERT_OK(Commit(table_path, {indexed_deletes, raw_deletes})); + + ASSERT_OK_AND_ASSIGN(pinned_result, pinned_read->Read(*pinned_plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + + auto search = [&](const std::map& mode_options, + const std::shared_ptr& filter, + int32_t limit = 100) -> Result { + std::map search_options = mode_options; + search_options.insert(index_options.begin(), index_options.end()); + return SearchTableWithFilter(table_path, query, limit, filter, {}, search_options); + }; + const std::map full = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, "full"}}; + ASSERT_OK_AND_ASSIGN(ScoredRows rows, search({}, nullptr)); + ASSERT_EQ(RowIdsOf(rows), std::vector({2, 3, 4})); + ASSERT_OK_AND_ASSIGN(ScoredRows raw_rows, search(full, nullptr)); + ASSERT_EQ(RowIdsOf(raw_rows), std::vector({2, 3, 4, 6, 8, 9})); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + const std::map mode_options = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}; + ASSERT_OK_AND_ASSIGN(rows, search(mode_options, nullptr)); + ASSERT_EQ(rows, raw_rows); + for (int64_t row_id : {6, 8, 9}) { + SCOPED_TRACE(row_id); + std::shared_ptr filter = PredicateBuilder::Equal( + 1, "f1", FieldType::INT, Literal(static_cast(row_id))); + ASSERT_OK_AND_ASSIGN(rows, search(mode_options, filter, 1)); + auto expected = + std::find_if(raw_rows.begin(), raw_rows.end(), + [row_id](const auto& row) { return row.first == row_id; }); + ASSERT_NE(expected, raw_rows.end()); + ASSERT_EQ(rows, ScoredRows({*expected})); + } + ASSERT_OK_AND_ASSIGN( + rows, search(mode_options, + PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(5)), 1)); + ASSERT_TRUE(rows.empty()); + } + std::shared_ptr f1_at_least_3 = + PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(3)); + ASSERT_OK_AND_ASSIGN(rows, search(full, f1_at_least_3)); + ASSERT_EQ(RowIdsOf(rows), std::vector({6, 8, 9})); + std::map scalar_full = full_options; + scalar_full[Options::SCALAR_INDEX_SEARCH_MODE] = "full"; + ASSERT_OK_AND_ASSIGN(rows, search(scalar_full, f1_at_least_3)); + ASSERT_EQ(RowIdsOf(rows), std::vector({3, 4, 6, 8, 9})); + ASSERT_OK_AND_ASSIGN( + rows, search({{Options::GLOBAL_INDEX_SEARCH_MODE, "detail"}}, f1_at_least_3)); + ASSERT_EQ(RowIdsOf(rows), std::vector({3, 4, 6, 8, 9})); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + scalar_full[Options::FULL_TEXT_INDEX_SEARCH_MODE] = mode; + scalar_full[Options::SCALAR_INDEX_SEARCH_MODE] = mode; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr serialized_result, + ReadSerializedSplits(table_path, query, 100, scalar_full, f1_at_least_3)); + ASSERT_OK_AND_ASSIGN(ScoredRows serialized_rows, ToScoredRows(serialized_result)); + ASSERT_EQ(serialized_rows, rows); + } + ASSERT_OK(WriteIndex(table_path, {}, "f1", "btree", {}, Range(0, 4))); + ASSERT_OK_AND_ASSIGN(rows, search(full, f1_at_least_3)); + ASSERT_EQ(RowIdsOf(rows), std::vector({3, 4, 6, 8, 9})); + + ASSERT_TRUE(pinned_plan->SnapshotId()); + int64_t pinned_snapshot_id = pinned_plan->SnapshotId().value(); + ASSERT_OK(TagSnapshot(table_path, pinned_snapshot_id, "before-deletion")); + SnapshotManager snapshots(dir_->GetFileSystem(), table_path); + ASSERT_OK(dir_->GetFileSystem()->Delete(snapshots.SnapshotPath(pinned_snapshot_id))); + + ASSERT_OK_AND_ASSIGN(split, ScanData(table_path, {})); + auto deletion_files = + DeletionVector::CreateDeletionFileMap(split->DataFiles(), split->DeletionFiles()); + auto raw_deletion_file = deletion_files.find(anchor_file_names[1]); + ASSERT_NE(raw_deletion_file, deletion_files.end()); + ASSERT_OK(dir_->GetFileSystem()->Delete(raw_deletion_file->second.path)); + ASSERT_OK_AND_ASSIGN(rows, search({}, nullptr)); + ASSERT_EQ(RowIdsOf(rows), std::vector({2, 3, 4})); + + options[Options::BLOB_VIEW_FIELD] = "f0"; + ASSERT_OK(TestHelper::WriteNextSchema( + dir_->GetFileSystem(), table_path, + {DataField(0, arrow::field("old_text", arrow::utf8())), DataField(1, fields[1]), + DataField(2, BlobUtils::ToArrowField("f0", true))}, + 2, options)); + pinned_builder->AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(pinned_snapshot_id)); + ASSERT_OK_AND_ASSIGN(pinned_read, pinned_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(pinned_result, pinned_read->Read(*pinned_plan)); + ASSERT_OK_AND_ASSIGN(pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + full_options[Options::SCAN_TAG_NAME] = "before-deletion"; + ASSERT_OK_AND_ASSIGN(pinned_result, + ReadSerializedSplits(table_path, query, 100, full_options)); + ASSERT_OK_AND_ASSIGN(pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + } + + void CheckRawSearchAcrossPartitions(const std::string& index_type, + const std::map& index_options, + const std::string& query) { + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {"orc.dictionary-key-size-threshold", "1"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"parquet.read.enable-dictionary-passthrough", "true"}}; + CreateTable({"f1"}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto write_partition = [&](const std::string& json, + const std::map& partition) { + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, partition, + schema->field_names(), src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + }; + write_partition(R"([["apple apple", 10, 1], ["apple", 10, 2]])", {{"f1", "10"}}); + ASSERT_OK( + WriteIndex(table_path, {{{"f1", "10"}}}, "f0", index_type, index_options, Range(0, 1))); + ASSERT_OK(WriteIndex(table_path, {{{"f1", "10"}}}, "f2", "btree", {}, Range(0, 1))); + write_partition(R"([["apple apple", 20, 1], [null, 20, 2]])", {{"f1", "20"}}); + + auto check = [&](std::map search_options, + const std::vector>& partitions, + const std::shared_ptr& filter, + const std::vector& expected_raw_ranges, + const std::vector& expected_row_ids) { + search_options.insert(index_options.begin(), index_options.end()); + search_options[Options::READ_BATCH_SIZE] = "1"; + ASSERT_OK_AND_ASSIGN( + std::vector raw_ranges, + PlanRawRanges(table_path, query, search_options, partitions, filter)); + ASSERT_EQ(raw_ranges, expected_raw_ranges); + ASSERT_OK_AND_ASSIGN( + ScoredRows rows, + SearchTableWithFilter(table_path, query, 100, filter, partitions, search_options)); + ASSERT_EQ(RowIdsOf(rows), expected_row_ids); + }; + const std::map fast = {}; + const std::map full = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, "full"}}; + const std::map detail = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, "detail"}}; + ASSERT_OK_AND_ASSIGN(std::vector partition_gaps, + PlanRawRanges(table_path, query, full, {{{"f1", "10"}}})); + ASSERT_EQ(partition_gaps, std::vector({Range(2, 3)})); + auto unavailable_writer_options = full; + if (index_type == "full-text") { + unavailable_writer_options["full-text.tokenizer"] = "unavailable"; + } + ASSERT_OK_AND_ASSIGN( + ScoredRows indexed_rows, + SearchTable(table_path, query, 100, {{{"f1", "10"}}}, unavailable_writer_options)); + ASSERT_EQ(RowIdsOf(indexed_rows), std::vector({0, 1})); + check(detail, {{{"f1", "10"}}}, nullptr, {}, {0, 1}); + + write_partition(R"([["apple apple apple", 10, 1], [null, 10, 2]])", {{"f1", "10"}}); + check(fast, {}, nullptr, {}, {0, 1}); + check(full, {}, nullptr, {Range(2, 5)}, {0, 1, 2, 4}); + check(detail, {}, nullptr, {Range(2, 5)}, {0, 1, 2, 4}); + check(full, {{{"f1", "10"}}}, nullptr, {Range(2, 5)}, {0, 1, 4}); + check(detail, {{{"f1", "10"}}}, nullptr, {Range(4, 5)}, {0, 1, 4}); + check(detail, {{{"f1", "20"}}}, nullptr, {}, {}); + + std::shared_ptr f2_is_1 = + PredicateBuilder::Equal(2, "f2", FieldType::INT, Literal(1)); + check(full, {}, f2_is_1, {Range(2, 5)}, {0, 2, 4}); + check(detail, {{{"f1", "10"}}}, f2_is_1, {Range(4, 5)}, {0, 4}); + auto worker_options = full; + worker_options.insert(index_options.begin(), index_options.end()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr distributed, + ReadSerializedSplits(table_path, query, 3, worker_options, f2_is_1, {{{"f1", "10"}}})); + ASSERT_OK_AND_ASSIGN(ScoredRows distributed_rows, ToScoredRows(distributed)); + ASSERT_EQ(RowIdsOf(distributed_rows), std::vector({0, 4})); + std::shared_ptr f2_is_2 = + PredicateBuilder::Equal(2, "f2", FieldType::INT, Literal(2)); + check(full, {}, f2_is_2, {Range(2, 5)}, {1}); + std::shared_ptr text_is_three_apples = PredicateBuilder::Equal( + 0, "f0", FieldType::STRING, Literal(FieldType::STRING, "apple apple apple", 17)); + check(detail, {}, text_is_three_apples, {Range(2, 5)}, {4}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr partition_20_and_f2_is_1, + PredicateBuilder::And( + {PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(20)), f2_is_1})); + check(full, {}, partition_20_and_f2_is_1, {}, {}); + + write_partition(R"([["apple", 20, 2], ["kiwi", 20, 2], ["apple", 20, 1]])", {{"f1", "20"}}); + ASSERT_OK( + WriteIndex(table_path, {{{"f1", "20"}}}, "f0", index_type, index_options, Range(6, 6))); + ASSERT_OK( + WriteIndex(table_path, {{{"f1", "10"}}}, "f0", index_type, index_options, Range(4, 4))); + auto update_column = [&](const std::shared_ptr& field, + const std::string& json) { + auto values = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({field}), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto updates, + WriteArray(table_path, {{"f1", "20"}}, {field->name()}, values)); + ASSERT_EQ(updates.size(), 1u); + auto update = std::dynamic_pointer_cast(updates.front()); + ASSERT_TRUE(update); + const auto& files = update->GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(files.size(), 1u); + files.front()->AssignFirstRowId(6); + ASSERT_OK(Commit(table_path, updates)); + }; + update_column(fields[2], "[[2], [1], [2]]"); + for (auto search_options : {full, detail}) { + check(search_options, {}, nullptr, {Range(2, 3), Range(5, 5), Range(7, 8)}, + {0, 1, 2, 4, 6, 8}); + search_options.insert(index_options.begin(), index_options.end()); + ASSERT_OK_AND_ASSIGN(ScoredRows raw_hit, + SearchTableWithFilter(table_path, R"({"match":{"query":"kiwi"}})", + 1, f2_is_1, {}, search_options)); + ASSERT_EQ(RowIdsOf(raw_hit), std::vector({7})); + } + check(detail, {{{"f1", "10"}}}, nullptr, {Range(5, 5)}, {0, 1, 4}); + check(detail, {{{"f1", "20"}}}, nullptr, {Range(2, 3), Range(7, 8)}, {2, 6, 8}); + + update_column(fields[0], R"([["kiwi"], ["kiwi"], ["apple"]])"); + auto builder = NewFullTextSearchBuilder(table_path, query, 1, index_options); + builder->SetPartitionFilter({{{"f1", "20"}}}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1u); + ASSERT_TRUE(plan->SnapshotId()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(*plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows stale_hits, ToScoredRows(result)); + ASSERT_EQ(RowIdsOf(stale_hits), std::vector({6})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr updated_text, + RawFullTextSearchSplitImpl::Create(*plan->SnapshotId(), "f0", + index_type, {Range(6, 6)}, 0, 0)); + ASSERT_OK_AND_ASSIGN(result, read->Read({plan->Splits().front(), updated_text})); + ASSERT_OK_AND_ASSIGN(ScoredRows current_hits, ToScoredRows(result)); + ASSERT_TRUE(current_hits.empty()); + } + + void CheckSplitUsesPlannedFilterColumn( + const std::string& index_type, const std::map& index_options) { + const arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + const std::map options = { + {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable({}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN( + auto indexed, WriteArray(table_path, schema->field_names(), AppleRows(fields, 0, 3))); + ASSERT_OK(Commit(table_path, indexed)); + ASSERT_OK(WriteIndex(table_path, {}, "f0", index_type, index_options, Range(0, 2))); + ASSERT_OK(WriteIndex(table_path, {}, "f1", "btree", {}, Range(0, 1))); + ASSERT_OK_AND_ASSIGN( + auto appended, WriteArray(table_path, schema->field_names(), AppleRows(fields, 3, 5))); + ASSERT_OK(Commit(table_path, appended)); + + auto fs = dir_->GetFileSystem(); + const auto category = arrow::field("category", arrow::int32()); + ASSERT_OK(TestHelper::WriteNextSchema( + fs, table_path, {DataField(0, fields[0]), DataField(1, category)}, 1, options)); + std::map search_options = index_options; + search_options[Options::GLOBAL_INDEX_SEARCH_MODE] = "full"; + const std::string query = R"({"match":{"query":"apple"}})"; + auto builder = NewFullTextSearchBuilder(table_path, query, 10, search_options); + builder->WithFilter(PredicateBuilder::In(1, "category", FieldType::INT, + {Literal(1), Literal(2), Literal(4)})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 2u); + std::vector serialized; + for (const auto& split : plan->Splits()) { + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + serialized.push_back(std::move(bytes)); + } + auto check_planned_rows = [&]() { + ASSERT_OK_AND_ASSIGN(std::unique_ptr worker, builder->NewFullTextRead()); + for (const auto& bytes : serialized) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr split, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + worker->Read({split})); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, ToScoredRows(result)); + if (std::dynamic_pointer_cast(split)) { + ASSERT_EQ(RowIdsOf(rows), std::vector({4})); + } else { + ASSERT_EQ(RowIdsOf(rows), std::vector({1, 2})); + } + } + }; + const DataField old_category(1, arrow::field("old_category", arrow::int32())); + ASSERT_OK(TestHelper::WriteNextSchema( + fs, table_path, {DataField(0, fields[0]), old_category, DataField(2, category)}, 2, + options)); + ASSERT_NO_FATAL_FAILURE(check_planned_rows()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr replanned, + builder->ExecuteLocal()); + ASSERT_OK_AND_ASSIGN(ScoredRows new_rows, ToScoredRows(replanned)); + ASSERT_TRUE(new_rows.empty()); + + const std::vector current_fields = { + DataField(0, fields[0]), old_category, + DataField(3, BlobUtils::ToArrowField("category", true))}; + auto current_options = options; + current_options[Options::BLOB_VIEW_FIELD] = "category"; + ASSERT_OK(TestHelper::WriteNextSchema(fs, table_path, current_fields, 3, current_options)); + ASSERT_NO_FATAL_FAILURE(check_planned_rows()); + ASSERT_NOK_WITH_MSG(builder->NewFullTextScan(), "Invalid type large_binary for predicate"); + + ASSERT_TRUE(plan->SnapshotId()); + search_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(*plan->SnapshotId()); + auto original_filter = + PredicateBuilder::In(1, "f1", FieldType::INT, {Literal(1), Literal(2), Literal(4)}); + ASSERT_OK_AND_ASSIGN( + ScoredRows historical_rows, + SearchTableWithFilter(table_path, query, 10, original_filter, {}, search_options)); + ASSERT_EQ(RowIdsOf(historical_rows), std::vector({1, 2, 4})); + + current_options[Options::QUERY_AUTH_ENABLED] = "true"; + ASSERT_OK(TestHelper::WriteNextSchema(fs, table_path, current_fields, 3, current_options)); + ASSERT_NOK_WITH_MSG( + SearchTableWithFilter(table_path, query, 10, original_filter, {}, search_options), + "Full-text search is not supported"); + } + + void CheckRawSearchWithoutTmpDir(const std::string& index_type, + const std::map& index_options) { + const arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + CreateTable({}, schema, + {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN( + auto indexed, WriteArray(table_path, schema->field_names(), AppleRows(fields, 0, 1))); + ASSERT_OK(Commit(table_path, indexed)); + ASSERT_OK(WriteIndex(table_path, {}, "f0", index_type, index_options, Range(0, 0))); + ASSERT_OK_AND_ASSIGN( + auto appended, WriteArray(table_path, schema->field_names(), AppleRows(fields, 1, 2))); + ASSERT_OK(Commit(table_path, appended)); + + const char* tmpdir = std::getenv("TMPDIR"); + std::optional saved; + if (tmpdir) { + saved = tmpdir; + } + ASSERT_EQ(setenv("TMPDIR", PathUtil::JoinPath(dir_->Str(), "missing").c_str(), 1), 0); + ScopeGuard restore([&]() { + if (saved) { + setenv("TMPDIR", saved->c_str(), 1); + } else { + unsetenv("TMPDIR"); + } + }); + std::map search_options = index_options; + search_options.erase("lucene-fts.write.tmp.directory"); + for (const char* mode : {"full", "detail"}) { + search_options[Options::FULL_TEXT_INDEX_SEARCH_MODE] = mode; + ASSERT_OK_AND_ASSIGN( + ScoredRows rows, + SearchTable(table_path, R"({"match":{"query":"apple"}})", 10, {}, search_options)); + ASSERT_EQ(RowIdsOf(rows), std::vector({0, 1})) << mode; + } + } + void CheckTableSearchWithPartitionFilter(const std::string& table_path, const std::string& query) const { ASSERT_OK_AND_ASSIGN(ScoredRows rows, @@ -4218,6 +4936,39 @@ TEST_P(GlobalIndexTest, TestLuceneTableFullTextSearchWithPartitionAndDataFilter) "lucene-fts", {{"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); } + +TEST_P(GlobalIndexTest, TestLuceneRawSearchSkipsDeletedRows) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckRawSearchSkipsDeletedRows("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} + +TEST_P(GlobalIndexTest, TestLuceneRawSearchAcrossPartitions) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckRawSearchAcrossPartitions("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}, + R"({"match":{"query":"apple"}})"); +} +TEST_P(GlobalIndexTest, TestLuceneSplitUsesPlannedFilterColumn) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckSplitUsesPlannedFilterColumn("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} + +TEST_P(GlobalIndexTest, TestLuceneRawSearchWithoutTmpDir) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckRawSearchWithoutTmpDir("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} + #endif #ifdef PAIMON_ENABLE_FULL_TEXT @@ -4375,6 +5126,151 @@ TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearch) { CheckTableFullTextSearch("full-text", {{"full-text.tokenizer", "default"}}); } +TEST_P(GlobalIndexTest, TestFullTextDistributedRawRangesOverrideStoredIndex) { + const arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::utf8())}; + const auto schema = arrow::schema(fields); + CreateTable({}, schema, + {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}); + const std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + const auto rows = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(fields), + R"([["apple", "apple"], ["apple apple apple apple", "x"], + ["apple apple", "x"], ["apple", "x"], + ["apple apple apple", "apple apple apple"], + ["apple apple apple apple apple", "x"]])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + WriteArray(table_path, schema->field_names(), rows)); + ASSERT_OK(Commit(table_path, messages)); + ASSERT_OK(WriteIndex(table_path, {}, "f0", "extra-field-full-text-test", {}, Range(1, 4))); + + auto update_text = [&](const std::string& json) { + const auto updates = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0]}), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> update_messages, + WriteArray(table_path, {"f0"}, updates)); + ASSERT_EQ(update_messages.size(), 1u); + const auto update = std::dynamic_pointer_cast(update_messages.front()); + ASSERT_TRUE(update); + const auto& update_files = update->GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(update_files.size(), 1u); + update_files.front()->AssignFirstRowId(0); + ASSERT_OK(Commit(table_path, update_messages)); + }; + ASSERT_NO_FATAL_FAILURE(update_text( + R"([["kiwi"], ["kiwi"], ["apple apple"], ["apple"], [null], ["apple apple apple apple apple"]])")); + + const std::string query = R"({"match":{"query":"apple"}})"; + auto check_plan = [&](const std::string& column, const std::string& mode, + const std::vector& expected_raw_ranges, + const std::vector& expected_index_ranges) { + SCOPED_TRACE(column + " " + mode); + auto builder = NewFullTextSearchBuilder( + table_path, query, 10, {{Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}, column); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), static_cast(!expected_raw_ranges.empty()) + + static_cast(!expected_index_ranges.empty())); + if (!expected_index_ranges.empty()) { + auto index_split = + std::dynamic_pointer_cast(plan->Splits().front()); + ASSERT_TRUE(index_split); + ASSERT_EQ(index_split->FullTextIndexFiles().size(), 1u); + ASSERT_EQ(index_split->FullTextIndexFiles().front()->IndexType(), + "extra-field-full-text-test"); + ASSERT_EQ(index_split->RowRangeStart(), 1); + ASSERT_EQ(index_split->RowRangeEnd(), 4); + ASSERT_EQ(index_split->SearchRowRanges(), expected_index_ranges); + } + if (!expected_raw_ranges.empty()) { + auto raw_split = + std::dynamic_pointer_cast(plan->Splits().back()); + ASSERT_TRUE(raw_split); + ASSERT_EQ(raw_split->RowRanges(), expected_raw_ranges); + } + }; + for (const char* column : {"f0", "f1"}) { + ASSERT_NO_FATAL_FAILURE(check_plan(column, "fast", {}, {Range(1, 4)})); + } + + auto check_search = [&](const std::string& column, + const std::vector& expected_row_ids) { + SCOPED_TRACE(column); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + const std::map options = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}; + ScoredRows all_rows; + for (int32_t limit : {10, 1}) { + SCOPED_TRACE(limit); + auto builder = NewFullTextSearchBuilder(table_path, query, limit, options, column); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + builder->ExecuteLocal()); + ASSERT_OK_AND_ASSIGN(ScoredRows local, ToScoredRows(result)); + if (limit == 10) { + ASSERT_EQ(RowIdsOf(local), expected_row_ids); + all_rows = local; + } else { + ASSERT_EQ(local.size(), 1u); + const auto best = std::max_element(all_rows.begin(), all_rows.end(), + [](const auto& left, const auto& right) { + return left.second < right.second; + }); + ASSERT_NE(best, all_rows.end()); + ASSERT_EQ(local.front().second, best->second); + ASSERT_NE(std::find(all_rows.begin(), all_rows.end(), local.front()), + all_rows.end()); + } + ASSERT_OK_AND_ASSIGN( + std::shared_ptr distributed, + ReadSerializedSplits(table_path, query, limit, options, nullptr, {}, column)); + ASSERT_OK_AND_ASSIGN(ScoredRows distributed_rows, ToScoredRows(distributed)); + ASSERT_EQ(distributed_rows, local); + } + } + }; + for (const char* mode : {"full", "detail"}) { + ASSERT_NO_FATAL_FAILURE(check_plan("f0", mode, {Range(0, 5)}, {})); + ASSERT_NO_FATAL_FAILURE(check_plan("f1", mode, {Range(0, 5)}, {})); + } + ASSERT_NO_FATAL_FAILURE(check_search("f0", {2, 3, 5})); + ASSERT_NO_FATAL_FAILURE(check_search("f1", {0, 4})); + ASSERT_OK(WriteIndex(table_path, {}, "f1", "bitmap", {}, Range(1, 1))); + auto filter = + PredicateBuilder::Equal(1, "f1", FieldType::STRING, Literal(FieldType::STRING, "x", 1)); + auto fast = NewFullTextSearchBuilder(table_path, query, 10); + fast->WithFilter(filter); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fast_scan, fast->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fast_plan, fast_scan->CreatePlan()); + ASSERT_EQ(fast_plan->Splits().size(), 1u); + auto shard = std::dynamic_pointer_cast(fast_plan->Splits()[0]); + ASSERT_TRUE(shard); + ASSERT_EQ(shard->ScalarIndexFiles().size(), 1u); + ASSERT_OK(dir_->GetFileSystem()->Delete(PathUtil::JoinPath( + FileStorePathFactory::IndexPath(table_path), shard->ScalarIndexFiles()[0]->FileName()))); + ASSERT_OK(WriteIndex(table_path, {}, "f1", "bitmap", {}, Range(2, 3))); + ASSERT_OK(WriteIndex(table_path, {}, "f0", "full-text", {}, Range(2, 3))); + for (const char* mode : {"full", "detail"}) { + ASSERT_NO_FATAL_FAILURE(check_plan("f0", mode, {Range(0, 1), Range(4, 5)}, {Range(2, 3)})); + ASSERT_NO_FATAL_FAILURE(check_plan("f1", mode, {Range(0, 5)}, {})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result, + ReadSerializedSplits(table_path, query, 10, {{Options::GLOBAL_INDEX_SEARCH_MODE, mode}}, + filter)); + ASSERT_OK_AND_ASSIGN(ScoredRows filtered_rows, ToScoredRows(result)); + ASSERT_EQ(RowIdsOf(filtered_rows), std::vector({2, 3, 5})) << mode; + } + ASSERT_NO_FATAL_FAILURE(check_search("f0", {2, 3, 5})); + ASSERT_NO_FATAL_FAILURE( + update_text(R"([["kiwi"], ["kiwi"], ["apple apple"], ["apple"], [null], ["kiwi"]])")); + ASSERT_NO_FATAL_FAILURE(check_search("f0", {2, 3})); +} + TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearchWithFilter) { CheckTableSearchWithFilter("full-text", {{"full-text.tokenizer", "default"}}); } @@ -4382,6 +5278,24 @@ TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearchWithFilter) { TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearchWithPartitionAndDataFilter) { CheckTableSearchWithPartitionAndDataFilter("full-text", {{"full-text.tokenizer", "default"}}); } + +TEST_P(GlobalIndexTest, TestFullTextRawSearchSkipsDeletedRows) { + CheckRawSearchSkipsDeletedRows("full-text", {{"full-text.tokenizer", "default"}}); +} + +TEST_P(GlobalIndexTest, TestFullTextRawSearchAcrossPartitions) { + CheckRawSearchAcrossPartitions("full-text", {{"full-text.tokenizer", "ngram"}}, + R"({"match":{"query":"ppl"}})"); +} + +TEST_P(GlobalIndexTest, TestFullTextSplitUsesPlannedFilterColumn) { + CheckSplitUsesPlannedFilterColumn("full-text", {}); +} + +TEST_P(GlobalIndexTest, TestFullTextRawSearchWithoutTmpDir) { + CheckRawSearchWithoutTmpDir("full-text", {}); +} + #endif TEST_P(GlobalIndexTest, TestFilteredRowIdReader) { @@ -4414,7 +5328,7 @@ TEST_P(GlobalIndexTest, TestFilteredRowIdReader) { std::shared_ptr filter = PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(1)); std::shared_ptr executor = CreateDefaultExecutor(); - auto matching_row_ids = [&](int64_t snapshot_id, const RoaringBitmap64& rows, + auto matching_row_ids = [&](int64_t snapshot_id, const auto& rows, const std::map& search_options = {}) -> Result { std::map merged_options = table_schema.value()->Options(); @@ -4452,6 +5366,10 @@ TEST_P(GlobalIndexTest, TestFilteredRowIdReader) { snapshot_manager.LatestSnapshotId()); ASSERT_TRUE(second_snapshot_id); + ASSERT_OK_AND_ASSIGN(RowRangeIndex wide_range, + RowRangeIndex::Create({Range(1, int64_t{1} << 40)})); + ASSERT_OK_AND_ASSIGN(rows, matching_row_ids(second_snapshot_id.value(), wide_range)); + ASSERT_EQ(rows, RoaringBitmap64::From({2, 3})); ASSERT_OK_AND_ASSIGN( rows, matching_row_ids(second_snapshot_id.value(), RoaringBitmap64::From({1, 3}))); ASSERT_EQ(rows, RoaringBitmap64::From({3})); From e0581892c5c9e9105349d6498f8491c5e270063e Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Fri, 20 Nov 2026 18:30:59 +0800 Subject: [PATCH 2/2] fix(reader): convert factory exceptions before publishing lazy readers --- .../common/reader/concat_batch_reader.cpp | 36 ++++++++++--------- .../reader/concat_batch_reader_test.cpp | 10 +++++- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/paimon/common/reader/concat_batch_reader.cpp b/src/paimon/common/reader/concat_batch_reader.cpp index fcdfeee74..2200dd519 100644 --- a/src/paimon/common/reader/concat_batch_reader.cpp +++ b/src/paimon/common/reader/concat_batch_reader.cpp @@ -35,6 +35,19 @@ namespace paimon { class MemoryPool; +template +static auto CatchPreparationErrors(Func&& func) -> decltype(func()) { + try { + return func(); + } catch (const std::bad_alloc&) { + return Status::OutOfMemory("out of memory preparing file reader"); + } catch (const std::exception& e) { + return Status::UnknownError("file reader preparation failed: ", e.what()); + } catch (...) { + return Status::UnknownError("file reader preparation failed with an unknown exception"); + } +} + ConcatBatchReader::ConcatBatchReader(std::vector>&& readers, const std::shared_ptr& arrow_pool) : arrow_pool_(arrow_pool), finished_reader_metrics_(std::make_shared()) { @@ -94,7 +107,7 @@ void ConcatBatchReader::FillWindow() { if (retained->cancelled.load()) { return std::unique_ptr(); } - return retained->factory(index); + return CatchPreparationErrors([&]() { return retained->factory(index); }); }); } } @@ -104,22 +117,11 @@ Status ConcatBatchReader::ResolveReader(ReaderState* state, bool wait) { (!wait && state->future.wait_for(std::chrono::seconds(0)) != std::future_status::ready)) { return state->failure; } - try { - auto result = state->future.get(); - if (!result.ok()) { - state->failure = result.status(); - } else { - state->reader = std::move(result).value(); - state->file_reader = dynamic_cast(state->reader.get()); - } - } catch (const std::bad_alloc&) { - state->failure = Status::OutOfMemory("out of memory preparing file reader"); - } catch (const std::exception& e) { - state->failure = Status::UnknownError("file reader preparation failed: ", e.what()); - } catch (...) { - state->failure = - Status::UnknownError("file reader preparation failed with an unknown exception"); - } + state->failure = CatchPreparationErrors([state]() -> Status { + PAIMON_ASSIGN_OR_RAISE(state->reader, state->future.get()); + state->file_reader = dynamic_cast(state->reader.get()); + return Status::OK(); + }); return state->failure; } diff --git a/src/paimon/common/reader/concat_batch_reader_test.cpp b/src/paimon/common/reader/concat_batch_reader_test.cpp index 0fa287919..e63d1bfff 100644 --- a/src/paimon/common/reader/concat_batch_reader_test.cpp +++ b/src/paimon/common/reader/concat_batch_reader_test.cpp @@ -691,7 +691,6 @@ TEST_F(ConcatBatchReaderTest, LazyEarlyCloseAndDestructionDrainExceptionalBuilds ConcatBatchReader::kDefaultPreparationWindow, GetArrowPool(pool_), executor)); ScopeGuard unblock([&release]() { release.set_value(); }); lifetime.reset(); - // Keep the failed build pending until we stop reading, so cleanup consumes its exception. for (size_t i = 0; i < ConcatBatchReader::kDefaultPreparationWindow; i++) { ASSERT_OK_AND_ASSIGN(auto batch, reader->NextBatch()); ASSERT_FALSE(BatchReader::IsEofBatch(batch)); @@ -719,6 +718,15 @@ TEST_F(ConcatBatchReaderTest, LazyEarlyCloseAndDestructionDrainExceptionalBuilds } } +TEST_F(ConcatBatchReaderTest, LazyCreationReportsAbandonedBuild) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(1)); + executor->ShutdownNow(); + auto result = ConcatBatchReader::CreateLazy( + 1, [](size_t) { return std::unique_ptr(); }, 1, GetArrowPool(pool_), executor); + ASSERT_NOK_WITH_MSG(result, "file reader preparation failed"); + ASSERT_TRUE(result.status().IsUnknownError()); +} + TEST_F(ConcatBatchReaderTest, LazyInitialFailureDrainsPartialWindow) { auto counts = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(2));