diff --git a/LICENSE b/LICENSE index e986c187b..98523fe93 100644 --- a/LICENSE +++ b/LICENSE @@ -414,6 +414,26 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product includes code from Apache Paimon Full Text 0.1.0. + +* Upstream source patch in cmake_modules/full_text.diff + +Copyright: 2026 The Apache Software Foundation. +Home page: https://github.com/apache/paimon-full-text +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Lucene++ 3.0.9. + +* Upstream source patch in cmake_modules/lucene.diff + +Copyright (c) 2009-2014 Alan Wright. All rights reserved. +Home page: https://github.com/luceneplusplus/LucenePlusPlus +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This product includes code from Vortex 0.75.0. * Callback I/O sink utilities in crates/vortex_callback_io/callback_io.rs diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 4b78d0e74..c29988aef 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -887,10 +887,14 @@ macro(build_lucene) "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}") set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a") + set(LUCENE_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/lucene.diff") + paimon_patch_command(lucene_ep PATCH_COMMAND) externalproject_add(lucene_ep ${EP_COMMON_OPTIONS} URL ${LUCENE_SOURCE_URL} URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}" + LOG_PATCH ON + PATCH_COMMAND ${PATCH_COMMAND} CMAKE_ARGS ${LUCENE_CMAKE_ARGS} BUILD_BYPRODUCTS ${LUCENE_LIB} DEPENDS zlib @@ -903,6 +907,21 @@ macro(build_lucene) boost_chrono boost_atomic) + paimon_register_patched_project(lucene_ep + VERSION + "${PAIMON_LUCENE_BUILD_VERSION}" + PATCHES + "${LUCENE_PATCH_FILE}" + INSTALL_DIR + "${LUCENE_PREFIX}" + VERSION_VARIABLES + PAIMON_LUCENE_BUILD_VERSION + PAIMON_LUCENE_BUILD_SHA256_CHECKSUM + PAIMON_BOOST_BUILD_VERSION + PAIMON_BOOST_BUILD_SHA256_CHECKSUM + PAIMON_ZLIB_BUILD_VERSION + PAIMON_ZLIB_BUILD_SHA256_CHECKSUM) + set(LUCENE_INCLUDE_DIR "${LUCENE_PREFIX}/include") # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${LUCENE_INCLUDE_DIR}") @@ -912,6 +931,12 @@ macro(build_lucene) PROPERTIES IMPORTED_LOCATION "${LUCENE_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${LUCENE_INCLUDE_DIR}") + if(APPLE) + # Boost 1.66 headers still use the function base classes removed in C++17. + target_compile_definitions(lucene + INTERFACE _LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION + ) + endif() target_link_libraries(lucene INTERFACE zlib @@ -1094,6 +1119,16 @@ macro(build_boost) file(WRITE ${BOOST_USER_CONFIG} "using ${BOOST_TOOLSET} : : \"${CMAKE_CXX_COMPILER}\" ;\n") + set(BOOST_CXX_FLAGS "${EP_CXX_FLAGS}") + if(APPLE) + string(APPEND BOOST_CXX_FLAGS + " -D_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION") + endif() + if(APPLE AND CMAKE_OSX_SYSROOT) + # b2 does not inherit the SDK flags CMake adds to native targets. + string(APPEND BOOST_CXX_FLAGS " -isysroot '${CMAKE_OSX_SYSROOT}'") + endif() + externalproject_add(boost_ep URL ${BOOST_SOURCE_URL} URL_HASH "SHA256=${PAIMON_BOOST_BUILD_SHA256_CHECKSUM}" @@ -1109,8 +1144,8 @@ macro(build_boost) --user-config=${BOOST_USER_CONFIG} toolset=${BOOST_TOOLSET} cxxstd=${CMAKE_CXX_STANDARD} - cxxflags=${EP_CXX_FLAGS} linkflags=${EP_CXX_FLAGS} - install + cxxflags=${BOOST_CXX_FLAGS} + linkflags=${BOOST_CXX_FLAGS} install INSTALL_COMMAND "" DEPENDS zlib BUILD_BYPRODUCTS ${BOOST_BYPRODUCTS} @@ -1460,10 +1495,8 @@ macro(build_full_text) set(FULL_TEXT_DYNAMIC_LIB "${FULL_TEXT_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" ) - set(FULL_TEXT_CARGO_TARGET_DIR - "${CMAKE_CURRENT_BINARY_DIR}/full_text_ep-prefix/src/full_text_ep-build") set(FULL_TEXT_CARGO_DYNAMIC_LIB - "${FULL_TEXT_CARGO_TARGET_DIR}/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" + "/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" ) file(MAKE_DIRECTORY "${FULL_TEXT_INCLUDE_DIR}") @@ -1472,13 +1505,17 @@ macro(build_full_text) # Build with the lock file shipped in the source release: a reader rejects index files whose # recorded Tantivy version differs from the linked one, so the Tantivy version must match the # one used by Java Paimon. + set(FULL_TEXT_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/full_text.diff") + paimon_patch_command(full_text_ep PATCH_COMMAND) externalproject_add(full_text_ep URL ${FULL_TEXT_SOURCE_URL} URL_HASH "SHA256=${PAIMON_FULL_TEXT_BUILD_SHA256_CHECKSUM}" ${THIRDPARTY_LOG_OPTIONS} + LOG_PATCH ON + PATCH_COMMAND ${PATCH_COMMAND} CONFIGURE_COMMAND "" BUILD_COMMAND ${CMAKE_COMMAND} -E env - CARGO_TARGET_DIR=${FULL_TEXT_CARGO_TARGET_DIR} + "CARGO_TARGET_DIR=" ${PAIMON_CARGO_EXECUTABLE} build --release --locked --manifest-path /ffi/Cargo.toml COMMAND ${CMAKE_COMMAND} -E copy_if_different @@ -1487,6 +1524,17 @@ macro(build_full_text) /include ${FULL_TEXT_INCLUDE_DIR} BUILD_BYPRODUCTS "${FULL_TEXT_DYNAMIC_LIB}") + paimon_register_patched_project(full_text_ep + VERSION + "${PAIMON_FULL_TEXT_BUILD_VERSION}" + PATCHES + "${FULL_TEXT_PATCH_FILE}" + INSTALL_DIR + "${FULL_TEXT_PREFIX}" + VERSION_VARIABLES + PAIMON_FULL_TEXT_BUILD_VERSION + PAIMON_FULL_TEXT_BUILD_SHA256_CHECKSUM) + add_library(paimon_ftindex_ffi SHARED IMPORTED GLOBAL) set_target_properties(paimon_ftindex_ffi PROPERTIES IMPORTED_LOCATION "${FULL_TEXT_DYNAMIC_LIB}" diff --git a/cmake_modules/full_text.diff b/cmake_modules/full_text.diff new file mode 100644 index 000000000..dceb482ee --- /dev/null +++ b/cmake_modules/full_text.diff @@ -0,0 +1,237 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Paimon Full Text 0.1.0 uses a temporary directory for its Tantivy writer. +# Provide a RAM-backed writer for unindexed rows with the same archive and reader. + +diff --git a/core/src/index.rs b/core/src/index.rs +--- a/core/src/index.rs ++++ b/core/src/index.rs +@@ -20,7 +20,9 @@ + use crate::error::{FtIndexError, Result}; + use crate::io::{FullTextReadMetrics, ReadMetrics, ReadRequest, SeekRead, SeekWrite}; + use crate::query::{BooleanOccur, MatchOperator, QuerySpec}; +-use crate::storage::{read_header, write_envelope_from_paths, ArchiveFileEntry, IndexHeader}; ++use crate::storage::{ ++ read_header, write_envelope_from_paths, write_envelope_header, ArchiveFileEntry, IndexHeader, ++}; + use crate::tokenizer::{TokenizerConfig, TokenizerKind}; + use levenshtein_automata::{Distance, LevenshteinAutomatonBuilder, DFA, SINK_STATE}; + use roaring::RoaringTreemap; +@@ -30,6 +32,7 @@ + use std::path::{Path, PathBuf}; + use std::sync::{Arc, Mutex}; + use tantivy::collector::{FilterCollector, TopDocs}; ++use tantivy::directory::{Directory, RamDirectory}; + use tantivy::query::{ + BooleanQuery, BoostQuery, ConstScorer, EmptyQuery, EnableScoring, Explanation, Occur, Query, + QueryParser, Scorer, TermQuery, Weight, +@@ -64,7 +67,12 @@ + + struct FullTextIndexWriterState { + index_writer: IndexWriter, +- temp_dir: TempDir, ++ directory: WriterDirectory, ++} ++ ++enum WriterDirectory { ++ Disk(TempDir), ++ Memory(RamDirectory), + } + + #[derive(Clone, Debug, PartialEq, Eq)] +@@ -77,8 +85,23 @@ + pub fn new(config: FullTextIndexConfig) -> Result { + config.validate()?; + let temp_dir = TempDir::new()?; +- let schema = build_schema(&config); +- let mut index = Index::create_in_dir(temp_dir.path(), schema.clone())?; ++ let index = Index::create_in_dir(temp_dir.path(), build_schema(&config))?; ++ Self::with_index(config, index, WriterDirectory::Disk(temp_dir)) ++ } ++ ++ pub fn new_in_memory(config: FullTextIndexConfig) -> Result { ++ config.validate()?; ++ let directory = RamDirectory::create(); ++ let index = Index::create(directory.clone(), build_schema(&config), Default::default())?; ++ Self::with_index(config, index, WriterDirectory::Memory(directory)) ++ } ++ ++ fn with_index( ++ config: FullTextIndexConfig, ++ mut index: Index, ++ directory: WriterDirectory, ++ ) -> Result { ++ let schema = index.schema(); + register_tokenizer(&mut index, &config.tokenizer)?; + let row_id_field = schema + .get_field(&config.row_id_field) +@@ -89,7 +112,7 @@ + config, + state: Some(FullTextIndexWriterState { + index_writer, +- temp_dir, ++ directory, + }), + row_id_field, + text_fields, +@@ -157,11 +180,23 @@ + })?; + let FullTextIndexWriterState { + mut index_writer, +- temp_dir, ++ directory, + } = state; + index_writer.commit()?; ++ let index = index_writer.index().clone(); + index_writer.wait_merging_threads()?; + ++ let metadata = FullTextIndexMetadata { ++ config: self.config.clone(), ++ document_count: self.document_count, ++ tantivy_version: tantivy::version().to_string(), ++ }; ++ let temp_dir = match directory { ++ WriterDirectory::Disk(temp_dir) => temp_dir, ++ WriterDirectory::Memory(ram) => { ++ return write_ram_archive(output, metadata, index, ram); ++ } ++ }; + let files = collect_index_files(temp_dir.path())?; + let mut offset = 0u64; + let mut entries = Vec::with_capacity(files.len()); +@@ -177,11 +212,7 @@ + } + + let header = IndexHeader { +- metadata: FullTextIndexMetadata { +- config: self.config.clone(), +- document_count: self.document_count, +- tantivy_version: tantivy::version().to_string(), +- }, ++ metadata, + files: entries, + }; + let paths = files.iter().map(|file| &file.path).collect::>(); +@@ -485,6 +516,43 @@ + "document field '{field}' is not configured for this index" + ))) + } ++} ++ ++/// Write the same archive as the on-disk writer, including Tantivy's segment footers. ++fn write_ram_archive( ++ output: &mut W, ++ metadata: FullTextIndexMetadata, ++ index: Index, ++ directory: RamDirectory, ++) -> Result<()> { ++ let mut paths: Vec<_> = index.directory().list_managed_files().into_iter().collect(); ++ paths.push(PathBuf::from(".managed.json")); ++ paths.sort(); ++ let mut files = Vec::with_capacity(paths.len()); ++ let mut contents = Vec::with_capacity(paths.len()); ++ let mut offset = 0u64; ++ for path in paths { ++ let bytes = directory ++ .open_read(&path) ++ .map_err(tantivy::TantivyError::from)? ++ .read_bytes()?; ++ let length = bytes.len() as u64; ++ files.push(ArchiveFileEntry { ++ name: path.to_string_lossy().into_owned(), ++ offset, ++ length, ++ }); ++ offset = offset.checked_add(length).ok_or_else(|| { ++ FtIndexError::InvalidStorage("full-text archive size overflow".to_string()) ++ })?; ++ contents.push(bytes); ++ } ++ write_envelope_header(output, &IndexHeader { metadata, files })?; ++ for bytes in contents { ++ output.write_all(bytes.as_slice())?; ++ } ++ output.flush()?; ++ Ok(()) + } + + struct IndexFile { +diff --git a/core/src/storage.rs b/core/src/storage.rs +--- a/core/src/storage.rs ++++ b/core/src/storage.rs +@@ -87,7 +87,10 @@ + Ok(()) + } + +-fn write_envelope_header(output: &mut W, header: &IndexHeader) -> Result<()> { ++pub(crate) fn write_envelope_header( ++ output: &mut W, ++ header: &IndexHeader, ++) -> Result<()> { + let header_json = serde_json::to_vec(header)?; + if header_json.len() > MAX_HEADER_BYTES { + return Err(FtIndexError::InvalidStorage(format!( +diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs +--- a/ffi/src/lib.rs ++++ b/ffi/src/lib.rs +@@ -220,10 +220,28 @@ + values: *const *const c_char, + len: usize, + ) -> *mut PaimonFtindexWriterHandle { ++ open_writer(keys, values, len, FullTextIndexWriter::new) ++} ++ ++#[no_mangle] ++pub unsafe extern "C" fn paimon_ftindex_writer_open_in_memory( ++ keys: *const *const c_char, ++ values: *const *const c_char, ++ len: usize, ++) -> *mut PaimonFtindexWriterHandle { ++ open_writer(keys, values, len, FullTextIndexWriter::new_in_memory) ++} ++ ++unsafe fn open_writer( ++ keys: *const *const c_char, ++ values: *const *const c_char, ++ len: usize, ++ create: fn(FullTextIndexConfig) -> paimon_ftindex_core::Result, ++) -> *mut PaimonFtindexWriterHandle { + ffi_ptr(|| { + let options = options_from_raw(keys, values, len)?; + let config = FullTextIndexConfig::from_options(&options).map_err(|e| e.to_string())?; +- let writer = FullTextIndexWriter::new(config).map_err(|e| e.to_string())?; ++ let writer = create(config).map_err(|e| e.to_string())?; + Ok(Box::into_raw(Box::new(PaimonFtindexWriterHandle { + inner: writer, + }))) +diff --git a/include/paimon_ftindex.h b/include/paimon_ftindex.h +--- a/include/paimon_ftindex.h ++++ b/include/paimon_ftindex.h +@@ -59,6 +59,12 @@ + const char **values, + size_t len); + ++/* Builds the index segments in memory for a temporary search index. */ ++PaimonFtindexWriterHandle *paimon_ftindex_writer_open_in_memory( ++ const char **keys, ++ const char **values, ++ size_t len); ++ + int paimon_ftindex_writer_add_document( + PaimonFtindexWriterHandle *writer, + int64_t row_id, diff --git a/cmake_modules/lucene.diff b/cmake_modules/lucene.diff new file mode 100644 index 000000000..32ba1570f --- /dev/null +++ b/cmake_modules/lucene.diff @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Lucene++ 3.0.9 uses pointer member access on reference parameters in these templates. +# Apple Clang rejects the definitions while parsing Lucene.h via LuceneInc.h, including +# when building the production library's precompiled headers, without any swap() calls. + +diff --git a/include/lucene++/Collection.h b/include/lucene++/Collection.h +--- a/include/lucene++/Collection.h ++++ b/include/lucene++/Collection.h +@@ -167,7 +167,7 @@ + } + + void swap(this_type& other) { +- container.swap(other->container); ++ container.swap(other.container); + } + + TYPE& operator[] (int32_t pos) { +diff --git a/include/lucene++/Set.h b/include/lucene++/Set.h +--- a/include/lucene++/Set.h ++++ b/include/lucene++/Set.h +@@ -108,7 +108,7 @@ + } + + void swap(this_type& other) { +- setContainer.swap(other->setContainer); ++ setContainer.swap(other.setContainer); + } + + operator bool() const { diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 5fcc3b1e2..38fd8be11 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -140,7 +140,7 @@ a column name containing only such whitespace selects the indexed field. - **Option**: ``lucene-fts.write.tmp.directory`` - - **Description**: Specifies the temporary directory used during Lucene index writing. No default value; must be explicitly set. + - **Description**: Required when building a persistent Lucene index. Full-text searches build their temporary indexes in memory. - **Environment Variable**: ``PAIMON_JIEBA_DICT_DIR`` @@ -298,12 +298,8 @@ Table-Level Full-Text Search full-text indexes of one column of a data-evolution table, as Java Paimon's ``FullTextSearchBuilder`` does, so a caller does not create index readers or choose index types and shard ranges. It searches the index files whose indexer supports full-text search -(``GlobalIndexer::SupportsFullTextSearch()``): ``full-text`` and ``lucene-fts``. Only the rows that -an index shard covers are searched, as in the default ``fast`` ``full-text-index.search-mode`` of -Java Paimon, so rows written since the index was last built are not found until it is built for -them. A search whose ``full-text-index.search-mode``, or else ``global-index.search-mode``, is -``full`` or ``detail``, which also search the rows that no index covers, fails with -``NotImplemented``. +(``GlobalIndexer::SupportsFullTextSearch()``): ``full-text`` and ``lucene-fts``. By default it +searches only indexed rows. To include rows appended since the last build, see `Search Modes`_. .. code-block:: cpp @@ -330,11 +326,10 @@ them. A search whose ``full-text-index.search-mode``, or else ``global-index.sea PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(data_plan->Splits())); -The result holds the ``limit`` highest-scoring rows among the candidates of all index shards, and -a read of the rows returns their scores in the ``_INDEX_SCORE`` field. Each shard returns at most -``limit`` candidates, chosen by its index, so ties are broken by ascending row id among the -candidates only: which of the rows tied at the ``limit``-th score of a shard become candidates is up -to its index. +The search keeps the ``limit`` highest-scoring candidates across the stored shards and any +temporary index built for unindexed rows. Read ``_INDEX_SCORE`` to retrieve their scores. +Each index returns up to ``limit`` candidates and decides which rows to return when scores tie +at its cutoff. The merge breaks ties among the returned candidates by ascending row id. - ``SetPartitionFilter()`` restricts the search to some partitions, as in ``ScanContextBuilder``. - ``WithFilter()`` restricts the search to the rows matching a predicate; see @@ -345,8 +340,13 @@ to its index. options may be set, and it must agree with an explicit ``scan.mode``: ``from-snapshot`` needs ``scan.snapshot-id`` or ``scan.tag-name``, ``from-snapshot-full`` needs ``scan.snapshot-id``, ``from-timestamp`` needs ``scan.timestamp`` or ``scan.timestamp-millis``, and ``latest-full`` and - ``latest`` take none of them. The search still uses the latest schema of the table, not the - schema of that snapshot. + ``latest`` take none of them. For an explicit snapshot, tag or timestamp, the search resolves + text and filter columns in the selected snapshot's schema. BLOB field declarations also + follow that schema unless explicitly overridden. Each call to ``CreatePlan()`` resolves the + selected tag or timestamp again. Splits from earlier plans still refer to their original + snapshot. A current search uses the latest table schema. +- Tables with ``query-auth.enabled=true`` require catalog row filters and column masks. Full-text + search rejects these tables, including for historical queries. - A shard whose indexed field is the column or that carries it as an extra field is searched. When shards overlap, each row is ranked by one of them: an index whose indexed field is the column comes first, then the order of the index type and fields, then the start and end of the @@ -360,6 +360,53 @@ to its index. - A column with a primary-key full-text index (``pk-full-text.index.columns``) cannot be searched yet. +Search Modes +~~~~~~~~~~~~ + +``full-text-index.search-mode``, or else ``global-index.search-mode``, selects the rows that a +search covers, as in Java Paimon: + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Mode + - Rows searched + * - ``fast`` (default) + - Only the rows that an index shard covers. + * - ``full`` + - Also the rows below the next row id of the snapshot that no single-field full-text index + covers. Planning reads no data manifest, and the read skips the rows outside the searched + partitions. + * - ``detail`` + - Also the rows of the data files in the searched partitions that no single-field full-text + index covers. Planning reads the data manifests to find them. + +If the selected partitions have no full-text index for the column, the search returns no rows. +Otherwise, ``full`` and ``detail`` add one ``RawFullTextSearchSplit`` for the gaps in coverage when +the snapshot has a positive next row id. +As in Java Paimon, this coverage counts only single-field full-text index definitions. Multi-field +indexes can serve the searched column as their primary or extra field, but do not reduce the raw +ranges. Coverage includes every single-field definition, even if shard selection chooses another +index to rank its rows. +The worker combines the raw ranges and reads the text column, ``_ROW_ID``, and any filter columns +from the planned snapshot. If the ranges contain live, non-null text, the worker builds one +in-memory index with the first selected shard's backend and searches it with the same query and +limit. Both ``full-text`` and ``lucene-fts`` keep the temporary index segments and archive in memory. +A worker can search these rows without a writable local temporary directory. +Set analyzer options such as ``full-text.tokenizer`` in the table or search options to +match the existing index. + +The temporary index scores against its own corpus of live, non-null text. Planning excludes raw +ranges from every stored split's search ranges and omits stored splits with no remaining search +rows. It preserves physical shard ranges for row id translation. A row in a raw range therefore +appears only if its current text matches, even when workers read the splits independently. +The final top-k combines hits from the temporary index and the remaining indexed rows. + +.. code-block:: cpp + + search_builder.AddOption("full-text-index.search-mode", "full"); + Row Filters ~~~~~~~~~~~ @@ -378,20 +425,21 @@ implies: ``(p = 1 AND a > 0) OR (p = 2 AND a > 0)`` searches only partitions ``p paimon::FieldType::INT, paimon::Literal(7)); search_builder.WithFilter(filter); -The scan keeps the scalar global index files that index a field of the filter, and the read -resolves the filter with them before ranking: - -- If the indexes decide the whole filter, the rows they return are searched. -- If they can only narrow the filter down to candidates, because a conjunct has no index or the - filter has a ``Contains``, ``EndsWith`` or ``Like`` predicate, the candidates are excluded and a - warning is logged. The search then never returns a non-matching row, but may return fewer rows - than ``limit``. With ``global-index.filter.refine-from-data=true``, the read instead verifies the - candidates by reading their filter columns and ``_ROW_ID`` in the planned snapshot, on the caller - and possibly for every candidate. -- Rows that no scalar index covers are excluded. If no index can evaluate the filter at all, no row - is searched and a warning is logged. This is the default ``fast`` ``scalar-index.search-mode``, - which falls back on ``global-index.search-mode``. A row filter in ``full`` or ``detail`` mode, - which decide such rows by reading their filter columns, fails with ``NotImplemented``. +On a full-text index shard, ``scalar-index.search-mode`` controls how the reader finds rows that +satisfy the filter before ranking. It falls back to ``global-index.search-mode``, then to ``fast``. + +- In ``fast`` mode, only rows identified by a scalar index can qualify. If no index can evaluate + the filter, the reader skips the shard's rows and logs a warning. +- In ``full`` and ``detail``, the reader checks the filter directly against the data wherever a + filter column lacks index coverage. If no index can evaluate the filter, the reader checks the + covered rows against the data as well. See `Search Modes`_ for how each mode finds gaps. +- In any mode, an index may return candidates that still need verification, for example for a + ``Contains``, ``EndsWith`` or ``Like`` predicate. By default the reader excludes those candidates + and logs a warning. Set ``global-index.filter.refine-from-data=true`` to check them against the + planned snapshot. This option does not affect rows read directly to fill gaps. + +A raw split always evaluates the filter on its data. All its live, non-null text contributes to +the temporary index's scores, even when the filter excludes a row from the results. Only single-field indexes that do not support full-text search evaluate a filter. A filter with a ``PredicateBuilder::FullTextMatch`` predicate fails with ``NotImplemented``. @@ -399,11 +447,11 @@ Only single-field indexes that do not support full-text search evaluate a filter Pinning the Data Read ~~~~~~~~~~~~~~~~~~~~~ -The row ids of the result refer to the snapshot the search was planned from. If the table may -change before the rows are read, plan with ``NewFullTextScan()`` and pin the data scan to the -snapshot of the plan, setting ``scan.mode`` too so that a ``scan.mode`` of the table cannot override -it. For a search of ``scan.tag-name``, pin the data scan with the same tag instead, as the tag may -outlive the snapshot. +The row ids in a search result belong to the planning snapshot. Use ``NewFullTextScan()`` to get +its ID and set both ``scan.mode`` and ``scan.snapshot-id`` on the data scan. If a tag retains an +expired snapshot, set ``scan.mode`` to ``from-snapshot`` and select the tag with ``scan.tag-name``. +A tag can move between the search and the data scan. Check that both plans have the same +``SnapshotId()`` before reading the rows. .. code-block:: cpp @@ -426,13 +474,22 @@ outlive the snapshot. ``ExecuteLocal()`` plans and searches in one process. To plan in one process and search in others, plan with ``NewFullTextScan()``, ship the splits with ``FullTextSearchSplit::Serialize()`` and ``FullTextSearchSplit::Deserialize()``, and search them with the ``NewFullTextRead()`` of a builder -for the same table, column, query and filter. The serialized form is versioned and specific to -Paimon C++: Java Paimon serializes its splits with Java serialization, so splits cannot be exchanged -with it. Each split records the snapshot it was planned from, and the read ranks the rows live in -that snapshot. The read rejects a split whose index files do not index the column of the latest -schema, such as after the column was replaced. A read returns the top rows of the splits it is -given, so keep the ``limit`` highest-scoring rows of all reads if the splits are searched by several -of them. +for the same table, column, query, limit, partitions, row filter and search options. The serialized +form is versioned and specific to Paimon C++. Each split records its snapshot. A worker configured +with an explicit snapshot ID can use a tag that retains that snapshot after its file expires. +Stored index splits from version 3 and all raw splits carry the planning schema ID. A worker +rejects these splits if the searched column name now refers to a different field ID, even when +both fields belong to the same index. Raw split version 1 records the text field ID explicitly. +Workers use the planning schema for index fields, row filters and BLOB storage declarations. +For example, if an integer filter column is replaced by a BLOB view with the same name, the +worker still reads the original integer column. Explicit BLOB field options take precedence. +When all supplied splits lack a schema ID, workers use their own schema. +Stored splits carry scalar index files only for their assigned search ranges. A worker also +reads deletion vectors only for data files that intersect those ranges. +When several workers search the splits, merge their results and keep the ``limit`` highest-scoring +rows. The planner assigns disjoint search ranges to stored and raw splits before serialization. +If you supply overlapping splits to one read, it excludes the raw ranges from the stored search +before ranking. .. code-block:: cpp diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 9b96d7220..a39a2e128 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -637,20 +637,21 @@ struct PAIMON_EXPORT Options { /// and checkpoints use independent round robin. If not set, the global index /// files will be stored under the index directory. static const char GLOBAL_INDEX_EXTERNAL_PATH[]; - /// "global-index.search-mode" - Fallback search mode for global index queries, used when the - /// search mode of the index family is not set: "fast", "full" or "detail". No default value. - /// Paimon C++ reads it only for full-text search so far. + /// "global-index.search-mode" - Fallback for full-text, scalar and vector index search modes. + /// Accepts "fast", "full" or "detail". Families default to "fast" when neither option is set. static const char GLOBAL_INDEX_SEARCH_MODE[]; - /// "full-text-index.search-mode" - Search mode for full-text index queries: "fast" searches - /// only the rows that an index covers, while "full" and "detail" also search the rows that no - /// index covers. Default value is "global-index.search-mode", or else "fast". Paimon C++ - /// supports only "fast" so far, and a full-text search in another mode fails. + /// "full-text-index.search-mode" - "fast" searches indexed rows. "full" finds gaps below the + /// snapshot's next row id. "detail" finds gaps in the selected partitions' data files. Both + /// search the gaps if a selected partition has a full-text index for the column. + /// Defaults to "global-index.search-mode", then "fast". static const char FULL_TEXT_INDEX_SEARCH_MODE[]; - /// "scalar-index.search-mode" - Search mode for scalar index queries: "fast" evaluates only the - /// rows that an index covers, while "full" and "detail" also evaluate the rows that no index - /// covers. Default value is "global-index.search-mode", or else "fast". Paimon C++ supports - /// only "fast" so far, and a full-text search with a row filter in another mode fails. + /// "scalar-index.search-mode" - Controls row filtering on full-text index shards. "fast" + /// uses scalar index candidates. "full" and "detail" also read the filter columns for gaps + /// in scalar index coverage. Defaults to "global-index.search-mode", then "fast". static const char SCALAR_INDEX_SEARCH_MODE[]; + /// "vector-index.search-mode" - Reserved for table-level vector search. Accepts "fast", + /// "full" or "detail". Defaults to "global-index.search-mode", then "fast". + static const char VECTOR_INDEX_SEARCH_MODE[]; /// "global-index.filter.refine-from-data" - Whether a vector, hybrid or full-text search may /// read the filter columns of candidate rows to verify a row filter that the scalar global /// index can only answer with a superset, such as contains, ends-with or like on a BTree index diff --git a/include/paimon/table/source/full_text_read.h b/include/paimon/table/source/full_text_read.h index 1d4a6f718..ff71652dc 100644 --- a/include/paimon/table/source/full_text_read.h +++ b/include/paimon/table/source/full_text_read.h @@ -40,28 +40,22 @@ class PAIMON_EXPORT FullTextRead { return Read(plan.Splits()); } - /// Searches `splits` and returns the `limit` highest-scoring rows among the candidates that - /// they return. The result holds global row ids and iterates them in ascending order; pass it - /// to `ScanContextBuilder::SetGlobalIndexResult()` to read the rows, whose scores the read - /// returns in the `_INDEX_SCORE` field. + /// Returns the top `limit` hits across `splits`, in ascending global row id order. Pass the + /// result to `ScanContextBuilder::SetGlobalIndexResult()` to read the rows with `_INDEX_SCORE`. /// - /// The splits are searched in parallel, and each one returns at most `limit` candidates, chosen - /// by its index. Ties among the candidates are broken by ascending row id, but which of the - /// rows tied at the `limit`-th score of a split become candidates is up to its index. A split - /// ranks only its search row ranges, and with deletion vectors enabled only the rows that are - /// live in the snapshot the splits were planned from; BM25 statistics still come from its whole - /// index shard. + /// Row filters apply before ranking. Each stored shard scores against its whole corpus. + /// The temporary index scores against all live, non-null text in the raw ranges. + /// Each backend resolves ties at its cutoff. The merge breaks ties by ascending row id. /// - /// With a row filter (`FullTextSearchBuilder::WithFilter()`), a split ranks only the rows that - /// satisfy it, as resolved before ranking by the scalar indexes that the splits carry. Rows - /// that no scalar index can evaluate are not ranked. Candidates that the indexes cannot decide - /// exactly, as for `Contains`, `EndsWith` and `Like`, are excluded too, so the result may hold - /// fewer than `limit` rows, unless `global-index.filter.refine-from-data` verifies them against - /// the data of the planned snapshot on this thread. - /// @param splits Splits planned by a `FullTextScan` of the same column and filter, all from one - /// snapshot. - /// @return The rows found, or an error if a split is invalid, its index files do not index the - /// column of the latest schema (for example, after the column was replaced), or an + /// Planned splits can be read independently. When raw and stored splits overlap, stored + /// shards exclude the raw ranges before selecting their top hits. + /// + /// On stored shards, `full` and `detail` scalar modes read filter columns wherever scalar + /// indexes lack coverage or cannot answer the predicate. + /// Inexact candidates rank only after verification with `global-index.filter.refine-from-data`. + /// Workers bind row-filter fields to the planning schema when the splits carry its ID. + /// @param splits Splits planned for the same snapshot, schema, column and filters. + /// @return The rows found, or an error if a split is invalid, its column was replaced, or an /// index or data file cannot be read. virtual Result> Read( const std::vector>& splits) = 0; diff --git a/include/paimon/table/source/full_text_scan.h b/include/paimon/table/source/full_text_scan.h index fc78111e1..2e32901da 100644 --- a/include/paimon/table/source/full_text_scan.h +++ b/include/paimon/table/source/full_text_scan.h @@ -60,28 +60,24 @@ class PAIMON_EXPORT FullTextScan { virtual ~FullTextScan() = default; - /// Plans the search on the snapshot that the options select, as Java - /// `TimeTravelUtil#tryTravelOrLatest` does: snapshot `scan.snapshot-id`, the snapshot of tag - /// `scan.tag-name`, the latest snapshot at or before `scan.timestamp` or - /// `scan.timestamp-millis`, or else the latest snapshot. As in Java, at most one of these - /// options may be set, and it must agree with an explicit `scan.mode`: `from-snapshot` needs - /// `scan.snapshot-id` or `scan.tag-name`, `from-snapshot-full` needs `scan.snapshot-id`, - /// `from-timestamp` needs `scan.timestamp` or `scan.timestamp-millis`, and `latest-full` and - /// `latest` take none of them. + /// Plans the search on the latest snapshot or the one selected by `scan.snapshot-id`, + /// `scan.tag-name`, `scan.timestamp` or `scan.timestamp-millis`. A timestamp selects the + /// latest snapshot at or before that time. Set at most one selector, consistent with + /// `scan.mode`. Each plan resolves its tag or timestamp anew. /// - /// Each index shard of the searched column becomes one split. The scan considers only the - /// index files that are in the partitions of the partition filter, that index the column as - /// their indexed field or as one of their extra fields, and whose indexer supports full-text - /// search (`GlobalIndexer::SupportsFullTextSearch()`). The plan fails if Paimon C++ is built - /// without the backend of a `full-text` or `lucene-fts` index it finds. When the row ranges of - /// index shards overlap, each row is ranked by one shard only: an index whose indexed field is - /// the column comes before one that carries it as an extra field, and further ties are broken - /// by index type and fields, then by the start and end of the shard. With a row filter, each - /// split also carries the files of single-field scalar indexes that intersect its shard and - /// index a field of the filter. - /// @return The plan, or an error if more than one time-travel option is set, the option does - /// not agree with `scan.mode`, the selected snapshot or tag does not exist, or reading - /// the index manifests fails; `NotImplemented` for an index whose backend is not built. + /// Each full-text shard assigned rows in the selected partitions becomes one split. For + /// overlapping shards, the planner prefers an index on the searched column over one that + /// carries it as an extra field. Ties are broken by index type, field IDs and shard range. + /// A stored split carries single-field scalar indexes for its row filter only where those + /// indexes intersect its assigned search ranges. + /// + /// When a selected partition has a full-text index for the column, `full` and `detail` assign + /// gaps in single-field index coverage to a `RawFullTextSearchSplit`. `full` checks all row + /// ids below the snapshot's next row id. `detail` checks data files in the selected partitions. + /// Raw and stored splits rank disjoint rows and can be read independently. + /// @return The plan, or an error for conflicting scan options, a missing snapshot or tag, or + /// unreadable manifests. A missing `full-text` or `lucene-fts` backend returns + /// `NotImplemented`. virtual Result> CreatePlan() = 0; }; diff --git a/include/paimon/table/source/full_text_search_builder.h b/include/paimon/table/source/full_text_search_builder.h index 2d07d9829..df80bbfca 100644 --- a/include/paimon/table/source/full_text_search_builder.h +++ b/include/paimon/table/source/full_text_search_builder.h @@ -50,13 +50,14 @@ class Executor; /// /// `ExecuteLocal()` plans and searches in this process. To run them in different processes, plan /// with `NewFullTextScan()`, serialize the splits with `FullTextSearchSplit::Serialize()`, and -/// search them with the `NewFullTextRead()` of a builder for the same table, column, query and -/// filter. +/// search them with `NewFullTextRead()` on each worker using the same table, column, query, +/// limit, partition and row filters, and search options. +/// Full-text search returns `NotImplemented` on tables that require catalog query authorization. /// /// The row ids of the result refer to the snapshot the search was planned from, which -/// `FullTextScan::Plan::SnapshotId()` returns. If the table may change in between, pin the data -/// scan to it with `scan.mode` = `from-snapshot` and `scan.snapshot-id`, or with the same -/// `scan.tag-name` for a search of a tag, as the tag may outlive the snapshot. +/// `FullTextScan::Plan::SnapshotId()` returns. Pin the data scan with `scan.mode` = `from-snapshot` +/// and `scan.snapshot-id`. If a tag retains an expired snapshot, use `scan.tag-name` and check +/// that the data plan's snapshot ID still matches the search plan's. class PAIMON_EXPORT FullTextSearchBuilder { public: /// @param table_path The root path of the table. @@ -111,17 +112,16 @@ class PAIMON_EXPORT FullTextSearchBuilder { FullTextSearchBuilder& WithFileSystem(const std::shared_ptr& file_system); /// Creates a scan that plans the search on the current settings. The query must be set, and - /// its column must exist in the latest schema of the table. + /// its column must exist in the selected snapshot's schema for time travel, or in the latest + /// table schema for a current search. /// @return The scan, or an error if the settings are invalid or the table cannot be loaded; - /// `NotImplemented` for a primary-key full-text index (`pk-full-text.index.columns`), a - /// search mode other than `fast` (`full-text-index.search-mode`, or else - /// `global-index.search-mode`), a row filter whose scalar search mode is not `fast` - /// (`scalar-index.search-mode`, or else `global-index.search-mode`), or a filter with a - /// full-text match. + /// `NotImplemented` for a primary-key full-text index (`pk-full-text.index.columns`) or + /// a filter with a full-text match. Result> NewFullTextScan() const; - /// Creates a read that searches the splits of a scan on the current settings. In addition to - /// the requirements of `NewFullTextScan()`, the limit must be positive. + /// Creates a worker for the configured query and positive limit. The worker validates the + /// row filter against the planning schema when it receives the splits in + /// `FullTextRead::Read()`. /// @return The read, or an error if the settings are invalid or the table cannot be loaded. Result> NewFullTextRead() const; diff --git a/include/paimon/table/source/full_text_search_split.h b/include/paimon/table/source/full_text_search_split.h index 839c12e6a..0918f28da 100644 --- a/include/paimon/table/source/full_text_search_split.h +++ b/include/paimon/table/source/full_text_search_split.h @@ -81,4 +81,18 @@ class PAIMON_EXPORT IndexFullTextSearchSplit : public FullTextSearchSplit { virtual const std::vector& SearchRowRanges() const = 0; }; +/// Ranges of unindexed rows that a full or detail search reads from the planned snapshot. +class PAIMON_EXPORT RawFullTextSearchSplit : public FullTextSearchSplit { + public: + /// @return The planning snapshot ID. + virtual int64_t SnapshotId() const = 0; + + /// @return The name of the searched table column. + virtual const std::string& ColumnName() const = 0; + + /// @return Sorted, disjoint and non-adjacent global row id ranges. Only live rows in the + /// selected partitions can match. + virtual const std::vector& RowRanges() const = 0; +}; + } // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 7b8d58aeb..e0954f778 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -283,6 +283,7 @@ set(PAIMON_CORE_SRCS core/deletionvectors/bitmap64_deletion_vector.cpp core/deletionvectors/deletion_vectors_index_file.cpp core/deletionvectors/deletion_vector_index_file_writer.cpp + core/global_index/data_evolution_global_index_coverage.cpp core/global_index/global_index_evaluator_impl.cpp core/global_index/global_index_live_row_filter.cpp core/global_index/global_index_scan.cpp @@ -512,6 +513,7 @@ set(PAIMON_CORE_SRCS core/table/source/filtered_row_id_reader.cpp core/table/source/full_text_search_builder.cpp core/table/source/full_text_search_split.cpp + core/table/source/raw_full_text_read_impl.cpp core/table/source/primary_key_sorted_index_scan.cpp core/table/source/primary_key_sorted_index_result.cpp core/table/source/primary_key_index_batch_scan.cpp @@ -917,6 +919,7 @@ if(PAIMON_BUILD_TESTS) core/io/rolling_blob_file_writer_test.cpp core/io/rolling_vector_file_writer_test.cpp core/global_index/global_index_file_manager_test.cpp + core/global_index/data_evolution_global_index_coverage_test.cpp core/global_index/global_index_evaluator_impl_test.cpp core/global_index/indexed_split_test.cpp core/global_index/live_row_global_index_reader_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 9fef4fe1c..703d13bda 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -168,6 +168,7 @@ const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; const char Options::GLOBAL_INDEX_SEARCH_MODE[] = "global-index.search-mode"; const char Options::FULL_TEXT_INDEX_SEARCH_MODE[] = "full-text-index.search-mode"; const char Options::SCALAR_INDEX_SEARCH_MODE[] = "scalar-index.search-mode"; +const char Options::VECTOR_INDEX_SEARCH_MODE[] = "vector-index.search-mode"; const char Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA[] = "global-index.filter.refine-from-data"; const char Options::PK_BTREE_INDEX_COLUMNS[] = "pk-btree.index.columns"; const char Options::PK_BITMAP_INDEX_COLUMNS[] = "pk-bitmap.index.columns"; diff --git a/src/paimon/common/reader/concat_batch_reader.cpp b/src/paimon/common/reader/concat_batch_reader.cpp index fcdfeee74..2200dd519 100644 --- a/src/paimon/common/reader/concat_batch_reader.cpp +++ b/src/paimon/common/reader/concat_batch_reader.cpp @@ -35,6 +35,19 @@ namespace paimon { class MemoryPool; +template +static auto CatchPreparationErrors(Func&& func) -> decltype(func()) { + try { + return func(); + } catch (const std::bad_alloc&) { + return Status::OutOfMemory("out of memory preparing file reader"); + } catch (const std::exception& e) { + return Status::UnknownError("file reader preparation failed: ", e.what()); + } catch (...) { + return Status::UnknownError("file reader preparation failed with an unknown exception"); + } +} + ConcatBatchReader::ConcatBatchReader(std::vector>&& readers, const std::shared_ptr& arrow_pool) : arrow_pool_(arrow_pool), finished_reader_metrics_(std::make_shared()) { @@ -94,7 +107,7 @@ void ConcatBatchReader::FillWindow() { if (retained->cancelled.load()) { return std::unique_ptr(); } - return retained->factory(index); + return CatchPreparationErrors([&]() { return retained->factory(index); }); }); } } @@ -104,22 +117,11 @@ Status ConcatBatchReader::ResolveReader(ReaderState* state, bool wait) { (!wait && state->future.wait_for(std::chrono::seconds(0)) != std::future_status::ready)) { return state->failure; } - try { - auto result = state->future.get(); - if (!result.ok()) { - state->failure = result.status(); - } else { - state->reader = std::move(result).value(); - state->file_reader = dynamic_cast(state->reader.get()); - } - } catch (const std::bad_alloc&) { - state->failure = Status::OutOfMemory("out of memory preparing file reader"); - } catch (const std::exception& e) { - state->failure = Status::UnknownError("file reader preparation failed: ", e.what()); - } catch (...) { - state->failure = - Status::UnknownError("file reader preparation failed with an unknown exception"); - } + state->failure = CatchPreparationErrors([state]() -> Status { + PAIMON_ASSIGN_OR_RAISE(state->reader, state->future.get()); + state->file_reader = dynamic_cast(state->reader.get()); + return Status::OK(); + }); return state->failure; } diff --git a/src/paimon/common/reader/concat_batch_reader_test.cpp b/src/paimon/common/reader/concat_batch_reader_test.cpp index 0fa287919..e63d1bfff 100644 --- a/src/paimon/common/reader/concat_batch_reader_test.cpp +++ b/src/paimon/common/reader/concat_batch_reader_test.cpp @@ -691,7 +691,6 @@ TEST_F(ConcatBatchReaderTest, LazyEarlyCloseAndDestructionDrainExceptionalBuilds ConcatBatchReader::kDefaultPreparationWindow, GetArrowPool(pool_), executor)); ScopeGuard unblock([&release]() { release.set_value(); }); lifetime.reset(); - // Keep the failed build pending until we stop reading, so cleanup consumes its exception. for (size_t i = 0; i < ConcatBatchReader::kDefaultPreparationWindow; i++) { ASSERT_OK_AND_ASSIGN(auto batch, reader->NextBatch()); ASSERT_FALSE(BatchReader::IsEofBatch(batch)); @@ -719,6 +718,15 @@ TEST_F(ConcatBatchReaderTest, LazyEarlyCloseAndDestructionDrainExceptionalBuilds } } +TEST_F(ConcatBatchReaderTest, LazyCreationReportsAbandonedBuild) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(1)); + executor->ShutdownNow(); + auto result = ConcatBatchReader::CreateLazy( + 1, [](size_t) { return std::unique_ptr(); }, 1, GetArrowPool(pool_), executor); + ASSERT_NOK_WITH_MSG(result, "file reader preparation failed"); + ASSERT_TRUE(result.status().IsUnknownError()); +} + TEST_F(ConcatBatchReaderTest, LazyInitialFailureDrainsPartialWindow) { auto counts = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(2)); diff --git a/src/paimon/common/utils/range.cpp b/src/paimon/common/utils/range.cpp index 753b80106..986ef5dc5 100644 --- a/src/paimon/common/utils/range.cpp +++ b/src/paimon/common/utils/range.cpp @@ -45,12 +45,9 @@ std::vector Range::SortAndMergeOverlap(const std::vector& ranges, for (size_t i = 1; i < sorted_ranges.size(); ++i) { Range next = sorted_ranges[i]; - // Check if current and next overlap (not just adjacent) - if (current.to + (adjacent ? 1 : 0) >= next.from) { - // Merge: extend current range + if (current.to >= next.from || (adjacent && current.to + 1 == next.from)) { current = Range(current.from, std::max(current.to, next.to)); } else { - // No overlap: add current to result and move to next results.push_back(current); current = next; } @@ -127,16 +124,12 @@ std::vector Range::Exclude(const std::vector& ranges) const { if (current < intersect.value().from) { result.emplace_back(current, intersect.value().from - 1); } - // Move current position past the intersection - current = intersect.value().to + 1; - if (current > to) { - break; + if (intersect->to == to) { + return result; } + current = intersect->to + 1; } - // Add the remaining part after all exclusions (if any) - if (current <= to) { - result.emplace_back(current, to); - } + result.emplace_back(current, to); return result; } diff --git a/src/paimon/common/utils/range_test.cpp b/src/paimon/common/utils/range_test.cpp index 65ee22eeb..69654b853 100644 --- a/src/paimon/common/utils/range_test.cpp +++ b/src/paimon/common/utils/range_test.cpp @@ -19,6 +19,8 @@ #include "paimon/utils/range.h" +#include + #include "gtest/gtest.h" namespace paimon::test { @@ -97,6 +99,9 @@ TEST(RangeTest, TestCompare) { } TEST(RangeTest, TestSortAndMergeOverlap) { + const int64_t last = std::numeric_limits::max(); + ASSERT_EQ(Range::SortAndMergeOverlap({Range(last - 2, last), Range(last, last)}, true), + std::vector({Range(last - 2, last)})); { // test simple std::vector ranges = {Range(0, 10), Range(5, 15)}; @@ -258,6 +263,9 @@ TEST(RangeTest, TestAnd) { } TEST(RangeTest, TestExclude) { + const int64_t last = std::numeric_limits::max(); + ASSERT_EQ(Range(last - 3, last).Exclude({Range(last - 2, last)}), + std::vector({Range(last - 3, last - 3)})); { // test basic // [0, 10000] exclude [1000,2000],[3000,4000],[5000,6000] diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 3a87f6616..2e92d7c98 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -454,6 +454,7 @@ struct CoreOptions::Impl { std::optional global_index_thread_num; GlobalIndexSearchMode full_text_index_search_mode = GlobalIndexSearchMode::FAST; GlobalIndexSearchMode scalar_index_search_mode = GlobalIndexSearchMode::FAST; + GlobalIndexSearchMode vector_index_search_mode = GlobalIndexSearchMode::FAST; bool global_index_filter_refine_from_data = false; bool realtime_enabled = false; @@ -942,12 +943,12 @@ struct CoreOptions::Impl { // index, no default value PAIMON_RETURN_NOT_OK( parser.Parse(Options::GLOBAL_INDEX_THREAD_NUM, &global_index_thread_num)); - // Parse full-text-index.search-mode, falling back to global-index.search-mode, default fast PAIMON_RETURN_NOT_OK(parser.ParseIndexSearchMode(Options::FULL_TEXT_INDEX_SEARCH_MODE, &full_text_index_search_mode)); - // Parse scalar-index.search-mode, falling back to global-index.search-mode, default fast PAIMON_RETURN_NOT_OK(parser.ParseIndexSearchMode(Options::SCALAR_INDEX_SEARCH_MODE, &scalar_index_search_mode)); + PAIMON_RETURN_NOT_OK(parser.ParseIndexSearchMode(Options::VECTOR_INDEX_SEARCH_MODE, + &vector_index_search_mode)); // Parse global-index.filter.refine-from-data, default false PAIMON_RETURN_NOT_OK(parser.Parse(Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, &global_index_filter_refine_from_data)); @@ -1976,6 +1977,10 @@ GlobalIndexSearchMode CoreOptions::GetScalarIndexSearchMode() const { return impl_->scalar_index_search_mode; } +GlobalIndexSearchMode CoreOptions::GetVectorIndexSearchMode() const { + return impl_->vector_index_search_mode; +} + bool CoreOptions::GlobalIndexFilterRefineFromData() const { return impl_->global_index_filter_refine_from_data; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index c478b2c1c..e513d5d93 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -307,12 +307,9 @@ class PAIMON_EXPORT CoreOptions { BucketFunctionType GetBucketFunctionType() const; std::optional GetGlobalIndexThreadNum() const; - /// Returns `full-text-index.search-mode`, or else `global-index.search-mode`, or else - /// `GlobalIndexSearchMode::FAST`, as Java `CoreOptions#fullTextIndexSearchMode` does. GlobalIndexSearchMode GetFullTextIndexSearchMode() const; - /// Returns `scalar-index.search-mode`, or else `global-index.search-mode`, or else - /// `GlobalIndexSearchMode::FAST`, as Java `CoreOptions#scalarIndexSearchMode` does. GlobalIndexSearchMode GetScalarIndexSearchMode() const; + GlobalIndexSearchMode GetVectorIndexSearchMode() const; bool GlobalIndexFilterRefineFromData() const; const std::vector& GetBlobFields() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index c3b78a696..0a6709471 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -1164,7 +1164,8 @@ TEST(CoreOptionsTest, TestParseIndexSearchMode) { }; const std::vector families = { {Options::FULL_TEXT_INDEX_SEARCH_MODE, &CoreOptions::GetFullTextIndexSearchMode}, - {Options::SCALAR_INDEX_SEARCH_MODE, &CoreOptions::GetScalarIndexSearchMode}}; + {Options::SCALAR_INDEX_SEARCH_MODE, &CoreOptions::GetScalarIndexSearchMode}, + {Options::VECTOR_INDEX_SEARCH_MODE, &CoreOptions::GetVectorIndexSearchMode}}; for (const IndexFamily& family : families) { SCOPED_TRACE(family.key); auto search_mode = [&family](const std::map& options) diff --git a/src/paimon/core/global_index/data_evolution_global_index_coverage.cpp b/src/paimon/core/global_index/data_evolution_global_index_coverage.cpp new file mode 100644 index 000000000..50a6226f6 --- /dev/null +++ b/src/paimon/core/global_index/data_evolution_global_index_coverage.cpp @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/global_index/data_evolution_global_index_coverage.h" + +#include +#include +#include + +namespace paimon { + +std::vector DataEvolutionGlobalIndexCoverage::IndexedRanges( + const std::vector>& index_files, + const std::set& field_ids) { + std::map> coverage_by_field; + for (const auto& file : index_files) { + const std::optional& meta = file->GetGlobalIndexMeta(); + if (!meta || (meta->extra_field_ids && !meta->extra_field_ids->empty())) { + continue; + } + coverage_by_field[meta->index_field_id].emplace_back(meta->row_range_start, + meta->row_range_end); + } + + std::vector indexed; + for (int32_t field_id : field_ids) { + auto iter = coverage_by_field.find(field_id); + if (iter == coverage_by_field.end()) { + return {}; + } + std::vector field_ranges = Range::SortAndMergeOverlap(iter->second, true); + indexed = indexed.empty() ? std::move(field_ranges) : Range::And(indexed, field_ranges); + if (indexed.empty()) { + break; + } + } + return indexed; +} + +Result> DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode mode, std::optional next_row_id, + const std::vector& indexed_ranges, const DataRangesSupplier& data_ranges) { + if (mode == GlobalIndexSearchMode::FAST || !next_row_id || next_row_id.value() <= 0) { + return std::vector(); + } + std::vector all_data_ranges; + if (mode == GlobalIndexSearchMode::DETAIL) { + PAIMON_ASSIGN_OR_RAISE(all_data_ranges, data_ranges()); + } else { + all_data_ranges.emplace_back(0, next_row_id.value() - 1); + } + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex covered, RowRangeIndex::Create(indexed_ranges)); + return ExcludeRanges(Range::SortAndMergeOverlap(all_data_ranges, /*adjacent=*/true), covered); +} + +std::vector DataEvolutionGlobalIndexCoverage::ExcludeRanges(const std::vector& ranges, + const RowRangeIndex& covered) { + std::vector uncovered; + for (const Range& range : ranges) { + std::vector remaining = + range.Exclude(covered.IntersectedRanges(range.from, range.to)); + uncovered.insert(uncovered.end(), remaining.begin(), remaining.end()); + } + return uncovered; +} + +} // namespace paimon diff --git a/src/paimon/core/global_index/data_evolution_global_index_coverage.h b/src/paimon/core/global_index/data_evolution_global_index_coverage.h new file mode 100644 index 000000000..e31c60378 --- /dev/null +++ b/src/paimon/core/global_index/data_evolution_global_index_coverage.h @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/options/global_index_search_mode.h" +#include "paimon/result.h" +#include "paimon/utils/range.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon { +class DataEvolutionGlobalIndexCoverage { + public: + using DataRangesSupplier = std::function>()>; + + DataEvolutionGlobalIndexCoverage() = delete; + ~DataEvolutionGlobalIndexCoverage() = delete; + + /// Returns the ranges where every requested field has a single-field index. + /// An empty field set yields no ranges. + static std::vector IndexedRanges( + const std::vector>& index_files, + const std::set& field_ids); + + /// Subtracts covered rows from sorted, disjoint ranges. + static std::vector ExcludeRanges(const std::vector& ranges, + const RowRangeIndex& covered); + + /// Returns sorted, disjoint gaps below the snapshot's next row id in FULL mode or within + /// the supplied data file ranges in DETAIL mode. Only DETAIL loads the data ranges. + /// FAST and snapshots without a positive next row id yield no gaps. + static Result> UnindexedRanges(GlobalIndexSearchMode mode, + std::optional next_row_id, + const std::vector& indexed_ranges, + const DataRangesSupplier& data_ranges); +}; + +} // namespace paimon diff --git a/src/paimon/core/global_index/data_evolution_global_index_coverage_test.cpp b/src/paimon/core/global_index/data_evolution_global_index_coverage_test.cpp new file mode 100644 index 000000000..fe3e85d22 --- /dev/null +++ b/src/paimon/core/global_index/data_evolution_global_index_coverage_test.cpp @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/global_index/data_evolution_global_index_coverage.h" + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/range.h" + +namespace paimon::test { +namespace { +std::shared_ptr NewIndexFile( + const std::string& file_name, const Range& range, int32_t index_field_id, + const std::optional>& extra_field_ids = std::nullopt) { + GlobalIndexMeta meta(range.from, range.to, index_field_id, extra_field_ids, nullptr); + return std::make_shared("btree", file_name, 128, range.Count(), std::nullopt, + std::nullopt, meta); +} + +DataEvolutionGlobalIndexCoverage::DataRangesSupplier Ranges(const std::vector& ranges) { + return [ranges]() -> Result> { return ranges; }; +} + +DataEvolutionGlobalIndexCoverage::DataRangesSupplier Unused() { + return []() -> Result> { + return Status::Invalid("data ranges are not needed"); + }; +} +} // namespace + +TEST(DataEvolutionGlobalIndexCoverageTest, TestIndexedRangesOfOneField) { + std::vector> files = { + NewIndexFile("a", Range(10, 19), 1), + NewIndexFile("b", Range(0, 9), 1), + NewIndexFile("overlap", Range(5, 14), 1), + NewIndexFile("c", Range(30, 39), 1), + NewIndexFile("other", Range(20, 29), 2), + std::make_shared("deletion-vector", "dv", 128, 10, std::nullopt, + std::nullopt)}; + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {1}), + std::vector({Range(0, 19), Range(30, 39)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {3}), std::vector()); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {}), std::vector()); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestIndexedRangesIntersectFields) { + std::vector> files = {NewIndexFile("disjoint", Range(50, 59), 0), + NewIndexFile("a", Range(0, 49), 1), + NewIndexFile("b", Range(20, 79), 2)}; + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {1, 2}), + std::vector({Range(20, 49)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {2}), + std::vector({Range(20, 79)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {1, 4}), std::vector()); + ASSERT_TRUE(DataEvolutionGlobalIndexCoverage::IndexedRanges(files, {0, 1, 2}).empty()); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestIndexedRangesIgnoreMultiFieldIndexes) { + const std::vector> multi_field_files = { + NewIndexFile("multi", Range(0, 9), 1, std::vector({2}))}; + for (const std::set& field_ids : + {std::set{1}, std::set{2}, std::set{1, 2}}) { + SCOPED_TRACE(::testing::PrintToString(field_ids)); + ASSERT_TRUE( + DataEvolutionGlobalIndexCoverage::IndexedRanges(multi_field_files, field_ids).empty()); + } + + std::vector> mixed_files = multi_field_files; + mixed_files.push_back(NewIndexFile("primary", Range(0, 5), 1)); + mixed_files.push_back(NewIndexFile("extra", Range(3, 9), 2, std::vector())); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(mixed_files, {1}), + std::vector({Range(0, 5)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(mixed_files, {2}), + std::vector({Range(3, 9)})); + ASSERT_EQ(DataEvolutionGlobalIndexCoverage::IndexedRanges(mixed_files, {1, 2}), + std::vector({Range(3, 5)})); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestFullExcludesIndexedRangesFromNextRowId) { + ASSERT_OK_AND_ASSIGN( + std::vector ranges, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::FULL, 100, + {Range(50, 59), Range(0, 9), Range(10, 19), Range(95, 120)}, Unused())); + ASSERT_EQ(ranges, std::vector({Range(20, 49), Range(60, 94)})); + + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::FULL, 100, {Range(0, 99)}, Unused())); + ASSERT_TRUE(ranges.empty()); + + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::FULL, 100, {}, Unused())); + ASSERT_EQ(ranges, std::vector({Range(0, 99)})); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestDetailExcludesIndexedRangesFromDataFiles) { + ASSERT_OK_AND_ASSIGN(std::vector ranges, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {Range(0, 14)}, + Ranges({Range(60, 69), Range(0, 9), Range(10, 19), Range(65, 79)}))); + ASSERT_EQ(ranges, std::vector({Range(15, 19), Range(60, 79)})); + + ASSERT_OK_AND_ASSIGN( + ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {}, Ranges({Range(2, 3), Range(7, 9)}))); + ASSERT_EQ(ranges, std::vector({Range(2, 3), Range(7, 9)})); + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {}, Ranges({}))); + ASSERT_TRUE(ranges.empty()); + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {Range(0, 99)}, + Ranges({Range(0, 9), Range(60, 79)}))); + ASSERT_TRUE(ranges.empty()); + + ASSERT_NOK_WITH_MSG(DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::DETAIL, 100, {}, Unused()), + "data ranges are not needed"); +} + +TEST(DataEvolutionGlobalIndexCoverageTest, TestSkipUnindexedRanges) { + ASSERT_OK_AND_ASSIGN(std::vector ranges, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + GlobalIndexSearchMode::FAST, 100, {}, Unused())); + ASSERT_TRUE(ranges.empty()); + for (const std::optional& next_row_id : + {std::optional(), std::optional(0), std::optional(-1)}) { + for (GlobalIndexSearchMode mode : + {GlobalIndexSearchMode::FULL, GlobalIndexSearchMode::DETAIL}) { + ASSERT_OK_AND_ASSIGN(ranges, DataEvolutionGlobalIndexCoverage::UnindexedRanges( + mode, next_row_id, {}, Unused())); + ASSERT_TRUE(ranges.empty()); + } + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/global_index/global_index_live_row_filter.cpp b/src/paimon/core/global_index/global_index_live_row_filter.cpp index 95c22d721..a34186415 100644 --- a/src/paimon/core/global_index/global_index_live_row_filter.cpp +++ b/src/paimon/core/global_index/global_index_live_row_filter.cpp @@ -217,4 +217,30 @@ Result> GlobalIndexLiveRowFilter::LiveRows( return std::optional(std::move(live_rows)); } +Result> GlobalIndexLiveRowFilter::DataRowRanges( + const std::string& root_path, const std::shared_ptr& table_schema, + const Snapshot& snapshot, const std::shared_ptr& partition_filter, + const CoreOptions& options, const std::shared_ptr& executor, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr snapshot_reader, + CreateSnapshotReader(root_path, table_schema, partition_filter, options, executor, pool)); + snapshot_reader->WithMode(ScanMode::ALL)->WithSnapshot(snapshot); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, snapshot_reader->Read()); + std::vector ranges; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("cannot cast split to DataSplitImpl when reading row ranges"); + } + for (const auto& file : data_split->DataFiles()) { + if (file->first_row_id && file->row_count > 0) { + PAIMON_ASSIGN_OR_RAISE(Range row_id_range, file->NonNullRowIdRange()); + ranges.push_back(row_id_range); + } + } + } + return ranges; +} + } // namespace paimon diff --git a/src/paimon/core/global_index/global_index_live_row_filter.h b/src/paimon/core/global_index/global_index_live_row_filter.h index 13cd15869..6523361db 100644 --- a/src/paimon/core/global_index/global_index_live_row_filter.h +++ b/src/paimon/core/global_index/global_index_live_row_filter.h @@ -22,6 +22,7 @@ #include #include #include +#include #include "paimon/common/predicate/predicate_filter.h" #include "paimon/core/core_options.h" @@ -30,6 +31,7 @@ #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" +#include "paimon/utils/range.h" #include "paimon/utils/roaring_bitmap64.h" #include "paimon/utils/row_range_index.h" @@ -62,6 +64,15 @@ class GlobalIndexLiveRowFilter { const Snapshot& snapshot, const std::shared_ptr& partition_filter, const std::optional& row_range_index, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool); + + /// Returns the global row id ranges of the data files of `snapshot` in the partitions accepted + /// by `partition_filter`. Each non-empty file with a first row id contributes its range, + /// including rows marked by deletion vectors. + static Result> DataRowRanges( + const std::string& root_path, const std::shared_ptr& table_schema, + const Snapshot& snapshot, const std::shared_ptr& partition_filter, + const CoreOptions& options, const std::shared_ptr& executor, + const std::shared_ptr& pool); }; } // namespace paimon diff --git a/src/paimon/core/table/source/abstract_table_scan.h b/src/paimon/core/table/source/abstract_table_scan.h index f8dd15aac..6da732e9c 100644 --- a/src/paimon/core/table/source/abstract_table_scan.h +++ b/src/paimon/core/table/source/abstract_table_scan.h @@ -43,6 +43,15 @@ class AbstractTableScan : public TableScan { return snapshot_reader_->GetMetrics(); } + /// Uses the supplied snapshot even after its snapshot file expires. + Result> PlanSnapshotRanges(const Snapshot& snapshot, + const RowRangeIndex& row_ranges) { + return snapshot_reader_->WithMode(ScanMode::ALL) + ->WithSnapshot(snapshot) + ->WithRowRangeIndex(row_ranges) + ->Read(); + } + protected: Result> CreateStartingScanner(bool is_streaming) const { const auto& snapshot_manager = snapshot_reader_->GetSnapshotManager(); diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.cpp b/src/paimon/core/table/source/data_evolution_full_text_read.cpp index 465b2e37f..57c6e7ff1 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_read.cpp @@ -19,9 +19,13 @@ #include "paimon/core/table/source/data_evolution_full_text_read.h" +#include #include #include +#include #include +#include +#include #include #include #include @@ -34,18 +38,22 @@ #include "paimon/common/global_index/global_index_utils.h" #include "paimon/common/global_index/offset_global_index_reader.h" #include "paimon/common/global_index/scored_global_index_result_utils.h" +#include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/global_index/data_evolution_global_index_coverage.h" #include "paimon/core/global_index/global_index_evaluator_impl.h" #include "paimon/core/global_index/global_index_live_row_filter.h" #include "paimon/core/global_index/global_index_scan_impl.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_evolution_full_text_scan.h" #include "paimon/core/table/source/filtered_row_id_reader.h" -#include "paimon/core/tag/tag.h" +#include "paimon/core/table/source/raw_full_text_read_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" #include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/core/utils/tag_manager.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" #include "paimon/global_index/global_index_io_meta.h" @@ -65,45 +73,7 @@ Logger* FullTextReadLogger() { /// live rows: at most 65536 run containers, cheaper than copying the live rows of a large table for /// every split. A wider span trims the live rows instead. constexpr int64_t kMaxExpandedSearchRows = static_cast(1) << 32; - -/// Loads snapshot `snapshot_id`, or the copy that a tag keeps of it once the snapshot has expired, -/// since a plan of `scan.tag-name` may come from a tag that outlives its snapshot. -Result LoadPlannedSnapshot(const CoreOptions& options, const std::string& root_path, - int64_t snapshot_id) { - std::string branch = BranchManager::NormalizeBranch(options.GetBranch()); - SnapshotManager snapshot_manager(options.GetFileSystem(), root_path, branch); - Result snapshot = snapshot_manager.LoadSnapshot(snapshot_id); - if (snapshot.ok()) { - return snapshot; - } - // The snapshot may expire between planning and loading, or even while it is loaded, so a - // failed load falls back to the tags unless the snapshot is still there. - PAIMON_ASSIGN_OR_RAISE(bool exists, snapshot_manager.SnapshotExists(snapshot_id)); - if (exists) { - return snapshot.status(); - } - TagManager tag_manager(options.GetFileSystem(), root_path, branch); - PAIMON_ASSIGN_OR_RAISE(std::vector tag_names, tag_manager.ListTagNames()); - for (const std::string& tag_name : tag_names) { - // A tag deleted after it was listed, even while it is read, is skipped, as it no longer - // keeps the snapshot; any other failure to read a tag is an error. - Result> tag = tag_manager.Get(tag_name); - if (!tag.ok()) { - PAIMON_ASSIGN_OR_RAISE(bool tag_exists, - options.GetFileSystem()->Exists(tag_manager.TagPath(tag_name))); - if (tag_exists) { - return tag.status(); - } - continue; - } - if (tag.value() && tag.value()->Id() == snapshot_id) { - return tag.value()->TrimToSnapshot(); - } - } - return Status::NotExist(fmt::format( - "snapshot {} that the full-text search splits were planned from no longer exists", - snapshot_id)); -} +constexpr size_t kMaxReportedRanges = 3; Result ToBitmap(const std::shared_ptr& result) { if (auto bitmap_result = std::dynamic_pointer_cast(result)) { @@ -119,24 +89,76 @@ Result ToBitmap(const std::shared_ptr& resul return bitmap; } -/// Describes `split` for the errors of its search. std::string Describe(const IndexFullTextSearchSplitImpl& split) { std::vector file_names; file_names.reserve(split.FullTextIndexFiles().size()); for (const auto& file : split.FullTextIndexFiles()) { file_names.push_back(file->FileName()); } + const auto& ranges = split.SearchRowRanges(); + std::vector sample(ranges.begin(), + ranges.begin() + std::min(ranges.size(), kMaxReportedRanges)); return fmt::format( "while searching the split of column {} in snapshot {} over rows [{}, {}] with {} index " - "files [{}]", + "files [{}], schema {}, {} search ranges, first {}", split.ColumnName(), split.SnapshotId(), split.RowRangeStart(), split.RowRangeEnd(), - split.FullTextIndexFiles().front()->IndexType(), fmt::join(file_names, ", ")); + split.FullTextIndexFiles().front()->IndexType(), fmt::join(file_names, ", "), + split.SchemaId() ? fmt::format("{}", *split.SchemaId()) : "unknown", ranges.size(), + StringUtils::VectorToString(sample)); +} + +Result> ScalarIndexCoverage( + const std::vector>& files, const std::set& field_ids, + const CoreOptions& options) { + // A worker may have fewer index backends than the process that planned its splits. + std::map readable_types; + std::vector> readable_files; + for (const auto& file : files) { + const auto& meta = file->GetGlobalIndexMeta().value(); + if (meta.extra_field_ids && !meta.extra_field_ids->empty()) { + continue; + } + auto [type, inserted] = readable_types.try_emplace(file->IndexType(), false); + if (inserted) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(type->first, options.ToMap())); + type->second = indexer && !indexer->SupportsFullTextSearch(); + } + if (type->second) { + readable_files.push_back(file); + } + } + return DataEvolutionGlobalIndexCoverage::IndexedRanges(readable_files, field_ids); +} + +Status ExcludeRawRanges(const RowRangeIndex& raw_rows, + std::vector>* index_splits) { + for (auto& split : *index_splits) { + if (!raw_rows.Intersects(split->RowRangeStart(), split->RowRangeEnd())) { + continue; + } + std::vector remaining = + DataEvolutionGlobalIndexCoverage::ExcludeRanges(split->SearchRowRanges(), raw_rows); + if (remaining.empty()) { + split.reset(); + } else if (remaining != split->SearchRowRanges()) { + PAIMON_ASSIGN_OR_RAISE( + split, IndexFullTextSearchSplitImpl::Create( + split->SnapshotId(), split->ColumnName(), split->RowRangeStart(), + split->RowRangeEnd(), split->FullTextIndexFiles(), std::move(remaining), + split->ScalarIndexFiles(), split->SchemaId())); + } + } + index_splits->erase(std::remove(index_splits->begin(), index_splits->end(), nullptr), + index_splits->end()); + return Status::OK(); } } // namespace DataEvolutionFullTextRead::DataEvolutionFullTextRead( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& index_path_factory, + const CoreOptions& options, const std::map& option_overrides, + const std::shared_ptr& index_path_factory, const std::shared_ptr& index_file_manager, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, const std::string& query, @@ -145,6 +167,7 @@ DataEvolutionFullTextRead::DataEvolutionFullTextRead( : root_path_(root_path), table_schema_(table_schema), options_(options), + option_overrides_(option_overrides), index_path_factory_(index_path_factory), index_file_manager_(index_file_manager), partition_filter_(partition_filter), @@ -157,7 +180,8 @@ DataEvolutionFullTextRead::DataEvolutionFullTextRead( Result> DataEvolutionFullTextRead::Create( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& path_factory, + const CoreOptions& options, const std::map& option_overrides, + const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, const std::string& query, int32_t limit, const std::shared_ptr& executor, @@ -172,8 +196,39 @@ Result> DataEvolutionFullTextRead::Cr auto index_file_manager = std::make_shared( options.GetFileSystem(), index_path_factory, /*checkpoint_path_factory=*/nullptr); return std::unique_ptr(new DataEvolutionFullTextRead( - root_path, table_schema, options, index_path_factory, index_file_manager, partition_filter, - filter, text_field, query, limit, final_executor, pool)); + root_path, table_schema, options, option_overrides, index_path_factory, index_file_manager, + partition_filter, filter, text_field, query, limit, final_executor, pool)); +} + +Status DataEvolutionFullTextRead::BindSchemaOptions( + const TableSchema& schema, const std::map& option_overrides, + CoreOptions* options) { + PAIMON_RETURN_NOT_OK( + SchemaValidation::RejectQueryAuth(schema, option_overrides, "Full-text search")); + auto merged = options->ToMap(); + auto restore = [&](std::initializer_list keys) { + if (std::any_of(keys.begin(), keys.end(), + [&](const char* key) { return option_overrides.count(key) > 0; })) { + return; + } + for (const char* key : keys) { + merged.erase(key); + auto entry = schema.Options().find(key); + if (entry != schema.Options().end()) { + merged.insert(*entry); + } + } + }; + restore({Options::BLOB_FIELD}); + restore({Options::BLOB_DESCRIPTOR_FIELD, Options::FALLBACK_BLOB_DESCRIPTOR_FIELD}); + restore({Options::BLOB_VIEW_FIELD}); + if (merged != options->ToMap()) { + PAIMON_ASSIGN_OR_RAISE(CoreOptions bound, + CoreOptions::FromMap(merged, options->GetFileSystem())); + bound.WithCache(options->GetCache()); + *options = bound; + } + return Status::OK(); } std::optional DataEvolutionFullTextRead::IncludeRowIds( @@ -208,23 +263,25 @@ std::optional DataEvolutionFullTextRead::IncludeRowIds( } Result> DataEvolutionFullTextRead::LiveRows( + const std::vector>& index_splits, int64_t snapshot_id) const { - // Without deletion vectors or row tracking every row is live, so the planned snapshot, which - // may be gone, is not loaded. - if (!options_.DeletionVectorsEnabled() || !options_.RowTrackingEnabled()) { + if (index_splits.empty() || !options_.DeletionVectorsEnabled() || + !options_.RowTrackingEnabled()) { return std::optional(); } PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, - LoadPlannedSnapshot(options_, root_path_, snapshot_id)); + FilteredRowIdReader::LoadSnapshot(options_, root_path_, snapshot_id)); + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex search_rows, + RowRangeIndex::Create(CollectSearchRanges(index_splits))); return GlobalIndexLiveRowFilter::LiveRows(root_path_, table_schema_, snapshot, - partition_filter_, /*row_range_index=*/std::nullopt, - options_, executor_, pool_); + partition_filter_, std::move(search_rows), options_, + executor_, pool_); } Result> DataEvolutionFullTextRead::MatchedRows( const std::vector>& index_splits, int64_t snapshot_id, const std::optional& live_rows) const { - if (!filter_) { + if (!filter_ || index_splits.empty() || (live_rows && live_rows->IsEmpty())) { return std::optional(); } // Keyed by file name: a scalar index file may intersect more than one split. @@ -239,60 +296,131 @@ Result> DataEvolutionFullTextRead::MatchedRows( for (const auto& [file_name, file] : files_by_name) { scalar_index_files.push_back(file); } - std::unique_ptr index_scan = GlobalIndexScanImpl::CreateForScalarFilters( - table_schema_, options_, index_path_factory_, scalar_index_files, executor_, pool_); - PAIMON_ASSIGN_OR_RAISE(std::optional evaluation, - index_scan->ScanWithCoverage(filter_)); - if (!evaluation) { - PAIMON_LOG_WARN(FullTextReadLogger(), - "Full-text search on table %s in snapshot %ld has a row filter %s that no " - "scalar global index can evaluate; indexed rows are excluded from the " - "result because %s is fast. Build a scalar index on the filtered columns.", - root_path_.c_str(), snapshot_id, filter_->ToString().c_str(), - Options::SCALAR_INDEX_SEARCH_MODE); - return std::optional(RoaringBitmap64()); + + const GlobalIndexSearchMode search_mode = options_.GetScalarIndexSearchMode(); + std::vector search_ranges; + std::vector unindexed_ranges; + bool use_scalar_indexes = !scalar_index_files.empty(); + if (search_mode != GlobalIndexSearchMode::FAST) { + search_ranges = Range::SortAndMergeOverlap(CollectSearchRanges(index_splits), true); + PAIMON_ASSIGN_OR_RAISE(std::set filter_field_ids, + GlobalIndexEvaluatorImpl::CollectFieldIds(*table_schema_, filter_)); + PAIMON_ASSIGN_OR_RAISE(std::vector indexed_ranges, + ScalarIndexCoverage(scalar_index_files, filter_field_ids, options_)); + const std::vector covered = Range::And(search_ranges, indexed_ranges); + use_scalar_indexes = + std::any_of(covered.begin(), covered.end(), [&live_rows](const Range& range) { + if (!live_rows) { + return true; + } + auto first = live_rows->EqualOrLarger(range.from); + return first != live_rows->End() && *first <= range.to; + }); + if (use_scalar_indexes && covered != search_ranges) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, FilteredRowIdReader::LoadSnapshot( + options_, root_path_, snapshot_id)); + auto data_ranges = [&]() { + return GlobalIndexLiveRowFilter::DataRowRanges(root_path_, table_schema_, snapshot, + partition_filter_, options_, + executor_, pool_); + }; + PAIMON_ASSIGN_OR_RAISE(std::vector gaps, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + search_mode, snapshot.NextRowId(), covered, data_ranges)); + unindexed_ranges = Range::And(search_ranges, gaps); + } } + std::optional evaluation; + if (use_scalar_indexes) { + std::unique_ptr index_scan = + GlobalIndexScanImpl::CreateForScalarFilters( + table_schema_, options_, index_path_factory_, scalar_index_files, executor_, pool_); + PAIMON_ASSIGN_OR_RAISE(evaluation, index_scan->ScanWithCoverage(filter_)); + } + FilteredRowIdReader row_id_reader(root_path_, table_schema_, options_, partition_filter_, + filter_, snapshot_id, executor_, pool_); + auto read_matches = [&](const std::vector& ranges) -> Result { + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex rows, RowRangeIndex::Create(ranges)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 matches, row_id_reader.MatchingRowIds(rows)); + if (live_rows) { + matches &= *live_rows; + } + return matches; + }; + if (!evaluation) { + if (search_mode == GlobalIndexSearchMode::FAST) { + PAIMON_LOG_WARN( + FullTextReadLogger(), + "Full-text search on table %s in snapshot %ld excluded indexed rows because no " + "scalar global index can evaluate the row filter %s in fast mode. Build a scalar " + "index on the filtered columns or set %s to full or detail.", + root_path_.c_str(), snapshot_id, filter_->ToString().c_str(), + Options::SCALAR_INDEX_SEARCH_MODE); + return std::optional(RoaringBitmap64()); + } + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 matches, read_matches(search_ranges)); + return std::optional(std::move(matches)); + } PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 index_rows, ToBitmap(evaluation->result)); if (live_rows) { index_rows &= live_rows.value(); } + for (const Range& range : unindexed_ranges) { + if (range.to == RoaringBitmap64::MAX_VALUE) { + index_rows.RemoveRange(range.from, range.to); + index_rows -= RoaringBitmap64::From({range.to}); + } else { + index_rows.RemoveRange(range.from, range.to + 1); + } + } PAIMON_ASSIGN_OR_RAISE( bool exact, FilteredRowIdReader::IsExact(*table_schema_, filter_, evaluation->contributing_field_ids)); - if (exact) { - return std::optional(std::move(index_rows)); - } - - // Only the candidates that a split ranks are worth verifying. - RoaringBitmap64 candidates; - for (const auto& split : index_splits) { - std::optional split_rows = IncludeRowIds(*split, &index_rows); - if (split_rows) { - candidates |= split_rows.value(); + if (!exact) { + RoaringBitmap64 candidates; + for (const auto& split : index_splits) { + std::optional split_rows = IncludeRowIds(*split, &index_rows); + if (split_rows) { + candidates |= split_rows.value(); + } else { + candidates |= + GlobalIndexUtils::RowIdsOf(Range(split->RowRangeStart(), split->RowRangeEnd())); + } + } + if (candidates.IsEmpty()) { + index_rows = std::move(candidates); + } else if (!options_.GlobalIndexFilterRefineFromData()) { + PAIMON_LOG_WARN(FullTextReadLogger(), + "The scalar global index can only answer the row filter %s on table " + "%s in snapshot %ld with candidates, and %s is false, so those %ld " + "rows are excluded from the search; the result may hold fewer rows " + "than requested. Set the option to true to verify the candidates " + "against the data, or build an index that answers the predicate " + "exactly.", + filter_->ToString().c_str(), root_path_.c_str(), snapshot_id, + Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, + candidates.Cardinality()); + index_rows = RoaringBitmap64(); } else { - candidates |= - GlobalIndexUtils::RowIdsOf(Range(split->RowRangeStart(), split->RowRangeEnd())); + PAIMON_ASSIGN_OR_RAISE(index_rows, row_id_reader.MatchingRowIds(std::move(candidates))); } } - if (candidates.IsEmpty()) { - return std::optional(std::move(candidates)); + if (!unindexed_ranges.empty()) { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 unindexed_matches, read_matches(unindexed_ranges)); + index_rows |= unindexed_matches; } - if (!options_.GlobalIndexFilterRefineFromData()) { - PAIMON_LOG_WARN(FullTextReadLogger(), - "The scalar global index can only answer the row filter %s on table %s " - "in snapshot %ld with candidates, and %s is false, so those %ld rows " - "are excluded from the search; the result may hold fewer rows than " - "requested. Set the option to true to verify the candidates against the " - "data, or build an index that answers the predicate exactly.", - filter_->ToString().c_str(), root_path_.c_str(), snapshot_id, - Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, candidates.Cardinality()); - return std::optional(RoaringBitmap64()); + return std::optional(std::move(index_rows)); +} + +std::vector DataEvolutionFullTextRead::CollectSearchRanges( + const std::vector>& index_splits) { + std::vector ranges; + for (const auto& split : index_splits) { + const std::vector& search_ranges = split->SearchRowRanges(); + ranges.insert(ranges.end(), search_ranges.begin(), search_ranges.end()); } - FilteredRowIdReader row_id_reader(root_path_, table_schema_, options_, partition_filter_, - filter_, snapshot_id, executor_, pool_); - PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 matched_rows, row_id_reader.MatchingRowIds(candidates)); - return std::optional(std::move(matched_rows)); + return ranges; } Result> DataEvolutionFullTextRead::SearchSplit( @@ -355,17 +483,62 @@ Result> DataEvolutionFullTextRead::Read const std::vector>& splits) { std::vector> index_splits; index_splits.reserve(splits.size()); + std::string raw_index_type; + std::vector raw_ranges; std::optional snapshot_id; + std::optional schema_id; + auto check_split = [&](const std::string& column_name, int64_t split_snapshot_id, + std::optional split_schema_id) -> Status { + if (column_name != text_field_.Name()) { + return Status::Invalid(fmt::format( + "a split of column {} cannot be read by a full-text search of column {}", + column_name, text_field_.Name())); + } + if (snapshot_id && snapshot_id.value() != split_snapshot_id) { + return Status::Invalid(fmt::format( + "full-text search splits planned from snapshots {} and {} cannot be read together", + snapshot_id.value(), split_snapshot_id)); + } + snapshot_id = split_snapshot_id; + if (split_schema_id) { + if (schema_id && schema_id != split_schema_id) { + return Status::Invalid( + fmt::format("full-text search splits planned with schemas {} and {} cannot be " + "read together", + *schema_id, *split_schema_id)); + } + schema_id = split_schema_id; + } + return Status::OK(); + }; for (const auto& split : splits) { + if (auto raw_split = std::dynamic_pointer_cast(split)) { + PAIMON_RETURN_NOT_OK(check_split(raw_split->ColumnName(), raw_split->SnapshotId(), + raw_split->SchemaId())); + if (raw_split->FieldId() != text_field_.Id()) { + return Status::Invalid(fmt::format( + "a raw split of column {} was planned for field id {}, but the column now has " + "field id {}", + text_field_.Name(), raw_split->FieldId(), text_field_.Id())); + } + if (!raw_index_type.empty() && raw_index_type != raw_split->IndexType()) { + return Status::Invalid( + fmt::format("cannot combine raw full-text splits with " + "index types {} and {}", + raw_index_type, raw_split->IndexType())); + } + raw_index_type = raw_split->IndexType(); + const auto& ranges = raw_split->RowRanges(); + raw_ranges.insert(raw_ranges.end(), ranges.begin(), ranges.end()); + continue; + } auto index_split = std::dynamic_pointer_cast(split); if (!index_split) { - return Status::Invalid("invalid full-text search split, cannot cast to an index split"); - } - if (index_split->ColumnName() != text_field_.Name()) { - return Status::Invalid(fmt::format( - "a split of column {} cannot be read by a full-text search of column {}", - index_split->ColumnName(), text_field_.Name())); + return Status::Invalid( + "invalid full-text search split, cannot cast to an index or raw split"); } + PAIMON_RETURN_NOT_OK(check_split(index_split->ColumnName(), index_split->SnapshotId(), + index_split->SchemaId())); // A column of the same name may have replaced the planned one, so match field ids too. const GlobalIndexMeta& meta = index_split->FullTextIndexFiles().front()->GetGlobalIndexMeta().value(); @@ -375,27 +548,51 @@ Result> DataEvolutionFullTextRead::Read "replaced after the split was planned", text_field_.Name(), text_field_.Id())); } - if (snapshot_id && snapshot_id.value() != index_split->SnapshotId()) { - return Status::Invalid(fmt::format( - "full-text search splits planned from snapshots {} and {} cannot be read together", - snapshot_id.value(), index_split->SnapshotId())); - } - snapshot_id = index_split->SnapshotId(); index_splits.push_back(std::move(index_split)); } - if (index_splits.empty()) { + if (!snapshot_id) { return std::shared_ptr( std::make_shared(RoaringBitmap64(), std::vector())); } - // Live rows are read on this thread before the searches fan out, so a read of manifests on - // `executor_` never waits for it from one of its own threads. They are pinned to the snapshot - // the splits were planned from (apache/paimon#9031). - PAIMON_ASSIGN_OR_RAISE(std::optional live_rows, LiveRows(snapshot_id.value())); - // Likewise for the row filter, which reads indexes and possibly data. + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex raw_rows, RowRangeIndex::Create(raw_ranges)); + if (!raw_ranges.empty()) { + PAIMON_RETURN_NOT_OK(ExcludeRawRanges(raw_rows, &index_splits)); + } + if (schema_id && *schema_id != table_schema_->Id()) { + SchemaManager schemas(options_.GetFileSystem(), root_path_, + BranchManager::NormalizeBranch(options_.GetBranch())); + DataEvolutionFullTextRead planned_read(*this); + PAIMON_ASSIGN_OR_RAISE(planned_read.table_schema_, schemas.ReadSchema(*schema_id)); + PAIMON_RETURN_NOT_OK(BindSchemaOptions(*planned_read.table_schema_, option_overrides_, + &planned_read.options_)); + PAIMON_ASSIGN_OR_RAISE(planned_read.text_field_, + planned_read.table_schema_->GetField(text_field_.Name())); + if (planned_read.text_field_.Id() != text_field_.Id()) { + return Status::Invalid(fmt::format( + "a split of column {} in schema {} was planned for field id {}, but the column " + "now has field id {}", + text_field_.Name(), *schema_id, planned_read.text_field_.Id(), text_field_.Id())); + } + return planned_read.SearchSplits(index_splits, *snapshot_id, raw_rows, raw_index_type); + } + return SearchSplits(index_splits, *snapshot_id, raw_rows, raw_index_type); +} + +Result> DataEvolutionFullTextRead::SearchSplits( + const std::vector>& index_splits, + int64_t snapshot_id, const RowRangeIndex& raw_rows, const std::string& raw_index_type) const { + if (filter_) { + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()), filter_, + /*validate_field_idx=*/false)); + } + // Keep data reads on the caller: they can submit work to the same executor as the shards. + PAIMON_ASSIGN_OR_RAISE(std::optional live_rows, + LiveRows(index_splits, snapshot_id)); PAIMON_ASSIGN_OR_RAISE(std::optional matched_rows, - MatchedRows(index_splits, snapshot_id.value(), live_rows)); + MatchedRows(index_splits, snapshot_id, live_rows)); const std::optional& eligible_rows = matched_rows ? matched_rows : live_rows; using SearchResult = Result>; std::vector> searches; @@ -416,20 +613,50 @@ Result> DataEvolutionFullTextRead::Read }); } std::vector results; + std::vector> futures; + // Drain tasks that borrow this reader if submission, the raw scan or a shard throws. + ScopeGuard drain([&futures] { + for (auto& future : futures) { + if (future.valid()) { + future.wait(); + } + } + }); if (searches.size() <= 1) { for (const auto& search : searches) { results.push_back(search()); } } else { - std::vector> futures; futures.reserve(searches.size()); for (const auto& search : searches) { futures.push_back(Via(executor_.get(), search)); } + } + std::optional raw_result; + if (!raw_rows.Ranges().empty()) { + RawFullTextReadImpl raw_read(root_path_, table_schema_, options_, partition_filter_, + filter_, text_field_, query_, limit_, executor_, pool_); + raw_result = raw_read.Search(snapshot_id, raw_index_type, raw_rows); + } + if (!futures.empty()) { results = CollectAll(futures); } std::vector> scored_results; - scored_results.reserve(results.size()); + scored_results.reserve(results.size() + 1); + if (raw_result) { + if (!raw_result->ok()) { + const Status& status = raw_result->status(); + const auto& ranges = raw_rows.Ranges(); + std::vector sample(ranges.begin(), + ranges.begin() + std::min(ranges.size(), kMaxReportedRanges)); + return status.WithMessage(fmt::format( + "{}; while searching column {} in snapshot {} with a temporary {} index " + "over {} unindexed row ranges, first {}, schema {}, field id {}", + status.message(), text_field_.Name(), snapshot_id, raw_index_type, ranges.size(), + StringUtils::VectorToString(sample), table_schema_->Id(), text_field_.Id())); + } + scored_results.push_back(std::move(*raw_result).value()); + } for (auto& result : results) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scored_result, std::move(result)); diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.h b/src/paimon/core/table/source/data_evolution_full_text_read.h index 62e1a99b4..91f4c6ab2 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read.h +++ b/src/paimon/core/table/source/data_evolution_full_text_read.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -37,7 +38,9 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate.h" #include "paimon/table/source/full_text_read.h" +#include "paimon/utils/range.h" #include "paimon/utils/roaring_bitmap64.h" +#include "paimon/utils/row_range_index.h" namespace paimon { /// `FullTextRead` of a data-evolution table, aligned with Java `DataEvolutionFullTextRead`. @@ -46,6 +49,7 @@ class DataEvolutionFullTextRead : public FullTextRead { /// @param root_path Root directory of the table. /// @param table_schema Schema of the table. /// @param options Options of the table, merged with those of the search. + /// @param option_overrides Options explicitly set on the search builder. /// @param path_factory Path factory of the table. /// @param partition_filter Predicate over the partition fields used to read live rows and /// candidate rows, or null for all partitions. @@ -58,12 +62,18 @@ class DataEvolutionFullTextRead : public FullTextRead { /// @param pool Memory pool for reading indexes, manifests and candidate rows. static Result> Create( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& path_factory, + const CoreOptions& options, const std::map& option_overrides, + const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, const std::string& query, int32_t limit, const std::shared_ptr& executor, const std::shared_ptr& pool); + /// Checks query authorization and binds BLOB field options to the selected schema. + static Status BindSchemaOptions(const TableSchema& schema, + const std::map& option_overrides, + CoreOptions* options); + using FullTextRead::Read; Result> Read( @@ -82,6 +92,7 @@ class DataEvolutionFullTextRead : public FullTextRead { DataEvolutionFullTextRead(const std::string& root_path, const std::shared_ptr& table_schema, const CoreOptions& options, + const std::map& option_overrides, const std::shared_ptr& index_path_factory, const std::shared_ptr& index_file_manager, const std::shared_ptr& partition_filter, @@ -90,19 +101,23 @@ class DataEvolutionFullTextRead : public FullTextRead { const std::shared_ptr& executor, const std::shared_ptr& pool); - /// Returns the rows live in snapshot `snapshot_id`, or nullopt if every row counts as live. - Result> LiveRows(int64_t snapshot_id) const; + Result> SearchSplits( + const std::vector>& index_splits, + int64_t snapshot_id, const RowRangeIndex& raw_rows, + const std::string& raw_index_type) const; - /// Resolves the row filter into the rows of `index_splits` that satisfy it, as Java - /// `DataEvolutionFullTextRead#matchedRows` does in the `fast` `scalar-index.search-mode`, the - /// only one supported so far: rows that no scalar index can evaluate are excluded. Candidates - /// that the indexes cannot decide exactly are verified against the data with - /// `global-index.filter.refine-from-data`, and excluded without it. - /// @return The matched rows, all of them live, or nullopt if there is no row filter. + Result> LiveRows( + const std::vector>& index_splits, + int64_t snapshot_id) const; + + /// Returns nullopt when the live-row filter alone determines which indexed rows can rank. Result> MatchedRows( const std::vector>& index_splits, int64_t snapshot_id, const std::optional& live_rows) const; + static std::vector CollectSearchRanges( + const std::vector>& index_splits); + /// Searches the index shard of `split` among `include_row_ids`, or all of its rows if not set, /// and returns at most `limit_` rows with global row ids. Result> SearchSplit( @@ -112,6 +127,7 @@ class DataEvolutionFullTextRead : public FullTextRead { std::string root_path_; std::shared_ptr table_schema_; CoreOptions options_; + std::map option_overrides_; std::shared_ptr index_path_factory_; std::shared_ptr index_file_manager_; std::shared_ptr partition_filter_; diff --git a/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp b/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp index 9d8507cd6..aafc9a3a6 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp @@ -20,11 +20,15 @@ #include "paimon/core/table/source/data_evolution_full_text_read.h" #include +#include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -43,7 +47,12 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/table/source/raw_full_text_read_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" +#include "paimon/executor.h" #include "paimon/factories/factory.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" @@ -54,6 +63,8 @@ #include "paimon/predicate/predicate_builder.h" #include "paimon/table/source/full_text_search_builder.h" #include "paimon/testing/mock/mock_global_index_reader.h" +#include "paimon/testing/utils/snapshot_test_helper.h" +#include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" #include "paimon/utils/range.h" @@ -64,6 +75,53 @@ constexpr char kFakeScalarIndexType[] = "fake-scalar"; constexpr char kOtherFakeScalarIndexType[] = "other-fake-scalar"; constexpr char kQuery[] = R"({"match":{"query":"paimon"}})"; +class RejectingSearchExecutor : public Executor { + public: + explicit RejectingSearchExecutor(uint32_t rejected_submission) + : rejected_submission_(rejected_submission) {} + + void Add(std::function task) override { + std::lock_guard lock(mutex_); + if (++submissions_ == rejected_submission_) { + rejected_.set_value(); + throw std::bad_alloc(); + } + tasks_.push_back(std::move(task)); + } + + void ShutdownNow() override { + std::lock_guard lock(mutex_); + tasks_.clear(); + } + + uint32_t GetThreadNum() const override { + return 2; + } + + bool WaitUntilRejected() { + return rejected_.get_future().wait_for(std::chrono::seconds(5)) == + std::future_status::ready; + } + + void RunAcceptedTasks() { + std::vector> tasks; + { + std::lock_guard lock(mutex_); + tasks.swap(tasks_); + } + for (const auto& task : tasks) { + task(); + } + } + + private: + uint32_t rejected_submission_; + uint32_t submissions_ = 0; + std::mutex mutex_; + std::promise rejected_; + std::vector> tasks_; +}; + struct RecordedSearch { std::string file_name; std::vector schema_fields; @@ -299,6 +357,50 @@ ScoredRows ToRows(const ScoredGlobalIndexResult& result) { } } // namespace +TEST(InMemoryGlobalIndexFilesTest, TestTemporaryFileLifetimeAndIsolation) { + const std::string contents("a\0b", 3); + std::shared_ptr pool = GetMemoryPool(); + std::unique_ptr input; + { + InMemoryGlobalIndexFiles files(pool); + ASSERT_OK_AND_ASSIGN(std::string path, files.NewPath("full-text")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, files.NewOutputStream(path)); + ASSERT_NOK(output->Write(contents.data(), -1)); + ASSERT_NOK(output->Write(nullptr, 1)); + ASSERT_OK_AND_ASSIGN(int64_t written, output->Write(contents.data(), contents.size())); + ASSERT_EQ(written, 3); + ASSERT_NOK(files.GetInputStream(path)); + ASSERT_OK(output->Close()); + ASSERT_OK(output->Close()); + ASSERT_NOK(output->Write(contents.data(), contents.size())); + ASSERT_OK_AND_ASSIGN(int64_t position, output->GetPos()); + ASSERT_EQ(position, written); + ASSERT_OK_AND_ASSIGN(int64_t size, files.GetFileSize(path)); + ASSERT_EQ(size, written); + ASSERT_OK_AND_ASSIGN(input, files.GetInputStream(path)); + + InMemoryGlobalIndexFiles other_search(pool); + ASSERT_OK_AND_ASSIGN(std::string other_path, other_search.NewPath("full-text")); + ASSERT_NE(path, other_path); + ASSERT_NOK(other_search.GetInputStream(path)); + ASSERT_OK_AND_ASSIGN(output, other_search.NewOutputStream(other_path)); + ASSERT_OK_AND_ASSIGN(written, output->Write(nullptr, 0)); + ASSERT_EQ(written, 0); + ASSERT_OK(output->Close()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr empty_file, + other_search.GetInputStream(other_path)); + ASSERT_OK_AND_ASSIGN(size, empty_file->Length()); + ASSERT_EQ(size, 0); + } + ASSERT_GT(pool->CurrentUsage(), 0u); + std::string bytes(contents.size(), '\0'); + ASSERT_OK_AND_ASSIGN(int64_t read, input->Read(bytes.data(), bytes.size(), 0)); + ASSERT_EQ(read, 3); + ASSERT_EQ(bytes, contents); + input.reset(); + ASSERT_EQ(pool->CurrentUsage(), 0u); +} + class DataEvolutionFullTextReadTest : public ::testing::Test { protected: void SetUp() override { @@ -335,7 +437,8 @@ class DataEvolutionFullTextReadTest : public ::testing::Test { index_type, file_name, /*file_size=*/128, /*row_count=*/range.Count(), /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, meta); return IndexFullTextSearchSplitImpl::Create(snapshot_id, "content", range.from, range.to, - {file}, search_row_ranges, scalar_index_files) + {file}, search_row_ranges, scalar_index_files, + TableSchema::FIRST_SCHEMA_ID) .value(); } @@ -411,6 +514,56 @@ TEST_F(DataEvolutionFullTextReadTest, TestMergesShardResultsIntoGlobalTopK) { ASSERT_EQ(ToRows(*result).size(), 10u); } +TEST_F(DataEvolutionFullTextReadTest, TestSubmissionFailureDrainsShardTasks) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(table_path_, {}, false)); + auto row_type = + arrow::struct_({arrow::field("id", arrow::int32()), arrow::field("content", arrow::utf8()), + arrow::field("title", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(row_type, + R"([[0, "paimon", "title"], [1, "paimon", "title"], + [2, "paimon", "title"], [3, "paimon", "title"]])", + {}, 0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), 0, std::nullopt)); + + constexpr uint32_t kSecondShard = 2; + constexpr uint32_t kFirstRawManifest = 3; + for (uint32_t rejected_submission : {kSecondShard, kFirstRawManifest}) { + SCOPED_TRACE(rejected_submission); + FakeIndexState::Instance().Reset(); + std::vector> splits = { + NewSplit("a.index", Range(0, 0), {0.1f}, {Range(0, 0)}), + NewSplit("b.index", Range(1, 1), {0.2f}, {Range(1, 1)})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr raw_split, + RawFullTextSearchSplitImpl::Create(1, "content", kFakeIndexType, {Range(2, 3)}, 1, 0)); + splits.push_back(raw_split); + auto executor = std::make_shared(rejected_submission); + auto builder = NewBuilder(3); + builder->WithExecutor(executor); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, builder->NewFullTextRead()); + auto reading = std::async(std::launch::async, [&read, &splits] { + try { + (void)read->Read(splits); + return false; + } catch (const std::bad_alloc&) { + return true; + } + }); + bool rejected = executor->WaitUntilRejected(); + bool returned_early = + reading.wait_for(std::chrono::milliseconds(200)) == std::future_status::ready; + executor->RunAcceptedTasks(); + bool threw = reading.get(); + ASSERT_TRUE(rejected); + ASSERT_FALSE(returned_early) << "Read exited while shard tasks were still pending"; + ASSERT_TRUE(threw); + ASSERT_EQ(FakeIndexState::Instance().Searches().size(), rejected_submission - 1); + } +} + TEST_F(DataEvolutionFullTextReadTest, TestRanksOnlySearchRowRanges) { std::vector> splits = { NewSplit("a.index", Range(0, 4), {0.1f, 0.9f, 0.5f, 0.3f, 0.7f}, @@ -597,6 +750,14 @@ TEST_F(DataEvolutionFullTextReadTest, TestReadsEachSplitWithItsOwnIndexFields) { NewSplit("dedicated.index", Range(100, 102), {0.1f, 0.5f, 0.2f}, {Range(100, 102)}), NewSplit("multi.index", Range(200, 202), {0.3f, 0.4f, 0.6f}, {Range(200, 202)}, /*index_field_id=*/2, std::vector({1}))}; + SchemaManager schemas(dir_->GetFileSystem(), table_path_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr planned_schema, + schemas.ReadSchema(TableSchema::FIRST_SCHEMA_ID)); + auto fields = planned_schema->Fields(); + fields[2] = DataField(2, arrow::field("renamed_title", arrow::utf8())); + ASSERT_OK(TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_, fields, + planned_schema->HighestFieldId(), + planned_schema->Options())); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(splits)); ASSERT_EQ(ToRows(*result), ScoredRows({{101, 0.5f}, {201, 0.4f}, {202, 0.6f}})); @@ -621,7 +782,24 @@ TEST_F(DataEvolutionFullTextReadTest, TestRejectsInvalidSplits) { /*index_field_id=*/1, std::nullopt, /*snapshot_id=*/2); ASSERT_NOK_WITH_MSG(read->Read({split, other_snapshot}), "planned from snapshots 1 and 2 cannot be read together"); - ASSERT_NOK_WITH_MSG(read->Read({nullptr}), "cannot cast to an index split"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr raw_split, + RawFullTextSearchSplitImpl::Create(1, "content", kFakeIndexType, {Range(2, 3)}, 1, 0)); + ASSERT_NOK_WITH_MSG(read->Read({raw_split, other_snapshot}), + "planned from snapshots 1 and 2 cannot be read together"); + ASSERT_NOK_WITH_MSG(read->Read({other_snapshot, raw_split}), + "planned from snapshots 2 and 1 cannot be read together"); + ASSERT_OK_AND_ASSIGN( + auto second_schema_split, + RawFullTextSearchSplitImpl::Create(1, "content", kFakeIndexType, {Range(4, 5)}, 1, 1)); + ASSERT_NOK_WITH_MSG(read->Read({raw_split, second_schema_split}), + "planned with schemas 0 and 1 cannot be read together"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr other_backend, + RawFullTextSearchSplitImpl::Create(1, "content", "lucene-fts", {Range(4, 5)}, 1, 0)); + ASSERT_NOK_WITH_MSG(read->Read({raw_split, other_backend}), + "cannot combine raw full-text splits with index types"); + ASSERT_NOK_WITH_MSG(read->Read({nullptr}), "cannot cast to an index or raw split"); FullTextSearchBuilder title_builder(table_path_); title_builder.SetQuery("title", kQuery).SetLimit(3); @@ -629,14 +807,34 @@ TEST_F(DataEvolutionFullTextReadTest, TestRejectsInvalidSplits) { ASSERT_NOK_WITH_MSG(title_read->Read({split}), "a split of column content cannot be read by a full-text search of " "column title"); + ASSERT_NOK_WITH_MSG(title_read->Read({raw_split}), + "a split of column content cannot be read by a full-text search of " + "column title"); - auto unknown_type = NewSplit("c.index", Range(0, 1), {0.1f, 0.2f}, {Range(0, 1)}, + auto unknown_type = NewSplit("c.index", Range(0, 15), std::vector(16, 0.1f), + {Range(0, 1), Range(4, 5), Range(8, 9), Range(12, 13)}, /*index_field_id=*/1, std::nullopt, /*snapshot_id=*/1, /*index_type=*/"no-such-index"); - ASSERT_NOK_WITH_MSG(read->Read({unknown_type}), + auto failed = read->Read({unknown_type}); + ASSERT_NOK_WITH_MSG(failed, "no global indexer is registered for index type no-such-index; while " - "searching the split of column content in snapshot 1 over rows [0, 1] " + "searching the split of column content in snapshot 1 over rows [0, 15] " "with no-such-index index files [c.index]"); + ASSERT_NE( + failed.status().message().find("schema 0, 4 search ranges, first [[0, 1], [4, 5], [8, 9]]"), + std::string::npos); + ASSERT_EQ(failed.status().message().find("[12, 13]"), std::string::npos); + ASSERT_OK_AND_ASSIGN(raw_split, + RawFullTextSearchSplitImpl::Create( + 1, "content", "no-such-index", + {Range(2, 3), Range(6, 7), Range(10, 11), Range(14, 15)}, 1, 0)); + failed = read->Read({raw_split}); + ASSERT_FALSE(failed.ok()); + ASSERT_NE(failed.status().message().find( + "while searching column content in snapshot 1 with a temporary no-such-index " + "index over 4 unindexed row ranges, first [[2, 3], [6, 7], [10, 11]]"), + std::string::npos); + ASSERT_EQ(failed.status().message().find("[14, 15]"), std::string::npos); } TEST_F(DataEvolutionFullTextReadTest, TestRejectsSplitOfReplacedColumn) { @@ -647,29 +845,39 @@ TEST_F(DataEvolutionFullTextReadTest, TestRejectsSplitOfReplacedColumn) { ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); ASSERT_OK(read->Read(splits)); - SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); - ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, - schema_manager.Latest()); - ASSERT_TRUE(latest_schema); - const std::shared_ptr& old_schema = latest_schema.value(); - auto replaced_content = arrow::schema( - {arrow::field("id", arrow::int32()), arrow::field("old_content", arrow::utf8()), - arrow::field("title", arrow::utf8()), arrow::field("content", arrow::utf8())}); ASSERT_OK_AND_ASSIGN( - std::unique_ptr new_schema, - TableSchema::Create(old_schema->Id() + 1, replaced_content, old_schema->PartitionKeys(), - old_schema->PrimaryKeys(), old_schema->Options())); - ASSERT_OK_AND_ASSIGN(std::string new_schema_json, new_schema->ToJsonString()); - ASSERT_OK(dir_->GetFileSystem()->AtomicStore( - PathUtil::JoinPath(schema_manager.SchemaDirectory(), - "schema-" + std::to_string(new_schema->Id())), - new_schema_json)); + std::shared_ptr raw_split, + RawFullTextSearchSplitImpl::Create(1, "content", kFakeIndexType, {Range(4, 5)}, 1, 0)); + ASSERT_OK_AND_ASSIGN(std::string bytes, + FullTextSearchSplit::Serialize(raw_split, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr received_raw_split, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), GetDefaultPool())); + + auto fs = dir_->GetFileSystem(); + SchemaManager schemas(fs, table_path_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr planned_schema, + schemas.ReadSchema(TableSchema::FIRST_SCHEMA_ID)); + auto fields = planned_schema->Fields(); + const auto content = fields[1].ArrowField(); + fields[1] = DataField(1, arrow::field("old_content", arrow::utf8())); + fields.emplace_back(3, content); + ASSERT_OK(TestHelper::WriteNextSchema(fs, table_path_, fields, 3, planned_schema->Options())); ASSERT_OK_AND_ASSIGN(read, NewBuilder(3)->NewFullTextRead()); for (const auto& split : splits) { ASSERT_NOK_WITH_MSG(read->Read({split}), "a split of column content does not index its field id 3"); } + ASSERT_NOK_WITH_MSG(read->Read({received_raw_split}), + "a raw split of column content was planned for field id 1"); + + fields.pop_back(); + fields[2] = DataField(2, content); + ASSERT_OK(TestHelper::WriteNextSchema(fs, table_path_, fields, 3, planned_schema->Options())); + ASSERT_OK_AND_ASSIGN(read, NewBuilder(3)->NewFullTextRead()); + ASSERT_NOK_WITH_MSG(read->Read({splits.back()}), + "planned for field id 1, but the column now has field id 2"); } TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { @@ -691,8 +899,15 @@ TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { std::unique_ptr full_mode = NewBuilder(3); full_mode->AddOption(Options::GLOBAL_INDEX_SEARCH_MODE, "full"); - ASSERT_NOK_WITH_MSG(full_mode->NewFullTextScan(), "full and detail are not supported yet"); - ASSERT_NOK_WITH_MSG(full_mode->NewFullTextRead(), "full and detail are not supported yet"); + ASSERT_OK(full_mode->NewFullTextScan()); + ASSERT_OK(full_mode->NewFullTextRead()); + full_mode->AddOption(Options::QUERY_AUTH_ENABLED, "true"); + ASSERT_NOK_WITH_MSG(full_mode->NewFullTextScan(), "Full-text search is not supported"); + ASSERT_NOK_WITH_MSG(full_mode->NewFullTextRead(), "Full-text search is not supported"); + std::unique_ptr invalid_mode = NewBuilder(3); + invalid_mode->AddOption(Options::FULL_TEXT_INDEX_SEARCH_MODE, "invalid"); + ASSERT_NOK_WITH_MSG(invalid_mode->NewFullTextScan(), + "invalid full-text-index.search-mode: invalid"); std::unique_ptr no_thread = NewBuilder(3); no_thread->AddOption(Options::GLOBAL_INDEX_THREAD_NUM, "0"); @@ -710,7 +925,9 @@ TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { missing_field->WithFilter( PredicateBuilder::Equal(/*field_index=*/0, "missing", FieldType::INT, Literal(1))); ASSERT_NOK_WITH_MSG(missing_field->NewFullTextScan(), "field missing does not exist in schema"); - ASSERT_NOK_WITH_MSG(missing_field->NewFullTextRead(), "field missing does not exist in schema"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr worker, missing_field->NewFullTextRead()); + auto split = NewSplit("a.index", Range(0, 0), {1.0f}, {Range(0, 0)}); + ASSERT_NOK_WITH_MSG(worker->Read({split}), "field missing does not exist in schema"); std::unique_ptr row_id_field = NewBuilder(3); row_id_field->WithFilter(PredicateBuilder::Equal( @@ -734,10 +951,8 @@ TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { scalar_full_mode->AddOption(Options::SCALAR_INDEX_SEARCH_MODE, "full"); ASSERT_OK(scalar_full_mode->NewFullTextScan()); scalar_full_mode->WithFilter(IdEquals(1)); - ASSERT_NOK_WITH_MSG(scalar_full_mode->NewFullTextScan(), - "full and detail are not supported yet"); - ASSERT_NOK_WITH_MSG(scalar_full_mode->NewFullTextRead(), - "full and detail are not supported yet"); + ASSERT_OK(scalar_full_mode->NewFullTextScan()); + ASSERT_OK(scalar_full_mode->NewFullTextRead()); std::unique_ptr null_filter = NewBuilder(3); null_filter->WithFilter(nullptr); @@ -775,7 +990,7 @@ TEST_F(DataEvolutionFullTextReadTest, TestResolvesPlanSnapshot) { NewBuilder(3)->ExecuteLocal()); ASSERT_TRUE(ToRows(*result).empty()); - std::vector> time_travel_errors = { + std::vector> unavailable_snapshots = { {"", Options::SCAN_SNAPSHOT_ID, "snapshot-1"}, {"", Options::SCAN_TAG_NAME, "Tag '1' doesn't exist."}, {"", Options::SCAN_TIMESTAMP, @@ -785,10 +1000,8 @@ TEST_F(DataEvolutionFullTextReadTest, TestResolvesPlanSnapshot) { {"from-snapshot", Options::SCAN_TAG_NAME, "Tag '1' doesn't exist."}, {"from-snapshot-full", Options::SCAN_SNAPSHOT_ID, "snapshot-1"}, {"from-timestamp", Options::SCAN_TIMESTAMP_MILLIS, - "There is currently no snapshot earlier than or equal to timestamp [1]"}, - {"latest-full", Options::SCAN_SNAPSHOT_ID, - "scan.snapshot-id must be null when you use latest-full for scan.mode"}}; - for (const auto& [mode, key, error] : time_travel_errors) { + "There is currently no snapshot earlier than or equal to timestamp [1]"}}; + for (const auto& [mode, key, error] : unavailable_snapshots) { SCOPED_TRACE(mode + " " + key); std::unique_ptr builder = NewBuilder(3); if (!mode.empty()) { @@ -797,7 +1010,47 @@ TEST_F(DataEvolutionFullTextReadTest, TestResolvesPlanSnapshot) { builder->AddOption(key, key == Options::SCAN_TIMESTAMP ? "2023-06-01 00:00:00" : "1"); ASSERT_OK_AND_ASSIGN(scan, builder->NewFullTextScan()); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), error); + ASSERT_OK(builder->NewFullTextRead()); } + + auto builder = NewBuilder(3); + builder->AddOption(Options::SCAN_MODE, "latest-full").AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + const std::string error = + "scan.snapshot-id must be null when you use latest-full for scan.mode"; + ASSERT_OK_AND_ASSIGN(scan, builder->NewFullTextScan()); + ASSERT_NOK_WITH_MSG(scan->CreatePlan(), error); + ASSERT_NOK_WITH_MSG(builder->NewFullTextRead(), error); +} + +TEST_F(DataEvolutionFullTextReadTest, TestReplansTagAndTimestamp) { + const auto fs = dir_->GetFileSystem(); + SnapshotManager snapshots(fs, table_path_); + const auto tag_path = TagManager(fs, table_path_).TagPath("current"); + auto publish = [&](int64_t id) { + ASSERT_OK_AND_ASSIGN(std::string json, BuildTestSnapshot(id).ToJsonString()); + ASSERT_OK(fs->WriteFile(snapshots.SnapshotPath(id), json, false)); + ASSERT_OK(fs->WriteFile(tag_path, json, true)); + }; + ASSERT_NO_FATAL_FAILURE(publish(1)); + auto new_scan = [&](const char* key, const std::string& value) { + return NewBuilder(3)->AddOption(key, value).NewFullTextScan(); + }; + ASSERT_OK_AND_ASSIGN(auto tag_scan, new_scan(Options::SCAN_TAG_NAME, "current")); + ASSERT_OK_AND_ASSIGN(auto timestamp_scan, + new_scan(Options::SCAN_TIMESTAMP_MILLIS, + std::to_string(BuildTestSnapshot(1).TimeMillis()))); + ASSERT_OK_AND_ASSIGN(auto fixed_scan, new_scan(Options::SCAN_SNAPSHOT_ID, "1")); + auto check_plans = [&](int64_t latest) { + ASSERT_OK_AND_ASSIGN(auto tag_plan, tag_scan->CreatePlan()); + EXPECT_EQ(tag_plan->SnapshotId(), latest); + ASSERT_OK_AND_ASSIGN(auto timestamp_plan, timestamp_scan->CreatePlan()); + EXPECT_EQ(timestamp_plan->SnapshotId(), latest); + }; + ASSERT_NO_FATAL_FAILURE(check_plans(1)); + ASSERT_NO_FATAL_FAILURE(publish(2)); + ASSERT_NO_FATAL_FAILURE(check_plans(2)); + ASSERT_OK_AND_ASSIGN(auto fixed_plan, fixed_scan->CreatePlan()); + ASSERT_EQ(fixed_plan->SnapshotId(), 1); } } // namespace paimon::test diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan.cpp b/src/paimon/core/table/source/data_evolution_full_text_scan.cpp index 983f58a29..0a0ed6723 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_scan.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -34,23 +33,24 @@ #include "fmt/format.h" #include "fmt/ranges.h" +#include "paimon/core/global_index/data_evolution_global_index_coverage.h" #include "paimon/core/global_index/global_index_evaluator_impl.h" +#include "paimon/core/global_index/global_index_live_row_filter.h" +#include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/manifest/index_manifest_entry.h" #include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/schema/schema_validation.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" -#include "paimon/core/tag/tag.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/core/utils/tag_manager.h" -#include "paimon/defs.h" #include "paimon/global_index/full_text/full_text_defs.h" #include "paimon/global_index/global_indexer.h" #include "paimon/global_index/global_indexer_factory.h" #include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/utils/row_range_index.h" namespace paimon { namespace { @@ -65,63 +65,6 @@ struct IndexRangeCandidate { std::vector assigned; }; -/// Resolves the snapshot to search as Java `TimeTravelUtil#tryTravelOrLatest` does: the one that -/// the time-travel option selects, or else the latest one. -/// -/// TODO: take the column, the partition keys and the options from the schema of the selected -/// snapshot, as Java time travel does. They still come from the latest schema, so a column dropped -/// and added again with the same name since that snapshot finds no index files. -Result> ResolveSnapshot(const CoreOptions& options, - const SnapshotManager& snapshot_manager, - const TagManager& tag_manager) { - // Java checks the options against `scan.mode` when it loads the table with them, which leaves - // at most one time-travel option set. - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateStartupMode(options)); - // Java `TimeTravelUtil#SCAN_KEYS`, without `scan.watermark`, which Paimon C++ does not define. - const char* const time_travel_keys[] = {Options::SCAN_SNAPSHOT_ID, Options::SCAN_TAG_NAME, - Options::SCAN_TIMESTAMP, - Options::SCAN_TIMESTAMP_MILLIS}; - const std::map& raw_options = options.ToMap(); - auto key_iter = - std::find_if(std::begin(time_travel_keys), std::end(time_travel_keys), - [&raw_options](const char* key) { return raw_options.count(key) > 0; }); - if (key_iter == std::end(time_travel_keys)) { - return snapshot_manager.LatestSnapshot(); - } - const std::string key = *key_iter; - if (key == Options::SCAN_SNAPSHOT_ID) { - const std::optional& snapshot_id = options.GetScanSnapshotId(); - if (!snapshot_id) { - return Status::Invalid(fmt::format("invalid {}", key)); - } - PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, - snapshot_manager.LoadSnapshot(snapshot_id.value())); - return std::optional(std::move(snapshot)); - } - if (key == Options::SCAN_TAG_NAME) { - std::optional tag_name = options.GetScanTagName(); - if (!tag_name) { - return Status::Invalid(fmt::format("invalid {}", key)); - } - PAIMON_ASSIGN_OR_RAISE(Tag tag, tag_manager.GetOrThrow(tag_name.value())); - PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, tag.TrimToSnapshot()); - return std::optional(std::move(snapshot)); - } - // `CoreOptions` parses both `scan.timestamp` and `scan.timestamp-millis` into milliseconds. - std::optional timestamp_millis = options.GetScanTimestampMillis(); - if (!timestamp_millis) { - return Status::Invalid(fmt::format("invalid {}", key)); - } - PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, - snapshot_manager.EarlierOrEqualTimeMillis(timestamp_millis.value())); - if (!snapshot) { - return Status::Invalid( - fmt::format("There is currently no snapshot earlier than or equal to timestamp [{}]", - timestamp_millis.value())); - } - return snapshot; -} - /// Returns whether `left` is chosen before `right` for a row that both cover, as Java /// `DataEvolutionFullTextScan#compareCandidates` does. struct CandidateRanksBefore { @@ -137,17 +80,20 @@ struct CandidateRanksBefore { DataEvolutionFullTextScan::DataEvolutionFullTextScan( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& path_factory, + const std::optional& time_travel_snapshot, const CoreOptions& options, + const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, - const std::shared_ptr& pool) + const std::shared_ptr& executor, const std::shared_ptr& pool) : root_path_(root_path), table_schema_(table_schema), + time_travel_snapshot_(time_travel_snapshot), options_(options), path_factory_(path_factory), partition_filter_(partition_filter), filter_(filter), text_field_(text_field), + executor_(executor), pool_(pool) {} bool DataEvolutionFullTextScan::IndexesField(const GlobalIndexMeta& meta, int32_t field_id) { @@ -167,20 +113,6 @@ bool DataEvolutionFullTextScan::IsScalarPreFilterFile(const GlobalIndexMeta& met filter_field_ids.count(meta.index_field_id) > 0; } -Result>> DataEvolutionFullTextScan::IntersectingFiles( - const std::vector>& index_files, const Range& row_range) { - std::vector> files; - for (const auto& file : index_files) { - // Checked before a `Range` is built from it, as a reversed range breaks its invariant. - PAIMON_RETURN_NOT_OK(IndexFullTextSearchSplitImpl::CheckScalarIndexFile(file)); - const GlobalIndexMeta& meta = file->GetGlobalIndexMeta().value(); - if (Range::HasIntersection(Range(meta.row_range_start, meta.row_range_end), row_range)) { - files.push_back(file); - } - } - return files; -} - std::string DataEvolutionFullTextScan::IndexIdentity(const IndexFileMeta& file) { const std::optional& meta = file.GetGlobalIndexMeta(); assert(meta); @@ -189,7 +121,7 @@ std::string DataEvolutionFullTextScan::IndexIdentity(const IndexFileMeta& file) fmt::join(extra_field_ids, ", ")); } -std::vector +Result> DataEvolutionFullTextScan::ChooseIndexRanges( const std::vector>& index_files, int32_t field_id) { // (physical range, identity) -> candidate @@ -199,6 +131,7 @@ DataEvolutionFullTextScan::ChooseIndexRanges( if (!meta || !IndexesField(meta.value(), field_id)) { continue; } + PAIMON_RETURN_NOT_OK(IndexFullTextSearchSplitImpl::CheckIndexFile(file, "full-text")); Range file_range(meta->row_range_start, meta->row_range_end); std::string identity = IndexIdentity(*file); auto iter = candidates @@ -282,12 +215,41 @@ Result DataEvolutionFullTextScan::ServesFullTextSearch(const std::string& return false; } +Result> DataEvolutionFullTextScan::PlanRawSplit( + const Snapshot& snapshot, + const std::vector>& full_text_index_files, + const std::vector& selections) const { + GlobalIndexSearchMode search_mode = options_.GetFullTextIndexSearchMode(); + if (search_mode == GlobalIndexSearchMode::FAST || selections.empty()) { + return std::shared_ptr(); + } + // Coverage counts every single-field definition, including unselected shards. + std::vector indexed_ranges = + DataEvolutionGlobalIndexCoverage::IndexedRanges(full_text_index_files, {text_field_.Id()}); + auto data_ranges = [&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, + GlobalIndexScanImpl::GetOrCreateExecutor(executor_, options_)); + return GlobalIndexLiveRowFilter::DataRowRanges( + root_path_, table_schema_, snapshot, partition_filter_, options_, executor, pool_); + }; + PAIMON_ASSIGN_OR_RAISE(std::vector raw_ranges, + DataEvolutionGlobalIndexCoverage::UnindexedRanges( + search_mode, snapshot.NextRowId(), indexed_ranges, data_ranges)); + if (raw_ranges.empty()) { + return std::shared_ptr(); + } + return RawFullTextSearchSplitImpl::Create( + snapshot.Id(), text_field_.Name(), selections.front().files.front()->IndexType(), + std::move(raw_ranges), text_field_.Id(), table_schema_->Id()); +} + Result> DataEvolutionFullTextScan::CreatePlan() { - std::string branch = BranchManager::NormalizeBranch(options_.GetBranch()); - SnapshotManager snapshot_manager(options_.GetFileSystem(), root_path_, branch); - TagManager tag_manager(options_.GetFileSystem(), root_path_, branch); - PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, - ResolveSnapshot(options_, snapshot_manager, tag_manager)); + std::optional snapshot = time_travel_snapshot_; + if (!snapshot) { + SnapshotManager snapshots(options_.GetFileSystem(), root_path_, + BranchManager::NormalizeBranch(options_.GetBranch())); + PAIMON_ASSIGN_OR_RAISE(snapshot, snapshots.LatestSnapshot()); + } if (!snapshot) { return std::make_shared(std::nullopt, std::vector>()); @@ -344,24 +306,41 @@ Result> DataEvolutionFullTextScan::CreatePla full_text_index_files.push_back(entry.index_file); } } - bool supports_full_text = indexer != nullptr && indexer->SupportsFullTextSearch(); - if (IsScalarPreFilterFile(meta, filter_field_ids, supports_full_text)) { + if (indexer && + IsScalarPreFilterFile(meta, filter_field_ids, indexer->SupportsFullTextSearch())) { scalar_index_files.push_back(entry.index_file); } } std::vector> splits; - for (auto& selection : ChooseIndexRanges(full_text_index_files, field_id)) { - PAIMON_ASSIGN_OR_RAISE(std::vector> split_scalar_files, - IntersectingFiles(scalar_index_files, selection.file_range)); + PAIMON_ASSIGN_OR_RAISE(std::vector selections, + ChooseIndexRanges(full_text_index_files, field_id)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr raw_split, + PlanRawSplit(*snapshot, full_text_index_files, selections)); + std::optional raw_rows; + if (raw_split) { + PAIMON_ASSIGN_OR_RAISE(raw_rows, RowRangeIndex::Create(raw_split->RowRanges())); + } + for (auto& selection : selections) { + if (raw_rows) { + // Keep the physical shard range for translating index-local row ids. + selection.search_ranges = + DataEvolutionGlobalIndexCoverage::ExcludeRanges(selection.search_ranges, *raw_rows); + if (selection.search_ranges.empty()) { + continue; + } + } PAIMON_ASSIGN_OR_RAISE( std::shared_ptr split, IndexFullTextSearchSplitImpl::Create( snapshot->Id(), text_field_.Name(), selection.file_range.from, selection.file_range.to, std::move(selection.files), - std::move(selection.search_ranges), std::move(split_scalar_files))); + std::move(selection.search_ranges), scalar_index_files, table_schema_->Id())); splits.push_back(std::move(split)); } + if (raw_split) { + splits.push_back(std::move(raw_split)); + } return std::make_shared(snapshot->Id(), std::move(splits)); } diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan.h b/src/paimon/core/table/source/data_evolution_full_text_scan.h index b439582dd..443e9eb4a 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan.h +++ b/src/paimon/core/table/source/data_evolution_full_text_scan.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -31,7 +32,9 @@ #include "paimon/core/index/global_index_meta.h" #include "paimon/core/index/index_file_meta.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/executor.h" #include "paimon/global_index/global_indexer.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate.h" @@ -39,9 +42,8 @@ #include "paimon/utils/range.h" namespace paimon { -/// `FullTextScan` of a data-evolution table, aligned with Java `DataEvolutionFullTextScan`: one -/// split per index shard of the searched column, planned from the global index files of one -/// snapshot. +class RawFullTextSearchSplitImpl; + class DataEvolutionFullTextScan : public FullTextScan { public: /// The index shard chosen to rank some rows of the searched column. @@ -58,18 +60,24 @@ class DataEvolutionFullTextScan : public FullTextScan { /// @param root_path Root directory of the table. /// @param table_schema Schema of the table. + /// @param time_travel_snapshot Explicit snapshot used to bind the schema, or nullopt to plan + /// the latest snapshot. /// @param options Options of the table, merged with those of the search. /// @param path_factory Path factory of the table. /// @param partition_filter Predicate over the partition fields, or null for all partitions. /// @param filter Row filter beyond the partition filter, or null for all rows. /// @param text_field The searched column. + /// @param executor Executor to read manifests on in the `detail` search mode, or null + /// to create one with `global-index.thread-num` threads if needed. /// @param pool Memory pool for reading index manifests. DataEvolutionFullTextScan(const std::string& root_path, const std::shared_ptr& table_schema, + const std::optional& time_travel_snapshot, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, const std::shared_ptr& filter, const DataField& text_field, + const std::shared_ptr& executor, const std::shared_ptr& pool); Result> CreatePlan() override; @@ -88,12 +96,6 @@ class DataEvolutionFullTextScan : public FullTextScan { const std::set& filter_field_ids, bool supports_full_text); - /// Returns the files of `index_files` whose row range intersects `row_range`, or `Invalid` if - /// a split cannot carry one of them; see - /// `IndexFullTextSearchSplitImpl::CheckScalarIndexFile()`. - static Result>> IntersectingFiles( - const std::vector>& index_files, const Range& row_range); - /// Returns the identity of the index that `file` belongs to: its index type, indexed field id /// and extra field ids, as `type|index_field_id|[extra_field_ids]`. Files of one identity /// differ only by shard, while files of different identities are never read together. @@ -108,7 +110,7 @@ class DataEvolutionFullTextScan : public FullTextScan { /// @param index_files Global index files; those that do not index `field_id` are ignored. /// @param field_id The id of the searched field. /// @return The chosen shards, ordered by start, end and identity. - static std::vector ChooseIndexRanges( + static Result> ChooseIndexRanges( const std::vector>& index_files, int32_t field_id); /// Returns whether the index files of `index_type` serve a full-text search. A `full-text` or @@ -123,13 +125,20 @@ class DataEvolutionFullTextScan : public FullTextScan { const std::string& table_path, int64_t snapshot_id); private: + Result> PlanRawSplit( + const Snapshot& snapshot, + const std::vector>& full_text_index_files, + const std::vector& selections) const; + std::string root_path_; std::shared_ptr table_schema_; + std::optional time_travel_snapshot_; CoreOptions options_; std::shared_ptr path_factory_; std::shared_ptr partition_filter_; std::shared_ptr filter_; DataField text_field_; + std::shared_ptr executor_; std::shared_ptr pool_; }; diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp b/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp index 47e8ae789..73019b9bb 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp @@ -75,17 +75,19 @@ std::ostream& operator<<(std::ostream& os, const Selection& selection) { return os << "]}"; } -std::vector Choose(const std::vector>& files) { +void CheckSelections(const std::vector>& files, + const std::vector& expected) { + ASSERT_OK_AND_ASSIGN(auto chosen, + DataEvolutionFullTextScan::ChooseIndexRanges(files, kTextFieldId)); std::vector selections; - for (const auto& selection : - DataEvolutionFullTextScan::ChooseIndexRanges(files, kTextFieldId)) { + for (const auto& selection : chosen) { std::vector file_names; for (const auto& file : selection.files) { file_names.push_back(file->FileName()); } selections.push_back(Selection{selection.file_range, file_names, selection.search_ranges}); } - return selections; + ASSERT_EQ(selections, expected); } } // namespace @@ -120,8 +122,8 @@ TEST(DataEvolutionFullTextScanTest, TestDisjointShards) { {Range(100, 199), {"shard-1"}, {Range(100, 199)}}, {Range(300, 399), {"shard-3"}, {Range(300, 399)}}, {Range(400, max_row_id), {"shard-4"}, {Range(400, max_row_id)}}}; - ASSERT_EQ(Choose(files), expected); - ASSERT_TRUE(Choose({}).empty()); + CheckSelections(files, expected); + CheckSelections({}, {}); } TEST(DataEvolutionFullTextScanTest, TestOverlappingShardsOfOneIndex) { @@ -131,7 +133,7 @@ TEST(DataEvolutionFullTextScanTest, TestOverlappingShardsOfOneIndex) { NewIndexFile("1-3", "full-text", Range(1, 3), kTextFieldId)}; std::vector expected = {{Range(0, 2), {"0-2"}, {Range(0, 2)}}, {Range(1, 3), {"1-3"}, {Range(3, 3)}}}; - ASSERT_EQ(Choose(files), expected); + CheckSelections(files, expected); } TEST(DataEvolutionFullTextScanTest, TestDedicatedIndexBeforeExtraField) { @@ -141,31 +143,27 @@ TEST(DataEvolutionFullTextScanTest, TestDedicatedIndexBeforeExtraField) { NewIndexFile("dedicated", "test-full-text", Range(50, 149), kTextFieldId)}; std::vector expected = {{Range(0, 99), {"multi"}, {Range(0, 49)}}, {Range(50, 149), {"dedicated"}, {Range(50, 149)}}}; - ASSERT_EQ(Choose(files), expected); - - ASSERT_EQ(Choose({files[0]}), - std::vector({{Range(0, 99), {"multi"}, {Range(0, 99)}}})); + CheckSelections(files, expected); + CheckSelections({files[0]}, {{Range(0, 99), {"multi"}, {Range(0, 99)}}}); } TEST(DataEvolutionFullTextScanTest, TestIdentityOrderBreaksTies) { std::vector> same_range = { NewIndexFile("lucene", "lucene-fts", Range(0, 9), kTextFieldId), NewIndexFile("full-text", "full-text", Range(0, 9), kTextFieldId)}; - ASSERT_EQ(Choose(same_range), - std::vector({{Range(0, 9), {"full-text"}, {Range(0, 9)}}})); + CheckSelections(same_range, {{Range(0, 9), {"full-text"}, {Range(0, 9)}}}); std::vector> lucene_inside = { NewIndexFile("full-text", "full-text", Range(0, 99), kTextFieldId), NewIndexFile("lucene", "lucene-fts", Range(50, 59), kTextFieldId)}; - ASSERT_EQ(Choose(lucene_inside), - std::vector({{Range(0, 99), {"full-text"}, {Range(0, 99)}}})); + CheckSelections(lucene_inside, {{Range(0, 99), {"full-text"}, {Range(0, 99)}}}); std::vector> full_text_inside = { NewIndexFile("lucene", "lucene-fts", Range(0, 99), kTextFieldId), NewIndexFile("full-text", "full-text", Range(40, 59), kTextFieldId)}; std::vector expected = {{Range(0, 99), {"lucene"}, {Range(0, 39), Range(60, 99)}}, {Range(40, 59), {"full-text"}, {Range(40, 59)}}}; - ASSERT_EQ(Choose(full_text_inside), expected); + CheckSelections(full_text_inside, expected); } TEST(DataEvolutionFullTextScanTest, TestScalarPreFilterFiles) { @@ -191,21 +189,7 @@ TEST(DataEvolutionFullTextScanTest, TestScalarPreFilterFiles) { ASSERT_FALSE(is_pre_filter_file(1, std::vector({2}), /*supports_full_text=*/true)); } -TEST(DataEvolutionFullTextScanTest, TestIntersectingFiles) { - auto first = NewIndexFile("first.index", "btree", Range(0, 9), /*index_field_id=*/2); - auto second = NewIndexFile("second.index", "btree", Range(10, 19), /*index_field_id=*/2); - auto third = NewIndexFile("third.index", "btree", Range(20, 29), /*index_field_id=*/2); - std::vector> files = {first, second, third}; - ASSERT_OK_AND_ASSIGN(std::vector> intersecting, - DataEvolutionFullTextScan::IntersectingFiles(files, Range(9, 10))); - ASSERT_EQ(intersecting, std::vector>({first, second})); - ASSERT_OK_AND_ASSIGN(intersecting, - DataEvolutionFullTextScan::IntersectingFiles(files, Range(25, 40))); - ASSERT_EQ(intersecting, std::vector>({third})); - ASSERT_OK_AND_ASSIGN(intersecting, - DataEvolutionFullTextScan::IntersectingFiles(files, Range(30, 40))); - ASSERT_TRUE(intersecting.empty()); - +TEST(DataEvolutionFullTextScanTest, TestRejectsInvalidIndexRanges) { std::vector> invalid_ranges = { {-1, 9}, {9, 0}, {0, std::numeric_limits::max()}}; for (const auto& [start, end] : invalid_ranges) { @@ -214,11 +198,9 @@ TEST(DataEvolutionFullTextScanTest, TestIntersectingFiles) { auto invalid = std::make_shared( "btree", "invalid.index", /*file_size=*/128, /*row_count=*/10, /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, meta); - std::vector> with_invalid = {first, invalid}; - ASSERT_NOK_WITH_MSG(DataEvolutionFullTextScan::IntersectingFiles(with_invalid, Range(0, 9)), - fmt::format("scalar index file invalid.index covers an invalid row " - "range [{}, {}]", - start, end)); + const std::string error = fmt::format( + "index file invalid.index covers an invalid row range [{}, {}]", start, end); + ASSERT_NOK_WITH_MSG(DataEvolutionFullTextScan::ChooseIndexRanges({invalid}, 2), error); } } diff --git a/src/paimon/core/table/source/filtered_row_id_reader.cpp b/src/paimon/core/table/source/filtered_row_id_reader.cpp index 6c7d54f1a..78a674439 100644 --- a/src/paimon/core/table/source/filtered_row_id_reader.cpp +++ b/src/paimon/core/table/source/filtered_row_id_reader.cpp @@ -20,7 +20,6 @@ #include "paimon/core/table/source/filtered_row_id_reader.h" #include -#include #include #include #include @@ -36,14 +35,14 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/global_index/global_index_evaluator_impl.h" -#include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/source/abstract_table_scan.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" #include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" -#include "paimon/format/file_format.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/leaf_predicate.h" @@ -52,7 +51,6 @@ #include "paimon/reader/batch_reader.h" #include "paimon/scan_context.h" #include "paimon/table/source/plan.h" -#include "paimon/table/source/startup_mode.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" @@ -71,65 +69,52 @@ bool HasCandidateOnlyLeaf(const std::shared_ptr& predicate) { } return false; } +} // namespace -/// Returns `options` with the scan pinned to snapshot `snapshot_id`, as Java does. A tag may -/// outlive its snapshot, so once the snapshot has expired a search of a tag reads the tag instead, -/// which the caller must check still refers to the snapshot. -Result> PinToSnapshot(const std::string& root_path, - const CoreOptions& options, - int64_t snapshot_id) { - std::map pinned = options.ToMap(); - pinned[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); - pinned.erase(Options::SCAN_TIMESTAMP); - pinned.erase(Options::SCAN_TIMESTAMP_MILLIS); - if (pinned.count(Options::SCAN_TAG_NAME) > 0) { - SnapshotManager snapshot_manager(options.GetFileSystem(), root_path, - BranchManager::NormalizeBranch(options.GetBranch())); - PAIMON_ASSIGN_OR_RAISE(bool exists, snapshot_manager.SnapshotExists(snapshot_id)); - if (!exists) { - pinned.erase(Options::SCAN_SNAPSHOT_ID); - return pinned; +Result FilteredRowIdReader::LoadSnapshot(const CoreOptions& options, + const std::string& root_path, + int64_t snapshot_id) { + std::string branch = BranchManager::NormalizeBranch(options.GetBranch()); + SnapshotManager snapshot_manager(options.GetFileSystem(), root_path, branch); + Result snapshot = snapshot_manager.LoadSnapshot(snapshot_id); + if (snapshot.ok()) { + return snapshot; + } + PAIMON_ASSIGN_OR_RAISE(bool exists, snapshot_manager.SnapshotExists(snapshot_id)); + if (exists) { + return snapshot.status(); + } + TagManager tag_manager(options.GetFileSystem(), root_path, branch); + if (std::optional name = options.GetScanTagName()) { + PAIMON_ASSIGN_OR_RAISE(Tag tag, tag_manager.GetOrThrow(*name)); + if (tag.Id() != snapshot_id) { + return Status::Invalid(fmt::format( + "cannot read the rows of snapshot {}, as the data scan resolved another one " + "({} from tag {})", + snapshot_id, tag.Id(), *name)); } - pinned.erase(Options::SCAN_TAG_NAME); + return tag.TrimToSnapshot(); } - pinned[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); - return pinned; -} - -/// Creates a read of `read_context` with `table_schema` rather than the latest schema, as Java -/// `FilteredRowIdReader` reads with the table that the search loaded, so that a column renamed or -/// replaced since then cannot point the filter at another field. -Result> CreateReadWithSchema( - std::unique_ptr read_context, const std::shared_ptr& table_schema, - const std::map& options, const std::shared_ptr& pool, - const std::shared_ptr& executor) { - // Row ids come from row tracking, which a primary-key table does not support. - if (!table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented("cannot evaluate a row filter on a primary-key table"); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_names, tag_manager.ListTagNames()); + for (const std::string& name : tag_names) { + // A tag may disappear after listing. Propagate read errors for tags that still exist. + Result> tag = tag_manager.Get(name); + if (!tag.ok()) { + PAIMON_ASSIGN_OR_RAISE(bool tag_exists, + options.GetFileSystem()->Exists(tag_manager.TagPath(name))); + if (tag_exists) { + return tag.status(); + } + continue; + } + if (tag.value() && tag.value()->Id() == snapshot_id) { + return tag.value()->TrimToSnapshot(); + } } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, options)); - const CoreOptions& core_options = internal_context->GetCoreOptions(); - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, - core_options.GetExternalPathWeights()); - PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, - core_options.CreateGlobalIndexExternalPaths()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), - external_path_weights, pool)); - return std::unique_ptr( - std::make_unique(path_factory, internal_context, pool, executor)); + return Status::NotExist(fmt::format( + "snapshot {} that the full-text search splits were planned from no longer exists", + snapshot_id)); } -} // namespace FilteredRowIdReader::FilteredRowIdReader( const std::string& root_path, const std::shared_ptr& table_schema, @@ -157,38 +142,18 @@ Result FilteredRowIdReader::IsExact(const TableSchema& table_schema, return !HasCandidateOnlyLeaf(filter); } -Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap64& rows) const { - RoaringBitmap64 matching; - if (rows.IsEmpty()) { - return matching; - } - std::map pinned_options; - PAIMON_ASSIGN_OR_RAISE(pinned_options, PinToSnapshot(root_path_, options_, snapshot_id_)); +Result FilteredRowIdReader::MatchingRowIds(RoaringBitmap64 rows) const { + BitmapGlobalIndexResult result(std::move(rows)); + PAIMON_ASSIGN_OR_RAISE(std::vector ranges, result.ToRanges()); + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex row_ranges, RowRangeIndex::Create(ranges)); + return MatchingRowIds(row_ranges); +} - // Unlike the read, the scan may load the latest schema, as it selects files by partition and - // row id alone. - ScanContextBuilder scan_builder(root_path_); - scan_builder.SetOptions(pinned_options) - .SetPredicate(partition_filter_) - .SetGlobalIndexResult(std::make_shared(rows)) - .WithFileSystem(options_.GetFileSystem()) - .WithMemoryPool(pool_) - .WithExecutor(executor_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - // An empty plan of another snapshot would drop every candidate, so it is checked too. - if (plan->SnapshotId() != snapshot_id_) { - return Status::Invalid(fmt::format( - "cannot evaluate a row filter on snapshot {}, as the data scan resolved another one", - snapshot_id_)); - } - if (plan->Splits().empty()) { +Result FilteredRowIdReader::MatchingRowIds(const RowRangeIndex& row_ranges) const { + RoaringBitmap64 matching; + if (row_ranges.Ranges().empty()) { return matching; } - - // The read schema decides the field indexes that a predicate of the read must refer to. std::set filter_fields; PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(filter_, &filter_fields)); std::vector read_fields; @@ -197,24 +162,82 @@ Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap6 read_fields.push_back(field.Name()); } } - read_fields.push_back(SpecialFields::RowId().Name()); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_filter, - PredicateUtils::CreatePickedFieldFilter( - filter_, ObjectUtils::CreateIdentifierToIndexMap(read_fields))); + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, LoadSnapshot(options_, root_path_, snapshot_id_)); + PAIMON_RETURN_NOT_OK( + ReadRows(root_path_, table_schema_, options_, partition_filter_, snapshot, row_ranges, + read_fields, filter_, executor_, pool_, + [&row_ranges, &matching](const arrow::StructArray& /*batch*/, + const arrow::Int64Array& row_ids) -> Status { + for (int64_t i = 0; i < row_ids.length(); ++i) { + int64_t row_id = row_ids.Value(i); + if (row_ranges.Intersects(row_id, row_id)) { + matching.Add(row_id); + } + } + return Status::OK(); + })); + return matching; +} + +Status FilteredRowIdReader::ReadRows( + const std::string& root_path, const std::shared_ptr& table_schema, + const CoreOptions& options, const std::shared_ptr& partition_filter, + const Snapshot& snapshot, const RowRangeIndex& row_ranges, + const std::vector& read_fields, const std::shared_ptr& predicate, + const std::shared_ptr& executor, const std::shared_ptr& pool, + const BatchConsumer& consumer) { + const std::string branch = BranchManager::NormalizeBranch(options.GetBranch()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(options.GetFileSystem(), root_path, table_schema, + CatalogUtils::IdentifierFromPath(root_path, branch), /*uuid=*/"", + branch)); + ScanContextBuilder scan_builder(root_path); + scan_builder.SetOptions(options.ToMap()) + .SetPredicate(partition_filter) + .WithFileSystem(options.GetFileSystem()) + .WithCache(options.GetCache()) + .WithMemoryPool(pool) + .WithExecutor(executor); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + table->NewScan(std::move(scan_context))); + auto* data_scan = dynamic_cast(table_scan.get()); + if (!data_scan) { + return Status::Invalid("reading row id ranges requires a native table scan"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + data_scan->PlanSnapshotRanges(snapshot, row_ranges)); + if (plan->Splits().empty()) { + return Status::OK(); + } + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented("cannot read full-text row id ranges on a primary-key table"); + } + if (options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE(plan, DataEvolutionBatchScan::WrapToIndexedSplits( + plan, row_ranges, /*id_to_score=*/{})); + } - ReadContextBuilder read_builder(root_path_); - read_builder.SetOptions(pinned_options) - .SetReadFieldNames(read_fields) - .SetPredicate(read_filter) - .EnablePredicateFilter(true) + std::vector fields = read_fields; + fields.push_back(SpecialFields::RowId().Name()); + ReadContextBuilder read_builder(root_path); + read_builder.SetOptions(options.ToMap()) + .SetReadFieldNames(fields) .EnablePrefetch(true) - .WithFileSystem(options_.GetFileSystem()) - .WithMemoryPool(pool_) - .WithExecutor(executor_); + .WithFileSystem(options.GetFileSystem()) + .WithCache(options.GetCache()) + .WithMemoryPool(pool) + .WithExecutor(executor); + if (predicate) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_predicate, + PredicateUtils::CreatePickedFieldFilter( + predicate, ObjectUtils::CreateIdentifierToIndexMap(fields))); + read_builder.SetPredicate(read_predicate).EnablePredicateFilter(true); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - CreateReadWithSchema(std::move(read_context), table_schema_, - pinned_options, pool_, executor_)); + table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, table_read->CreateReader(plan->Splits())); ScopeGuard guard([&reader]() { reader->Close(); }); @@ -227,24 +250,19 @@ Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap6 PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, arrow::ImportArray(c_array.get(), c_schema.get())); if (array->type_id() != arrow::Type::STRUCT) { - return Status::Invalid("cannot evaluate a row filter on a batch that is not a struct"); + return Status::Invalid("cannot read rows from a batch that is not a struct"); } + auto struct_array = checked_pointer_cast(array); std::shared_ptr row_id_array = - checked_pointer_cast(array)->GetFieldByName( - SpecialFields::RowId().Name()); + struct_array->GetFieldByName(SpecialFields::RowId().Name()); if (!row_id_array || row_id_array->type_id() != arrow::Type::INT64) { - return Status::Invalid(fmt::format("cannot evaluate a row filter without an int64 {}", - SpecialFields::RowId().Name())); - } - auto row_ids = checked_pointer_cast(row_id_array); - for (int64_t i = 0; i < row_ids->length(); ++i) { - int64_t row_id = row_ids->Value(i); - if (rows.Contains(row_id)) { - matching.Add(row_id); - } + return Status::Invalid( + fmt::format("cannot read rows without an int64 {}", SpecialFields::RowId().Name())); } + PAIMON_RETURN_NOT_OK( + consumer(*struct_array, *checked_pointer_cast(row_id_array))); } - return matching; + return Status::OK(); } } // namespace paimon diff --git a/src/paimon/core/table/source/filtered_row_id_reader.h b/src/paimon/core/table/source/filtered_row_id_reader.h index e70d5d26e..bd10f4d4f 100644 --- a/src/paimon/core/table/source/filtered_row_id_reader.h +++ b/src/paimon/core/table/source/filtered_row_id_reader.h @@ -20,24 +20,26 @@ #pragma once #include +#include #include #include #include +#include +#include "arrow/api.h" #include "paimon/common/predicate/predicate_filter.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/utils/roaring_bitmap64.h" +#include "paimon/utils/row_range_index.h" namespace paimon { -/// Evaluates a row filter on the data of the given row ids and returns those that satisfy it, -/// aligned with Java `FilteredRowIdReader`. Only the filter columns and `_ROW_ID` are read, so this -/// is the exact counterpart of a scalar global index lookup: a search can hand the result to its -/// index as include row ids without touching the index corpus. +/// Verifies a row filter against the data of the planned snapshot. class FilteredRowIdReader { public: /// @param root_path Root directory of the table. @@ -45,9 +47,7 @@ class FilteredRowIdReader { /// @param options Options of the table, merged with those of the search. /// @param partition_filter Predicate over the partition fields, or null for all partitions. /// @param filter The row filter; must not be null. - /// @param snapshot_id The snapshot to read. Once it has expired, a `scan.tag-name` in - /// `options` is read instead, as a tag may outlive its snapshot, and - /// must still refer to it. + /// @param snapshot_id The planned snapshot, which a tag may retain after expiry. /// @param executor Executor to read data on. /// @param pool Memory pool for reading data. FilteredRowIdReader(const std::string& root_path, @@ -67,7 +67,28 @@ class FilteredRowIdReader { const std::set& contributing_field_ids); /// Returns the subset of `rows` whose data satisfies the filter. - Result MatchingRowIds(const RoaringBitmap64& rows) const; + Result MatchingRowIds(RoaringBitmap64 rows) const; + Result MatchingRowIds(const RowRangeIndex& row_ranges) const; + + /// Loads the planned snapshot, including a copy retained by a tag on the same branch. + static Result LoadSnapshot(const CoreOptions& options, const std::string& root_path, + int64_t snapshot_id); + + using BatchConsumer = + std::function; + + /// Reads live rows of files intersecting `row_ranges` in the given snapshot. + /// The consumer must check row ids if a file reader returns rows outside the ranges. + /// `read_fields` excludes `_ROW_ID`, which is appended to each batch. + static Status ReadRows(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& partition_filter, + const Snapshot& snapshot, const RowRangeIndex& row_ranges, + const std::vector& read_fields, + const std::shared_ptr& predicate, + const std::shared_ptr& executor, + const std::shared_ptr& pool, const BatchConsumer& consumer); private: std::string root_path_; diff --git a/src/paimon/core/table/source/full_text_search_builder.cpp b/src/paimon/core/table/source/full_text_search_builder.cpp index 5f81c282f..98b51c3e9 100644 --- a/src/paimon/core/table/source/full_text_search_builder.cpp +++ b/src/paimon/core/table/source/full_text_search_builder.cpp @@ -33,11 +33,17 @@ #include "paimon/core/index/pk/primary_key_index_definitions.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_evolution_full_text_read.h" #include "paimon/core/table/source/data_evolution_full_text_scan.h" +#include "paimon/core/table/source/filtered_row_id_reader.h" +#include "paimon/core/tag/tag.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/format/file_format.h" @@ -52,19 +58,30 @@ namespace paimon { class FullTextSearchBuilder::Impl { public: - /// What a scan and a read of the search are created from. - struct ResolvedSearch { + class TimeTravelScan; + + struct LoadedTable { std::string root_path; - std::shared_ptr table_schema; + std::string branch; + std::shared_ptr schema; CoreOptions options; + + Result> TimeTravelSnapshot(bool for_worker) const; + }; + + struct ResolvedSearch { + LoadedTable table; + std::optional time_travel_snapshot; std::shared_ptr path_factory; std::shared_ptr partition_filter; std::shared_ptr filter; DataField text_field; }; - /// Loads the table and checks the settings shared by scans and reads. - Result Resolve() const; + Result LoadTable() const; + Result Bind(LoadedTable table, std::optional snapshot, + bool validate_data_filter) const; + Result ResolveForPlan() const; Status CheckLimit() const; @@ -85,6 +102,51 @@ class FullTextSearchBuilder::Impl { std::shared_ptr file_system_; }; +class FullTextSearchBuilder::Impl::TimeTravelScan : public FullTextScan { + public: + explicit TimeTravelScan(const Impl& builder) : builder_(builder) {} + + Result> CreatePlan() override { + PAIMON_ASSIGN_OR_RAISE(ResolvedSearch search, builder_.ResolveForPlan()); + return builder_.CreateScan(search)->CreatePlan(); + } + + private: + Impl builder_; +}; + +Result> FullTextSearchBuilder::Impl::LoadedTable::TimeTravelSnapshot( + bool for_worker) const { + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateStartupMode(options)); + if (std::optional tag_name = options.GetScanTagName()) { + TagManager tags(options.GetFileSystem(), root_path, branch); + PAIMON_ASSIGN_OR_RAISE(Tag tag, tags.GetOrThrow(*tag_name)); + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, tag.TrimToSnapshot()); + return std::optional(std::move(snapshot)); + } + SnapshotManager snapshots(options.GetFileSystem(), root_path, branch); + if (std::optional id = options.GetScanSnapshotId()) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, for_worker ? FilteredRowIdReader::LoadSnapshot( + options, root_path, *id) + : snapshots.LoadSnapshot(*id)); + return std::optional(std::move(snapshot)); + } + if (std::optional timestamp = options.GetScanTimestampMillis()) { + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, + snapshots.EarlierOrEqualTimeMillis(*timestamp)); + if (!snapshot && !for_worker) { + return Status::Invalid(fmt::format( + "There is currently no snapshot earlier than or equal to timestamp [{}]", + *timestamp)); + } + return snapshot; + } + if (options.ToMap().count(Options::SCAN_TIMESTAMP) > 0) { + return Status::Invalid(fmt::format("invalid {}", Options::SCAN_TIMESTAMP)); + } + return std::optional(); +} + namespace { /// Rejects a column with a primary-key full-text index, which Java searches with /// `PrimaryKeyFullTextScan` and `PrimaryKeyFullTextRead` instead. @@ -151,10 +213,10 @@ struct SearchFilters { std::shared_ptr data_filter; }; -/// Validates `filters` and splits their conjunction. Result ResolveFilters(const std::vector>& filters, const arrow::Schema& schema, - const std::vector& partition_keys) { + const std::vector& partition_keys, + bool validate_data_filter) { if (filters.empty()) { return SearchFilters(); } @@ -162,9 +224,6 @@ Result ResolveFilters(const std::vector partition_key_to_idx = ObjectUtils::CreateIdentifierToIndexMap(partition_keys); std::vector> partition_conjuncts; @@ -192,11 +251,16 @@ Result ResolveFilters(const std::vector FullTextSearchBuilder::Impl::Resolve() const { +Result FullTextSearchBuilder::Impl::LoadTable() const { if (!field_name_) { return Status::Invalid("Query must be set via SetQuery()"); } @@ -217,18 +281,26 @@ Result FullTextSearchBuilder::Impl: fmt::format("schema file not found in path {}, branch {}", root_path, branch)); } std::shared_ptr table_schema = latest_schema.value(); + PAIMON_RETURN_NOT_OK( + SchemaValidation::RejectQueryAuth(*table_schema, options_, "Full-text search")); std::map merged_options = table_schema->Options(); for (const auto& [key, value] : options_) { merged_options[key] = value; } PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(merged_options, file_system_)); - if (options.GetFullTextIndexSearchMode() != GlobalIndexSearchMode::FAST) { - return Status::NotImplemented(fmt::format( - "full-text search only supports the fast {} (or {}), which searches only the rows " - "that an index covers; full and detail are not supported yet", - Options::FULL_TEXT_INDEX_SEARCH_MODE, Options::GLOBAL_INDEX_SEARCH_MODE)); - } + return LoadedTable{std::move(root_path), std::move(branch), std::move(table_schema), + std::move(options)}; +} +Result FullTextSearchBuilder::Impl::Bind( + LoadedTable table, std::optional snapshot, bool validate_data_filter) const { + auto& [root_path, branch, table_schema, options] = table; + if (snapshot && snapshot->SchemaId() != table_schema->Id()) { + SchemaManager schema_manager(options.GetFileSystem(), root_path, branch); + PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(snapshot->SchemaId())); + PAIMON_RETURN_NOT_OK( + DataEvolutionFullTextRead::BindSchemaOptions(*table_schema, options_, &options)); + } Result text_field = table_schema->GetField(field_name_.value()); if (!text_field.ok()) { return Status::Invalid( @@ -239,15 +311,8 @@ Result FullTextSearchBuilder::Impl: auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(SearchFilters search_filters, - ResolveFilters(filters_, *arrow_schema, table_schema->PartitionKeys())); - if (search_filters.data_filter && - options.GetScalarIndexSearchMode() != GlobalIndexSearchMode::FAST) { - return Status::NotImplemented(fmt::format( - "a full-text search with a row filter only supports the fast {} (or {}), which " - "evaluates only the rows that a scalar index covers; full and detail are not " - "supported yet", - Options::SCALAR_INDEX_SEARCH_MODE, Options::GLOBAL_INDEX_SEARCH_MODE)); - } + ResolveFilters(filters_, *arrow_schema, table_schema->PartitionKeys(), + validate_data_filter)); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, options.GetExternalPathWeights()); @@ -279,13 +344,17 @@ Result FullTextSearchBuilder::Impl: return Status::Invalid("invalid partition predicate, cannot cast to predicate filter"); } } - return ResolvedSearch{root_path, - table_schema, - std::move(options), - path_factory, - partition_filter, - search_filters.data_filter, - std::move(text_field).value()}; + return ResolvedSearch{ + std::move(table), std::move(snapshot), path_factory, + partition_filter, search_filters.data_filter, std::move(text_field).value()}; +} + +Result FullTextSearchBuilder::Impl::ResolveForPlan() + const { + PAIMON_ASSIGN_OR_RAISE(LoadedTable table, LoadTable()); + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, + table.TimeTravelSnapshot(/*for_worker=*/false)); + return Bind(std::move(table), std::move(snapshot), /*validate_data_filter=*/true); } Status FullTextSearchBuilder::Impl::CheckLimit() const { @@ -298,17 +367,19 @@ Status FullTextSearchBuilder::Impl::CheckLimit() const { std::unique_ptr FullTextSearchBuilder::Impl::CreateScan( const ResolvedSearch& search) const { return std::make_unique( - search.root_path, search.table_schema, search.options, search.path_factory, - search.partition_filter, search.filter, search.text_field, memory_pool_); + search.table.root_path, search.table.schema, search.time_travel_snapshot, + search.table.options, search.path_factory, search.partition_filter, search.filter, + search.text_field, executor_, memory_pool_); } Result> FullTextSearchBuilder::Impl::CreateRead( const ResolvedSearch& search) const { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - DataEvolutionFullTextRead::Create( - search.root_path, search.table_schema, search.options, - search.path_factory, search.partition_filter, search.filter, - search.text_field, query_, limit_.value(), executor_, memory_pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr read, + DataEvolutionFullTextRead::Create(search.table.root_path, search.table.schema, + search.table.options, options_, search.path_factory, + search.partition_filter, search.filter, search.text_field, + query_, limit_.value(), executor_, memory_pool_)); return std::unique_ptr(std::move(read)); } @@ -376,20 +447,39 @@ FullTextSearchBuilder& FullTextSearchBuilder::WithFileSystem( } Result> FullTextSearchBuilder::NewFullTextScan() const { - PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); - return impl_->CreateScan(search); + PAIMON_ASSIGN_OR_RAISE(Impl::LoadedTable table, impl_->LoadTable()); + Result> snapshot = table.TimeTravelSnapshot(/*for_worker=*/false); + if (snapshot.ok()) { + // Report schema and filter errors at construction even for selectors resolved again later. + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, + impl_->Bind(std::move(table), std::move(snapshot).value(), + /*validate_data_filter=*/true)); + if (!search.table.options.GetScanTagName() && + !search.table.options.GetScanTimestampMillis()) { + return impl_->CreateScan(search); + } + } + return std::make_unique(*impl_); } Result> FullTextSearchBuilder::NewFullTextRead() const { PAIMON_RETURN_NOT_OK(impl_->CheckLimit()); - PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); + PAIMON_ASSIGN_OR_RAISE(Impl::LoadedTable table, impl_->LoadTable()); + Result> snapshot = table.TimeTravelSnapshot(/*for_worker=*/true); + // A stored split can outlive its snapshot if the search needs no table data. + if (!snapshot.ok() && !snapshot.status().IsNotExist()) { + return snapshot.status(); + } + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, + impl_->Bind(std::move(table), std::move(snapshot).value_or(std::nullopt), + /*validate_data_filter=*/false)); return impl_->CreateRead(search); } Result> FullTextSearchBuilder::ExecuteLocal() const { PAIMON_RETURN_NOT_OK(impl_->CheckLimit()); // The scan and the read share one load of the table, so they see the same schema. - PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->ResolveForPlan()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, impl_->CreateRead(search)); std::unique_ptr scan = impl_->CreateScan(search); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); diff --git a/src/paimon/core/table/source/full_text_search_split.cpp b/src/paimon/core/table/source/full_text_search_split.cpp index 7f52cde4d..4cf6a97bd 100644 --- a/src/paimon/core/table/source/full_text_search_split.cpp +++ b/src/paimon/core/table/source/full_text_search_split.cpp @@ -19,6 +19,7 @@ #include "paimon/table/source/full_text_search_split.h" +#include #include #include #include @@ -37,8 +38,10 @@ #include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/index/index_file_meta_serializer.h" +#include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_evolution_full_text_scan.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" @@ -75,6 +78,68 @@ Status CheckEntryCount(const DataInputStream& in, int32_t count, int64_t min_ent return Status::OK(); } +Status CheckSerializableName(const std::string& value, const char* what) { + if (value.empty()) { + return Status::Invalid(fmt::format("full-text search split requires {}", what)); + } + if (value.size() > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("full-text search split requires {} of at most {} bytes, but got {} bytes", + what, std::numeric_limits::max(), value.size())); + } + return Status::OK(); +} + +Status CheckSplitMetadata(int64_t snapshot_id, const std::string& column_name, + std::optional schema_id) { + if (snapshot_id < Snapshot::FIRST_SNAPSHOT_ID) { + return Status::Invalid(fmt::format( + "full-text search split snapshot id must be positive, got {}", snapshot_id)); + } + if (schema_id && *schema_id < 0) { + return Status::Invalid("full-text search split requires a non-negative schema id"); + } + return CheckSerializableName(column_name, "a column name"); +} + +void WriteRowRanges(const std::vector& ranges, MemorySegmentOutputStream* out) { + out->WriteValue(static_cast(ranges.size())); + for (const Range& range : ranges) { + out->WriteValue(range.from); + out->WriteValue(range.to); + } +} + +Result> ReadRowRanges(DataInputStream* in, const std::string& entry) { + std::string entries = entry + "s"; + PAIMON_ASSIGN_OR_RAISE(int32_t range_count, in->ReadValue()); + PAIMON_RETURN_NOT_OK(CheckEntryCount(*in, range_count, kSearchRowRangeBytes, entries.c_str())); + std::vector ranges; + ranges.reserve(range_count); + for (int32_t i = 0; i < range_count; ++i) { + PAIMON_ASSIGN_OR_RAISE(int64_t from, in->ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int64_t to, in->ReadValue()); + if (from > to) { + return Status::Invalid( + fmt::format("invalid full-text search split, {} [{}, {}] ends before it starts", + entry, from, to)); + } + ranges.emplace_back(from, to); + } + return ranges; +} + +Status CheckFullyRead(const DataInputStream& in) { + PAIMON_ASSIGN_OR_RAISE(int64_t pos, in.GetPos()); + PAIMON_ASSIGN_OR_RAISE(int64_t stream_length, in.Length()); + if (pos != stream_length) { + return Status::Invalid( + fmt::format("invalid full-text search split, remaining {} bytes after deserializing", + stream_length - pos)); + } + return Status::OK(); +} + Status CheckBinaryRowLayout(const MemorySegment& segment, int64_t offset, int64_t size, const arrow::StructType& type); @@ -213,16 +278,16 @@ Result>> ReadIndexFiles( } } // namespace -Status IndexFullTextSearchSplitImpl::CheckScalarIndexFile( - const std::shared_ptr& file) { +Status IndexFullTextSearchSplitImpl::CheckIndexFile(const std::shared_ptr& file, + const char* kind) { if (!file || !file->GetGlobalIndexMeta()) { return Status::Invalid("full-text search split requires global index files"); } const GlobalIndexMeta& meta = file->GetGlobalIndexMeta().value(); if (!IsValidRowRange(meta.row_range_start, meta.row_range_end)) { - return Status::Invalid( - fmt::format("scalar index file {} covers an invalid row range [{}, {}]", - file->FileName(), meta.row_range_start, meta.row_range_end)); + return Status::Invalid(fmt::format("{} index file {} covers an invalid row range [{}, {}]", + kind, file->FileName(), meta.row_range_start, + meta.row_range_end)); } return Status::OK(); } @@ -231,16 +296,9 @@ Result> IndexFullTextSearchSplitIm int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, std::vector search_row_ranges, - std::vector> scalar_index_files) { - if (column_name.empty()) { - return Status::Invalid("full-text search split requires a column name"); - } - // `Serialize()` writes the column name with a 16-bit length. - if (column_name.size() > std::numeric_limits::max()) { - return Status::Invalid(fmt::format( - "full-text search split requires a column name of at most {} bytes, but got {} bytes", - std::numeric_limits::max(), column_name.size())); - } + const std::vector>& scalar_index_files, + std::optional schema_id) { + PAIMON_RETURN_NOT_OK(CheckSplitMetadata(snapshot_id, column_name, schema_id)); if (!IsValidRowRange(row_range_start, row_range_end)) { return Status::Invalid( fmt::format("invalid physical index range [{}, {}]", row_range_start, row_range_end)); @@ -292,12 +350,20 @@ Result> IndexFullTextSearchSplitIm previous.ToString(), range.ToString())); } } + std::vector> intersecting_scalar_files; for (const auto& file : scalar_index_files) { - PAIMON_RETURN_NOT_OK(CheckScalarIndexFile(file)); + PAIMON_RETURN_NOT_OK(CheckIndexFile(file, "scalar")); + const GlobalIndexMeta& meta = *file->GetGlobalIndexMeta(); + auto first = std::lower_bound( + search_row_ranges.begin(), search_row_ranges.end(), meta.row_range_start, + [](const Range& range, int64_t start) { return range.to < start; }); + if (first != search_row_ranges.end() && first->from <= meta.row_range_end) { + intersecting_scalar_files.push_back(file); + } } return std::shared_ptr(new IndexFullTextSearchSplitImpl( snapshot_id, column_name, row_range_start, row_range_end, std::move(full_text_index_files), - std::move(search_row_ranges), std::move(scalar_index_files))); + std::move(search_row_ranges), std::move(intersecting_scalar_files), schema_id)); } bool IndexFullTextSearchSplitImpl::operator==(const IndexFullTextSearchSplitImpl& other) const { @@ -306,7 +372,7 @@ bool IndexFullTextSearchSplitImpl::operator==(const IndexFullTextSearchSplitImpl } return snapshot_id_ == other.snapshot_id_ && column_name_ == other.column_name_ && row_range_start_ == other.row_range_start_ && row_range_end_ == other.row_range_end_ && - search_row_ranges_ == other.search_row_ranges_ && + schema_id_ == other.schema_id_ && search_row_ranges_ == other.search_row_ranges_ && ObjectUtils::Equal(full_text_index_files_, other.full_text_index_files_) && ObjectUtils::Equal(scalar_index_files_, other.scalar_index_files_); } @@ -314,35 +380,91 @@ bool IndexFullTextSearchSplitImpl::operator==(const IndexFullTextSearchSplitImpl std::string IndexFullTextSearchSplitImpl::ToString() const { return fmt::format( "IndexFullTextSearchSplit{{snapshotId={}, columnName={}, rowRangeStart={}, " - "rowRangeEnd={}, searchRowRanges={}, fullTextIndexFiles={}, scalarIndexFiles={}}}", + "rowRangeEnd={}, searchRowRanges={}, fullTextIndexFiles={}, scalarIndexFiles={}, " + "schemaId={}}}", snapshot_id_, column_name_, row_range_start_, row_range_end_, StringUtils::VectorToString(search_row_ranges_), StringUtils::VectorToString(full_text_index_files_), - StringUtils::VectorToString(scalar_index_files_)); + StringUtils::VectorToString(scalar_index_files_), + schema_id_ ? fmt::format("{}", *schema_id_) : "unknown"); +} + +Result> RawFullTextSearchSplitImpl::Create( + int64_t snapshot_id, const std::string& column_name, const std::string& index_type, + std::vector row_ranges, int32_t field_id, int64_t schema_id) { + PAIMON_RETURN_NOT_OK(CheckSplitMetadata(snapshot_id, column_name, schema_id)); + if (field_id < 0) { + return Status::Invalid("a raw full-text split requires a non-negative field id"); + } + PAIMON_RETURN_NOT_OK(CheckSerializableName(index_type, "an index type")); + if (row_ranges.empty()) { + return Status::Invalid("raw full-text search split requires at least one row range"); + } + for (size_t i = 0; i < row_ranges.size(); ++i) { + const Range& range = row_ranges[i]; + if (!IsValidRowRange(range.from, range.to)) { + return Status::Invalid( + fmt::format("invalid raw full-text search row range {}", range.ToString())); + } + if (i == 0) { + continue; + } + const Range& previous = row_ranges[i - 1]; + if (range.from <= previous.to || range.from - 1 == previous.to) { + return Status::Invalid( + fmt::format("row ranges {} and {} are not sorted, disjoint and non-adjacent", + previous.ToString(), range.ToString())); + } + } + return std::shared_ptr(new RawFullTextSearchSplitImpl( + snapshot_id, column_name, index_type, std::move(row_ranges), field_id, schema_id)); +} + +bool RawFullTextSearchSplitImpl::operator==(const RawFullTextSearchSplitImpl& other) const { + return snapshot_id_ == other.snapshot_id_ && column_name_ == other.column_name_ && + index_type_ == other.index_type_ && row_ranges_ == other.row_ranges_ && + field_id_ == other.field_id_ && schema_id_ == other.schema_id_; +} + +std::string RawFullTextSearchSplitImpl::ToString() const { + return fmt::format( + "RawFullTextSearchSplit{{snapshotId={}, columnName={}, indexType={}, rowRanges={}, " + "fieldId={}, schemaId={}}}", + snapshot_id_, column_name_, index_type_, StringUtils::VectorToString(row_ranges_), + field_id_, schema_id_); } Result FullTextSearchSplit::Serialize( const std::shared_ptr& split, const std::shared_ptr& pool) { - auto index_split = std::dynamic_pointer_cast(split); - if (!index_split) { - return Status::Invalid("invalid full-text search split, cannot cast to an index split"); - } MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); - out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); - out.WriteValue(IndexFullTextSearchSplitImpl::VERSION); - out.WriteValue(index_split->SnapshotId()); - out.WriteString(index_split->ColumnName()); - out.WriteValue(index_split->RowRangeStart()); - out.WriteValue(index_split->RowRangeEnd()); - const std::vector& search_row_ranges = index_split->SearchRowRanges(); - out.WriteValue(static_cast(search_row_ranges.size())); - for (const auto& range : search_row_ranges) { - out.WriteValue(range.from); - out.WriteValue(range.to); + if (auto raw_split = std::dynamic_pointer_cast(split)) { + out.WriteValue(RawFullTextSearchSplitImpl::MAGIC); + out.WriteValue(RawFullTextSearchSplitImpl::VERSION); + out.WriteValue(raw_split->SnapshotId()); + out.WriteString(raw_split->ColumnName()); + out.WriteValue(raw_split->FieldId()); + out.WriteString(raw_split->IndexType()); + WriteRowRanges(raw_split->RowRanges(), &out); + out.WriteValue(raw_split->SchemaId()); + } else if (auto index_split = std::dynamic_pointer_cast(split)) { + out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); + out.WriteValue(index_split->SchemaId() ? IndexFullTextSearchSplitImpl::VERSION + : 2); + out.WriteValue(index_split->SnapshotId()); + out.WriteString(index_split->ColumnName()); + out.WriteValue(index_split->RowRangeStart()); + out.WriteValue(index_split->RowRangeEnd()); + WriteRowRanges(index_split->SearchRowRanges(), &out); + IndexFileMetaSerializer serializer(pool); + PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->FullTextIndexFiles(), &out)); + PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->ScalarIndexFiles(), &out)); + if (index_split->SchemaId()) { + out.WriteValue(*index_split->SchemaId()); + } + } else { + return Status::Invalid( + "invalid full-text search split, cannot cast to an index or raw split"); } - IndexFileMetaSerializer serializer(pool); - PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->FullTextIndexFiles(), &out)); - PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->ScalarIndexFiles(), &out)); PAIMON_UNIQUE_PTR bytes = MemorySegmentUtils::CopyToBytes(out.Segments(), 0, out.CurrentSize(), pool.get()); return std::string(bytes->data(), bytes->size()); @@ -362,6 +484,25 @@ Result> FullTextSearchSplit::Deserialize( std::make_shared(buffer, static_cast(length)); DataInputStream in(input_stream); PAIMON_ASSIGN_OR_RAISE(int64_t magic, in.ReadValue()); + if (magic == RawFullTextSearchSplitImpl::MAGIC) { + PAIMON_ASSIGN_OR_RAISE(int32_t version, in.ReadValue()); + if (version != RawFullTextSearchSplitImpl::VERSION) { + return Status::Invalid( + fmt::format("unsupported RawFullTextSearchSplit version: {}", version)); + } + PAIMON_ASSIGN_OR_RAISE(int64_t snapshot_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(std::string column_name, in.ReadString()); + PAIMON_ASSIGN_OR_RAISE(int32_t field_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(std::string index_type, in.ReadString()); + PAIMON_ASSIGN_OR_RAISE(std::vector row_ranges, ReadRowRanges(&in, "row range")); + PAIMON_ASSIGN_OR_RAISE(int64_t schema_id, in.ReadValue()); + PAIMON_RETURN_NOT_OK(CheckFullyRead(in)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr split, + RawFullTextSearchSplitImpl::Create(snapshot_id, column_name, index_type, + std::move(row_ranges), field_id, schema_id)); + return std::shared_ptr(std::move(split)); + } if (magic != IndexFullTextSearchSplitImpl::MAGIC) { return Status::Invalid("invalid full-text search split, unknown magic number"); } @@ -374,39 +515,24 @@ Result> FullTextSearchSplit::Deserialize( PAIMON_ASSIGN_OR_RAISE(std::string column_name, in.ReadString()); PAIMON_ASSIGN_OR_RAISE(int64_t row_range_start, in.ReadValue()); PAIMON_ASSIGN_OR_RAISE(int64_t row_range_end, in.ReadValue()); - PAIMON_ASSIGN_OR_RAISE(int32_t range_count, in.ReadValue()); - PAIMON_RETURN_NOT_OK( - CheckEntryCount(in, range_count, kSearchRowRangeBytes, "search row ranges")); - std::vector search_row_ranges; - search_row_ranges.reserve(range_count); - for (int32_t i = 0; i < range_count; ++i) { - PAIMON_ASSIGN_OR_RAISE(int64_t from, in.ReadValue()); - PAIMON_ASSIGN_OR_RAISE(int64_t to, in.ReadValue()); - if (from > to) { - return Status::Invalid(fmt::format( - "invalid full-text search split, search row range [{}, {}] ends before it starts", - from, to)); - } - search_row_ranges.emplace_back(from, to); - } + PAIMON_ASSIGN_OR_RAISE(std::vector search_row_ranges, + ReadRowRanges(&in, "search row range")); PAIMON_ASSIGN_OR_RAISE(std::vector> full_text_index_files, ReadIndexFiles(&in, pool)); std::vector> scalar_index_files; if (version >= 2) { PAIMON_ASSIGN_OR_RAISE(scalar_index_files, ReadIndexFiles(&in, pool)); } - PAIMON_ASSIGN_OR_RAISE(int64_t pos, in.GetPos()); - PAIMON_ASSIGN_OR_RAISE(int64_t stream_length, in.Length()); - if (pos != stream_length) { - return Status::Invalid( - fmt::format("invalid full-text search split, remaining {} bytes after deserializing", - stream_length - pos)); + std::optional schema_id; + if (version >= 3) { + PAIMON_ASSIGN_OR_RAISE(schema_id, in.ReadValue()); } + PAIMON_RETURN_NOT_OK(CheckFullyRead(in)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr split, IndexFullTextSearchSplitImpl::Create( snapshot_id, column_name, row_range_start, row_range_end, std::move(full_text_index_files), std::move(search_row_ranges), - std::move(scalar_index_files))); + scalar_index_files, schema_id)); return std::shared_ptr(std::move(split)); } diff --git a/src/paimon/core/table/source/full_text_search_split_test.cpp b/src/paimon/core/table/source/full_text_search_split_test.cpp index f94673668..68e533dfb 100644 --- a/src/paimon/core/table/source/full_text_search_split_test.cpp +++ b/src/paimon/core/table/source/full_text_search_split_test.cpp @@ -38,6 +38,7 @@ #include "paimon/core/index/index_file_meta.h" #include "paimon/core/index/index_file_meta_serializer.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" @@ -79,14 +80,15 @@ std::string FromHex(const std::string& hex) { return bytes; } -std::string SplitBytes(const std::function& write_body, - const std::shared_ptr& pool) { +std::string Version2IndexSplitBytes( + const std::function& write_body, + const std::shared_ptr& pool) { const int64_t snapshot_id = 1; const int64_t row_range_start = 0; const int64_t row_range_end = 9; MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); - out.WriteValue(IndexFullTextSearchSplitImpl::VERSION); + out.WriteValue(2); out.WriteValue(snapshot_id); out.WriteString("f0"); out.WriteValue(row_range_start); @@ -98,6 +100,23 @@ std::string SplitBytes(const std::function& wr class FullTextSearchSplitTest : public ::testing::Test { protected: + void CheckInvalidSplitBytes(const std::string& bytes) const { + std::string trailing = bytes + std::string(1, '\0'); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(trailing.data(), trailing.size(), pool_), + "remaining 1 bytes after deserializing"); + for (size_t size = 0; size < bytes.size(); ++size) { + ASSERT_NOK(FullTextSearchSplit::Deserialize(bytes.data(), size, pool_)) << size; + } + for (const auto& snapshot_id : {FromHex("0000000000000000"), FromHex("FFFFFFFFFFFFFFFF")}) { + std::string invalid = bytes; + invalid.replace(sizeof(int64_t) + sizeof(int32_t), sizeof(int64_t), snapshot_id); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(invalid.data(), invalid.size(), pool_), + "snapshot id must be positive"); + } + } + std::shared_ptr pool_ = GetDefaultPool(); }; @@ -107,10 +126,11 @@ TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { /*extra_field_ids=*/std::vector({1, 3}), pool_), NewIndexFile("full-text-1.index", "full-text", Range(100, 199), /*index_field_id=*/2, /*extra_field_ids=*/std::vector({1, 3}), pool_)}; - ASSERT_OK_AND_ASSIGN(std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create( - /*snapshot_id=*/7, "content", /*row_range_start=*/100, - /*row_range_end=*/199, files, {Range(100, 119), Range(150, 199)})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create( + /*snapshot_id=*/7, "content", /*row_range_start=*/100, + /*row_range_end=*/199, files, {Range(100, 119), Range(150, 199)}, {}, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); const std::string index_files = FromHex( @@ -172,10 +192,18 @@ TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { ASSERT_TRUE(version_1_impl); ASSERT_EQ(*version_1_impl, *split) << version_1_impl->ToString() << " vs " << split->ToString(); + ASSERT_OK_AND_ASSIGN(split, + IndexFullTextSearchSplitImpl::Create(7, "content", 100, 199, files, + split->SearchRowRanges(), {}, 5)); + ASSERT_OK_AND_ASSIGN(bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_EQ(bytes, bytes_of_version("00000003") + FromHex("00000000" + "0000000000000005")); + std::string longest_column(std::numeric_limits::max(), 'c'); - ASSERT_OK_AND_ASSIGN(split, IndexFullTextSearchSplitImpl::Create( - /*snapshot_id=*/7, longest_column, /*row_range_start=*/100, - /*row_range_end=*/199, files, {Range(100, 199)})); + ASSERT_OK_AND_ASSIGN(split, + IndexFullTextSearchSplitImpl::Create( + /*snapshot_id=*/7, longest_column, /*row_range_start=*/100, + /*row_range_end=*/199, files, {Range(100, 199)}, {}, std::nullopt)); ASSERT_OK_AND_ASSIGN(bytes, FullTextSearchSplit::Serialize(split, pool_)); ASSERT_OK_AND_ASSIGN(result, FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); @@ -188,45 +216,47 @@ TEST_F(FullTextSearchSplitTest, TestSerializeScalarIndexFiles) { std::vector> files = {NewIndexFile( "full-text-0.index", "full-text", Range(0, 99), /*index_field_id=*/1, std::nullopt, pool_)}; std::vector> scalar_files = { - NewIndexFile("btree-0.index", "btree", Range(0, 49), /*index_field_id=*/2, std::nullopt, - pool_), - NewIndexFile("bitmap-0.index", "bitmap", Range(50, 149), /*index_field_id=*/3, std::nullopt, - pool_)}; - ASSERT_OK_AND_ASSIGN(std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 99, files, - {Range(0, 99)}, scalar_files)); - ASSERT_EQ(split->ScalarIndexFiles().size(), 2u); + NewIndexFile("btree-0.index", "btree", Range(0, 10), 2, std::nullopt, pool_), + NewIndexFile("gap.index", "btree", Range(20, 59), 2, std::nullopt, pool_), + NewIndexFile("bitmap-0.index", "bitmap", Range(69, 149), 3, std::nullopt, pool_), + NewIndexFile("outside.index", "bitmap", Range(100, 109), 3, std::nullopt, pool_)}; + const std::vector search_ranges = {Range(10, 19), Range(60, 69)}; + for (const auto& schema_id : {std::optional(), std::optional(5)}) { + SCOPED_TRACE(::testing::PrintToString(schema_id)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create( + 1, "f0", 0, 99, files, search_ranges, scalar_files, schema_id)); + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + auto impl = std::dynamic_pointer_cast(result); + ASSERT_TRUE(impl); + ASSERT_EQ(*impl, *split) << impl->ToString() << " vs " << split->ToString(); + ASSERT_EQ(impl->SchemaId(), schema_id); + ASSERT_EQ(impl->ScalarIndexFiles().size(), 2u); + ASSERT_EQ(impl->ScalarIndexFiles()[0]->FileName(), "btree-0.index"); + ASSERT_EQ(impl->ScalarIndexFiles()[1]->FileName(), "bitmap-0.index"); + ASSERT_NE(impl->ToString().find("scalarIndexFiles=["), std::string::npos); + ASSERT_NOK(FullTextSearchSplit::Deserialize(bytes.data(), bytes.size() - 1, pool_)); - ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); - auto impl = std::dynamic_pointer_cast(result); - ASSERT_TRUE(impl); - ASSERT_EQ(*impl, *split) << impl->ToString() << " vs " << split->ToString(); - ASSERT_EQ(impl->ScalarIndexFiles()[0]->FileName(), "btree-0.index"); - ASSERT_EQ(impl->ScalarIndexFiles()[1]->FileName(), "bitmap-0.index"); - ASSERT_NE(impl->ToString().find("scalarIndexFiles=["), std::string::npos); - - ASSERT_OK_AND_ASSIGN(std::shared_ptr no_scalar_files, - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 99, files, - {Range(0, 99)})); - ASSERT_FALSE(*no_scalar_files == *split); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr no_scalar_files, + IndexFullTextSearchSplitImpl::Create(1, "f0", 0, 99, files, search_ranges, + {scalar_files[1]}, schema_id)); + ASSERT_TRUE(no_scalar_files->ScalarIndexFiles().empty()); + ASSERT_FALSE(*no_scalar_files == *split); + } } TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsInvalidBytes) { std::vector> files = {NewIndexFile( "full-text-0.index", "full-text", Range(0, 9), /*index_field_id=*/1, std::nullopt, pool_)}; - ASSERT_OK_AND_ASSIGN( - std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, {Range(0, 9)})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, + {Range(0, 9)}, {}, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - std::string trailing = bytes + std::string(1, '\0'); - ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Deserialize(trailing.data(), trailing.size(), pool_), - "remaining 1 bytes after deserializing"); - for (size_t size = 0; size < bytes.size(); ++size) { - ASSERT_NOK(FullTextSearchSplit::Deserialize(bytes.data(), size, pool_)) << size; - } + ASSERT_NO_FATAL_FAILURE(CheckInvalidSplitBytes(bytes)); std::string too_many_scalar_files = bytes; too_many_scalar_files.replace(too_many_scalar_files.size() - sizeof(int32_t), sizeof(int32_t), FromHex("7FFFFFFF")); @@ -249,22 +279,22 @@ TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsInvalidBytes) { MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); out.WriteValue(IndexFullTextSearchSplitImpl::VERSION + 1); - std::string newer_version = ToBytes(&out, pool_); + std::string unsupported = ToBytes(&out, pool_); ASSERT_NOK_WITH_MSG( - FullTextSearchSplit::Deserialize(newer_version.data(), newer_version.size(), pool_), - "unsupported IndexFullTextSearchSplit version: 3"); + FullTextSearchSplit::Deserialize(unsupported.data(), unsupported.size(), pool_), + "unsupported IndexFullTextSearchSplit version: 4"); } ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Serialize(nullptr, pool_), - "cannot cast to an index split"); + "cannot cast to an index or raw split"); } TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptFileLayout) { std::vector> files = { NewIndexFile("full-text-0.index", "full-text", Range(0, 9), /*index_field_id=*/2, /*extra_field_ids=*/std::vector({1}), pool_)}; - ASSERT_OK_AND_ASSIGN( - std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, {Range(0, 9)})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, + {Range(0, 9)}, {}, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); ASSERT_OK(FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); @@ -316,7 +346,7 @@ TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptFileLayout) { TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptEntries) { auto deserialize = [this](const std::function& write_body) { - std::string bytes = SplitBytes(write_body, pool_); + std::string bytes = Version2IndexSplitBytes(write_body, pool_); return FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_); }; auto write_one_range = [](MemorySegmentOutputStream* out) { @@ -387,9 +417,12 @@ TEST_F(FullTextSearchSplitTest, TestCreateValidatesArguments) { auto create = [&](const std::vector>& index_files, const std::vector& search_row_ranges) { return IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 10, 19, index_files, - search_row_ranges); + search_row_ranges, {}, std::nullopt); }; ASSERT_OK(create(files, {Range(10, 11), Range(13, 19)})); + ASSERT_NOK_WITH_MSG( + IndexFullTextSearchSplitImpl::Create(1, "f0", 10, 19, files, {Range(10, 19)}, {}, -1), + "non-negative schema id"); ASSERT_NOK_WITH_MSG(create({}, {Range(10, 19)}), "at least one index file"); ASSERT_NOK_WITH_MSG(create(files, {}), "at least one search row range"); ASSERT_NOK_WITH_MSG(create(files, {Range(9, 19)}), "outside physical index range [10, 19]"); @@ -418,27 +451,145 @@ TEST_F(FullTextSearchSplitTest, TestCreateValidatesArguments) { "full-text", "full-text-3.index", /*file_size=*/128, /*row_count=*/10, /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt); ASSERT_NOK_WITH_MSG(create({no_global_meta}, {Range(10, 19)}), "global index files"); - ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 10, 19, files, - {Range(10, 19)}, {no_global_meta}), - "global index files"); ASSERT_NOK_WITH_MSG( - IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "", 10, 19, files, {Range(10, 19)}), - "requires a column name"); + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 10, 19, files, + {Range(10, 19)}, {no_global_meta}, std::nullopt), + "global index files"); + ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "", 10, 19, files, + {Range(10, 19)}, {}, std::nullopt), + "requires a column name"); std::string too_long_column(std::numeric_limits::max() + 1, 'c'); - ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, too_long_column, 10, - 19, files, {Range(10, 19)}), - "column name of at most 65535 bytes, but got 65536 bytes"); + ASSERT_NOK_WITH_MSG( + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, too_long_column, 10, 19, files, + {Range(10, 19)}, {}, std::nullopt), + "column name of at most 65535 bytes, but got 65536 bytes"); std::vector> invalid_ranges = { {-1, 19}, {19, 10}, {0, std::numeric_limits::max()}}; for (const auto& [start, end] : invalid_ranges) { - ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", start, - end, files, {Range(10, 19)}), - "invalid physical index range"); + ASSERT_NOK_WITH_MSG( + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", start, end, files, + {Range(10, 19)}, {}, std::nullopt), + "invalid physical index range"); ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create( /*snapshot_id=*/1, "f0", 10, 19, files, {Range(10, 19)}, - {NewScalarIndexFile(start, end, pool_)}), + {NewScalarIndexFile(start, end, pool_)}, std::nullopt), "scalar index file btree-0.index covers an invalid row range"); } } +TEST_F(FullTextSearchSplitTest, TestRawSplitSerializeRoundTrip) { + const std::vector ranges = {Range(0, 4), Range(10, 19)}; + const std::string range_bytes = FromHex( + "00000002" + "0000000000000000" + "0000000000000004" + "000000000000000A" + "0000000000000013"); + const std::string bytes = "PFTRAWS1" + FromHex("0000000100000000000000070002") + "f0" + + FromHex("000000030009") + "full-text" + range_bytes + + FromHex("0000000000000005"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + auto split = std::dynamic_pointer_cast(result); + ASSERT_TRUE(split); + ASSERT_EQ(split->SnapshotId(), 7); + ASSERT_EQ(split->ColumnName(), "f0"); + ASSERT_EQ(split->IndexType(), "full-text"); + ASSERT_EQ(split->RowRanges(), ranges); + ASSERT_EQ(split->FieldId(), 3); + ASSERT_EQ(split->SchemaId(), 5); + ASSERT_OK_AND_ASSIGN(std::string serialized, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_EQ(serialized, bytes); + ASSERT_OK_AND_ASSIGN(std::shared_ptr created, + RawFullTextSearchSplitImpl::Create(7, "f0", "full-text", ranges, 3, 5)); + ASSERT_EQ(*created, *split); + ASSERT_OK_AND_ASSIGN(std::shared_ptr other_type, + RawFullTextSearchSplitImpl::Create(7, "f0", "lucene-fts", ranges, 3, 5)); + ASSERT_FALSE(*other_type == *split); +} + +TEST_F(FullTextSearchSplitTest, TestRawSplitCreateValidatesArguments) { + ASSERT_NOK_WITH_MSG( + RawFullTextSearchSplitImpl::Create(1, "f0", "full-text", {Range(0, 4)}, 0, -1), + "non-negative schema id"); + ASSERT_NOK_WITH_MSG( + RawFullTextSearchSplitImpl::Create(1, "f0", "full-text", {Range(0, 4)}, -1, 0), + "non-negative field id"); + auto create = [](const std::vector& row_ranges) { + return RawFullTextSearchSplitImpl::Create(1, "f0", "full-text", row_ranges, 0, 0); + }; + ASSERT_OK(create({Range(0, 0), Range(2, std::numeric_limits::max() - 1)})); + ASSERT_NOK_WITH_MSG(create({}), "at least one row range"); + ASSERT_NOK_WITH_MSG(create({Range(-1, 3)}), "invalid raw full-text search row range"); + ASSERT_NOK_WITH_MSG(create({Range(0, std::numeric_limits::max())}), + "invalid raw full-text search row range"); + ASSERT_NOK_WITH_MSG(create({Range(5, 9), Range(0, 3)}), + "not sorted, disjoint and non-adjacent"); + ASSERT_NOK_WITH_MSG(create({Range(0, 4), Range(4, 9)}), + "not sorted, disjoint and non-adjacent"); + ASSERT_NOK_WITH_MSG(create({Range(0, 4), Range(5, 9)}), + "not sorted, disjoint and non-adjacent"); + ASSERT_NOK_WITH_MSG(RawFullTextSearchSplitImpl::Create(1, "", "full-text", {Range(0, 4)}, 0, 0), + "requires a column name"); + ASSERT_NOK_WITH_MSG(RawFullTextSearchSplitImpl::Create(1, "f0", "", {Range(0, 4)}, 0, 0), + "requires an index type"); + std::string too_long(std::numeric_limits::max() + 1, 'c'); + ASSERT_NOK_WITH_MSG(RawFullTextSearchSplitImpl::Create(1, "f0", too_long, {Range(0, 4)}, 0, 0), + "index type of at most 65535 bytes, but got 65536 bytes"); +} + +TEST_F(FullTextSearchSplitTest, TestRawSplitDeserializeRejectsInvalidBytes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + RawFullTextSearchSplitImpl::Create(1, "f0", "full-text", + {Range(0, 4), Range(10, 19)}, 0, 5)); + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_NO_FATAL_FAILURE(CheckInvalidSplitBytes(bytes)); + auto write_header = [](MemorySegmentOutputStream* out, int32_t version) { + out->WriteValue(RawFullTextSearchSplitImpl::MAGIC); + out->WriteValue(version); + out->WriteValue(1); + out->WriteString("f0"); + out->WriteValue(0); + out->WriteString("full-text"); + }; + for (int32_t version : {0, 2, 3}) { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + write_header(&out, version); + std::string newer_version = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(newer_version.data(), newer_version.size(), pool_), + "unsupported RawFullTextSearchSplit version"); + } + { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + write_header(&out, RawFullTextSearchSplitImpl::VERSION); + out.WriteValue(std::numeric_limits::max()); + std::string too_many_ranges = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(too_many_ranges.data(), too_many_ranges.size(), pool_), + "2147483647 row ranges exceed the remaining 0 bytes"); + } + { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + write_header(&out, RawFullTextSearchSplitImpl::VERSION); + out.WriteValue(1); + out.WriteValue(9); + out.WriteValue(0); + std::string reversed_range = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(reversed_range.data(), reversed_range.size(), pool_), + "row range [9, 0] ends before it starts"); + } + { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + write_header(&out, RawFullTextSearchSplitImpl::VERSION); + out.WriteValue(0); + out.WriteValue(5); + std::string no_ranges = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(no_ranges.data(), no_ranges.size(), pool_), + "at least one row range"); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/table/source/index_full_text_search_split_impl.h b/src/paimon/core/table/source/index_full_text_search_split_impl.h index eaaac04aa..32bc586f0 100644 --- a/src/paimon/core/table/source/index_full_text_search_split_impl.h +++ b/src/paimon/core/table/source/index_full_text_search_split_impl.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -37,9 +38,8 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { public: /// "PFTSPLT1" in ASCII. static constexpr int64_t MAGIC = 0x50465453504C5431L; - /// Version 2 appends the scalar index files that pre-filter rows, as version 2 of Java - /// `IndexFullTextSearchSplit` does. - static constexpr int32_t VERSION = 2; + /// Version 2 adds scalar index files. Version 3 adds the planning schema ID. + static constexpr int32_t VERSION = 3; /// Creates a split. /// @param snapshot_id The snapshot the split is planned from. @@ -52,18 +52,19 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { /// global index meta of the shard range. /// @param search_row_ranges The sorted, disjoint and non-adjacent row id ranges to rank, at /// least one and all within the shard range. - /// @param scalar_index_files The global index files that intersect the shard range and can - /// pre-filter its rows, each accepted by `CheckScalarIndexFile()`. + /// @param scalar_index_files Scalar indexes for the row filter. The split keeps the files + /// that intersect its assigned search ranges. + /// @param schema_id The schema used when planning, absent from versions 1 and 2. /// @return The split, or `Invalid` if an argument breaks these rules. static Result> Create( int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, std::vector search_row_ranges, - std::vector> scalar_index_files = {}); + const std::vector>& scalar_index_files, + std::optional schema_id); - /// Checks that `file` has a global index meta whose row range is not negative, is ordered, and - /// has a row count that fits in int64, as a split requires of its scalar index files. - static Status CheckScalarIndexFile(const std::shared_ptr& file); + /// Checks that the global index range is ordered, non-negative and has a representable count. + static Status CheckIndexFile(const std::shared_ptr& file, const char* kind); int64_t SnapshotId() const override { return snapshot_id_; @@ -93,6 +94,10 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { return scalar_index_files_; } + std::optional SchemaId() const { + return schema_id_; + } + bool operator==(const IndexFullTextSearchSplitImpl& other) const; std::string ToString() const; @@ -102,14 +107,16 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, std::vector search_row_ranges, - std::vector> scalar_index_files) + std::vector> scalar_index_files, + std::optional schema_id) : snapshot_id_(snapshot_id), column_name_(column_name), row_range_start_(row_range_start), row_range_end_(row_range_end), full_text_index_files_(std::move(full_text_index_files)), search_row_ranges_(std::move(search_row_ranges)), - scalar_index_files_(std::move(scalar_index_files)) {} + scalar_index_files_(std::move(scalar_index_files)), + schema_id_(schema_id) {} int64_t snapshot_id_; std::string column_name_; @@ -118,6 +125,7 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { std::vector> full_text_index_files_; std::vector search_row_ranges_; std::vector> scalar_index_files_; + std::optional schema_id_; }; } // namespace paimon diff --git a/src/paimon/core/table/source/raw_full_text_read_impl.cpp b/src/paimon/core/table/source/raw_full_text_read_impl.cpp new file mode 100644 index 000000000..977ba521b --- /dev/null +++ b/src/paimon/core/table/source/raw_full_text_read_impl.cpp @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/raw_full_text_read_impl.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "arrow/compute/api_vector.h" +#include "arrow/io/memory.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/math.h" +#include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/uuid.h" +#include "paimon/core/table/source/filtered_row_id_reader.h" +#include "paimon/core/utils/nested_projection_utils.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/utils/roaring_bitmap64.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon { +struct InMemoryGlobalIndexFiles::Storage { + explicit Storage(const std::shared_ptr& pool) : arrow_pool(GetArrowPool(pool)) {} + + std::shared_ptr arrow_pool; + std::mutex mutex; + std::map> files; +}; + +class InMemoryGlobalIndexFiles::IndexOutputStream : public OutputStream { + public: + IndexOutputStream(const std::string& file_path, const std::shared_ptr& storage, + const std::shared_ptr& output) + : file_path_(file_path), storage_(storage), output_(output) {} + + Result Write(const char* buffer, int64_t size) override { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "write length")); + if (size > 0 && buffer == nullptr) { + return Status::Invalid("write buffer must not be null when size is positive"); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(output_->Write(buffer, size)); + return size; + } + + Status Flush() override { + return Status::OK(); + } + + Result GetPos() const override { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(int64_t position, output_->Tell()); + return position; + } + + Result GetUri() const override { + return file_path_; + } + + Status Close() override { + if (output_->closed()) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr bytes, output_->Finish()); + std::lock_guard lock(storage_->mutex); + storage_->files[file_path_] = std::move(bytes); + return Status::OK(); + } + + private: + std::string file_path_; + std::shared_ptr storage_; + std::shared_ptr output_; +}; + +namespace { +class SharedBytesInputStream : public ByteArrayInputStream { + public: + SharedBytesInputStream(const std::shared_ptr& pool, + const std::shared_ptr& bytes, + const std::string& file_path) + : ByteArrayInputStream(bytes->size() == 0 ? "" : bytes->data_as(), bytes->size()), + pool_(pool), + bytes_(bytes), + file_path_(file_path) {} + + Result GetUri() const override { + return file_path_; + } + + private: + // Arrow buffers free their memory through this pool, which must outlive bytes_. + std::shared_ptr pool_; + std::shared_ptr bytes_; + std::string file_path_; +}; + +std::shared_ptr EmptyResult() { + return std::make_shared(RoaringBitmap64(), std::vector()); +} +} // namespace + +InMemoryGlobalIndexFiles::InMemoryGlobalIndexFiles(const std::shared_ptr& pool) + : storage_(std::make_shared(pool)) {} + +Result InMemoryGlobalIndexFiles::NewPath(const std::string& prefix) const { + std::string uuid; + if (!UUID::Generate(&uuid)) { + return Status::IOError("failed to generate a UUID for a temporary full-text index file"); + } + return fmt::format("memory://global-index/{}-{}.index", prefix, uuid); +} + +Result> InMemoryGlobalIndexFiles::NewOutputStream( + const std::string& file_path) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr output, + arrow::io::BufferOutputStream::Create(0, storage_->arrow_pool.get())); + return std::unique_ptr( + std::make_unique(file_path, storage_, output)); +} + +Result InMemoryGlobalIndexFiles::GetFileSize(const std::string& file_path) const { + std::lock_guard lock(storage_->mutex); + auto iter = storage_->files.find(file_path); + if (iter == storage_->files.end()) { + return Status::NotExist(fmt::format("in-memory index file {} does not exist", file_path)); + } + return iter->second->size(); +} + +Result> InMemoryGlobalIndexFiles::GetInputStream( + const std::string& file_path) const { + std::shared_ptr bytes; + { + std::lock_guard lock(storage_->mutex); + auto iter = storage_->files.find(file_path); + if (iter == storage_->files.end()) { + return Status::NotExist( + fmt::format("in-memory index file {} does not exist", file_path)); + } + bytes = iter->second; + } + return std::unique_ptr( + std::make_unique(storage_->arrow_pool, bytes, file_path)); +} + +RawFullTextReadImpl::RawFullTextReadImpl(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, + const DataField& text_field, const std::string& query, + int32_t limit, const std::shared_ptr& executor, + const std::shared_ptr& pool) + : root_path_(root_path), + table_schema_(table_schema), + options_(options), + partition_filter_(partition_filter), + filter_(filter), + text_field_(text_field), + query_(query), + limit_(limit), + executor_(executor), + pool_(pool) {} + +Result> RawFullTextReadImpl::Search( + int64_t snapshot_id, const std::string& index_type, const RowRangeIndex& raw_rows) const { + if (raw_rows.Ranges().empty()) { + return EmptyResult(); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, options_.ToMap())); + if (!indexer || !indexer->SupportsFullTextSearch()) { + return Status::NotImplemented(fmt::format( + "cannot search unindexed rows with index type {}, which has no registered global " + "indexer that supports full-text search", + index_type)); + } + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, + FilteredRowIdReader::LoadSnapshot(options_, root_path_, snapshot_id)); + auto index_schema = DataField::ConvertDataFieldsToArrowSchema({text_field_}); + auto files = std::make_shared(pool_); + std::shared_ptr writer; + + std::vector read_fields = {text_field_.Name()}; + if (filter_) { + std::set filter_fields; + PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(filter_, &filter_fields)); + for (const DataField& field : table_schema_->Fields()) { + if (field.Name() != text_field_.Name() && filter_fields.count(field.Name()) > 0) { + read_fields.push_back(field.Name()); + } + } + } + + std::shared_ptr arrow_pool = GetArrowPool(pool_); + std::shared_ptr bound_filter; + const int64_t first_row_id = raw_rows.Ranges().front().from; + const int64_t last_relative_id = raw_rows.Ranges().back().to - first_row_id; + // Lucene requires contiguous document ids. Dense ids also keep the row count within int64_t. + const bool use_relative_ids = + index_type != lucene::kIdentifier && last_relative_id < std::numeric_limits::max(); + std::vector document_row_ids; + RoaringBitmap64 matching_documents; + auto consumer = [&](const arrow::StructArray& batch, + const arrow::Int64Array& row_ids) -> Status { + std::shared_ptr text_array = batch.GetFieldByName(text_field_.Name()); + if (!text_array) { + return Status::Invalid( + fmt::format("unindexed rows are missing text column {}", text_field_.Name())); + } + PAIMON_ASSIGN_OR_RAISE(text_array, NestedProjectionUtils::AlignArrayToReadType( + text_array, arrow::utf8(), arrow_pool.get())); + std::vector accepted; + if (filter_) { + if (!bound_filter) { + const auto& struct_type = checked_cast(*batch.type()); + std::map field_indexes = + ObjectUtils::CreateIdentifierToIndexMap( + struct_type.fields(), + [](const std::shared_ptr& field) { return field->name(); }); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr bound_predicate, + PredicateUtils::CreatePickedFieldFilter(filter_, field_indexes)); + bound_filter = std::dynamic_pointer_cast(bound_predicate); + if (!bound_filter) { + return Status::Invalid(fmt::format( + "cannot evaluate row filter {} on unindexed rows", filter_->ToString())); + } + } + PAIMON_ASSIGN_OR_RAISE(accepted, bound_filter->Test(batch, arrow_pool.get())); + } + + arrow::BooleanBuilder selected(arrow_pool.get()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(selected.Reserve(batch.length())); + std::vector document_ids; + document_ids.reserve(batch.length()); + for (int64_t i = 0; i < batch.length(); ++i) { + int64_t row_id = row_ids.Value(i); + bool include = text_array->IsValid(i) && raw_rows.Intersects(row_id, row_id); + selected.UnsafeAppend(include); + if (!include) { + continue; + } + int64_t document_id = row_id - first_row_id; + if (!use_relative_ids) { + document_id = static_cast(document_row_ids.size()); + document_row_ids.push_back(row_id); + } + if (filter_ && accepted[i]) { + matching_documents.Add(document_id); + } + document_ids.push_back(document_id); + } + if (document_ids.empty()) { + return Status::OK(); + } + if (!writer) { + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*index_schema, &c_schema)); + ScopeGuard guard([&c_schema]() { ArrowSchemaRelease(&c_schema); }); + PAIMON_ASSIGN_OR_RAISE( + writer, indexer->CreateWriter(text_field_.Name(), &c_schema, files, pool_)); + } + if (static_cast(document_ids.size()) != batch.length()) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr mask, + selected.Finish()); + arrow::compute::ExecContext context(arrow_pool.get()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Datum filtered, + arrow::compute::Filter(text_array, mask, arrow::compute::FilterOptions::Defaults(), + &context)); + text_array = filtered.make_array(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr document_batch, + arrow::StructArray::Make({text_array}, index_schema->fields())); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*document_batch, &c_array)); + ScopeGuard guard([&c_array]() { ArrowArrayRelease(&c_array); }); + return writer->AddBatch(&c_array, std::move(document_ids)); + }; + PAIMON_RETURN_NOT_OK(FilteredRowIdReader::ReadRows( + root_path_, table_schema_, options_, partition_filter_, snapshot, raw_rows, read_fields, + /*predicate=*/nullptr, executor_, pool_, consumer)); + if (!writer || (filter_ && matching_documents.IsEmpty())) { + return EmptyResult(); + } + + PAIMON_ASSIGN_OR_RAISE(std::vector index_metas, writer->Finish()); + if (index_metas.empty()) { + return EmptyResult(); + } + const int64_t reader_row_count = + use_relative_ids ? last_relative_id + 1 : static_cast(document_row_ids.size()); + std::shared_ptr reader; + { + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*index_schema, &c_schema)); + ScopeGuard guard([&c_schema]() { ArrowSchemaRelease(&c_schema); }); + PAIMON_ASSIGN_OR_RAISE( + reader, indexer->CreateReader(&c_schema, files, index_metas, reader_row_count, + std::nullopt, pool_)); + } + std::optional include_documents; + if (filter_) { + include_documents = std::move(matching_documents); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + reader->VisitFullTextSearch(std::make_shared( + text_field_.Name(), query_, limit_, std::move(include_documents)))); + if (!result) { + return Status::NotImplemented( + fmt::format("global index type {} does not answer full-text searches", index_type)); + } + + std::vector> hits; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + result->CreateScoredIterator()); + while (iterator->HasNext()) { + std::pair hit = iterator->NextWithScore(); + if (hit.first < 0 || hit.first >= reader_row_count) { + return Status::Invalid(fmt::format("temporary {} index returned row {} outside [0, {})", + index_type, hit.first, reader_row_count)); + } + hits.emplace_back(use_relative_ids ? first_row_id + hit.first : document_row_ids[hit.first], + hit.second); + } + if (!use_relative_ids) { + std::sort(hits.begin(), hits.end(), + [](const auto& left, const auto& right) { return left.first < right.first; }); + } + RoaringBitmap64 row_ids; + std::vector scores; + scores.reserve(hits.size()); + for (const auto& [row_id, score] : hits) { + row_ids.Add(row_id); + scores.push_back(score); + } + return std::shared_ptr( + std::make_shared(std::move(row_ids), std::move(scores))); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/raw_full_text_read_impl.h b/src/paimon/core/table/source/raw_full_text_read_impl.h new file mode 100644 index 000000000..47f8c9f64 --- /dev/null +++ b/src/paimon/core/table/source/raw_full_text_read_impl.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/executor.h" +#include "paimon/global_index/global_index_result.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/global_index/io/global_index_file_writer.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate.h" +#include "paimon/result.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon { +/// Files become visible to readers when their output streams close. +class InMemoryGlobalIndexFiles : public GlobalIndexFileWriter, public GlobalIndexFileReader { + public: + explicit InMemoryGlobalIndexFiles(const std::shared_ptr& pool); + + Result NewPath(const std::string& prefix) const override; + + Result> NewOutputStream( + const std::string& file_path) const override; + + Result GetFileSize(const std::string& file_path) const override; + + Result> GetInputStream( + const std::string& file_path) const override; + + private: + struct Storage; + class IndexOutputStream; + + std::shared_ptr storage_; +}; + +class RawFullTextReadImpl { + public: + RawFullTextReadImpl(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, const DataField& text_field, + const std::string& query, int32_t limit, + const std::shared_ptr& executor, + const std::shared_ptr& pool); + + Result> Search(int64_t snapshot_id, + const std::string& index_type, + const RowRangeIndex& raw_rows) const; + + private: + std::string root_path_; + std::shared_ptr table_schema_; + CoreOptions options_; + std::shared_ptr partition_filter_; + std::shared_ptr filter_; + DataField text_field_; + std::string query_; + int32_t limit_; + std::shared_ptr executor_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/raw_full_text_search_split_impl.h b/src/paimon/core/table/source/raw_full_text_search_split_impl.h new file mode 100644 index 000000000..2768311e1 --- /dev/null +++ b/src/paimon/core/table/source/raw_full_text_search_split_impl.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/table/source/full_text_search_split.h" +#include "paimon/utils/range.h" + +namespace paimon { +/// Carries the index type so a raw split can be read independently of its plan's index splits. +class RawFullTextSearchSplitImpl : public RawFullTextSearchSplit { + public: + /// "PFTRAWS1" in ASCII. + static constexpr int64_t MAGIC = 0x5046545241575331L; + static constexpr int32_t VERSION = 1; + + static Result> Create( + int64_t snapshot_id, const std::string& column_name, const std::string& index_type, + std::vector row_ranges, int32_t field_id, int64_t schema_id); + + int64_t SnapshotId() const override { + return snapshot_id_; + } + + const std::string& ColumnName() const override { + return column_name_; + } + + const std::vector& RowRanges() const override { + return row_ranges_; + } + + const std::string& IndexType() const { + return index_type_; + } + + int32_t FieldId() const { + return field_id_; + } + + int64_t SchemaId() const { + return schema_id_; + } + + bool operator==(const RawFullTextSearchSplitImpl& other) const; + + std::string ToString() const; + + private: + RawFullTextSearchSplitImpl(int64_t snapshot_id, const std::string& column_name, + const std::string& index_type, std::vector row_ranges, + int32_t field_id, int64_t schema_id) + : snapshot_id_(snapshot_id), + column_name_(column_name), + index_type_(index_type), + row_ranges_(std::move(row_ranges)), + field_id_(field_id), + schema_id_(schema_id) {} + + int64_t snapshot_id_; + std::string column_name_; + std::string index_type_; + std::vector row_ranges_; + int32_t field_id_; + int64_t schema_id_; +}; + +} // namespace paimon diff --git a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp index 0df69ac95..c1c49a04a 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/rapidjson_util.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/full_text/full_text_defs.h" #include "paimon/memory/bytes.h" @@ -105,23 +106,30 @@ Result> FullTextGlobalIndexWriter::Cr keys.push_back(key.c_str()); values.push_back(value.c_str()); } - FtindexWriterPtr writer(paimon_ftindex_writer_open(keys.data(), values.data(), keys.size())); + PAIMON_ASSIGN_OR_RAISE(std::string file_path, file_writer->NewPath(kFileNamePrefix)); + auto open_writer = StringUtils::StartsWith(file_path, "memory://") + ? paimon_ftindex_writer_open_in_memory + : paimon_ftindex_writer_open; + FtindexWriterPtr writer(open_writer(keys.data(), values.data(), keys.size())); if (!writer) { return LastFtindexError( fmt::format("open full-text index writer for field {}", field_name)); } - return std::shared_ptr(new FullTextGlobalIndexWriter( - field_name, arrow_type, std::move(writer), file_writer, options, pool)); + return std::shared_ptr( + new FullTextGlobalIndexWriter(field_name, arrow_type, std::move(writer), + std::move(file_path), file_writer, options, pool)); } FullTextGlobalIndexWriter::FullTextGlobalIndexWriter( const std::string& field_name, const std::shared_ptr& arrow_type, - FtindexWriterPtr writer, const std::shared_ptr& file_writer, + FtindexWriterPtr writer, std::string file_path, + const std::shared_ptr& file_writer, const std::map& options, const std::shared_ptr& pool) : pool_(pool ? pool : GetDefaultPool()), field_name_(field_name), arrow_type_(arrow_type), writer_(std::move(writer)), + file_path_(std::move(file_path)), file_writer_(file_writer), options_(options), logger_(Logger::GetLogger("FullTextGlobalIndexWriter")) {} @@ -194,15 +202,14 @@ Result> FullTextGlobalIndexWriter::Finish() { Result FullTextGlobalIndexWriter::WriteIndex( PaimonFtindexWriterHandle* writer) const { - PAIMON_ASSIGN_OR_RAISE(std::string file_path, file_writer_->NewPath(kFileNamePrefix)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr out, - file_writer_->NewOutputStream(file_path)); + file_writer_->NewOutputStream(file_path_)); // Closes the stream on every error path, keeping the original error. ScopeGuard close_guard([&]() { Status status = out->Close(); if (!status.ok()) { PAIMON_LOG_WARN(logger_, "failed to close full-text index file %s: %s", - file_path.c_str(), status.ToString().c_str()); + file_path_.c_str(), status.ToString().c_str()); } }); OutputContext ctx; @@ -210,17 +217,17 @@ Result FullTextGlobalIndexWriter::WriteIndex( PaimonFtindexOutputFile output{static_cast(&ctx), &WriteToOutputStream, &FlushOutputStream}; if (paimon_ftindex_writer_write_index(writer, output) != 0) { - return LastFtindexError(fmt::format("write full-text index {}", file_path), ctx.status); + return LastFtindexError(fmt::format("write full-text index {}", file_path_), ctx.status); } PAIMON_RETURN_NOT_OK(out->Flush()); close_guard.Release(); PAIMON_RETURN_NOT_OK(out->Close()); - PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_path)); + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_path_)); std::string options_json; PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto metadata = std::make_shared(options_json, pool_.get()); - return GlobalIndexIOMeta(file_path, file_size, row_count_, metadata); + return GlobalIndexIOMeta(file_path_, file_size, row_count_, metadata); } } // namespace paimon::full_text diff --git a/src/paimon/global_index/full_text/full_text_global_index_writer.h b/src/paimon/global_index/full_text/full_text_global_index_writer.h index d10da2b7a..f433c2127 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_writer.h +++ b/src/paimon/global_index/full_text/full_text_global_index_writer.h @@ -54,7 +54,7 @@ class FullTextGlobalIndexWriter : public GlobalIndexWriter { private: FullTextGlobalIndexWriter(const std::string& field_name, const std::shared_ptr& arrow_type, - FtindexWriterPtr writer, + FtindexWriterPtr writer, std::string file_path, const std::shared_ptr& file_writer, const std::map& options, const std::shared_ptr& pool); @@ -65,6 +65,7 @@ class FullTextGlobalIndexWriter : public GlobalIndexWriter { std::string field_name_; std::shared_ptr arrow_type_; FtindexWriterPtr writer_; + std::string file_path_; std::shared_ptr file_writer_; std::map options_; /// Number of rows received, including null values. diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index 36548daea..be91347ef 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -63,6 +63,7 @@ if(PAIMON_ENABLE_LUCENE) ${LUCENE_INCLUDE_DIR} STATIC_LINK_LIBS paimon_shared + lucene test_utils_static ${PAIMON_LUCENE_INDEX_STATIC_LINK_LIBS} ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 23e495979..933aed05d 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -18,8 +18,6 @@ #include "paimon/global_index/lucene/lucene_global_index_writer.h" -#include - #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "lucene++/FileUtils.h" @@ -29,6 +27,7 @@ #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/uuid.h" #include "paimon/global_index/lucene/jieba_analyzer.h" #include "paimon/global_index/lucene/lucene_defs.h" @@ -42,35 +41,39 @@ namespace paimon::lucene { } while (0) LuceneGlobalIndexWriter::LuceneWriteContext::LuceneWriteContext( - const std::string& _tmp_index_path, const Lucene::FSDirectoryPtr& _lucene_dir, - const Lucene::IndexWriterPtr& _index_writer, const Lucene::DocumentPtr& _doc, - const Lucene::FieldPtr& _field) - : tmp_index_path(_tmp_index_path), - lucene_dir(_lucene_dir), - index_writer(_index_writer), - doc(_doc), - field(_field) {} + const std::string& output_path, const std::string& tmp_index_path, + const Lucene::DirectoryPtr& lucene_dir, const Lucene::IndexWriterPtr& index_writer, + const Lucene::DocumentPtr& doc, const Lucene::FieldPtr& field) + : output_path(output_path), + tmp_index_path(tmp_index_path), + lucene_dir(lucene_dir), + index_writer(index_writer), + doc(doc), + field(field) {} Result> LuceneGlobalIndexWriter::Create( const std::string& field_name, const std::shared_ptr& arrow_type, const std::shared_ptr& file_writer, const std::map& options, const std::shared_ptr& pool) { try { - std::string uuid; - if (!UUID::Generate(&uuid)) { - return Status::Invalid("generate uuid for lucene tmp path failed."); + PAIMON_ASSIGN_OR_RAISE(std::string output_path, file_writer->NewPath(kIdentifier)); + std::string tmp_path; + Lucene::DirectoryPtr lucene_dir; + if (StringUtils::StartsWith(output_path, "memory://")) { + lucene_dir = Lucene::newLucene(); + } else { + PAIMON_ASSIGN_OR_RAISE(std::string tmp_dir, OptionsUtils::GetValueFromMap( + options, kLuceneWriteTmpDir)); + std::string uuid; + if (!UUID::Generate(&uuid)) { + return Status::Invalid("generate uuid for lucene tmp path failed."); + } + tmp_path = PathUtil::JoinPath(tmp_dir, "paimon-lucene-" + uuid); + // FSDirectory mutates the lock prefix, so each writer needs its own lock factory. + lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(tmp_path), + Lucene::newLucene()); } - // get local tmp path - PAIMON_ASSIGN_OR_RAISE(std::string tmp_dir, OptionsUtils::GetValueFromMap( - options, std::string(kLuceneWriteTmpDir))); - std::string tmp_path = PathUtil::JoinPath(tmp_dir, "paimon-lucene-" + uuid); - - // FSDirectory calls Directory::setLockFactory(), which mutates lockPrefix. Use a separate - // factory per directory to avoid racing on the NoLockFactory singleton across writers. - auto lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(tmp_path), - Lucene::newLucene()); // TODO(xinyu.lxy): support other tokenizer - // open lucene index writer PAIMON_ASSIGN_OR_RAISE( bool omit_term_freq_and_positions, OptionsUtils::GetValueFromMap(options, kLuceneWriteOmitTermFreqAndPositions, false)); @@ -96,8 +99,9 @@ Result> LuceneGlobalIndexWriter::Create field->setOmitTermFreqAndPositions(omit_term_freq_and_positions); doc->add(field); return std::shared_ptr(new LuceneGlobalIndexWriter( - field_name, arrow_type, LuceneWriteContext(tmp_path, lucene_dir, writer, doc, field), - file_writer, options, pool)); + field_name, arrow_type, + LuceneWriteContext(output_path, tmp_path, lucene_dir, writer, doc, field), file_writer, + options, pool)); } catch (const std::exception& e) { return Status::Invalid( fmt::format("create lucene global index writer failed, with {} error.", e.what())); @@ -119,6 +123,9 @@ LuceneGlobalIndexWriter::LuceneGlobalIndexWriter( options_(options) {} LuceneGlobalIndexWriter::~LuceneGlobalIndexWriter() { + if (write_context_.tmp_index_path.empty()) { + return; + } try { [[maybe_unused]] bool ec = Lucene::FileUtils::removeDirectory( LuceneUtils::StringToWstring(write_context_.tmp_index_path)); @@ -167,9 +174,8 @@ Status LuceneGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array, return Status::OK(); } -Result LuceneGlobalIndexWriter::FlushIndexToFinal() { +Status LuceneGlobalIndexWriter::FlushIndexToFinal() { try { - // flush index to tmp dir if (write_context_.index_writer->numDocs() != row_id_) { return Status::Invalid( fmt::format("lucene writer row count {} mismatch paimon inner row count {}", @@ -178,19 +184,15 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { write_context_.index_writer->optimize(); write_context_.index_writer->close(); - // list tmp dir - auto tmp_file_names = write_context_.lucene_dir->listAll(); - PAIMON_ASSIGN_OR_RAISE(std::string index_file_path, file_writer_->NewPath(kIdentifier)); - // prepare output from file_writer + auto file_names = write_context_.lucene_dir->listAll(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, - file_writer_->NewOutputStream(index_file_path)); + file_writer_->NewOutputStream(write_context_.output_path)); DataOutputStream data_output_stream(out); PAIMON_RETURN_NOT_OK(data_output_stream.WriteValue(kVersion)); PAIMON_RETURN_NOT_OK( - data_output_stream.WriteValue(static_cast(tmp_file_names.size()))); - // read all data from index files and write to target output + data_output_stream.WriteValue(static_cast(file_names.size()))); auto buffer = std::make_shared(kDefaultReadBufferSize, pool_.get()); - for (const auto& wfile_name : tmp_file_names) { + for (const auto& wfile_name : file_names) { auto file_name = LuceneUtils::WstringToString(wfile_name); PAIMON_RETURN_NOT_OK( data_output_stream.WriteValue(static_cast(file_name.size()))); @@ -220,7 +222,7 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); write_context_.lucene_dir->close(); - return index_file_path; + return Status::OK(); } catch (const std::exception& e) { return Status::Invalid( fmt::format("finish for lucene global index writer failed, with {} error.", e.what())); @@ -231,8 +233,8 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { } Result> LuceneGlobalIndexWriter::Finish() { - PAIMON_ASSIGN_OR_RAISE(std::string index_file_path, FlushIndexToFinal()); - // prepare global index meta + PAIMON_RETURN_NOT_OK(FlushIndexToFinal()); + const std::string& index_file_path = write_context_.output_path; PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_path)); std::string options_json; PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.h b/src/paimon/global_index/lucene/lucene_global_index_writer.h index e11be99cd..115191090 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.h +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.h @@ -28,16 +28,17 @@ namespace paimon::lucene { class LuceneGlobalIndexWriter : public GlobalIndexWriter { public: struct LuceneWriteContext { - LuceneWriteContext(const std::string& _tmp_index_path, - const Lucene::FSDirectoryPtr& _lucene_dir, - const Lucene::IndexWriterPtr& _index_writer, - const Lucene::DocumentPtr& _doc, const Lucene::FieldPtr& _field); + LuceneWriteContext(const std::string& output_path, const std::string& tmp_index_path, + const Lucene::DirectoryPtr& lucene_dir, + const Lucene::IndexWriterPtr& index_writer, + const Lucene::DocumentPtr& doc, const Lucene::FieldPtr& field); LuceneWriteContext(LuceneWriteContext&&) = default; LuceneWriteContext& operator=(LuceneWriteContext&&) = default; + std::string output_path; std::string tmp_index_path; - Lucene::FSDirectoryPtr lucene_dir; + Lucene::DirectoryPtr lucene_dir; Lucene::IndexWriterPtr index_writer; Lucene::DocumentPtr doc; Lucene::FieldPtr field; @@ -62,7 +63,7 @@ class LuceneGlobalIndexWriter : public GlobalIndexWriter { const std::map& options, const std::shared_ptr& pool); - Result FlushIndexToFinal(); + Status FlushIndexToFinal(); private: std::shared_ptr pool_; diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 25a9eceb3..045642760 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -18,17 +18,21 @@ #include #include +#include #include #include #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "arrow/type.h" #include "fmt/format.h" #include "fmt/ranges.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_factory.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_options.h" +#include "paimon/common/global_index/scored_global_index_result_utils.h" #include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/global_index/union_global_index_reader.h" #include "paimon/common/table/special_fields.h" @@ -36,6 +40,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/deletion_vector.h" #include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/schema/schema_manager.h" @@ -43,6 +48,8 @@ #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/filtered_row_id_reader.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/table/source/raw_full_text_search_split_impl.h" +#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" @@ -173,6 +180,58 @@ class NestedFieldsIndexFactory : public GlobalIndexerFactory { REGISTER_PAIMON_FACTORY(NestedFieldsIndexFactory); #endif +#ifdef PAIMON_ENABLE_FULL_TEXT +class ExtraFieldFullTextIndexer : public GlobalIndexer { + public: + Result>> GetExtraFieldNames() const override { + return std::optional>({"f1"}); + } + + Result> CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const override { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get("full-text", {})); + return indexer->CreateWriter(field_name, arrow_schema, file_writer, pool); + } + + Result> CreateReader( + ::ArrowSchema* c_arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, + const std::shared_ptr& pool) const override { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(c_arrow_schema)); + auto text_schema = arrow::schema({schema->field(0)}); + ::ArrowSchema c_text_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*text_schema, &c_text_schema)); + ScopeGuard guard([&c_text_schema]() { ArrowSchemaRelease(&c_text_schema); }); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get("full-text", {})); + return indexer->CreateReader(&c_text_schema, file_reader, files, total_row_count, + row_ranges, pool); + } + + bool SupportsFullTextSearch() const override { + return true; + } +}; + +class ExtraFieldFullTextIndexerFactory : public GlobalIndexerFactory { + public: + const char* Identifier() const override { + return "extra-field-full-text-test-global"; + } + + Result> Create( + const std::map& options) const override { + return std::make_unique(); + } +}; +REGISTER_PAIMON_FACTORY(ExtraFieldFullTextIndexerFactory); +#endif + } // namespace using ScoredRows = std::vector>; @@ -336,9 +395,10 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter std::unique_ptr NewFullTextSearchBuilder( const std::string& table_path, const std::string& query, int32_t limit, - const std::map& options = {}) const { + const std::map& options = {}, + const std::string& column = "f0") const { auto builder = std::make_unique(table_path); - builder->SetQuery("f0", query).SetLimit(limit).SetOptions(options).WithFileSystem(fs_); + builder->SetQuery(column, query).SetLimit(limit).SetOptions(options).WithFileSystem(fs_); return builder; } @@ -416,6 +476,61 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter return shards; } + Result> PlanRawRanges( + const std::string& table_path, const std::string& query, + const std::map& options, + const std::vector>& partition_filters = {}, + const std::shared_ptr& filter = nullptr) const { + std::unique_ptr builder = + NewFullTextSearchBuilder(table_path, query, 1, options); + builder->SetPartitionFilter(partition_filters).WithFilter(filter); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, builder->NewFullTextScan()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + std::vector raw_ranges; + int32_t raw_splits = 0; + for (const auto& split : plan->Splits()) { + auto raw_split = std::dynamic_pointer_cast(split); + if (!raw_split) { + continue; + } + if (++raw_splits > 1 || raw_split->ColumnName() != "f0" || + raw_split->SnapshotId() != plan->SnapshotId()) { + return Status::Invalid("unexpected raw full-text search split"); + } + raw_ranges = raw_split->RowRanges(); + } + return raw_ranges; + } + + Result> ReadSerializedSplits( + const std::string& table_path, const std::string& query, int32_t limit, + const std::map& options, + const std::shared_ptr& filter = nullptr, + const std::vector>& partitions = {}, + const std::string& column = "f0") const { + auto new_builder = [&]() { + auto builder = NewFullTextSearchBuilder(table_path, query, limit, options, column); + builder->SetPartitionFilter(partitions).WithFilter(filter); + return builder; + }; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + new_builder()->NewFullTextScan()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + std::vector> partial_results; + for (const auto& split : plan->Splits()) { + PAIMON_ASSIGN_OR_RAISE(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr received_split, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + new_builder()->NewFullTextRead()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partial_result, + read->Read({received_split})); + partial_results.push_back(std::move(partial_result)); + } + return ScoredGlobalIndexResultUtils::MergeTopK(partial_results, limit); + } + void CheckTableSearchMatchesReader(const std::string& table_path, const std::string& query, const std::shared_ptr& reader, int32_t max_limit) const { @@ -672,6 +787,11 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_FALSE(unreadable_snapshot_result.ok()); ASSERT_EQ(unreadable_snapshot_result.status().ToString().find("no longer exists"), std::string::npos); + auto worker_builder = NewFullTextSearchBuilder( + table_path, query, 100, + {{Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_before_deletion)}}); + ASSERT_NOK_WITH_MSG(worker_builder->NewFullTextRead(), + unreadable_snapshot_result.status().message()); ASSERT_OK(file_system->Delete(snapshot_path)); ASSERT_OK_AND_ASSIGN(std::shared_ptr index_result, @@ -719,13 +839,18 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, schema->field_names(), AppleRows(fields, 0, 5))); ASSERT_OK(Commit(table_path, commit_msgs1)); - ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, schema->field_names(), - AppleRows(fields, 5, 10))); - ASSERT_OK(Commit(table_path, commit_msgs2)); std::string query = R"({"match":{"query":"apple"}})"; - + for (const char* mode : {"fast", "full", "detail"}) { + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTable(table_path, query, 10, {}, + {{Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}})); + ASSERT_TRUE(rows.empty()); + } ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", index_type, index_options, Range(0, 4))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, schema->field_names(), + AppleRows(fields, 5, 10))); + ASSERT_OK(Commit(table_path, commit_msgs2)); ASSERT_OK_AND_ASSIGN( std::unique_ptr first_shard_scan, NewFullTextSearchBuilder(table_path, query, /*limit=*/10)->NewFullTextScan()); @@ -733,19 +858,105 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter first_shard_scan->CreatePlan()); ASSERT_TRUE(first_shard_plan->SnapshotId()); ASSERT_EQ(first_shard_plan->Splits().size(), 1u); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr raw_split, + RawFullTextSearchSplitImpl::Create(first_shard_plan->SnapshotId().value(), "f0", + index_type, {Range(0, 9)}, 0, 0)); + auto worker_options = index_options; + worker_options.erase("lucene-fts.write.tmp.directory"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr top_one_read, + NewFullTextSearchBuilder(table_path, query, 1, worker_options)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw_result, + top_one_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows raw_top_row, ToScoredRows(raw_result)); + ASSERT_EQ(raw_top_row.size(), 1u); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr top_ten_read, + NewFullTextSearchBuilder(table_path, query, 10, worker_options)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(raw_result, top_ten_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows all_raw_rows, ToScoredRows(raw_result)); + ASSERT_EQ(all_raw_rows.size(), 10u); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr raw_prefix, + RawFullTextSearchSplitImpl::Create(first_shard_plan->SnapshotId().value(), "f0", + index_type, {Range(0, 5)}, 0, 0)); + ASSERT_OK_AND_ASSIGN(raw_split, RawFullTextSearchSplitImpl::Create( + first_shard_plan->SnapshotId().value(), "f0", + index_type, {Range(3, int64_t{1} << 40)}, 0, 0)); + const std::vector> overlapping_splits = { + raw_split, first_shard_plan->Splits().front(), raw_prefix}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr overlapping_result, + top_ten_read->Read(overlapping_splits)); + ASSERT_OK_AND_ASSIGN(ScoredRows overlapping_rows, ToScoredRows(overlapping_result)); + ASSERT_EQ(overlapping_rows, all_raw_rows); + ASSERT_OK_AND_ASSIGN(overlapping_result, top_one_read->Read(overlapping_splits)); + ASSERT_OK_AND_ASSIGN(ScoredRows overlapping_top_row, ToScoredRows(overlapping_result)); + ASSERT_EQ(overlapping_top_row, raw_top_row); + ASSERT_OK_AND_ASSIGN(raw_result, top_ten_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows raw_suffix, ToScoredRows(raw_result)); ASSERT_OK_AND_ASSIGN(ScoredRows indexed_rows, SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, {{Options::FULL_TEXT_INDEX_SEARCH_MODE, "fast"}, {Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); - ASSERT_EQ(indexed_rows.size(), 5u); - for (const auto& [row_id, score] : indexed_rows) { - ASSERT_LT(row_id, 5) << score; - } - for (const char* mode : {"full", "detail"}) { - ASSERT_NOK_WITH_MSG( - SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, - {{Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}), - "full and detail are not supported yet"); + ASSERT_EQ(RowIdsOf(indexed_rows), std::vector({0, 1, 2, 3, 4})); + ScoredRows expected_overlap(indexed_rows.begin(), indexed_rows.begin() + 3); + expected_overlap.insert(expected_overlap.end(), raw_suffix.begin(), raw_suffix.end()); + ASSERT_OK_AND_ASSIGN(overlapping_result, + top_ten_read->Read({raw_split, first_shard_plan->Splits().front()})); + ASSERT_OK_AND_ASSIGN(overlapping_rows, ToScoredRows(overlapping_result)); + ASSERT_EQ(overlapping_rows, expected_overlap); + ASSERT_OK_AND_ASSIGN(overlapping_result, + top_one_read->Read({first_shard_plan->Splits().front(), raw_split})); + ASSERT_OK_AND_ASSIGN(overlapping_top_row, ToScoredRows(overlapping_result)); + ASSERT_EQ(overlapping_top_row.size(), 1u); + const auto& best = *std::max_element( + expected_overlap.begin(), expected_overlap.end(), + [](const auto& left, const auto& right) { return left.second < right.second; }); + ASSERT_EQ(overlapping_top_row.front().second, best.second); + ASSERT_NE(std::find(expected_overlap.begin(), expected_overlap.end(), + overlapping_top_row.front()), + expected_overlap.end()); + ASSERT_OK_AND_ASSIGN( + raw_split, RawFullTextSearchSplitImpl::Create( + first_shard_plan->SnapshotId().value(), "f0", index_type, + {Range(0, 4), Range(7, std::numeric_limits::max())}, 0, 0)); + ASSERT_OK_AND_ASSIGN(raw_result, top_ten_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows sparse_rows, ToScoredRows(raw_result)); + ASSERT_EQ(RowIdsOf(sparse_rows), std::vector({0, 1, 2, 3, 4, 7, 8, 9})); + auto sparse_builder = NewFullTextSearchBuilder(table_path, query, 1, worker_options); + sparse_builder->WithFilter( + PredicateBuilder::In(1, "f1", FieldType::INT, {Literal(5), Literal(7)})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr sparse_read, + sparse_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(raw_result, sparse_read->Read({raw_split})); + ASSERT_OK_AND_ASSIGN(ScoredRows filtered_sparse_rows, ToScoredRows(raw_result)); + auto expected_sparse_row = std::find_if(sparse_rows.begin(), sparse_rows.end(), + [](const auto& row) { return row.first == 7; }); + ASSERT_NE(expected_sparse_row, sparse_rows.end()); + ASSERT_EQ(filtered_sparse_rows, ScoredRows({*expected_sparse_row})); + const std::vector> raw_modes = { + {{Options::FULL_TEXT_INDEX_SEARCH_MODE, "full"}}, + {{Options::FULL_TEXT_INDEX_SEARCH_MODE, "detail"}}, + {{Options::GLOBAL_INDEX_SEARCH_MODE, "full"}}}; + std::vector raw_mode_rows; + std::vector raw_mode_top_rows; + for (std::map search_options : raw_modes) { + search_options.insert(index_options.begin(), index_options.end()); + ASSERT_OK_AND_ASSIGN(std::vector raw_ranges, + PlanRawRanges(table_path, query, search_options)); + ASSERT_EQ(raw_ranges, std::vector({Range(5, 9)})); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTable(table_path, query, 10, {}, search_options)); + ASSERT_EQ(RowIdsOf(rows), std::vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + raw_mode_rows.push_back(std::move(rows)); + ASSERT_OK_AND_ASSIGN(ScoredRows top_rows, + SearchTable(table_path, query, 3, {}, search_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr received_result, + ReadSerializedSplits(table_path, query, 3, search_options)); + ASSERT_OK_AND_ASSIGN(ScoredRows received_rows, ToScoredRows(received_result)); + ASSERT_EQ(received_rows, top_rows); + raw_mode_top_rows.push_back(std::move(top_rows)); } ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", index_type, index_options, Range(5, 9))); @@ -753,6 +964,20 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK_AND_ASSIGN(PlannedShards shards, PlanShards(table_path, query)); ASSERT_EQ(shards, PlannedShards({{Range(0, 4), {Range(0, 4)}}, {Range(5, 9), {Range(5, 9)}}})); + ASSERT_OK_AND_ASSIGN(ScoredRows fully_indexed_rows, SearchTable(table_path, query, 10)); + for (const ScoredRows& rows : raw_mode_rows) { + ASSERT_EQ(rows, fully_indexed_rows); + } + ASSERT_OK_AND_ASSIGN(ScoredRows fully_indexed_top_rows, SearchTable(table_path, query, 3)); + for (const ScoredRows& rows : raw_mode_top_rows) { + ASSERT_EQ(rows, fully_indexed_top_rows); + } + for (std::map search_options : raw_modes) { + search_options.insert(index_options.begin(), index_options.end()); + ASSERT_OK_AND_ASSIGN(std::vector raw_ranges, + PlanRawRanges(table_path, query, search_options)); + ASSERT_TRUE(raw_ranges.empty()); + } ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, /*partitions=*/std::nullopt, /*options=*/{}, @@ -762,28 +987,11 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_EQ(readers.size(), 1u); CheckTableSearchMatchesReader(table_path, query, readers[0], /*max_limit=*/11); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr scan, - NewFullTextSearchBuilder(table_path, query, /*limit=*/3)->NewFullTextScan()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); - std::vector> received_splits; - for (const auto& split : plan->Splits()) { - ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr received_split, - FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); - received_splits.push_back(std::move(received_split)); - } - ASSERT_OK_AND_ASSIGN( - std::unique_ptr read, - NewFullTextSearchBuilder(table_path, query, /*limit=*/3)->NewFullTextRead()); ASSERT_OK_AND_ASSIGN(std::shared_ptr top_result, - read->Read(received_splits)); + ReadSerializedSplits(table_path, query, 3, {})); ASSERT_OK_AND_ASSIGN(ScoredRows top_rows, ToScoredRows(top_result)); ASSERT_EQ(top_rows.size(), 3u); - ASSERT_OK_AND_ASSIGN(ScoredRows local_top_rows, - SearchTable(table_path, query, /*limit=*/3)); - ASSERT_EQ(top_rows, local_top_rows); + ASSERT_EQ(top_rows, fully_indexed_top_rows); std::string first_shard_snapshot_id = std::to_string(first_shard_plan->SnapshotId().value()); @@ -803,10 +1011,18 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter {{Options::SCAN_MODE, "latest-full"}, {Options::SCAN_SNAPSHOT_ID, first_shard_snapshot_id}}), "scan.snapshot-id must be null when you use latest-full for scan.mode"); + auto tag_builder = NewFullTextSearchBuilder(table_path, query, 10, + {{Options::SCAN_TAG_NAME, "first-shard"}}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr tag_scan, + tag_builder->NewFullTextScan()); + ASSERT_NOK_WITH_MSG(tag_scan->CreatePlan(), "Tag 'first-shard' doesn't exist."); ASSERT_OK(TagSnapshot(table_path, first_shard_plan->SnapshotId().value(), "first-shard")); - ASSERT_OK_AND_ASSIGN(ScoredRows tag_rows, - SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, - {{Options::SCAN_TAG_NAME, "first-shard"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr tag_plan, tag_scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr tag_read, + tag_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr tag_result, + tag_read->Read(*tag_plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows tag_rows, ToScoredRows(tag_result)); ASSERT_EQ(tag_rows, first_shard_rows); ASSERT_OK_AND_ASSIGN(ScoredRows latest_rows, SearchTable(table_path, query, /*limit=*/10)); ASSERT_EQ(latest_rows.size(), 10u); @@ -847,6 +1063,27 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK_AND_ASSIGN(ScoredRows rows_with_bitmap, SearchTable(table_path, query, /*limit=*/3)); ASSERT_EQ(rows_with_bitmap, top_rows); + + auto null_rows = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + "[[null, 10], [null, 11]]") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> null_messages, + WriteArray(table_path, schema->field_names(), null_rows)); + ASSERT_OK(Commit(table_path, null_messages)); + for (std::map search_options : raw_modes) { + SCOPED_TRACE(search_options.begin()->second); + search_options.insert(index_options.begin(), index_options.end()); + search_options.erase("lucene-fts.write.tmp.directory"); + if (index_type == "full-text") { + search_options["full-text.tokenizer"] = "unavailable"; + } + ASSERT_OK_AND_ASSIGN(std::vector raw_ranges, + PlanRawRanges(table_path, query, search_options)); + ASSERT_EQ(raw_ranges, std::vector({Range(10, 11)})); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTable(table_path, query, 10, {}, search_options)); + ASSERT_EQ(rows, fully_indexed_rows); + } } void CheckTableSearchWithFilter(const std::string& index_type, @@ -1025,21 +1262,13 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK_AND_ASSIGN(std::shared_ptr unfiltered_plan, unfiltered_scan->CreatePlan()); ASSERT_EQ(unfiltered_plan->Splits().size(), 1u); - index_split = + auto unfiltered_split = std::dynamic_pointer_cast(unfiltered_plan->Splits()[0]); - ASSERT_TRUE(index_split); - ASSERT_TRUE(index_split->ScalarIndexFiles().empty()); + ASSERT_TRUE(unfiltered_split); + ASSERT_TRUE(unfiltered_split->ScalarIndexFiles().empty()); - std::vector> splits; - for (const auto& split : plan->Splits()) { - ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr deserialized, - FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); - splits.push_back(std::move(deserialized)); - } - ASSERT_OK_AND_ASSIGN(std::unique_ptr read, builder->NewFullTextRead()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(splits)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadSerializedSplits(table_path, query, 100, {}, f1_is_1)); ASSERT_OK_AND_ASSIGN(ScoredRows rows, ToScoredRows(result)); ASSERT_EQ(rows, rows_of({2, 3})); @@ -1055,6 +1284,67 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter search_options.insert(time_travel_option); check_filter(partly_indexed, {3}, search_options); } + + ASSERT_OK(WriteIndex(table_path, {}, "f3", "btree", {}, Range(0, 2))); + ASSERT_OK(WriteIndex(table_path, {}, "f3", "bitmap", {}, Range(3, 4))); + auto f3_at_least_2 = PredicateBuilder::GreaterOrEqual(3, "f3", FieldType::INT, Literal(2)); + check_filter(f3_at_least_2, {2, 3, 4}); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + std::map scalar_options = { + {Options::SCALAR_INDEX_SEARCH_MODE, mode}}; + check_filter(f3_at_least_2, {2, 3, 4, 5}, scalar_options); + scalar_options[BitmapGlobalIndexOptions::kBitmapIndexFallbackScanMaxSize] = "1B"; + check_filter(f3_at_least_2, {2, 3, 4, 5}, scalar_options); + } + auto partial_builder = NewFullTextSearchBuilder(table_path, query, 100); + partial_builder->WithFilter(f3_at_least_2); + ASSERT_OK_AND_ASSIGN(std::unique_ptr partial_scan, + partial_builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr partial_plan, + partial_scan->CreatePlan()); + auto* factories = FactoryCreator::GetInstance(); + factories->TEST_Unregister("bitmap-global"); + ScopeGuard restore_bitmap([factories]() { + factories->Register("bitmap-global", new BitmapGlobalIndexFactory()); + }); + check_filter(f3_at_least_2, {2}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr contains_b, + PredicateBuilder::Contains(2, "f2", FieldType::STRING, + Literal(FieldType::STRING, "b", 1))); + auto f3_at_least_1 = PredicateBuilder::GreaterOrEqual(3, "f3", FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr partial_contains, + PredicateBuilder::And({contains_b, f3_at_least_1})); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + partial_builder->AddOption(Options::SCALAR_INDEX_SEARCH_MODE, mode); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + partial_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(result, read->Read(*partial_plan)); + ASSERT_OK_AND_ASSIGN(rows, ToScoredRows(result)); + ASSERT_EQ(rows, rows_of({2, 3, 4, 5})); + std::map scalar_options = { + {Options::SCALAR_INDEX_SEARCH_MODE, mode}}; + check_filter(partly_indexed, {3}, scalar_options); + check_filter(undecidable_or, {2, 3}, scalar_options); + check_filter(contains, {}, scalar_options); + check_filter(partial_contains, {3, 4, 5}, scalar_options); + scalar_options.insert(refine.begin(), refine.end()); + check_filter(partial_contains, {1, 3, 4, 5}, scalar_options); + } + + const auto& f1_index = index_split->ScalarIndexFiles()[0]; + ASSERT_OK(dir_->GetFileSystem()->Delete( + PathUtil::JoinPath(FileStorePathFactory::IndexPath(table_path), f1_index->FileName()))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr f1_and_f4, + PredicateBuilder::And({f1_is_1, f4_is_pear})); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + const std::map scalar_options = { + {Options::SCALAR_INDEX_SEARCH_MODE, mode}}; + check_filter(f1_and_f4, {2, 3}, scalar_options); + ASSERT_NOK(SearchTableWithFilter(table_path, query, 100, f1_is_1, {}, scalar_options)); + } } void CheckTableSearchWithPartitionAndDataFilter( @@ -1150,6 +1440,434 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter PlannedShards({{Range(0, 1), {Range(0, 1)}}, {Range(2, 3), {Range(2, 3)}}})); } + void CheckRawSearchSkipsDeletedRows(const std::string& index_type, + const std::map& index_options) { + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; + CreateTable({}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, schema->field_names(), + AppleRows(fields, 0, 5))); + ASSERT_OK(Commit(table_path, commit_msgs1)); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, schema->field_names(), + AppleRows(fields, 5, 10))); + ASSERT_OK(Commit(table_path, commit_msgs2)); + ASSERT_OK(WriteIndex(table_path, {}, "f0", index_type, index_options, Range(0, 4))); + + std::string query = R"({"match":{"query":"apple"}})"; + std::map full_options = index_options; + full_options[Options::FULL_TEXT_INDEX_SEARCH_MODE] = "full"; + std::unique_ptr pinned_builder = + NewFullTextSearchBuilder(table_path, query, 100, full_options); + ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_scan, + pinned_builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned_plan, + pinned_scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr pinned_read, + pinned_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned_result, + pinned_read->Read(*pinned_plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows rows_before_deletion, ToScoredRows(pinned_result)); + ASSERT_EQ(RowIdsOf(rows_before_deletion), + std::vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + + ASSERT_OK_AND_ASSIGN(auto split, ScanData(table_path, {})); + ASSERT_OK_AND_ASSIGN(std::vector anchor_file_names, + DeletionVectorTestHelper::RetrieveAnchorFileNames(split->DataFiles())); + ASSERT_EQ(anchor_file_names.size(), 2u); + ASSERT_OK_AND_ASSIGN(std::shared_ptr indexed_deletes, + DeletionVectorTestHelper::CreateDeletionVectorCommitMessage( + dir_->GetFileSystem(), table_path, file_format_, commit_msgs1[0], + {{anchor_file_names[0], {0, 1}}}, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr raw_deletes, + DeletionVectorTestHelper::CreateDeletionVectorCommitMessage( + dir_->GetFileSystem(), table_path, file_format_, commit_msgs2[0], + {{anchor_file_names[1], {0, 2}}}, pool_)); + ASSERT_OK(Commit(table_path, {indexed_deletes, raw_deletes})); + + ASSERT_OK_AND_ASSIGN(pinned_result, pinned_read->Read(*pinned_plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + + auto search = [&](const std::map& mode_options, + const std::shared_ptr& filter, + int32_t limit = 100) -> Result { + std::map search_options = mode_options; + search_options.insert(index_options.begin(), index_options.end()); + return SearchTableWithFilter(table_path, query, limit, filter, {}, search_options); + }; + const std::map full = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, "full"}}; + ASSERT_OK_AND_ASSIGN(ScoredRows rows, search({}, nullptr)); + ASSERT_EQ(RowIdsOf(rows), std::vector({2, 3, 4})); + ASSERT_OK_AND_ASSIGN(ScoredRows raw_rows, search(full, nullptr)); + ASSERT_EQ(RowIdsOf(raw_rows), std::vector({2, 3, 4, 6, 8, 9})); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + const std::map mode_options = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}; + ASSERT_OK_AND_ASSIGN(rows, search(mode_options, nullptr)); + ASSERT_EQ(rows, raw_rows); + for (int64_t row_id : {6, 8, 9}) { + SCOPED_TRACE(row_id); + std::shared_ptr filter = PredicateBuilder::Equal( + 1, "f1", FieldType::INT, Literal(static_cast(row_id))); + ASSERT_OK_AND_ASSIGN(rows, search(mode_options, filter, 1)); + auto expected = + std::find_if(raw_rows.begin(), raw_rows.end(), + [row_id](const auto& row) { return row.first == row_id; }); + ASSERT_NE(expected, raw_rows.end()); + ASSERT_EQ(rows, ScoredRows({*expected})); + } + ASSERT_OK_AND_ASSIGN( + rows, search(mode_options, + PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(5)), 1)); + ASSERT_TRUE(rows.empty()); + } + std::shared_ptr f1_at_least_3 = + PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(3)); + ASSERT_OK_AND_ASSIGN(rows, search(full, f1_at_least_3)); + ASSERT_EQ(RowIdsOf(rows), std::vector({6, 8, 9})); + std::map scalar_full = full_options; + scalar_full[Options::SCALAR_INDEX_SEARCH_MODE] = "full"; + ASSERT_OK_AND_ASSIGN(rows, search(scalar_full, f1_at_least_3)); + ASSERT_EQ(RowIdsOf(rows), std::vector({3, 4, 6, 8, 9})); + ASSERT_OK_AND_ASSIGN( + rows, search({{Options::GLOBAL_INDEX_SEARCH_MODE, "detail"}}, f1_at_least_3)); + ASSERT_EQ(RowIdsOf(rows), std::vector({3, 4, 6, 8, 9})); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + scalar_full[Options::FULL_TEXT_INDEX_SEARCH_MODE] = mode; + scalar_full[Options::SCALAR_INDEX_SEARCH_MODE] = mode; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr serialized_result, + ReadSerializedSplits(table_path, query, 100, scalar_full, f1_at_least_3)); + ASSERT_OK_AND_ASSIGN(ScoredRows serialized_rows, ToScoredRows(serialized_result)); + ASSERT_EQ(serialized_rows, rows); + } + ASSERT_OK(WriteIndex(table_path, {}, "f1", "btree", {}, Range(0, 4))); + ASSERT_OK_AND_ASSIGN(rows, search(full, f1_at_least_3)); + ASSERT_EQ(RowIdsOf(rows), std::vector({3, 4, 6, 8, 9})); + + ASSERT_TRUE(pinned_plan->SnapshotId()); + int64_t pinned_snapshot_id = pinned_plan->SnapshotId().value(); + ASSERT_OK(TagSnapshot(table_path, pinned_snapshot_id, "before-deletion")); + SnapshotManager snapshots(dir_->GetFileSystem(), table_path); + ASSERT_OK(dir_->GetFileSystem()->Delete(snapshots.SnapshotPath(pinned_snapshot_id))); + + ASSERT_OK_AND_ASSIGN(split, ScanData(table_path, {})); + auto deletion_files = + DeletionVector::CreateDeletionFileMap(split->DataFiles(), split->DeletionFiles()); + auto raw_deletion_file = deletion_files.find(anchor_file_names[1]); + ASSERT_NE(raw_deletion_file, deletion_files.end()); + ASSERT_OK(dir_->GetFileSystem()->Delete(raw_deletion_file->second.path)); + ASSERT_OK_AND_ASSIGN(rows, search({}, nullptr)); + ASSERT_EQ(RowIdsOf(rows), std::vector({2, 3, 4})); + + options[Options::BLOB_VIEW_FIELD] = "f0"; + ASSERT_OK(TestHelper::WriteNextSchema( + dir_->GetFileSystem(), table_path, + {DataField(0, arrow::field("old_text", arrow::utf8())), DataField(1, fields[1]), + DataField(2, BlobUtils::ToArrowField("f0", true))}, + 2, options)); + pinned_builder->AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(pinned_snapshot_id)); + ASSERT_OK_AND_ASSIGN(pinned_read, pinned_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(pinned_result, pinned_read->Read(*pinned_plan)); + ASSERT_OK_AND_ASSIGN(pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + full_options[Options::SCAN_TAG_NAME] = "before-deletion"; + ASSERT_OK_AND_ASSIGN(pinned_result, + ReadSerializedSplits(table_path, query, 100, full_options)); + ASSERT_OK_AND_ASSIGN(pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + } + + void CheckRawSearchAcrossPartitions(const std::string& index_type, + const std::map& index_options, + const std::string& query) { + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {"orc.dictionary-key-size-threshold", "1"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"parquet.read.enable-dictionary-passthrough", "true"}}; + CreateTable({"f1"}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto write_partition = [&](const std::string& json, + const std::map& partition) { + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, partition, + schema->field_names(), src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + }; + write_partition(R"([["apple apple", 10, 1], ["apple", 10, 2]])", {{"f1", "10"}}); + ASSERT_OK( + WriteIndex(table_path, {{{"f1", "10"}}}, "f0", index_type, index_options, Range(0, 1))); + ASSERT_OK(WriteIndex(table_path, {{{"f1", "10"}}}, "f2", "btree", {}, Range(0, 1))); + write_partition(R"([["apple apple", 20, 1], [null, 20, 2]])", {{"f1", "20"}}); + + auto check = [&](std::map search_options, + const std::vector>& partitions, + const std::shared_ptr& filter, + const std::vector& expected_raw_ranges, + const std::vector& expected_row_ids) { + search_options.insert(index_options.begin(), index_options.end()); + search_options[Options::READ_BATCH_SIZE] = "1"; + ASSERT_OK_AND_ASSIGN( + std::vector raw_ranges, + PlanRawRanges(table_path, query, search_options, partitions, filter)); + ASSERT_EQ(raw_ranges, expected_raw_ranges); + ASSERT_OK_AND_ASSIGN( + ScoredRows rows, + SearchTableWithFilter(table_path, query, 100, filter, partitions, search_options)); + ASSERT_EQ(RowIdsOf(rows), expected_row_ids); + }; + const std::map fast = {}; + const std::map full = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, "full"}}; + const std::map detail = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, "detail"}}; + ASSERT_OK_AND_ASSIGN(std::vector partition_gaps, + PlanRawRanges(table_path, query, full, {{{"f1", "10"}}})); + ASSERT_EQ(partition_gaps, std::vector({Range(2, 3)})); + auto unavailable_writer_options = full; + if (index_type == "full-text") { + unavailable_writer_options["full-text.tokenizer"] = "unavailable"; + } + ASSERT_OK_AND_ASSIGN( + ScoredRows indexed_rows, + SearchTable(table_path, query, 100, {{{"f1", "10"}}}, unavailable_writer_options)); + ASSERT_EQ(RowIdsOf(indexed_rows), std::vector({0, 1})); + check(detail, {{{"f1", "10"}}}, nullptr, {}, {0, 1}); + + write_partition(R"([["apple apple apple", 10, 1], [null, 10, 2]])", {{"f1", "10"}}); + check(fast, {}, nullptr, {}, {0, 1}); + check(full, {}, nullptr, {Range(2, 5)}, {0, 1, 2, 4}); + check(detail, {}, nullptr, {Range(2, 5)}, {0, 1, 2, 4}); + check(full, {{{"f1", "10"}}}, nullptr, {Range(2, 5)}, {0, 1, 4}); + check(detail, {{{"f1", "10"}}}, nullptr, {Range(4, 5)}, {0, 1, 4}); + check(detail, {{{"f1", "20"}}}, nullptr, {}, {}); + + std::shared_ptr f2_is_1 = + PredicateBuilder::Equal(2, "f2", FieldType::INT, Literal(1)); + check(full, {}, f2_is_1, {Range(2, 5)}, {0, 2, 4}); + check(detail, {{{"f1", "10"}}}, f2_is_1, {Range(4, 5)}, {0, 4}); + auto worker_options = full; + worker_options.insert(index_options.begin(), index_options.end()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr distributed, + ReadSerializedSplits(table_path, query, 3, worker_options, f2_is_1, {{{"f1", "10"}}})); + ASSERT_OK_AND_ASSIGN(ScoredRows distributed_rows, ToScoredRows(distributed)); + ASSERT_EQ(RowIdsOf(distributed_rows), std::vector({0, 4})); + std::shared_ptr f2_is_2 = + PredicateBuilder::Equal(2, "f2", FieldType::INT, Literal(2)); + check(full, {}, f2_is_2, {Range(2, 5)}, {1}); + std::shared_ptr text_is_three_apples = PredicateBuilder::Equal( + 0, "f0", FieldType::STRING, Literal(FieldType::STRING, "apple apple apple", 17)); + check(detail, {}, text_is_three_apples, {Range(2, 5)}, {4}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr partition_20_and_f2_is_1, + PredicateBuilder::And( + {PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(20)), f2_is_1})); + check(full, {}, partition_20_and_f2_is_1, {}, {}); + + write_partition(R"([["apple", 20, 2], ["kiwi", 20, 2], ["apple", 20, 1]])", {{"f1", "20"}}); + ASSERT_OK( + WriteIndex(table_path, {{{"f1", "20"}}}, "f0", index_type, index_options, Range(6, 6))); + ASSERT_OK( + WriteIndex(table_path, {{{"f1", "10"}}}, "f0", index_type, index_options, Range(4, 4))); + auto update_column = [&](const std::shared_ptr& field, + const std::string& json) { + auto values = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({field}), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto updates, + WriteArray(table_path, {{"f1", "20"}}, {field->name()}, values)); + ASSERT_EQ(updates.size(), 1u); + auto update = std::dynamic_pointer_cast(updates.front()); + ASSERT_TRUE(update); + const auto& files = update->GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(files.size(), 1u); + files.front()->AssignFirstRowId(6); + ASSERT_OK(Commit(table_path, updates)); + }; + update_column(fields[2], "[[2], [1], [2]]"); + for (auto search_options : {full, detail}) { + check(search_options, {}, nullptr, {Range(2, 3), Range(5, 5), Range(7, 8)}, + {0, 1, 2, 4, 6, 8}); + search_options.insert(index_options.begin(), index_options.end()); + ASSERT_OK_AND_ASSIGN(ScoredRows raw_hit, + SearchTableWithFilter(table_path, R"({"match":{"query":"kiwi"}})", + 1, f2_is_1, {}, search_options)); + ASSERT_EQ(RowIdsOf(raw_hit), std::vector({7})); + } + check(detail, {{{"f1", "10"}}}, nullptr, {Range(5, 5)}, {0, 1, 4}); + check(detail, {{{"f1", "20"}}}, nullptr, {Range(2, 3), Range(7, 8)}, {2, 6, 8}); + + update_column(fields[0], R"([["kiwi"], ["kiwi"], ["apple"]])"); + auto builder = NewFullTextSearchBuilder(table_path, query, 1, index_options); + builder->SetPartitionFilter({{{"f1", "20"}}}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1u); + ASSERT_TRUE(plan->SnapshotId()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(*plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows stale_hits, ToScoredRows(result)); + ASSERT_EQ(RowIdsOf(stale_hits), std::vector({6})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr updated_text, + RawFullTextSearchSplitImpl::Create(*plan->SnapshotId(), "f0", + index_type, {Range(6, 6)}, 0, 0)); + ASSERT_OK_AND_ASSIGN(result, read->Read({plan->Splits().front(), updated_text})); + ASSERT_OK_AND_ASSIGN(ScoredRows current_hits, ToScoredRows(result)); + ASSERT_TRUE(current_hits.empty()); + } + + void CheckSplitUsesPlannedFilterColumn( + const std::string& index_type, const std::map& index_options) { + const arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + const std::map options = { + {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable({}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN( + auto indexed, WriteArray(table_path, schema->field_names(), AppleRows(fields, 0, 3))); + ASSERT_OK(Commit(table_path, indexed)); + ASSERT_OK(WriteIndex(table_path, {}, "f0", index_type, index_options, Range(0, 2))); + ASSERT_OK(WriteIndex(table_path, {}, "f1", "btree", {}, Range(0, 1))); + ASSERT_OK_AND_ASSIGN( + auto appended, WriteArray(table_path, schema->field_names(), AppleRows(fields, 3, 5))); + ASSERT_OK(Commit(table_path, appended)); + + auto fs = dir_->GetFileSystem(); + const auto category = arrow::field("category", arrow::int32()); + ASSERT_OK(TestHelper::WriteNextSchema( + fs, table_path, {DataField(0, fields[0]), DataField(1, category)}, 1, options)); + std::map search_options = index_options; + search_options[Options::GLOBAL_INDEX_SEARCH_MODE] = "full"; + const std::string query = R"({"match":{"query":"apple"}})"; + auto builder = NewFullTextSearchBuilder(table_path, query, 10, search_options); + builder->WithFilter(PredicateBuilder::In(1, "category", FieldType::INT, + {Literal(1), Literal(2), Literal(4)})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 2u); + std::vector serialized; + for (const auto& split : plan->Splits()) { + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + serialized.push_back(std::move(bytes)); + } + auto check_planned_rows = [&]() { + ASSERT_OK_AND_ASSIGN(std::unique_ptr worker, builder->NewFullTextRead()); + for (const auto& bytes : serialized) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr split, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + worker->Read({split})); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, ToScoredRows(result)); + if (std::dynamic_pointer_cast(split)) { + ASSERT_EQ(RowIdsOf(rows), std::vector({4})); + } else { + ASSERT_EQ(RowIdsOf(rows), std::vector({1, 2})); + } + } + }; + const DataField old_category(1, arrow::field("old_category", arrow::int32())); + ASSERT_OK(TestHelper::WriteNextSchema( + fs, table_path, {DataField(0, fields[0]), old_category, DataField(2, category)}, 2, + options)); + ASSERT_NO_FATAL_FAILURE(check_planned_rows()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr replanned, + builder->ExecuteLocal()); + ASSERT_OK_AND_ASSIGN(ScoredRows new_rows, ToScoredRows(replanned)); + ASSERT_TRUE(new_rows.empty()); + + const std::vector current_fields = { + DataField(0, fields[0]), old_category, + DataField(3, BlobUtils::ToArrowField("category", true))}; + auto current_options = options; + current_options[Options::BLOB_VIEW_FIELD] = "category"; + ASSERT_OK(TestHelper::WriteNextSchema(fs, table_path, current_fields, 3, current_options)); + ASSERT_NO_FATAL_FAILURE(check_planned_rows()); + ASSERT_NOK_WITH_MSG(builder->NewFullTextScan(), "Invalid type large_binary for predicate"); + + ASSERT_TRUE(plan->SnapshotId()); + search_options[Options::SCAN_SNAPSHOT_ID] = std::to_string(*plan->SnapshotId()); + auto original_filter = + PredicateBuilder::In(1, "f1", FieldType::INT, {Literal(1), Literal(2), Literal(4)}); + ASSERT_OK_AND_ASSIGN( + ScoredRows historical_rows, + SearchTableWithFilter(table_path, query, 10, original_filter, {}, search_options)); + ASSERT_EQ(RowIdsOf(historical_rows), std::vector({1, 2, 4})); + + current_options[Options::QUERY_AUTH_ENABLED] = "true"; + ASSERT_OK(TestHelper::WriteNextSchema(fs, table_path, current_fields, 3, current_options)); + ASSERT_NOK_WITH_MSG( + SearchTableWithFilter(table_path, query, 10, original_filter, {}, search_options), + "Full-text search is not supported"); + } + + void CheckRawSearchWithoutTmpDir(const std::string& index_type, + const std::map& index_options) { + const arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + CreateTable({}, schema, + {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN( + auto indexed, WriteArray(table_path, schema->field_names(), AppleRows(fields, 0, 1))); + ASSERT_OK(Commit(table_path, indexed)); + ASSERT_OK(WriteIndex(table_path, {}, "f0", index_type, index_options, Range(0, 0))); + ASSERT_OK_AND_ASSIGN( + auto appended, WriteArray(table_path, schema->field_names(), AppleRows(fields, 1, 2))); + ASSERT_OK(Commit(table_path, appended)); + + const char* tmpdir = std::getenv("TMPDIR"); + std::optional saved; + if (tmpdir) { + saved = tmpdir; + } + ASSERT_EQ(setenv("TMPDIR", PathUtil::JoinPath(dir_->Str(), "missing").c_str(), 1), 0); + ScopeGuard restore([&]() { + if (saved) { + setenv("TMPDIR", saved->c_str(), 1); + } else { + unsetenv("TMPDIR"); + } + }); + std::map search_options = index_options; + search_options.erase("lucene-fts.write.tmp.directory"); + for (const char* mode : {"full", "detail"}) { + search_options[Options::FULL_TEXT_INDEX_SEARCH_MODE] = mode; + ASSERT_OK_AND_ASSIGN( + ScoredRows rows, + SearchTable(table_path, R"({"match":{"query":"apple"}})", 10, {}, search_options)); + ASSERT_EQ(RowIdsOf(rows), std::vector({0, 1})) << mode; + } + } + void CheckTableSearchWithPartitionFilter(const std::string& table_path, const std::string& query) const { ASSERT_OK_AND_ASSIGN(ScoredRows rows, @@ -4218,6 +4936,39 @@ TEST_P(GlobalIndexTest, TestLuceneTableFullTextSearchWithPartitionAndDataFilter) "lucene-fts", {{"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); } + +TEST_P(GlobalIndexTest, TestLuceneRawSearchSkipsDeletedRows) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckRawSearchSkipsDeletedRows("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} + +TEST_P(GlobalIndexTest, TestLuceneRawSearchAcrossPartitions) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckRawSearchAcrossPartitions("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}, + R"({"match":{"query":"apple"}})"); +} +TEST_P(GlobalIndexTest, TestLuceneSplitUsesPlannedFilterColumn) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckSplitUsesPlannedFilterColumn("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} + +TEST_P(GlobalIndexTest, TestLuceneRawSearchWithoutTmpDir) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckRawSearchWithoutTmpDir("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} + #endif #ifdef PAIMON_ENABLE_FULL_TEXT @@ -4375,6 +5126,151 @@ TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearch) { CheckTableFullTextSearch("full-text", {{"full-text.tokenizer", "default"}}); } +TEST_P(GlobalIndexTest, TestFullTextDistributedRawRangesOverrideStoredIndex) { + const arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::utf8())}; + const auto schema = arrow::schema(fields); + CreateTable({}, schema, + {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}); + const std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + const auto rows = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(fields), + R"([["apple", "apple"], ["apple apple apple apple", "x"], + ["apple apple", "x"], ["apple", "x"], + ["apple apple apple", "apple apple apple"], + ["apple apple apple apple apple", "x"]])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + WriteArray(table_path, schema->field_names(), rows)); + ASSERT_OK(Commit(table_path, messages)); + ASSERT_OK(WriteIndex(table_path, {}, "f0", "extra-field-full-text-test", {}, Range(1, 4))); + + auto update_text = [&](const std::string& json) { + const auto updates = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0]}), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> update_messages, + WriteArray(table_path, {"f0"}, updates)); + ASSERT_EQ(update_messages.size(), 1u); + const auto update = std::dynamic_pointer_cast(update_messages.front()); + ASSERT_TRUE(update); + const auto& update_files = update->GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(update_files.size(), 1u); + update_files.front()->AssignFirstRowId(0); + ASSERT_OK(Commit(table_path, update_messages)); + }; + ASSERT_NO_FATAL_FAILURE(update_text( + R"([["kiwi"], ["kiwi"], ["apple apple"], ["apple"], [null], ["apple apple apple apple apple"]])")); + + const std::string query = R"({"match":{"query":"apple"}})"; + auto check_plan = [&](const std::string& column, const std::string& mode, + const std::vector& expected_raw_ranges, + const std::vector& expected_index_ranges) { + SCOPED_TRACE(column + " " + mode); + auto builder = NewFullTextSearchBuilder( + table_path, query, 10, {{Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}, column); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), static_cast(!expected_raw_ranges.empty()) + + static_cast(!expected_index_ranges.empty())); + if (!expected_index_ranges.empty()) { + auto index_split = + std::dynamic_pointer_cast(plan->Splits().front()); + ASSERT_TRUE(index_split); + ASSERT_EQ(index_split->FullTextIndexFiles().size(), 1u); + ASSERT_EQ(index_split->FullTextIndexFiles().front()->IndexType(), + "extra-field-full-text-test"); + ASSERT_EQ(index_split->RowRangeStart(), 1); + ASSERT_EQ(index_split->RowRangeEnd(), 4); + ASSERT_EQ(index_split->SearchRowRanges(), expected_index_ranges); + } + if (!expected_raw_ranges.empty()) { + auto raw_split = + std::dynamic_pointer_cast(plan->Splits().back()); + ASSERT_TRUE(raw_split); + ASSERT_EQ(raw_split->RowRanges(), expected_raw_ranges); + } + }; + for (const char* column : {"f0", "f1"}) { + ASSERT_NO_FATAL_FAILURE(check_plan(column, "fast", {}, {Range(1, 4)})); + } + + auto check_search = [&](const std::string& column, + const std::vector& expected_row_ids) { + SCOPED_TRACE(column); + for (const char* mode : {"full", "detail"}) { + SCOPED_TRACE(mode); + const std::map options = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}; + ScoredRows all_rows; + for (int32_t limit : {10, 1}) { + SCOPED_TRACE(limit); + auto builder = NewFullTextSearchBuilder(table_path, query, limit, options, column); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + builder->ExecuteLocal()); + ASSERT_OK_AND_ASSIGN(ScoredRows local, ToScoredRows(result)); + if (limit == 10) { + ASSERT_EQ(RowIdsOf(local), expected_row_ids); + all_rows = local; + } else { + ASSERT_EQ(local.size(), 1u); + const auto best = std::max_element(all_rows.begin(), all_rows.end(), + [](const auto& left, const auto& right) { + return left.second < right.second; + }); + ASSERT_NE(best, all_rows.end()); + ASSERT_EQ(local.front().second, best->second); + ASSERT_NE(std::find(all_rows.begin(), all_rows.end(), local.front()), + all_rows.end()); + } + ASSERT_OK_AND_ASSIGN( + std::shared_ptr distributed, + ReadSerializedSplits(table_path, query, limit, options, nullptr, {}, column)); + ASSERT_OK_AND_ASSIGN(ScoredRows distributed_rows, ToScoredRows(distributed)); + ASSERT_EQ(distributed_rows, local); + } + } + }; + for (const char* mode : {"full", "detail"}) { + ASSERT_NO_FATAL_FAILURE(check_plan("f0", mode, {Range(0, 5)}, {})); + ASSERT_NO_FATAL_FAILURE(check_plan("f1", mode, {Range(0, 5)}, {})); + } + ASSERT_NO_FATAL_FAILURE(check_search("f0", {2, 3, 5})); + ASSERT_NO_FATAL_FAILURE(check_search("f1", {0, 4})); + ASSERT_OK(WriteIndex(table_path, {}, "f1", "bitmap", {}, Range(1, 1))); + auto filter = + PredicateBuilder::Equal(1, "f1", FieldType::STRING, Literal(FieldType::STRING, "x", 1)); + auto fast = NewFullTextSearchBuilder(table_path, query, 10); + fast->WithFilter(filter); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fast_scan, fast->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fast_plan, fast_scan->CreatePlan()); + ASSERT_EQ(fast_plan->Splits().size(), 1u); + auto shard = std::dynamic_pointer_cast(fast_plan->Splits()[0]); + ASSERT_TRUE(shard); + ASSERT_EQ(shard->ScalarIndexFiles().size(), 1u); + ASSERT_OK(dir_->GetFileSystem()->Delete(PathUtil::JoinPath( + FileStorePathFactory::IndexPath(table_path), shard->ScalarIndexFiles()[0]->FileName()))); + ASSERT_OK(WriteIndex(table_path, {}, "f1", "bitmap", {}, Range(2, 3))); + ASSERT_OK(WriteIndex(table_path, {}, "f0", "full-text", {}, Range(2, 3))); + for (const char* mode : {"full", "detail"}) { + ASSERT_NO_FATAL_FAILURE(check_plan("f0", mode, {Range(0, 1), Range(4, 5)}, {Range(2, 3)})); + ASSERT_NO_FATAL_FAILURE(check_plan("f1", mode, {Range(0, 5)}, {})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result, + ReadSerializedSplits(table_path, query, 10, {{Options::GLOBAL_INDEX_SEARCH_MODE, mode}}, + filter)); + ASSERT_OK_AND_ASSIGN(ScoredRows filtered_rows, ToScoredRows(result)); + ASSERT_EQ(RowIdsOf(filtered_rows), std::vector({2, 3, 5})) << mode; + } + ASSERT_NO_FATAL_FAILURE(check_search("f0", {2, 3, 5})); + ASSERT_NO_FATAL_FAILURE( + update_text(R"([["kiwi"], ["kiwi"], ["apple apple"], ["apple"], [null], ["kiwi"]])")); + ASSERT_NO_FATAL_FAILURE(check_search("f0", {2, 3})); +} + TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearchWithFilter) { CheckTableSearchWithFilter("full-text", {{"full-text.tokenizer", "default"}}); } @@ -4382,6 +5278,24 @@ TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearchWithFilter) { TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearchWithPartitionAndDataFilter) { CheckTableSearchWithPartitionAndDataFilter("full-text", {{"full-text.tokenizer", "default"}}); } + +TEST_P(GlobalIndexTest, TestFullTextRawSearchSkipsDeletedRows) { + CheckRawSearchSkipsDeletedRows("full-text", {{"full-text.tokenizer", "default"}}); +} + +TEST_P(GlobalIndexTest, TestFullTextRawSearchAcrossPartitions) { + CheckRawSearchAcrossPartitions("full-text", {{"full-text.tokenizer", "ngram"}}, + R"({"match":{"query":"ppl"}})"); +} + +TEST_P(GlobalIndexTest, TestFullTextSplitUsesPlannedFilterColumn) { + CheckSplitUsesPlannedFilterColumn("full-text", {}); +} + +TEST_P(GlobalIndexTest, TestFullTextRawSearchWithoutTmpDir) { + CheckRawSearchWithoutTmpDir("full-text", {}); +} + #endif TEST_P(GlobalIndexTest, TestFilteredRowIdReader) { @@ -4414,7 +5328,7 @@ TEST_P(GlobalIndexTest, TestFilteredRowIdReader) { std::shared_ptr filter = PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(1)); std::shared_ptr executor = CreateDefaultExecutor(); - auto matching_row_ids = [&](int64_t snapshot_id, const RoaringBitmap64& rows, + auto matching_row_ids = [&](int64_t snapshot_id, const auto& rows, const std::map& search_options = {}) -> Result { std::map merged_options = table_schema.value()->Options(); @@ -4452,6 +5366,10 @@ TEST_P(GlobalIndexTest, TestFilteredRowIdReader) { snapshot_manager.LatestSnapshotId()); ASSERT_TRUE(second_snapshot_id); + ASSERT_OK_AND_ASSIGN(RowRangeIndex wide_range, + RowRangeIndex::Create({Range(1, int64_t{1} << 40)})); + ASSERT_OK_AND_ASSIGN(rows, matching_row_ids(second_snapshot_id.value(), wide_range)); + ASSERT_EQ(rows, RoaringBitmap64::From({2, 3})); ASSERT_OK_AND_ASSIGN( rows, matching_row_ids(second_snapshot_id.value(), RoaringBitmap64::From({1, 3}))); ASSERT_EQ(rows, RoaringBitmap64::From({3}));