Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,26 @@ License: https://www.apache.org/licenses/LICENSE-2.0

--------------------------------------------------------------------------------

This product includes code from Apache Paimon Full Text 0.1.0.

* Upstream source patch in cmake_modules/full_text.diff

Copyright: 2026 The Apache Software Foundation.
Home page: https://github.com/apache/paimon-full-text
License: https://www.apache.org/licenses/LICENSE-2.0

--------------------------------------------------------------------------------

This product includes code from Lucene++ 3.0.9.

* Upstream source patch in cmake_modules/lucene.diff

Copyright (c) 2009-2014 Alan Wright. All rights reserved.
Home page: https://github.com/luceneplusplus/LucenePlusPlus
License: https://www.apache.org/licenses/LICENSE-2.0

--------------------------------------------------------------------------------

This product includes code from Vortex 0.75.0.

* Callback I/O sink utilities in crates/vortex_callback_io/callback_io.rs
Expand Down
60 changes: 54 additions & 6 deletions cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -887,10 +887,14 @@ macro(build_lucene)
"-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}")

set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a")
set(LUCENE_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/lucene.diff")
paimon_patch_command(lucene_ep PATCH_COMMAND)
externalproject_add(lucene_ep
${EP_COMMON_OPTIONS}
URL ${LUCENE_SOURCE_URL}
URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}"
LOG_PATCH ON
PATCH_COMMAND ${PATCH_COMMAND}
CMAKE_ARGS ${LUCENE_CMAKE_ARGS}
BUILD_BYPRODUCTS ${LUCENE_LIB}
DEPENDS zlib
Expand All @@ -903,6 +907,21 @@ macro(build_lucene)
boost_chrono
boost_atomic)

paimon_register_patched_project(lucene_ep
VERSION
"${PAIMON_LUCENE_BUILD_VERSION}"
PATCHES
"${LUCENE_PATCH_FILE}"
INSTALL_DIR
"${LUCENE_PREFIX}"
VERSION_VARIABLES
PAIMON_LUCENE_BUILD_VERSION
PAIMON_LUCENE_BUILD_SHA256_CHECKSUM
PAIMON_BOOST_BUILD_VERSION
PAIMON_BOOST_BUILD_SHA256_CHECKSUM
PAIMON_ZLIB_BUILD_VERSION
PAIMON_ZLIB_BUILD_SHA256_CHECKSUM)

set(LUCENE_INCLUDE_DIR "${LUCENE_PREFIX}/include")
# The include directory must exist before it is referenced by a target.
file(MAKE_DIRECTORY "${LUCENE_INCLUDE_DIR}")
Expand All @@ -912,6 +931,12 @@ macro(build_lucene)
PROPERTIES IMPORTED_LOCATION "${LUCENE_LIB}"
INTERFACE_INCLUDE_DIRECTORIES
"${LUCENE_INCLUDE_DIR}")
if(APPLE)
# Boost 1.66 headers still use the function base classes removed in C++17.
target_compile_definitions(lucene
INTERFACE _LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION
)
endif()

target_link_libraries(lucene
INTERFACE zlib
Expand Down Expand Up @@ -1094,6 +1119,16 @@ macro(build_boost)
file(WRITE ${BOOST_USER_CONFIG}
"using ${BOOST_TOOLSET} : : \"${CMAKE_CXX_COMPILER}\" ;\n")

set(BOOST_CXX_FLAGS "${EP_CXX_FLAGS}")
if(APPLE)
string(APPEND BOOST_CXX_FLAGS
" -D_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION")
endif()
if(APPLE AND CMAKE_OSX_SYSROOT)
# b2 does not inherit the SDK flags CMake adds to native targets.
string(APPEND BOOST_CXX_FLAGS " -isysroot '${CMAKE_OSX_SYSROOT}'")
endif()

externalproject_add(boost_ep
URL ${BOOST_SOURCE_URL}
URL_HASH "SHA256=${PAIMON_BOOST_BUILD_SHA256_CHECKSUM}"
Expand All @@ -1109,8 +1144,8 @@ macro(build_boost)
--user-config=${BOOST_USER_CONFIG}
toolset=${BOOST_TOOLSET}
cxxstd=${CMAKE_CXX_STANDARD}
cxxflags=${EP_CXX_FLAGS} linkflags=${EP_CXX_FLAGS}
install
cxxflags=${BOOST_CXX_FLAGS}
linkflags=${BOOST_CXX_FLAGS} install
INSTALL_COMMAND ""
DEPENDS zlib
BUILD_BYPRODUCTS ${BOOST_BYPRODUCTS}
Expand Down Expand Up @@ -1460,10 +1495,8 @@ macro(build_full_text)
set(FULL_TEXT_DYNAMIC_LIB
"${FULL_TEXT_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
)
set(FULL_TEXT_CARGO_TARGET_DIR
"${CMAKE_CURRENT_BINARY_DIR}/full_text_ep-prefix/src/full_text_ep-build")
set(FULL_TEXT_CARGO_DYNAMIC_LIB
"${FULL_TEXT_CARGO_TARGET_DIR}/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
"<BINARY_DIR>/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
)

file(MAKE_DIRECTORY "${FULL_TEXT_INCLUDE_DIR}")
Expand All @@ -1472,13 +1505,17 @@ macro(build_full_text)
# Build with the lock file shipped in the source release: a reader rejects index files whose
# recorded Tantivy version differs from the linked one, so the Tantivy version must match the
# one used by Java Paimon.
set(FULL_TEXT_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/full_text.diff")
paimon_patch_command(full_text_ep PATCH_COMMAND)
externalproject_add(full_text_ep
URL ${FULL_TEXT_SOURCE_URL}
URL_HASH "SHA256=${PAIMON_FULL_TEXT_BUILD_SHA256_CHECKSUM}"
${THIRDPARTY_LOG_OPTIONS}
LOG_PATCH ON
PATCH_COMMAND ${PATCH_COMMAND}
CONFIGURE_COMMAND ""
BUILD_COMMAND ${CMAKE_COMMAND} -E env
CARGO_TARGET_DIR=${FULL_TEXT_CARGO_TARGET_DIR}
"CARGO_TARGET_DIR=<BINARY_DIR>"
${PAIMON_CARGO_EXECUTABLE} build --release --locked
--manifest-path <SOURCE_DIR>/ffi/Cargo.toml
COMMAND ${CMAKE_COMMAND} -E copy_if_different
Expand All @@ -1487,6 +1524,17 @@ macro(build_full_text)
<SOURCE_DIR>/include ${FULL_TEXT_INCLUDE_DIR}
BUILD_BYPRODUCTS "${FULL_TEXT_DYNAMIC_LIB}")

paimon_register_patched_project(full_text_ep
VERSION
"${PAIMON_FULL_TEXT_BUILD_VERSION}"
PATCHES
"${FULL_TEXT_PATCH_FILE}"
INSTALL_DIR
"${FULL_TEXT_PREFIX}"
VERSION_VARIABLES
PAIMON_FULL_TEXT_BUILD_VERSION
PAIMON_FULL_TEXT_BUILD_SHA256_CHECKSUM)

add_library(paimon_ftindex_ffi SHARED IMPORTED GLOBAL)
set_target_properties(paimon_ftindex_ffi
PROPERTIES IMPORTED_LOCATION "${FULL_TEXT_DYNAMIC_LIB}"
Expand Down
237 changes: 237 additions & 0 deletions cmake_modules/full_text.diff
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Paimon Full Text 0.1.0 uses a temporary directory for its Tantivy writer.
# Provide a RAM-backed writer for unindexed rows with the same archive and reader.

diff --git a/core/src/index.rs b/core/src/index.rs
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -20,7 +20,9 @@
use crate::error::{FtIndexError, Result};
use crate::io::{FullTextReadMetrics, ReadMetrics, ReadRequest, SeekRead, SeekWrite};
use crate::query::{BooleanOccur, MatchOperator, QuerySpec};
-use crate::storage::{read_header, write_envelope_from_paths, ArchiveFileEntry, IndexHeader};
+use crate::storage::{
+ read_header, write_envelope_from_paths, write_envelope_header, ArchiveFileEntry, IndexHeader,
+};
use crate::tokenizer::{TokenizerConfig, TokenizerKind};
use levenshtein_automata::{Distance, LevenshteinAutomatonBuilder, DFA, SINK_STATE};
use roaring::RoaringTreemap;
@@ -30,6 +32,7 @@
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use tantivy::collector::{FilterCollector, TopDocs};
+use tantivy::directory::{Directory, RamDirectory};
use tantivy::query::{
BooleanQuery, BoostQuery, ConstScorer, EmptyQuery, EnableScoring, Explanation, Occur, Query,
QueryParser, Scorer, TermQuery, Weight,
@@ -64,7 +67,12 @@

struct FullTextIndexWriterState {
index_writer: IndexWriter<TantivyDocument>,
- temp_dir: TempDir,
+ directory: WriterDirectory,
+}
+
+enum WriterDirectory {
+ Disk(TempDir),
+ Memory(RamDirectory),
}

#[derive(Clone, Debug, PartialEq, Eq)]
@@ -77,8 +85,23 @@
pub fn new(config: FullTextIndexConfig) -> Result<Self> {
config.validate()?;
let temp_dir = TempDir::new()?;
- let schema = build_schema(&config);
- let mut index = Index::create_in_dir(temp_dir.path(), schema.clone())?;
+ let index = Index::create_in_dir(temp_dir.path(), build_schema(&config))?;
+ Self::with_index(config, index, WriterDirectory::Disk(temp_dir))
+ }
+
+ pub fn new_in_memory(config: FullTextIndexConfig) -> Result<Self> {
+ config.validate()?;
+ let directory = RamDirectory::create();
+ let index = Index::create(directory.clone(), build_schema(&config), Default::default())?;
+ Self::with_index(config, index, WriterDirectory::Memory(directory))
+ }
+
+ fn with_index(
+ config: FullTextIndexConfig,
+ mut index: Index,
+ directory: WriterDirectory,
+ ) -> Result<Self> {
+ let schema = index.schema();
register_tokenizer(&mut index, &config.tokenizer)?;
let row_id_field = schema
.get_field(&config.row_id_field)
@@ -89,7 +112,7 @@
config,
state: Some(FullTextIndexWriterState {
index_writer,
- temp_dir,
+ directory,
}),
row_id_field,
text_fields,
@@ -157,11 +180,23 @@
})?;
let FullTextIndexWriterState {
mut index_writer,
- temp_dir,
+ directory,
} = state;
index_writer.commit()?;
+ let index = index_writer.index().clone();
index_writer.wait_merging_threads()?;

+ let metadata = FullTextIndexMetadata {
+ config: self.config.clone(),
+ document_count: self.document_count,
+ tantivy_version: tantivy::version().to_string(),
+ };
+ let temp_dir = match directory {
+ WriterDirectory::Disk(temp_dir) => temp_dir,
+ WriterDirectory::Memory(ram) => {
+ return write_ram_archive(output, metadata, index, ram);
+ }
+ };
let files = collect_index_files(temp_dir.path())?;
let mut offset = 0u64;
let mut entries = Vec::with_capacity(files.len());
@@ -177,11 +212,7 @@
}

let header = IndexHeader {
- metadata: FullTextIndexMetadata {
- config: self.config.clone(),
- document_count: self.document_count,
- tantivy_version: tantivy::version().to_string(),
- },
+ metadata,
files: entries,
};
let paths = files.iter().map(|file| &file.path).collect::<Vec<_>>();
@@ -485,6 +516,43 @@
"document field '{field}' is not configured for this index"
)))
}
+}
+
+/// Write the same archive as the on-disk writer, including Tantivy's segment footers.
+fn write_ram_archive<W: SeekWrite>(
+ output: &mut W,
+ metadata: FullTextIndexMetadata,
+ index: Index,
+ directory: RamDirectory,
+) -> Result<()> {
+ let mut paths: Vec<_> = index.directory().list_managed_files().into_iter().collect();
+ paths.push(PathBuf::from(".managed.json"));
+ paths.sort();
+ let mut files = Vec::with_capacity(paths.len());
+ let mut contents = Vec::with_capacity(paths.len());
+ let mut offset = 0u64;
+ for path in paths {
+ let bytes = directory
+ .open_read(&path)
+ .map_err(tantivy::TantivyError::from)?
+ .read_bytes()?;
+ let length = bytes.len() as u64;
+ files.push(ArchiveFileEntry {
+ name: path.to_string_lossy().into_owned(),
+ offset,
+ length,
+ });
+ offset = offset.checked_add(length).ok_or_else(|| {
+ FtIndexError::InvalidStorage("full-text archive size overflow".to_string())
+ })?;
+ contents.push(bytes);
+ }
+ write_envelope_header(output, &IndexHeader { metadata, files })?;
+ for bytes in contents {
+ output.write_all(bytes.as_slice())?;
+ }
+ output.flush()?;
+ Ok(())
}

struct IndexFile {
diff --git a/core/src/storage.rs b/core/src/storage.rs
--- a/core/src/storage.rs
+++ b/core/src/storage.rs
@@ -87,7 +87,10 @@
Ok(())
}

-fn write_envelope_header<W: SeekWrite>(output: &mut W, header: &IndexHeader) -> Result<()> {
+pub(crate) fn write_envelope_header<W: SeekWrite>(
+ output: &mut W,
+ header: &IndexHeader,
+) -> Result<()> {
let header_json = serde_json::to_vec(header)?;
if header_json.len() > MAX_HEADER_BYTES {
return Err(FtIndexError::InvalidStorage(format!(
diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs
--- a/ffi/src/lib.rs
+++ b/ffi/src/lib.rs
@@ -220,10 +220,28 @@
values: *const *const c_char,
len: usize,
) -> *mut PaimonFtindexWriterHandle {
+ open_writer(keys, values, len, FullTextIndexWriter::new)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn paimon_ftindex_writer_open_in_memory(
+ keys: *const *const c_char,
+ values: *const *const c_char,
+ len: usize,
+) -> *mut PaimonFtindexWriterHandle {
+ open_writer(keys, values, len, FullTextIndexWriter::new_in_memory)
+}
+
+unsafe fn open_writer(
+ keys: *const *const c_char,
+ values: *const *const c_char,
+ len: usize,
+ create: fn(FullTextIndexConfig) -> paimon_ftindex_core::Result<FullTextIndexWriter>,
+) -> *mut PaimonFtindexWriterHandle {
ffi_ptr(|| {
let options = options_from_raw(keys, values, len)?;
let config = FullTextIndexConfig::from_options(&options).map_err(|e| e.to_string())?;
- let writer = FullTextIndexWriter::new(config).map_err(|e| e.to_string())?;
+ let writer = create(config).map_err(|e| e.to_string())?;
Ok(Box::into_raw(Box::new(PaimonFtindexWriterHandle {
inner: writer,
})))
diff --git a/include/paimon_ftindex.h b/include/paimon_ftindex.h
--- a/include/paimon_ftindex.h
+++ b/include/paimon_ftindex.h
@@ -59,6 +59,12 @@
const char **values,
size_t len);

+/* Builds the index segments in memory for a temporary search index. */
+PaimonFtindexWriterHandle *paimon_ftindex_writer_open_in_memory(
+ const char **keys,
+ const char **values,
+ size_t len);
+
int paimon_ftindex_writer_add_document(
PaimonFtindexWriterHandle *writer,
int64_t row_id,
Loading
Loading