diff --git a/docs/source/user_guide/realtime.rst b/docs/source/user_guide/realtime.rst index b32454148..5339aaa82 100644 --- a/docs/source/user_guide/realtime.rst +++ b/docs/source/user_guide/realtime.rst @@ -437,6 +437,33 @@ filtering. Keep predicate fields in the read schema and enable In-memory and disk indexes are configured independently. Use ``file-index..columns`` to enable indexes for committed files. +Real-time Vector Search +~~~~~~~~~~~~~~~~~~~~~~ + +Build with ``-DPAIMON_ENABLE_LUMINA=ON`` for ``LIST`` vector indexes +using ``bruteforce`` or ``flatnsw`` with ``rawf32`` encoding: + +.. code-block:: text + + realtime.index.lumina.columns = embedding + realtime.index.lumina.embedding.index.type = bruteforce + realtime.index.lumina.embedding.index.dimension = 2 + realtime.index.lumina.embedding.distance.metric = l2 + realtime.index.lumina.embedding.encoding.type = rawf32 + +For disk indexes, repeat these settings with the ``file-index.lumina`` prefix, +keeping dimension and distance metric consistent. Append deduplicate mode also +requires ``_REALTIME_OFFSET`` in ``file-index.bitmap.columns``. + +Use ``ReadContextBuilder::EnablePreciseFileIndexEvaluate(true)`` with +``SetVectorSearch()``. Keep scalar predicate fields in the read projection and +project ``_embedding_INDEX_SCORE`` for scores. Scalar filters, including those +without indexes, and committed or pending DVs are applied before vector TopK. + +TopK is per memory segment or data file; apply global ranking in the calling +engine. +Real-time vector tag predicates are unsupported. + Implementation by Mode ---------------------- @@ -634,8 +661,9 @@ Current restrictions include: * Native tables on the main branch with fixed positive bucket counts and full write schemas only. * Process-local batch union scans only. Streaming scans, limit pushdown, - global-index result splits, vector/full-text search, and count-only real-time - readers are unsupported. + global-index result splits, and count-only real-time readers are unsupported. + Precise index evaluation and Lumina vector search support append and append + deduplicate modes; primary-key real-time search is unsupported. * PK mode supports deduplicate with default delete handling, or partial update with deletion vectors disabled and its existing delete and sequence-group options. Both require ascending sequence order and ``changelog-producer=none``. Custom diff --git a/include/paimon/realtime/index/realtime_index.h b/include/paimon/realtime/index/realtime_index.h index 0a3717dd8..7c25b37b1 100644 --- a/include/paimon/realtime/index/realtime_index.h +++ b/include/paimon/realtime/index/realtime_index.h @@ -29,6 +29,7 @@ struct ArrowArray; namespace paimon { +class MemoryPool; class RealtimeIndexReader; /// Append-only in-memory index for one field, supporting one writer and concurrent readers. @@ -40,8 +41,9 @@ class PAIMON_EXPORT RealtimeIndex { virtual Status AddBatch(::ArrowArray* batch) = 0; /// Creates a reader for [begin, end), sharing postings without copying. + /// The pool belongs to the query and is used for reader/search allocations. virtual Result> CreateReader( - const OffsetRange& offset_range) const = 0; + const OffsetRange& offset_range, const std::shared_ptr& pool) const = 0; virtual uint64_t GetMemoryUsage() const = 0; }; diff --git a/include/paimon/realtime/index/realtime_index_read_source.h b/include/paimon/realtime/index/realtime_index_read_source.h index d78b9c59a..0fdcad812 100644 --- a/include/paimon/realtime/index/realtime_index_read_source.h +++ b/include/paimon/realtime/index/realtime_index_read_source.h @@ -41,8 +41,9 @@ class PAIMON_EXPORT RealtimeIndexReadSource { /// Returns readers bound to this source's offset range. A missing field has no readers. /// Readers must not expose writes published after the pinned snapshot. + /// The pool belongs to the query, rather than the retained index. virtual Result>> CreateIndexReaders( - const std::string& field_name) const = 0; + const std::string& field_name, const std::shared_ptr& pool) const = 0; }; /// Optional capability for precise real-time filtering and search. Ordinary query reads do not diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 88a9a36f8..aff6f4ab2 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -451,6 +451,7 @@ set(PAIMON_CORE_SRCS core/realtime/realtime_snapshot_offsets.cpp core/realtime/realtime_store_read_pipeline.cpp core/realtime/realtime_primary_key_writer.cpp + core/realtime/realtime_utils.cpp core/postpone/postpone_bucket_writer.cpp core/schema/arrow_schema_validator.cpp core/schema/schema_manager.cpp diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index be0f7a917..75f33565f 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -47,6 +47,7 @@ #include "paimon/core/realtime/realtime_utils.h" #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" diff --git a/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp index 1ba1698a4..cc60addb6 100644 --- a/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp +++ b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp @@ -259,8 +259,8 @@ class ArrowDeduplicateRealtimeStore::IndexReadSource final : public RealtimeInde } Result>> CreateIndexReaders( - const std::string& field_name) const override { - return source_->CreateIndexReaders(field_name); + const std::string& field_name, const std::shared_ptr& pool) const override { + return source_->CreateIndexReaders(field_name, pool); } private: diff --git a/src/paimon/core/realtime/arrow_realtime_store.cpp b/src/paimon/core/realtime/arrow_realtime_store.cpp index 298ac476f..85d956621 100644 --- a/src/paimon/core/realtime/arrow_realtime_store.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store.cpp @@ -585,7 +585,8 @@ class ArrowRealtimeStore::AppendQueryBatchReader : public BatchReader { RealtimeIndexEvaluator::Evaluate( segment->GetIndex(), predicate_, OffsetRange(std::max(segment_range.begin, offset_range_.begin), - std::min(segment_range.end, offset_range_.end)))); + std::min(segment_range.end, offset_range_.end)), + memory_pool_)); if (indexed) { if (candidates_) { *candidates_ &= *indexed; @@ -802,12 +803,12 @@ class ArrowRealtimeStore::IndexReadSource final : public RealtimeIndexReadSource } Result>> CreateIndexReaders( - const std::string& field_name) const override { + const std::string& field_name, const std::shared_ptr& pool) const override { std::vector> readers; if (segment_->GetIndex()) { for (const auto& index : segment_->GetIndex()->FindIndexes(field_name)) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr reader, - index->CreateReader(offset_range_)); + index->CreateReader(offset_range_, pool)); if (reader) { readers.push_back(std::move(reader)); } diff --git a/src/paimon/core/realtime/index/hash/hash_realtime_index.cpp b/src/paimon/core/realtime/index/hash/hash_realtime_index.cpp index 552d5e29e..a105a25ae 100644 --- a/src/paimon/core/realtime/index/hash/hash_realtime_index.cpp +++ b/src/paimon/core/realtime/index/hash/hash_realtime_index.cpp @@ -196,7 +196,7 @@ Status HashRealtimeIndex::AddBatch(::ArrowArray* batch) { } Result> HashRealtimeIndex::CreateReader( - const OffsetRange& offset_range) const { + const OffsetRange& offset_range, const std::shared_ptr&) const { if (offset_range.begin < 0 || offset_range.begin > offset_range.end) { return Status::Invalid("real-time index query offset range is invalid"); } diff --git a/src/paimon/core/realtime/index/hash/hash_realtime_index.h b/src/paimon/core/realtime/index/hash/hash_realtime_index.h index 44ae0b2da..3b92d1563 100644 --- a/src/paimon/core/realtime/index/hash/hash_realtime_index.h +++ b/src/paimon/core/realtime/index/hash/hash_realtime_index.h @@ -52,7 +52,7 @@ class HashRealtimeIndex final : public RealtimeIndex, Status AddBatch(::ArrowArray* batch) override; Result> CreateReader( - const OffsetRange& offset_range) const override; + const OffsetRange& offset_range, const std::shared_ptr& pool) const override; uint64_t GetMemoryUsage() const override; diff --git a/src/paimon/core/realtime/index/realtime_index_evaluator.cpp b/src/paimon/core/realtime/index/realtime_index_evaluator.cpp index e0c6b3806..8157fff32 100644 --- a/src/paimon/core/realtime/index/realtime_index_evaluator.cpp +++ b/src/paimon/core/realtime/index/realtime_index_evaluator.cpp @@ -34,7 +34,7 @@ namespace paimon { Result> RealtimeIndexEvaluator::Evaluate( const std::shared_ptr& indexes, const std::shared_ptr& predicate, - const OffsetRange& offset_range) { + const OffsetRange& offset_range, const std::shared_ptr& pool) { if (offset_range.begin < 0 || offset_range.begin > offset_range.end) { return Status::Invalid("real-time index query offset range is invalid"); } @@ -51,23 +51,24 @@ Result> RealtimeIndexEvaluator::Evaluate( if (!compound || compound->Children().empty()) { return Status::Invalid("invalid or empty real-time index compound predicate"); } - return EvaluateCompound(indexes, compound, offset_range); + return EvaluateCompound(indexes, compound, offset_range, pool); } const std::shared_ptr leaf = std::dynamic_pointer_cast(predicate); if (!leaf) { return std::optional(); } - return EvaluateLeaf(indexes, leaf, offset_range); + return EvaluateLeaf(indexes, leaf, offset_range, pool); } Result> RealtimeIndexEvaluator::EvaluateCompound( const std::shared_ptr& indexes, - const std::shared_ptr& compound, const OffsetRange& offset_range) { + const std::shared_ptr& compound, const OffsetRange& offset_range, + const std::shared_ptr& pool) { const Function::Type function = compound->GetFunction().GetType(); std::optional result; for (const std::shared_ptr& child : compound->Children()) { PAIMON_ASSIGN_OR_RAISE(std::optional candidates, - Evaluate(indexes, child, offset_range)); + Evaluate(indexes, child, offset_range, pool)); if (!candidates) { if (function == Function::Type::OR) { return std::optional(); @@ -90,12 +91,12 @@ Result> RealtimeIndexEvaluator::EvaluateCompound( Result> RealtimeIndexEvaluator::EvaluateLeaf( const std::shared_ptr& indexes, const std::shared_ptr& leaf, - const OffsetRange& offset_range) { + const OffsetRange& offset_range, const std::shared_ptr& pool) { std::optional result; for (const std::shared_ptr& index : indexes->FindIndexes(leaf->FieldName(), leaf->GetFieldType())) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr reader, - index->CreateReader(offset_range)); + index->CreateReader(offset_range, pool)); if (!reader) { continue; } diff --git a/src/paimon/core/realtime/index/realtime_index_evaluator.h b/src/paimon/core/realtime/index/realtime_index_evaluator.h index f749c0dce..9c6485b27 100644 --- a/src/paimon/core/realtime/index/realtime_index_evaluator.h +++ b/src/paimon/core/realtime/index/realtime_index_evaluator.h @@ -29,6 +29,7 @@ namespace paimon { class CompoundPredicate; class LeafPredicate; +class MemoryPool; class Predicate; class RealtimeIndexSet; @@ -39,16 +40,19 @@ class RealtimeIndexEvaluator { static Result> Evaluate( const std::shared_ptr& indexes, - const std::shared_ptr& predicate, const OffsetRange& offset_range); + const std::shared_ptr& predicate, const OffsetRange& offset_range, + const std::shared_ptr& pool); private: static Result> EvaluateCompound( const std::shared_ptr& indexes, - const std::shared_ptr& compound, const OffsetRange& offset_range); + const std::shared_ptr& compound, const OffsetRange& offset_range, + const std::shared_ptr& pool); static Result> EvaluateLeaf( const std::shared_ptr& indexes, - const std::shared_ptr& leaf, const OffsetRange& offset_range); + const std::shared_ptr& leaf, const OffsetRange& offset_range, + const std::shared_ptr& pool); }; } // namespace paimon diff --git a/src/paimon/core/realtime/index/realtime_index_test.cpp b/src/paimon/core/realtime/index/realtime_index_test.cpp index 8ae7152a3..57b623074 100644 --- a/src/paimon/core/realtime/index/realtime_index_test.cpp +++ b/src/paimon/core/realtime/index/realtime_index_test.cpp @@ -75,7 +75,7 @@ class RealtimeIndexTest : public testing::Test { Result> Evaluate(const std::shared_ptr& predicate, const OffsetRange& range) const { - return RealtimeIndexEvaluator::Evaluate(index_, predicate, range); + return RealtimeIndexEvaluator::Evaluate(index_, predicate, range, GetDefaultPool()); } Status Add(const std::string& json, const OffsetRange& range) const { @@ -240,9 +240,9 @@ TEST_F(RealtimeIndexTest, FieldReadersBindIndependentRangesAndShareAppendOnlyPos std::shared_ptr key_index = std::move(key_indexes[0]); ASSERT_NE(key_index, value_indexes[0]); ASSERT_OK_AND_ASSIGN(std::shared_ptr first, - key_index->CreateReader(OffsetRange(10, 14))); + key_index->CreateReader(OffsetRange(10, 14), GetDefaultPool())); ASSERT_OK_AND_ASSIGN(std::shared_ptr second, - key_index->CreateReader(OffsetRange(20, 21))); + key_index->CreateReader(OffsetRange(20, 21), GetDefaultPool())); ASSERT_OK(Add(R"([[25, 1, "d"]])", OffsetRange(25, 26))); ASSERT_OK_AND_ASSIGN(std::optional candidates, first->VisitEqual(Literal(int64_t{1}))); @@ -252,7 +252,7 @@ TEST_F(RealtimeIndexTest, FieldReadersBindIndependentRangesAndShareAppendOnlyPos ASSERT_TRUE(candidates.has_value()); ASSERT_EQ(RoaringBitmap64::From({20}), *candidates); ASSERT_OK_AND_ASSIGN(std::shared_ptr later, - key_index->CreateReader(OffsetRange(25, 26))); + key_index->CreateReader(OffsetRange(25, 26), GetDefaultPool())); ASSERT_OK_AND_ASSIGN(candidates, later->VisitEqual(Literal(int64_t{1}))); ASSERT_TRUE(candidates.has_value()); ASSERT_EQ(RoaringBitmap64::From({25}), *candidates); @@ -261,10 +261,10 @@ TEST_F(RealtimeIndexTest, FieldReadersBindIndependentRangesAndShareAppendOnlyPos ASSERT_EQ(RoaringBitmap64::From({10}), *candidates); ASSERT_OK_AND_ASSIGN(candidates, first->VisitGreaterThan(Literal(int64_t{0}))); ASSERT_FALSE(candidates.has_value()); - ASSERT_NOK(key_index->CreateReader(OffsetRange(-1, 20))); - ASSERT_NOK(key_index->CreateReader(OffsetRange(20, 10))); + ASSERT_NOK(key_index->CreateReader(OffsetRange(-1, 20), GetDefaultPool())); + ASSERT_NOK(key_index->CreateReader(OffsetRange(20, 10), GetDefaultPool())); ASSERT_OK_AND_ASSIGN(std::shared_ptr empty, - key_index->CreateReader(OffsetRange(10, 10))); + key_index->CreateReader(OffsetRange(10, 10), GetDefaultPool())); ASSERT_OK_AND_ASSIGN(candidates, empty->VisitEqual(Literal(int64_t{1}))); ASSERT_TRUE(candidates.has_value()); ASSERT_TRUE(candidates->IsEmpty()); @@ -325,7 +325,7 @@ TEST_F(RealtimeIndexTest, FactoryAcceptsOnlyOneIndexedFieldAndReaderNeedsNoField ASSERT_OK_AND_ASSIGN(std::shared_ptr index, RealtimeIndexFactory::Get("hash", &c_schema, {}, GetDefaultPool())); ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, - index->CreateReader(OffsetRange(0, 10))); + index->CreateReader(OffsetRange(0, 10), GetDefaultPool())); ASSERT_OK_AND_ASSIGN(std::optional candidates, reader->VisitEqual(Literal(int64_t{1}))); ASSERT_TRUE(candidates.has_value()); @@ -343,7 +343,7 @@ TEST_F(RealtimeIndexTest, FactoryAcceptsOnlyOneIndexedFieldAndReaderNeedsNoField ASSERT_TRUE(arrow::ExportArray(*data, &batch).ok()); ScopeGuard batch_guard([&batch]() { ArrowArrayRelease(&batch); }); ASSERT_OK(index->AddBatch(&batch)); - ASSERT_OK_AND_ASSIGN(reader, index->CreateReader(OffsetRange(11, 31))); + ASSERT_OK_AND_ASSIGN(reader, index->CreateReader(OffsetRange(11, 31), GetDefaultPool())); ASSERT_OK_AND_ASSIGN(candidates, reader->VisitEqual(Literal(int64_t{1}))); ASSERT_TRUE(candidates.has_value()); ASSERT_EQ(RoaringBitmap64::From({30}), *candidates); @@ -369,7 +369,7 @@ TEST_F(RealtimeIndexTest, IndexesTimeField) { ASSERT_OK(index->AddBatch(&batch)); ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, - index->CreateReader(OffsetRange(0, 41))); + index->CreateReader(OffsetRange(0, 41), GetDefaultPool())); ASSERT_OK_AND_ASSIGN(std::optional candidates, reader->VisitEqual(Literal(FieldType::TIME, 45296123))); ASSERT_TRUE(candidates.has_value()); diff --git a/src/paimon/core/realtime/precise_realtime_index_read.cpp b/src/paimon/core/realtime/precise_realtime_index_read.cpp index 4b02f4d21..367bc21da 100644 --- a/src/paimon/core/realtime/precise_realtime_index_read.cpp +++ b/src/paimon/core/realtime/precise_realtime_index_read.cpp @@ -214,8 +214,9 @@ class RealtimeSourceRead { const std::string& field_name) { auto found = readers_.find(field_name); if (found == readers_.end()) { - PAIMON_ASSIGN_OR_RAISE(std::vector> readers, - source_->CreateIndexReaders(field_name)); + PAIMON_ASSIGN_OR_RAISE( + std::vector> readers, + source_->CreateIndexReaders(field_name, context_->GetMemoryPool())); for (const auto& reader : readers) { if (!reader) { return Status::Invalid("Real-time source returned a null index reader"); diff --git a/src/paimon/core/realtime/realtime_utils.cpp b/src/paimon/core/realtime/realtime_utils.cpp new file mode 100644 index 000000000..89e1102b7 --- /dev/null +++ b/src/paimon/core/realtime/realtime_utils.cpp @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/realtime/realtime_utils.h" + +#include "arrow/api.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/core/io/file_index_options.h" +#include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/file_index/file_indexer.h" +#include "paimon/file_index/file_indexer_factory.h" +#include "paimon/macros.h" +#include "paimon/status.h" + +namespace paimon { + +Status RealtimeUtils::ValidateBranch(const std::string& branch) { + if (!BranchManager::IsMainBranch(BranchManager::NormalizeBranch(branch))) { + return Status::Invalid("real-time operations only support the main branch"); + } + return Status::OK(); +} + +Result RealtimeUtils::GetDeduplicateKeyPosition( + const std::shared_ptr& schema, + const std::vector& deduplicate_key_fields) { + if (!schema) { + return Status::Invalid("real-time deduplicate schema must not be null"); + } + // TODO(xinyu.lxy): Support multiple user-defined deduplicate key fields. This requires + // composite-key projection and serialization in the configured field order. + if (deduplicate_key_fields.size() != 1) { + return Status::Invalid( + "real-time deduplicate currently requires exactly one deduplicate key field"); + } + const int32_t key_position = schema->GetFieldIndex(deduplicate_key_fields[0]); + if (key_position < 0) { + return Status::Invalid("real-time deduplicate key field does not exist: ", + deduplicate_key_fields[0]); + } + return key_position; +} + +Status RealtimeUtils::ValidateOffsetField(const std::shared_ptr& schema) { + if (!schema) { + return Status::Invalid("real-time schema must not be null"); + } + const std::shared_ptr offset_field = + schema->GetFieldByName(SpecialFields::RealtimeOffset().Name()); + if (!offset_field || offset_field->type()->id() != arrow::Type::INT64 || + offset_field->nullable()) { + return Status::Invalid("real-time schema requires non-null int64 _REALTIME_OFFSET"); + } + return Status::OK(); +} + +Status RealtimeUtils::ValidateDeduplicateSchema( + const std::shared_ptr& schema, + const std::vector& deduplicate_key_fields) { + PAIMON_RETURN_NOT_OK(ValidateOffsetField(schema)); + PAIMON_ASSIGN_OR_RAISE(int32_t key_position, + GetDeduplicateKeyPosition(schema, deduplicate_key_fields)); + const std::shared_ptr& key_field = schema->field(key_position); + if (key_field->name() == SpecialFields::RealtimeOffset().Name()) { + return Status::Invalid("real-time offset field cannot be used as the deduplicate key"); + } + if (key_field->nullable()) { + return Status::Invalid("real-time deduplicate key field must be non-null: ", + key_field->name()); + } + if (ArrowSchemaValidator::IsNestedType(key_field->type())) { + return Status::Invalid("real-time deduplicate key field must not be nested: ", + key_field->name()); + } + return Status::OK(); +} + +Status RealtimeUtils::ValidateDeduplicateFileIndex(const CoreOptions& options) { + if (!options.FileIndexReadEnabled()) { + return Status::Invalid( + "real-time deduplicate mode requires 'file-index.read.enabled=true'"); + } + PAIMON_ASSIGN_OR_RAISE(FileIndexOptions file_indexes, + FileIndexOptions::FromCoreOptions(options)); + const std::string& offset_name = SpecialFields::RealtimeOffset().Name(); + for (const FileIndexDefinition& definition : file_indexes.Definitions()) { + if (definition.column_name != offset_name) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + FileIndexerFactory::Get(definition.index_type, definition.options)); + if (indexer && indexer->SupportsExactPredicate(Function::Type::IN, FieldType::BIGINT) && + indexer->SupportsExactPredicate(Function::Type::NOT_IN, FieldType::BIGINT)) { + return Status::OK(); + } + } + return Status::Invalid( + "real-time deduplicate mode requires an index supporting exact BIGINT IN and NOT_IN " + "row positions on ", + offset_name); +} + +} // namespace paimon diff --git a/src/paimon/core/realtime/realtime_utils.h b/src/paimon/core/realtime/realtime_utils.h index 6b28a04ce..a8c15d457 100644 --- a/src/paimon/core/realtime/realtime_utils.h +++ b/src/paimon/core/realtime/realtime_utils.h @@ -25,14 +25,9 @@ #include #include "arrow/api.h" -#include "paimon/common/table/special_fields.h" #include "paimon/core/core_options.h" -#include "paimon/core/io/file_index_options.h" -#include "paimon/core/schema/arrow_schema_validator.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/file_index/file_indexer.h" -#include "paimon/file_index/file_indexer_factory.h" #include "paimon/result.h" +#include "paimon/visibility.h" namespace paimon { @@ -41,92 +36,18 @@ class RealtimeUtils { RealtimeUtils() = delete; ~RealtimeUtils() = delete; - static Status ValidateBranch(const std::string& branch) { - if (!BranchManager::IsMainBranch(BranchManager::NormalizeBranch(branch))) { - return Status::Invalid("real-time operations only support the main branch"); - } - return Status::OK(); - } + static Status ValidateBranch(const std::string& branch); static Result GetDeduplicateKeyPosition( const std::shared_ptr& schema, - const std::vector& deduplicate_key_fields) { - if (!schema) { - return Status::Invalid("real-time deduplicate schema must not be null"); - } - // TODO(xinyu.lxy): Support multiple user-defined deduplicate key fields. This requires - // composite-key projection and serialization in the configured field order. - if (deduplicate_key_fields.size() != 1) { - return Status::Invalid( - "real-time deduplicate currently requires exactly one deduplicate key field"); - } - const int32_t key_position = schema->GetFieldIndex(deduplicate_key_fields[0]); - if (key_position < 0) { - return Status::Invalid("real-time deduplicate key field does not exist: ", - deduplicate_key_fields[0]); - } - return key_position; - } + const std::vector& deduplicate_key_fields); - static Status ValidateOffsetField(const std::shared_ptr& schema) { - if (!schema) { - return Status::Invalid("real-time schema must not be null"); - } - const std::shared_ptr offset_field = - schema->GetFieldByName(SpecialFields::RealtimeOffset().Name()); - if (!offset_field || offset_field->type()->id() != arrow::Type::INT64 || - offset_field->nullable()) { - return Status::Invalid("real-time schema requires non-null int64 _REALTIME_OFFSET"); - } - return Status::OK(); - } + PAIMON_EXPORT static Status ValidateOffsetField(const std::shared_ptr& schema); - static Status ValidateDeduplicateSchema( - const std::shared_ptr& schema, - const std::vector& deduplicate_key_fields) { - PAIMON_RETURN_NOT_OK(ValidateOffsetField(schema)); - PAIMON_ASSIGN_OR_RAISE(int32_t key_position, - GetDeduplicateKeyPosition(schema, deduplicate_key_fields)); - const std::shared_ptr& key_field = schema->field(key_position); - if (key_field->name() == SpecialFields::RealtimeOffset().Name()) { - return Status::Invalid("real-time offset field cannot be used as the deduplicate key"); - } - if (key_field->nullable()) { - return Status::Invalid("real-time deduplicate key field must be non-null: ", - key_field->name()); - } - if (ArrowSchemaValidator::IsNestedType(key_field->type())) { - return Status::Invalid("real-time deduplicate key field must not be nested: ", - key_field->name()); - } - return Status::OK(); - } + static Status ValidateDeduplicateSchema(const std::shared_ptr& schema, + const std::vector& deduplicate_key_fields); - static Status ValidateDeduplicateFileIndex(const CoreOptions& options) { - if (!options.FileIndexReadEnabled()) { - return Status::Invalid( - "real-time deduplicate mode requires 'file-index.read.enabled=true'"); - } - PAIMON_ASSIGN_OR_RAISE(FileIndexOptions file_indexes, - FileIndexOptions::FromCoreOptions(options)); - const std::string& offset_name = SpecialFields::RealtimeOffset().Name(); - for (const FileIndexDefinition& definition : file_indexes.Definitions()) { - if (definition.column_name != offset_name) { - continue; - } - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr indexer, - FileIndexerFactory::Get(definition.index_type, definition.options)); - if (indexer && indexer->SupportsExactPredicate(Function::Type::IN, FieldType::BIGINT) && - indexer->SupportsExactPredicate(Function::Type::NOT_IN, FieldType::BIGINT)) { - return Status::OK(); - } - } - return Status::Invalid( - "real-time deduplicate mode requires an index supporting exact BIGINT IN and NOT_IN " - "row positions on ", - offset_name); - } + static Status ValidateDeduplicateFileIndex(const CoreOptions& options); }; } // namespace paimon diff --git a/src/paimon/indexer/lumina/CMakeLists.txt b/src/paimon/indexer/lumina/CMakeLists.txt index 280956e7c..8b57b22e8 100644 --- a/src/paimon/indexer/lumina/CMakeLists.txt +++ b/src/paimon/indexer/lumina/CMakeLists.txt @@ -25,6 +25,8 @@ if(PAIMON_ENABLE_LUMINA) lumina_index_accumulator.cpp lumina_index_options.cpp lumina_index_searcher.cpp + lumina_realtime_index.cpp + lumina_realtime_index_factory.cpp lumina_tag_utils.cpp) add_paimon_lib(paimon_lumina_index SOURCES @@ -50,6 +52,7 @@ if(PAIMON_ENABLE_LUMINA) lumina_file_index_test.cpp lumina_file_io_test.cpp lumina_global_index_test.cpp + lumina_realtime_index_test.cpp EXTRA_INCLUDES ${LUMINA_INCLUDE_DIR} STATIC_LINK_LIBS diff --git a/src/paimon/indexer/lumina/lumina_api_test.cpp b/src/paimon/indexer/lumina/lumina_api_test.cpp index 8b7b6a480..3ea43d70a 100644 --- a/src/paimon/indexer/lumina/lumina_api_test.cpp +++ b/src/paimon/indexer/lumina/lumina_api_test.cpp @@ -36,7 +36,7 @@ class LuminaInterfaceTest : public ::testing::Test { const std::vector<::lumina::core::vector_id_t>& row_ids) const { auto fs = std::make_shared(); std::shared_ptr paimon_pool = GetMemoryPool(); - auto pool = std::make_shared(paimon_pool); + auto pool = std::make_shared(/*track_memory=*/false, paimon_pool); ::lumina::core::MemoryResourceConfig memory_resource(pool.get()); { // create writer ::lumina::api::BuilderOptions builder_options; @@ -74,7 +74,7 @@ class LuminaInterfaceTest : public ::testing::Test { ASSERT_EQ(expected_row_ids.size(), expected_distances.size()); auto fs = std::make_shared(); std::shared_ptr paimon_pool = GetMemoryPool(); - auto pool = std::make_shared(paimon_pool); + auto pool = std::make_shared(/*track_memory=*/false, paimon_pool); ::lumina::core::MemoryResourceConfig memory_resource(pool.get()); { // create reader diff --git a/src/paimon/indexer/lumina/lumina_file_index.cpp b/src/paimon/indexer/lumina/lumina_file_index.cpp index 2308bd7a6..e5d24d7d1 100644 --- a/src/paimon/indexer/lumina/lumina_file_index.cpp +++ b/src/paimon/indexer/lumina/lumina_file_index.cpp @@ -191,7 +191,7 @@ Result> LuminaFileIndexer::CreateWriter( PAIMON_RETURN_NOT_OK(LuminaTagUtils::ValidateTagFields(*struct_type, tag_fields)); PAIMON_ASSIGN_OR_RAISE(::lumina::api::BuilderOptions builder_options, LuminaIndexOptions::CreateBuilderOptions(options_)); - auto lumina_pool = std::make_shared(pool); + auto lumina_pool = std::make_shared(/*track_memory=*/false, pool); return std::make_shared(schema->field(0)->name(), std::move(struct_type), index_info, std::move(builder_options), std::move(tag_fields), std::move(lumina_pool)); @@ -210,7 +210,7 @@ Result> LuminaFileIndexer::CreateReader( PAIMON_ASSIGN_OR_RAISE(LuminaIndexInfo index_info, LuminaIndexOptions::GetIndexInfo(options_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr artifact_input, OffsetInputStream::Create(input_stream, length, start)); - auto lumina_pool = std::make_shared(pool); + auto lumina_pool = std::make_shared(/*track_memory=*/false, pool); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr searcher, LuminaIndexSearcher::Open(options_, index_info, artifact_input, lumina_pool)); diff --git a/src/paimon/indexer/lumina/lumina_global_index.cpp b/src/paimon/indexer/lumina/lumina_global_index.cpp index f544e871e..b46348aee 100644 --- a/src/paimon/indexer/lumina/lumina_global_index.cpp +++ b/src/paimon/indexer/lumina/lumina_global_index.cpp @@ -82,7 +82,7 @@ Result> LuminaGlobalIndex::CreateWriter( return Status::Invalid("Lumina checkpoint requires a checkpoint-capable file writer"); } } - auto lumina_pool = std::make_shared(pool); + auto lumina_pool = std::make_shared(/*track_memory=*/false, pool); return std::make_shared( field_name, arrow_type, dimension, file_writer, std::move(builder_options), ::lumina::api::IOOptions(), lumina_options, std::move(tag_fields), checkpoint_file_manager, @@ -128,7 +128,7 @@ Result> LuminaGlobalIndex::CreateReader( PAIMON_ASSIGN_OR_RAISE(LuminaIndexReader::IndexInfo index_info, LuminaIndexReader::GetIndexInfo(io_meta)); - auto lumina_pool = std::make_shared(pool); + auto lumina_pool = std::make_shared(/*track_memory=*/false, pool); auto lumina_options = OptionsUtils::FetchOptionsWithPrefix(LuminaDefines::kOptionKeyPrefix, options_); // get input stream and open index diff --git a/src/paimon/indexer/lumina/lumina_global_index_test.cpp b/src/paimon/indexer/lumina/lumina_global_index_test.cpp index 9e2020d6f..a6d6c6bb3 100644 --- a/src/paimon/indexer/lumina/lumina_global_index_test.cpp +++ b/src/paimon/indexer/lumina/lumina_global_index_test.cpp @@ -554,7 +554,7 @@ TEST_F(LuminaGlobalIndexTest, TestResumeFromCheckpoint) { .Set(::lumina::core::kEncodingType, ::lumina::core::kEncodingRawf32) .Set(::lumina::core::kExtensionCkptThreshold, static_cast(1)) .Set(::lumina::core::kExtensionCkptCount, static_cast(1)); - LuminaMemoryPool lumina_pool(pool_); + LuminaMemoryPool lumina_pool(/*track_memory=*/false, pool_); ::lumina::core::MemoryResourceConfig memory_resource(&lumina_pool); auto builder_result = ::lumina::api::LuminaBuilder::Create(builder_options, memory_resource); @@ -672,7 +672,7 @@ TEST_F(LuminaGlobalIndexTest, TestBuildFailureDiscardsCheckpointAndRetries) { .Set(::lumina::core::kEncodingType, ::lumina::core::kEncodingRawf32) .Set(::lumina::core::kExtensionCkptThreshold, static_cast(1)) .Set(::lumina::core::kExtensionCkptCount, static_cast(1)); - LuminaMemoryPool lumina_pool(pool_); + LuminaMemoryPool lumina_pool(/*track_memory=*/false, pool_); ::lumina::core::MemoryResourceConfig memory_resource(&lumina_pool); auto builder_result = ::lumina::api::LuminaBuilder::Create(builder_options, memory_resource); ASSERT_TRUE(builder_result.IsOk()) << builder_result.GetStatus().Message(); diff --git a/src/paimon/indexer/lumina/lumina_index_options.cpp b/src/paimon/indexer/lumina/lumina_index_options.cpp index 526982fbf..adf7c82b2 100644 --- a/src/paimon/indexer/lumina/lumina_index_options.cpp +++ b/src/paimon/indexer/lumina/lumina_index_options.cpp @@ -18,6 +18,7 @@ #include "paimon/indexer/lumina/lumina_index_options.h" +#include #include #include "fmt/format.h" @@ -71,6 +72,41 @@ Result<::lumina::api::BuilderOptions> LuminaIndexOptions::CreateBuilderOptions( return builder_options; } +Result<::lumina::api::StreamerOptions> LuminaIndexOptions::CreateStreamerOptions( + const std::map& lumina_options) { + // TODO(xinyu.lxy): Use NormalizeStreamerOptions when Lumina exposes it. Convert types here; + // leave option value validation and backend support checks to LuminaStreamer::Create. + static const std::set integer_keys = {"index.dimension", + "bruteforce.streamer.capacity", + "bruteforce.streamer.rows_per_block", + "flatnsw.streamer.capacity", + "flatnsw.streamer.rows_per_block", + "flatnsw.build.ef_construction", + "flatnsw.build.neighbor_count"}; + static const std::set double_keys = {"flatnsw.build.slack_pruning_factor", + "extension.build.tag.max_range_label_ratio", + "extension.build.tag.min_range_label_ratio"}; + ::lumina::api::StreamerOptions options; + for (const auto& [key, value] : lumina_options) { + if (integer_keys.count(key)) { + PAIMON_ASSIGN_OR_RAISE(int64_t parsed, + OptionsUtils::GetValueFromMap(lumina_options, key)); + options.Set(key, parsed); + } else if (double_keys.count(key)) { + PAIMON_ASSIGN_OR_RAISE(double parsed, + OptionsUtils::GetValueFromMap(lumina_options, key)); + options.Set(key, parsed); + } else if (key == ::lumina::core::kEnableExperimental) { + PAIMON_ASSIGN_OR_RAISE(bool parsed, + OptionsUtils::GetValueFromMap(lumina_options, key)); + options.Set(key, parsed); + } else { + options.Set(key, value); + } + } + return options; +} + bool LuminaIndexOptions::IsCheckpointEnabled( const std::map& lumina_options) { return lumina_options.count(std::string(::lumina::core::kExtensionCkptThreshold)) != 0 || diff --git a/src/paimon/indexer/lumina/lumina_index_options.h b/src/paimon/indexer/lumina/lumina_index_options.h index 1c3cc355e..c1156b57f 100644 --- a/src/paimon/indexer/lumina/lumina_index_options.h +++ b/src/paimon/indexer/lumina/lumina_index_options.h @@ -52,6 +52,9 @@ class LuminaIndexOptions { static Result<::lumina::api::BuilderOptions> CreateBuilderOptions( const std::map& lumina_options); + static Result<::lumina::api::StreamerOptions> CreateStreamerOptions( + const std::map& lumina_options); + static bool IsCheckpointEnabled(const std::map& lumina_options); static Result<::lumina::api::SearcherOptions> CreateSearcherOptions( diff --git a/src/paimon/indexer/lumina/lumina_index_searcher.cpp b/src/paimon/indexer/lumina/lumina_index_searcher.cpp index 33d4e86f6..9949e2d70 100644 --- a/src/paimon/indexer/lumina/lumina_index_searcher.cpp +++ b/src/paimon/indexer/lumina/lumina_index_searcher.cpp @@ -79,8 +79,8 @@ Result<::lumina::api::LuminaSearcher::SearchResult> LuminaIndexSearcher::Search( PAIMON_ASSIGN_OR_RAISE(::lumina::api::SearchOptions search_options, LuminaIndexOptions::CreateSearchOptions(*vector_search, index_info_)); ::lumina::api::Query query(vector_search->query.data(), vector_search->query.size()); - LuminaMemoryPool session_pool(vector_search->pool ? vector_search->pool - : pool_->GetPaimonPool()); + LuminaMemoryPool session_pool( + /*track_memory=*/false, vector_search->pool ? vector_search->pool : pool_->GetPaimonPool()); if (vector_search->predicate) { if (!searcher_with_tag_) { diff --git a/src/paimon/indexer/lumina/lumina_memory_pool.h b/src/paimon/indexer/lumina/lumina_memory_pool.h index e5dcee9d6..58bc35455 100644 --- a/src/paimon/indexer/lumina/lumina_memory_pool.h +++ b/src/paimon/indexer/lumina/lumina_memory_pool.h @@ -18,6 +18,8 @@ #pragma once +#include +#include #include #include @@ -25,7 +27,8 @@ namespace paimon::lumina { class LuminaMemoryPool : public std::pmr::memory_resource { public: - explicit LuminaMemoryPool(const std::shared_ptr& pool) : pool_(pool) {} + explicit LuminaMemoryPool(bool track_memory, const std::shared_ptr& pool) + : pool_(pool), track_memory_(track_memory) {} ~LuminaMemoryPool() override = default; @@ -36,17 +39,30 @@ class LuminaMemoryPool : public std::pmr::memory_resource { return pool_; } + /// Bytes currently allocated through this resource, excluding other users of the same pool. + /// Returns zero when memory tracking is disabled. + uint64_t GetMemoryUsage() const { + return track_memory_ ? memory_usage_.load(std::memory_order_relaxed) : 0; + } + private: void* do_allocate(std::size_t bytes, std::size_t alignment) override { static const size_t min_alignment = alignof(std::max_align_t); if (alignment < min_alignment) { alignment = min_alignment; } - return pool_->Malloc(bytes, alignment); + void* ptr = pool_->Malloc(bytes, alignment); + if (track_memory_) { + memory_usage_.fetch_add(bytes, std::memory_order_relaxed); + } + return ptr; } void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override { pool_->Free(p, bytes, alignment); + if (track_memory_) { + memory_usage_.fetch_sub(bytes, std::memory_order_relaxed); + } } bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { @@ -55,5 +71,7 @@ class LuminaMemoryPool : public std::pmr::memory_resource { private: std::shared_ptr pool_; + const bool track_memory_; + std::atomic memory_usage_{0}; }; } // namespace paimon::lumina diff --git a/src/paimon/indexer/lumina/lumina_realtime_index.cpp b/src/paimon/indexer/lumina/lumina_realtime_index.cpp new file mode 100644 index 000000000..b046d2b77 --- /dev/null +++ b/src/paimon/indexer/lumina/lumina_realtime_index.cpp @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/indexer/lumina/lumina_realtime_index.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "lumina/core/Constants.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/core/realtime/realtime_utils.h" +#include "paimon/indexer/lumina/lumina_memory_pool.h" +#include "paimon/indexer/lumina/lumina_utils.h" + +namespace paimon::lumina { + +LuminaRealtimeIndex::LuminaRealtimeIndex( + std::shared_ptr schema, LuminaIndexInfo index_info, + std::unique_ptr<::lumina::extensions::StreamWithFilterExtension> filter, + std::unique_ptr<::lumina::api::LuminaStreamer> streamer, std::unique_ptr pool) + : schema_(std::move(schema)), + index_info_(std::move(index_info)), + pool_(std::move(pool)), + filter_(std::move(filter)), + streamer_(std::move(streamer)) {} + +LuminaRealtimeIndex::~LuminaRealtimeIndex() = default; + +Result> LuminaRealtimeIndex::Create( + const std::shared_ptr& schema, const std::map& options, + const std::shared_ptr& pool) { + if (!pool) { + return Status::Invalid("Lumina realtime index requires a memory pool"); + } + PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(schema)); + if (schema->num_fields() != 2 || schema->field(1)->type()->id() != arrow::Type::LIST || + checked_pointer_cast(schema->field(1)->type())->value_type()->id() != + arrow::Type::FLOAT) { + return Status::Invalid("Lumina realtime index requires [offset, LIST]"); + } + const std::map lumina_options = + OptionsUtils::FetchOptionsWithPrefix(LuminaDefines::kOptionKeyPrefix, options); + if (lumina_options.count(std::string(::lumina::core::kExtensionTagSchema))) { + return Status::NotImplemented("Lumina realtime tags are not supported"); + } + PAIMON_ASSIGN_OR_RAISE(LuminaIndexInfo info, LuminaIndexOptions::GetIndexInfo(lumina_options)); + PAIMON_ASSIGN_OR_RAISE(::lumina::api::StreamerOptions streamer_options, + LuminaIndexOptions::CreateStreamerOptions(lumina_options)); + auto lumina_pool = std::make_unique(/*track_memory=*/true, pool); + auto filter = std::make_unique<::lumina::extensions::StreamWithFilterExtension>(); + PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( + ::lumina::api::LuminaStreamer native_streamer, + ::lumina::api::LuminaStreamer::Create( + streamer_options, ::lumina::core::MemoryResourceConfig(lumina_pool.get()))); + auto streamer = std::make_unique<::lumina::api::LuminaStreamer>(std::move(native_streamer)); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(streamer->Attach(*filter)); + return std::shared_ptr(new LuminaRealtimeIndex( + schema, std::move(info), std::move(filter), std::move(streamer), std::move(lumina_pool))); +} + +Status LuminaRealtimeIndex::AddBatch(::ArrowArray* batch) { + if (!batch || !batch->release) { + return Status::Invalid("Lumina realtime batch is null or released"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr imported, + arrow::ImportArray(batch, arrow::struct_(schema_->fields()))); + if (write_failed_) { + return Status::Invalid("Lumina realtime index cannot write after an insertion failure"); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(imported->ValidateFull()); + const auto data = checked_pointer_cast(imported); + const auto offsets = checked_pointer_cast(data->field(0)); + const auto vectors = checked_pointer_cast(data->field(1)); + const auto values = checked_pointer_cast(vectors->values()); + if (data->null_count() != 0 || offsets->null_count() != 0) { + return Status::Invalid("Lumina realtime rows and offsets must not be null"); + } + for (int64_t row = 0; row < data->length(); ++row) { + if (vectors->IsNull(row)) { + continue; + } + if (vectors->value_length(row) != static_cast(index_info_.dimension)) { + return Status::Invalid("Lumina realtime vector dimension mismatch"); + } + for (int64_t pos = vectors->value_offset(row); pos < vectors->value_offset(row + 1); + ++pos) { + if (values->IsNull(pos) || !std::isfinite(values->Value(pos))) { + return Status::Invalid( + "Lumina realtime vector elements must be finite non-null floats"); + } + } + } + for (int64_t row = 0; row < data->length(); ++row) { + if (vectors->IsNull(row)) { + continue; + } + const Status status = LuminaToPaimonStatus(streamer_->Insert( + values->raw_values() + vectors->value_offset(row), offsets->Value(row))); + if (!status.ok()) { + write_failed_ = true; + return status; + } + } + return Status::OK(); +} + +Result> LuminaRealtimeIndex::CreateReader( + const OffsetRange& offset_range, const std::shared_ptr& pool) const { + if (offset_range.begin < 0 || offset_range.begin > offset_range.end) { + return Status::Invalid("Lumina realtime query offset range is invalid"); + } + if (!pool) { + return Status::Invalid("Lumina realtime reader requires a query memory pool"); + } + return std::shared_ptr( + std::make_shared(shared_from_this(), offset_range, pool)); +} + +uint64_t LuminaRealtimeIndex::GetMemoryUsage() const { + return pool_->GetMemoryUsage(); +} + +LuminaRealtimeIndexReader::LuminaRealtimeIndexReader( + std::shared_ptr index, const OffsetRange& offset_range, + const std::shared_ptr& pool) + : index_(std::move(index)), offset_range_(offset_range), pool_(pool) {} + +bool LuminaRealtimeIndexReader::SupportsSearch(SearchType search_type) const { + return search_type == SearchType::VECTOR_SEARCH; +} + +Result> LuminaRealtimeIndexReader::VisitVectorSearch( + const std::shared_ptr& vector_search) { + if (!vector_search || vector_search->field_name != index_->schema_->field(1)->name() || + vector_search->limit <= 0) { + return Status::Invalid("invalid Lumina realtime vector search field or limit"); + } + if (vector_search->predicate) { + return Status::NotImplemented("Lumina realtime tags are not supported"); + } + for (float value : vector_search->query) { + if (!std::isfinite(value)) { + return Status::Invalid("Lumina realtime query vector must contain finite floats"); + } + } + PAIMON_ASSIGN_OR_RAISE( + ::lumina::api::SearchOptions options, + LuminaIndexOptions::CreateSearchOptions(*vector_search, index_->index_info_)); + if (offset_range_.Empty()) { + return ScoredRealtimeIndexResult::Create(RoaringBitmap64(), {}); + } + auto filter = [range = offset_range_, + user_filter = vector_search->pre_filter](::lumina::core::vector_id_t offset) { + return offset >= static_cast(range.begin) && + offset < static_cast(range.end) && + (!user_filter || user_filter(static_cast(offset))); + }; + ::lumina::api::Query query(vector_search->query.data(), vector_search->query.size()); + // Use the search session pool when provided, otherwise the reader's query pool. + LuminaMemoryPool session_pool(/*track_memory=*/false, + vector_search->pool ? vector_search->pool : pool_); + PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( + ::lumina::api::SearchResult hits, + index_->filter_->SearchWithFilter(query, std::move(filter), options, session_pool)); + std::sort(hits.topk.begin(), hits.topk.end(), + [](const auto& left, const auto& right) { return left.id < right.id; }); + RoaringBitmap64 offsets; + std::vector scores; + scores.reserve(hits.topk.size()); + int64_t previous = -1; + for (const auto& hit : hits.topk) { + if (hit.id > static_cast(std::numeric_limits::max())) { + return Status::Invalid("Lumina returned an out-of-range realtime offset"); + } + const int64_t offset = static_cast(hit.id); + if (offset < offset_range_.begin || offset >= offset_range_.end || offset <= previous) { + return Status::Invalid("Lumina returned invalid or duplicate realtime offsets"); + } + offsets.Add(offset); + scores.push_back(hit.distance); + previous = offset; + } + return ScoredRealtimeIndexResult::Create(std::move(offsets), std::move(scores)); +} + +// TODO(xinyu.lxy): Dump the memory index with offset-to-(file, row-id) remapping when Lumina +// supports exporting the index structure, excluding skipped rows and respecting file rolling. + +} // namespace paimon::lumina diff --git a/src/paimon/indexer/lumina/lumina_realtime_index.h b/src/paimon/indexer/lumina/lumina_realtime_index.h new file mode 100644 index 000000000..5f742941e --- /dev/null +++ b/src/paimon/indexer/lumina/lumina_realtime_index.h @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "lumina/api/LuminaStreamer.h" +#include "lumina/extensions/SearchWithFilterExtension.h" +#include "paimon/indexer/lumina/lumina_index_options.h" +#include "paimon/realtime/index/realtime_index.h" +#include "paimon/realtime/index/realtime_index_reader.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { +class MemoryPool; +} // namespace paimon + +namespace paimon::lumina { + +class LuminaRealtimeIndexReader; +class LuminaMemoryPool; + +/// Append-only vector index with one writer and concurrent readers. IDs are realtime offsets. +class LuminaRealtimeIndex final : public RealtimeIndex, + public std::enable_shared_from_this { + public: + /// Options use the lumina. prefix, removed internally. The vector field is LIST. + static Result> Create( + const std::shared_ptr& schema, + const std::map& options, const std::shared_ptr& pool); + + ~LuminaRealtimeIndex() override; + + /// Consumes [_REALTIME_OFFSET, vector]. Validates the whole batch before inserting. + /// The caller guarantees unique, increasing offsets and publishes query ranges only after + /// successful writes. An insertion failure disables subsequent writes. + Status AddBatch(::ArrowArray* batch) override; + + Result> CreateReader( + const OffsetRange& offset_range, const std::shared_ptr& pool) const override; + + /// Bytes currently allocated by this streamer's Lumina memory resource. + uint64_t GetMemoryUsage() const override; + + private: + friend class LuminaRealtimeIndexReader; + LuminaRealtimeIndex(std::shared_ptr schema, LuminaIndexInfo index_info, + std::unique_ptr<::lumina::extensions::StreamWithFilterExtension> filter, + std::unique_ptr<::lumina::api::LuminaStreamer> streamer, + std::unique_ptr pool); + + std::shared_ptr schema_; + LuminaIndexInfo index_info_; + bool write_failed_ = false; + // Destroy the streamer before its attached extension and memory resource. + std::unique_ptr pool_; + std::unique_ptr<::lumina::extensions::StreamWithFilterExtension> filter_; + std::unique_ptr<::lumina::api::LuminaStreamer> streamer_; +}; + +/// Shares the streamer while restricting every search to [begin, end). +/// Later appends cannot make offsets outside this range visible. Approximate search results +/// within the range may vary as the graph grows; this reader does not snapshot the graph. +class LuminaRealtimeIndexReader final : public RealtimeIndexReader { + public: + LuminaRealtimeIndexReader(std::shared_ptr index, + const OffsetRange& offset_range, + const std::shared_ptr& pool); + + bool SupportsSearch(SearchType search_type) const override; + + /// pre_filter receives realtime offsets and runs before TopK selection. Must be thread-safe. + /// Vector predicates (tags) are not supported. + Result> VisitVectorSearch( + const std::shared_ptr& vector_search) override; + + private: + std::shared_ptr index_; + OffsetRange offset_range_; + std::shared_ptr pool_; +}; + +} // namespace paimon::lumina diff --git a/src/paimon/indexer/lumina/lumina_realtime_index_factory.cpp b/src/paimon/indexer/lumina/lumina_realtime_index_factory.cpp new file mode 100644 index 000000000..bba3932e4 --- /dev/null +++ b/src/paimon/indexer/lumina/lumina_realtime_index_factory.cpp @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/indexer/lumina/lumina_realtime_index_factory.h" + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/indexer/lumina/lumina_realtime_index.h" +#include "paimon/indexer/lumina/lumina_utils.h" + +namespace paimon::lumina { + +const char LuminaRealtimeIndexFactory::IDENTIFIER[] = "lumina-realtime"; + +Result> LuminaRealtimeIndexFactory::Create( + ::ArrowSchema* write_schema, const std::map& options, + const std::shared_ptr& pool) const { + if (!write_schema || !write_schema->release) { + return Status::Invalid("Lumina realtime factory requires a schema"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(write_schema)); + if (schema->num_fields() != 2) { + return Status::Invalid("Lumina realtime factory requires offset and one vector field"); + } + const std::string prefix = "realtime.index.lumina." + schema->field(1)->name() + "."; + const std::map column_options = + OptionsUtils::FetchOptionsWithPrefix(prefix, options); + std::map lumina_options; + for (const auto& [key, value] : column_options) { + lumina_options.emplace(std::string(LuminaDefines::kOptionKeyPrefix) + key, value); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index, + LuminaRealtimeIndex::Create(schema, lumina_options, pool)); + return index; +} + +REGISTER_PAIMON_FACTORY(LuminaRealtimeIndexFactory); + +} // namespace paimon::lumina diff --git a/src/paimon/indexer/lumina/lumina_realtime_index_factory.h b/src/paimon/indexer/lumina/lumina_realtime_index_factory.h new file mode 100644 index 000000000..8ea03b7cb --- /dev/null +++ b/src/paimon/indexer/lumina/lumina_realtime_index_factory.h @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/realtime/index/realtime_index_factory.h" + +namespace paimon::lumina { + +/// Creates column indexes configured by realtime.index.lumina..