Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions docs/source/user_guide/realtime.rst
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,33 @@ filtering. Keep predicate fields in the read schema and enable
In-memory and disk indexes are configured independently. Use
``file-index.<type>.columns`` to enable indexes for committed files.

Real-time Vector Search
~~~~~~~~~~~~~~~~~~~~~~

Build with ``-DPAIMON_ENABLE_LUMINA=ON`` for ``LIST<FLOAT>`` vector indexes
using ``bruteforce`` or ``flatnsw`` with ``rawf32`` encoding:

.. code-block:: text

realtime.index.lumina.columns = embedding
realtime.index.lumina.embedding.index.type = bruteforce
realtime.index.lumina.embedding.index.dimension = 2
realtime.index.lumina.embedding.distance.metric = l2
realtime.index.lumina.embedding.encoding.type = rawf32

For disk indexes, repeat these settings with the ``file-index.lumina`` prefix,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not understand with the doc, if I need use the realtime vector index, I need to config "realtime.index.lumina.columns = xxx" and "file-index.lumina.columns = xxx" together and use the same param/encoding value?

keeping dimension and distance metric consistent. Append deduplicate mode also
requires ``_REALTIME_OFFSET`` in ``file-index.bitmap.columns``.

Use ``ReadContextBuilder::EnablePreciseFileIndexEvaluate(true)`` with
``SetVectorSearch()``. Keep scalar predicate fields in the read projection and
project ``_embedding_INDEX_SCORE`` for scores. Scalar filters, including those
without indexes, and committed or pending DVs are applied before vector TopK.

TopK is per memory segment or data file; apply global ranking in the calling
engine.
Real-time vector tag predicates are unsupported.

Implementation by Mode
----------------------

Expand Down Expand Up @@ -634,8 +661,9 @@ Current restrictions include:
* Native tables on the main branch with fixed positive bucket counts and full
write schemas only.
* Process-local batch union scans only. Streaming scans, limit pushdown,
global-index result splits, vector/full-text search, and count-only real-time
readers are unsupported.
global-index result splits, and count-only real-time readers are unsupported.
Precise index evaluation and Lumina vector search support append and append
deduplicate modes; primary-key real-time search is unsupported.
* PK mode supports deduplicate with default delete handling, or partial update
with deletion vectors disabled and its existing delete and sequence-group
options. Both require ascending sequence order and ``changelog-producer=none``. Custom
Expand Down
4 changes: 3 additions & 1 deletion include/paimon/realtime/index/realtime_index.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
struct ArrowArray;

namespace paimon {
class MemoryPool;
class RealtimeIndexReader;

/// Append-only in-memory index for one field, supporting one writer and concurrent readers.
Expand All @@ -40,8 +41,9 @@ class PAIMON_EXPORT RealtimeIndex {
virtual Status AddBatch(::ArrowArray* batch) = 0;

/// Creates a reader for [begin, end), sharing postings without copying.
/// The pool belongs to the query and is used for reader/search allocations.
virtual Result<std::shared_ptr<RealtimeIndexReader>> CreateReader(
const OffsetRange& offset_range) const = 0;
const OffsetRange& offset_range, const std::shared_ptr<MemoryPool>& pool) const = 0;

virtual uint64_t GetMemoryUsage() const = 0;
};
Expand Down
3 changes: 2 additions & 1 deletion include/paimon/realtime/index/realtime_index_read_source.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,9 @@ class PAIMON_EXPORT RealtimeIndexReadSource {

/// Returns readers bound to this source's offset range. A missing field has no readers.
/// Readers must not expose writes published after the pinned snapshot.
/// The pool belongs to the query, rather than the retained index.
virtual Result<std::vector<std::shared_ptr<RealtimeIndexReader>>> CreateIndexReaders(
const std::string& field_name) const = 0;
const std::string& field_name, const std::shared_ptr<MemoryPool>& pool) const = 0;
};

/// Optional capability for precise real-time filtering and search. Ordinary query reads do not
Expand Down
1 change: 1 addition & 0 deletions src/paimon/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,7 @@ set(PAIMON_CORE_SRCS
core/realtime/realtime_snapshot_offsets.cpp
core/realtime/realtime_store_read_pipeline.cpp
core/realtime/realtime_primary_key_writer.cpp
core/realtime/realtime_utils.cpp
core/postpone/postpone_bucket_writer.cpp
core/schema/arrow_schema_validator.cpp
core/schema/schema_manager.cpp
Expand Down
1 change: 1 addition & 0 deletions src/paimon/core/operation/internal_read_context.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include "paimon/core/realtime/realtime_utils.h"
#include "paimon/core/schema/arrow_schema_validator.h"
#include "paimon/core/schema/schema_manager.h"
#include "paimon/core/utils/branch_manager.h"
#include "paimon/core/utils/nested_projection_utils.h"
#include "paimon/predicate/compound_predicate.h"
#include "paimon/predicate/full_text_search.h"
Expand Down
4 changes: 2 additions & 2 deletions src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -259,8 +259,8 @@ class ArrowDeduplicateRealtimeStore::IndexReadSource final : public RealtimeInde
}

Result<std::vector<std::shared_ptr<RealtimeIndexReader>>> CreateIndexReaders(
const std::string& field_name) const override {
return source_->CreateIndexReaders(field_name);
const std::string& field_name, const std::shared_ptr<MemoryPool>& pool) const override {
return source_->CreateIndexReaders(field_name, pool);
}

private:
Expand Down
7 changes: 4 additions & 3 deletions src/paimon/core/realtime/arrow_realtime_store.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -585,7 +585,8 @@ class ArrowRealtimeStore::AppendQueryBatchReader : public BatchReader {
RealtimeIndexEvaluator::Evaluate(
segment->GetIndex(), predicate_,
OffsetRange(std::max(segment_range.begin, offset_range_.begin),
std::min(segment_range.end, offset_range_.end))));
std::min(segment_range.end, offset_range_.end)),
memory_pool_));
if (indexed) {
if (candidates_) {
*candidates_ &= *indexed;
Expand Down Expand Up @@ -802,12 +803,12 @@ class ArrowRealtimeStore::IndexReadSource final : public RealtimeIndexReadSource
}

Result<std::vector<std::shared_ptr<RealtimeIndexReader>>> CreateIndexReaders(
const std::string& field_name) const override {
const std::string& field_name, const std::shared_ptr<MemoryPool>& pool) const override {
std::vector<std::shared_ptr<RealtimeIndexReader>> readers;
if (segment_->GetIndex()) {
for (const auto& index : segment_->GetIndex()->FindIndexes(field_name)) {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeIndexReader> reader,
index->CreateReader(offset_range_));
index->CreateReader(offset_range_, pool));
if (reader) {
readers.push_back(std::move(reader));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ Status HashRealtimeIndex::AddBatch(::ArrowArray* batch) {
}

Result<std::shared_ptr<RealtimeIndexReader>> HashRealtimeIndex::CreateReader(
const OffsetRange& offset_range) const {
const OffsetRange& offset_range, const std::shared_ptr<MemoryPool>&) const {
if (offset_range.begin < 0 || offset_range.begin > offset_range.end) {
return Status::Invalid("real-time index query offset range is invalid");
}
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/core/realtime/index/hash/hash_realtime_index.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ class HashRealtimeIndex final : public RealtimeIndex,
Status AddBatch(::ArrowArray* batch) override;

Result<std::shared_ptr<RealtimeIndexReader>> CreateReader(
const OffsetRange& offset_range) const override;
const OffsetRange& offset_range, const std::shared_ptr<MemoryPool>& pool) const override;

uint64_t GetMemoryUsage() const override;

Expand Down
15 changes: 8 additions & 7 deletions src/paimon/core/realtime/index/realtime_index_evaluator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ namespace paimon {

Result<std::optional<RoaringBitmap64>> RealtimeIndexEvaluator::Evaluate(
const std::shared_ptr<RealtimeIndexSet>& indexes, const std::shared_ptr<Predicate>& predicate,
const OffsetRange& offset_range) {
const OffsetRange& offset_range, const std::shared_ptr<MemoryPool>& pool) {
if (offset_range.begin < 0 || offset_range.begin > offset_range.end) {
return Status::Invalid("real-time index query offset range is invalid");
}
Expand All @@ -51,23 +51,24 @@ Result<std::optional<RoaringBitmap64>> RealtimeIndexEvaluator::Evaluate(
if (!compound || compound->Children().empty()) {
return Status::Invalid("invalid or empty real-time index compound predicate");
}
return EvaluateCompound(indexes, compound, offset_range);
return EvaluateCompound(indexes, compound, offset_range, pool);
}
const std::shared_ptr<LeafPredicate> leaf = std::dynamic_pointer_cast<LeafPredicate>(predicate);
if (!leaf) {
return std::optional<RoaringBitmap64>();
}
return EvaluateLeaf(indexes, leaf, offset_range);
return EvaluateLeaf(indexes, leaf, offset_range, pool);
}

Result<std::optional<RoaringBitmap64>> RealtimeIndexEvaluator::EvaluateCompound(
const std::shared_ptr<RealtimeIndexSet>& indexes,
const std::shared_ptr<CompoundPredicate>& compound, const OffsetRange& offset_range) {
const std::shared_ptr<CompoundPredicate>& compound, const OffsetRange& offset_range,
const std::shared_ptr<MemoryPool>& pool) {
const Function::Type function = compound->GetFunction().GetType();
std::optional<RoaringBitmap64> result;
for (const std::shared_ptr<Predicate>& child : compound->Children()) {
PAIMON_ASSIGN_OR_RAISE(std::optional<RoaringBitmap64> candidates,
Evaluate(indexes, child, offset_range));
Evaluate(indexes, child, offset_range, pool));
if (!candidates) {
if (function == Function::Type::OR) {
return std::optional<RoaringBitmap64>();
Expand All @@ -90,12 +91,12 @@ Result<std::optional<RoaringBitmap64>> RealtimeIndexEvaluator::EvaluateCompound(

Result<std::optional<RoaringBitmap64>> RealtimeIndexEvaluator::EvaluateLeaf(
const std::shared_ptr<RealtimeIndexSet>& indexes, const std::shared_ptr<LeafPredicate>& leaf,
const OffsetRange& offset_range) {
const OffsetRange& offset_range, const std::shared_ptr<MemoryPool>& pool) {
std::optional<RoaringBitmap64> result;
for (const std::shared_ptr<RealtimeIndex>& index :
indexes->FindIndexes(leaf->FieldName(), leaf->GetFieldType())) {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeIndexReader> reader,
index->CreateReader(offset_range));
index->CreateReader(offset_range, pool));
if (!reader) {
continue;
}
Expand Down
10 changes: 7 additions & 3 deletions src/paimon/core/realtime/index/realtime_index_evaluator.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
namespace paimon {
class CompoundPredicate;
class LeafPredicate;
class MemoryPool;
class Predicate;
class RealtimeIndexSet;

Expand All @@ -39,16 +40,19 @@ class RealtimeIndexEvaluator {

static Result<std::optional<RoaringBitmap64>> Evaluate(
const std::shared_ptr<RealtimeIndexSet>& indexes,
const std::shared_ptr<Predicate>& predicate, const OffsetRange& offset_range);
const std::shared_ptr<Predicate>& predicate, const OffsetRange& offset_range,
const std::shared_ptr<MemoryPool>& pool);

private:
static Result<std::optional<RoaringBitmap64>> EvaluateCompound(
const std::shared_ptr<RealtimeIndexSet>& indexes,
const std::shared_ptr<CompoundPredicate>& compound, const OffsetRange& offset_range);
const std::shared_ptr<CompoundPredicate>& compound, const OffsetRange& offset_range,
const std::shared_ptr<MemoryPool>& pool);

static Result<std::optional<RoaringBitmap64>> EvaluateLeaf(
const std::shared_ptr<RealtimeIndexSet>& indexes,
const std::shared_ptr<LeafPredicate>& leaf, const OffsetRange& offset_range);
const std::shared_ptr<LeafPredicate>& leaf, const OffsetRange& offset_range,
const std::shared_ptr<MemoryPool>& pool);
};

} // namespace paimon
20 changes: 10 additions & 10 deletions src/paimon/core/realtime/index/realtime_index_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ class RealtimeIndexTest : public testing::Test {

Result<std::optional<RoaringBitmap64>> Evaluate(const std::shared_ptr<Predicate>& predicate,
const OffsetRange& range) const {
return RealtimeIndexEvaluator::Evaluate(index_, predicate, range);
return RealtimeIndexEvaluator::Evaluate(index_, predicate, range, GetDefaultPool());
}

Status Add(const std::string& json, const OffsetRange& range) const {
Expand Down Expand Up @@ -240,9 +240,9 @@ TEST_F(RealtimeIndexTest, FieldReadersBindIndependentRangesAndShareAppendOnlyPos
std::shared_ptr<RealtimeIndex> key_index = std::move(key_indexes[0]);
ASSERT_NE(key_index, value_indexes[0]);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeIndexReader> first,
key_index->CreateReader(OffsetRange(10, 14)));
key_index->CreateReader(OffsetRange(10, 14), GetDefaultPool()));
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeIndexReader> second,
key_index->CreateReader(OffsetRange(20, 21)));
key_index->CreateReader(OffsetRange(20, 21), GetDefaultPool()));
ASSERT_OK(Add(R"([[25, 1, "d"]])", OffsetRange(25, 26)));
ASSERT_OK_AND_ASSIGN(std::optional<RoaringBitmap64> candidates,
first->VisitEqual(Literal(int64_t{1})));
Expand All @@ -252,7 +252,7 @@ TEST_F(RealtimeIndexTest, FieldReadersBindIndependentRangesAndShareAppendOnlyPos
ASSERT_TRUE(candidates.has_value());
ASSERT_EQ(RoaringBitmap64::From({20}), *candidates);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeIndexReader> later,
key_index->CreateReader(OffsetRange(25, 26)));
key_index->CreateReader(OffsetRange(25, 26), GetDefaultPool()));
ASSERT_OK_AND_ASSIGN(candidates, later->VisitEqual(Literal(int64_t{1})));
ASSERT_TRUE(candidates.has_value());
ASSERT_EQ(RoaringBitmap64::From({25}), *candidates);
Expand All @@ -261,10 +261,10 @@ TEST_F(RealtimeIndexTest, FieldReadersBindIndependentRangesAndShareAppendOnlyPos
ASSERT_EQ(RoaringBitmap64::From({10}), *candidates);
ASSERT_OK_AND_ASSIGN(candidates, first->VisitGreaterThan(Literal(int64_t{0})));
ASSERT_FALSE(candidates.has_value());
ASSERT_NOK(key_index->CreateReader(OffsetRange(-1, 20)));
ASSERT_NOK(key_index->CreateReader(OffsetRange(20, 10)));
ASSERT_NOK(key_index->CreateReader(OffsetRange(-1, 20), GetDefaultPool()));
ASSERT_NOK(key_index->CreateReader(OffsetRange(20, 10), GetDefaultPool()));
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeIndexReader> empty,
key_index->CreateReader(OffsetRange(10, 10)));
key_index->CreateReader(OffsetRange(10, 10), GetDefaultPool()));
ASSERT_OK_AND_ASSIGN(candidates, empty->VisitEqual(Literal(int64_t{1})));
ASSERT_TRUE(candidates.has_value());
ASSERT_TRUE(candidates->IsEmpty());
Expand Down Expand Up @@ -325,7 +325,7 @@ TEST_F(RealtimeIndexTest, FactoryAcceptsOnlyOneIndexedFieldAndReaderNeedsNoField
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeIndex> index,
RealtimeIndexFactory::Get("hash", &c_schema, {}, GetDefaultPool()));
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeIndexReader> reader,
index->CreateReader(OffsetRange(0, 10)));
index->CreateReader(OffsetRange(0, 10), GetDefaultPool()));
ASSERT_OK_AND_ASSIGN(std::optional<RoaringBitmap64> candidates,
reader->VisitEqual(Literal(int64_t{1})));
ASSERT_TRUE(candidates.has_value());
Expand All @@ -343,7 +343,7 @@ TEST_F(RealtimeIndexTest, FactoryAcceptsOnlyOneIndexedFieldAndReaderNeedsNoField
ASSERT_TRUE(arrow::ExportArray(*data, &batch).ok());
ScopeGuard batch_guard([&batch]() { ArrowArrayRelease(&batch); });
ASSERT_OK(index->AddBatch(&batch));
ASSERT_OK_AND_ASSIGN(reader, index->CreateReader(OffsetRange(11, 31)));
ASSERT_OK_AND_ASSIGN(reader, index->CreateReader(OffsetRange(11, 31), GetDefaultPool()));
ASSERT_OK_AND_ASSIGN(candidates, reader->VisitEqual(Literal(int64_t{1})));
ASSERT_TRUE(candidates.has_value());
ASSERT_EQ(RoaringBitmap64::From({30}), *candidates);
Expand All @@ -369,7 +369,7 @@ TEST_F(RealtimeIndexTest, IndexesTimeField) {
ASSERT_OK(index->AddBatch(&batch));

ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeIndexReader> reader,
index->CreateReader(OffsetRange(0, 41)));
index->CreateReader(OffsetRange(0, 41), GetDefaultPool()));
ASSERT_OK_AND_ASSIGN(std::optional<RoaringBitmap64> candidates,
reader->VisitEqual(Literal(FieldType::TIME, 45296123)));
ASSERT_TRUE(candidates.has_value());
Expand Down
5 changes: 3 additions & 2 deletions src/paimon/core/realtime/precise_realtime_index_read.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,9 @@ class RealtimeSourceRead {
const std::string& field_name) {
auto found = readers_.find(field_name);
if (found == readers_.end()) {
PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<RealtimeIndexReader>> readers,
source_->CreateIndexReaders(field_name));
PAIMON_ASSIGN_OR_RAISE(
std::vector<std::shared_ptr<RealtimeIndexReader>> readers,
source_->CreateIndexReaders(field_name, context_->GetMemoryPool()));
for (const auto& reader : readers) {
if (!reader) {
return Status::Invalid("Real-time source returned a null index reader");
Expand Down
Loading
Loading