From 0e73d7755c494b0e6da2f493524ada96793b1161 Mon Sep 17 00:00:00 2001 From: ljyss9 <1057859151@qq.com> Date: Thu, 12 Nov 2026 18:41:58 +0800 Subject: [PATCH] feat(cache)!: reuse preloaded vector index readers with query pools Look up externally populated readers by immutable index URI, column and actual index type. A miss reuses the open format reader. Use the requested pool for Lumina searches and preserve it when replacing pre-filters. Include charged memory usage in cache value equality so replacing an entry with a different weight updates LRU accounting and eviction. BREAKING CHANGE: CacheValue is abstract. Use MemorySegmentCacheValue or ObjectCacheValue and CacheValue::Get. FileIndexFormat::Reader implementations must implement GetIndexTypes. Recompile downstream users. Generated-by: OpenAI Codex (GPT-6) --- docs/source/user_guide/manifest_cache.rst | 5 +- include/paimon/cache/cache.h | 90 ++- include/paimon/file_index/file_index_format.h | 7 + include/paimon/predicate/vector_search.h | 12 +- .../common/file_index/file_index_format.cpp | 12 + .../file_index/file_index_format_test.cpp | 24 + src/paimon/common/io/cache/cache.cpp | 31 +- src/paimon/common/io/cache/cache_key.cpp | 11 + src/paimon/common/io/cache/cache_manager.cpp | 8 +- src/paimon/common/io/cache/lru_cache_test.cpp | 319 ++++++++++- .../live_row_global_index_reader_test.cpp | 3 + src/paimon/core/io/file_index_evaluator.cpp | 59 +- src/paimon/core/io/file_index_evaluator.h | 9 +- .../core/io/file_index_evaluator_test.cpp | 519 ++++++++++++++++-- .../index_manifest_file_handler_test.cpp | 7 +- .../core/manifest/manifest_list_test.cpp | 7 +- .../core/manifest/manifest_row_range_test.cpp | 2 +- .../core/operation/abstract_split_read.cpp | 2 +- .../append_only_file_store_scan_test.cpp | 5 +- .../operation/data_evolution_split_read.cpp | 4 +- src/paimon/core/operation/file_store_scan.cpp | 16 +- .../operation/precise_file_index_read.cpp | 10 +- .../operation/raw_file_split_read_test.cpp | 110 +++- .../realtime_offset_file_index_lookup.cpp | 7 +- src/paimon/core/utils/objects_file.h | 11 +- .../format/parquet/parquet_input_stream.h | 7 +- .../format/parquet/parquet_reader_builder.h | 9 +- src/paimon/indexer/lumina/CMakeLists.txt | 1 + .../indexer/lumina/lumina_file_index_test.cpp | 337 ++++++++++++ .../indexer/lumina/lumina_index_searcher.cpp | 10 +- 30 files changed, 1487 insertions(+), 167 deletions(-) diff --git a/docs/source/user_guide/manifest_cache.rst b/docs/source/user_guide/manifest_cache.rst index f6dd034af..6b45fe00c 100644 --- a/docs/source/user_guide/manifest_cache.rst +++ b/docs/source/user_guide/manifest_cache.rst @@ -102,7 +102,8 @@ concurrent cache misses. Cache implementations should use ``CacheValue::GetMemoryUsage()`` for admission and eviction accounting. For manifest IPC, this reports the retained Arrow -buffer capacity, including unused space from growth. ``GetSegment().Size()`` -continues to describe the valid IPC byte length. The built-in ``LruCache`` uses +buffer capacity, including unused space from growth. +``CacheValue::Get(value)->Size()`` describes the valid IPC +byte length. The built-in ``LruCache`` uses the memory usage value; existing cache values without an explicit allocation size continue to charge their segment length. diff --git a/include/paimon/cache/cache.h b/include/paimon/cache/cache.h index e05b555e5..af290feb7 100644 --- a/include/paimon/cache/cache.h +++ b/include/paimon/cache/cache.h @@ -18,11 +18,13 @@ #pragma once +#include #include #include #include #include #include +#include #include "paimon/memory/memory_segment.h" #include "paimon/result.h" @@ -36,6 +38,7 @@ enum class CacheKind { MANIFEST, DATA_FILE_FOOTER, SNAPSHOT_LIVE_MANIFEST, + INDEX_READER, }; class PAIMON_EXPORT CacheKey { @@ -44,6 +47,13 @@ class PAIMON_EXPORT CacheKey { int32_t length, bool is_index); static std::shared_ptr ForKind(const std::string& file_path, int64_t position, int32_t length, CacheKind kind); + /// Identifies one index type for a column of an immutable index file. + /// Store ObjectCacheValue under this key. + /// Readers populated externally must keep their resources alive for the cached lifetime. + /// Paimon only looks up these entries; a miss creates a reader without populating the cache. + static std::shared_ptr ForIndexReader(const std::string& uri, + const std::string& column_name, + const std::string& index_type); public: virtual ~CacheKey() = default; @@ -88,28 +98,84 @@ class PAIMON_EXPORT Cache { class PAIMON_EXPORT CacheValue { public: - /// Charge the segment's logical size when no separate allocation size is supplied. - CacheValue(const MemorySegment& segment, CacheCallback callback); + virtual ~CacheValue(); - /// @param memory_usage Retained allocation size in bytes, including unused capacity. - /// The charge is at least the segment's logical size. - CacheValue(const MemorySegment& segment, CacheCallback callback, int64_t memory_usage); + /// Returns the typed payload, retaining its owner; null for a null value or type mismatch. + template + static std::shared_ptr Get(const std::shared_ptr& value); - ~CacheValue(); - - const MemorySegment& GetSegment() const; - - /// Bytes to charge against the cache budget. May exceed GetSegment().Size(). + /// Bytes charged against the cache budget. int64_t GetMemoryUsage() const; void OnEvict(const std::shared_ptr& key) const; - bool operator==(const CacheValue& other) const; + virtual bool operator==(const CacheValue& other) const = 0; + + protected: + CacheValue(CacheCallback callback, int64_t memory_usage); private: - MemorySegment segment_; CacheCallback callback_; int64_t memory_usage_; }; +template +class PAIMON_EXPORT TypedCacheValue : public CacheValue { + protected: + using CacheValue::CacheValue; + + private: + friend class CacheValue; + virtual T* GetValuePointer() = 0; +}; + +template +std::shared_ptr CacheValue::Get(const std::shared_ptr& value) { + auto* typed_value = dynamic_cast*>(value.get()); + if (typed_value) { + if (T* payload = typed_value->GetValuePointer()) { + return std::shared_ptr(value, payload); + } + } + return nullptr; +} + +class PAIMON_EXPORT MemorySegmentCacheValue final : public TypedCacheValue { + public: + MemorySegmentCacheValue(const MemorySegment& segment, CacheCallback callback); + + /// Charges retained allocation size, at least the segment's logical size. + MemorySegmentCacheValue(const MemorySegment& segment, CacheCallback callback, + int64_t memory_usage); + + bool operator==(const CacheValue& other) const override; + + private: + const MemorySegment* GetValuePointer() override { + return &segment_; + } + + MemorySegment segment_; +}; + +template +class PAIMON_EXPORT ObjectCacheValue final : public TypedCacheValue { + public: + ObjectCacheValue(std::shared_ptr value, CacheCallback callback, int64_t memory_usage) + : TypedCacheValue(std::move(callback), std::max(1, memory_usage)), + value_(std::move(value)) {} + + bool operator==(const CacheValue& other) const override { + const auto* rhs = dynamic_cast*>(&other); + return rhs && this->GetMemoryUsage() == rhs->GetMemoryUsage() && value_ == rhs->value_; + } + + private: + T* GetValuePointer() override { + return value_.get(); + } + + std::shared_ptr value_; +}; + } // namespace paimon diff --git a/include/paimon/file_index/file_index_format.h b/include/paimon/file_index/file_index_format.h index 3777df0a6..1aa1d04ba 100644 --- a/include/paimon/file_index/file_index_format.h +++ b/include/paimon/file_index/file_index_format.h @@ -108,6 +108,13 @@ class PAIMON_EXPORT FileIndexFormat::Writer { class PAIMON_EXPORT FileIndexFormat::Reader { public: virtual ~Reader() = default; + /// Returns index types for a specific column from the parsed file metadata without opening + /// index readers. + /// @param column_name Name of the column to retrieve index types for. + /// @return A vector of index types, including empty and unregistered indexes; or an empty + /// vector if the column has no indexes. + virtual std::vector GetIndexTypes(const std::string& column_name) const = 0; + /// Reads index data for a specific column from the index file. /// @param column_name Name of the column to retrieve index data for. /// @param arrow_schema Arrow schema that must contain a field corresponding to `column_name`. diff --git a/include/paimon/predicate/vector_search.h b/include/paimon/predicate/vector_search.h index f41556328..7d01e03f4 100644 --- a/include/paimon/predicate/vector_search.h +++ b/include/paimon/predicate/vector_search.h @@ -22,12 +22,15 @@ #include #include #include +#include #include #include "paimon/predicate/predicate.h" #include "paimon/visibility.h" namespace paimon { +class MemoryPool; + /// `VectorSearch` to perform vector similarity search. struct PAIMON_EXPORT VectorSearch { /// `PreFilter`: A lightweight pre-filtering function applied **before** similarity scoring. @@ -56,8 +59,9 @@ struct PAIMON_EXPORT VectorSearch { options(_options) {} std::shared_ptr ReplacePreFilter(PreFilter _pre_filter) const { - return std::make_shared(field_name, limit, query, _pre_filter, predicate, - distance_type, options); + auto search = std::make_shared(*this); + search->pre_filter = std::move(_pre_filter); + return search; } /// Search field name. @@ -91,5 +95,9 @@ struct PAIMON_EXPORT VectorSearch { /// A key-value map of query-specific runtime options. /// Such as the size of candidate list in approximate search or parallelism for this query. std::map options; + /// Optional memory pool for temporary allocations during a Lumina search. + /// If null, the search uses the reader's pool. Must support concurrent allocations when + /// the search runs in parallel or the pool is shared by concurrent queries. + std::shared_ptr pool; }; } // namespace paimon diff --git a/src/paimon/common/file_index/file_index_format.cpp b/src/paimon/common/file_index/file_index_format.cpp index 2585f1b1d..44d31b584 100644 --- a/src/paimon/common/file_index/file_index_format.cpp +++ b/src/paimon/common/file_index/file_index_format.cpp @@ -68,6 +68,18 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader { input_stream, std::move(header), pool, std::move(file_index_options))); } + std::vector GetIndexTypes(const std::string& column_name) const override { + std::vector types; + auto iter = header_.find(column_name); + if (iter != header_.end()) { + types.reserve(iter->second.size()); + for (const auto& entry : iter->second) { + types.push_back(entry.first); + } + } + return types; + } + Result>> ReadColumnIndex( const std::string& column_name, ::ArrowSchema* c_arrow_schema) const override { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp index e9ea6cf56..c8a739a8b 100644 --- a/src/paimon/common/file_index/file_index_format_test.cpp +++ b/src/paimon/common/file_index/file_index_format_test.cpp @@ -180,6 +180,30 @@ class FileIndexFormatTest : public ::testing::Test { std::shared_ptr pool_; }; +TEST_F(FileIndexFormatTest, TestGetIndexTypesFromMetadata) { + for (int32_t version : {FileIndexFormat::V_1, FileIndexFormat::V_2}) { + auto payload = std::make_shared(std::string("not an index payload"), pool_.get()); + FileIndexFormat::ColumnIndexes indexes = { + {"embedding", {{"lumina", payload}}}, + {"mixed", {{"bitmap", nullptr}, {"unregistered-index", payload}}}}; + auto output = NewIndexOutput(pool_); + ASSERT_OK_AND_ASSIGN(auto writer, FileIndexFormat::CreateWriter(output, version, pool_)); + ASSERT_OK(WriteIndexes(indexes, writer.get())); + ASSERT_OK(writer->Finish()); + ASSERT_OK_AND_ASSIGN(auto bytes, output->Finish(pool_.get())); + auto input = std::make_shared(bytes->data(), bytes->size()); + ASSERT_OK_AND_ASSIGN(auto reader, + FileIndexFormat::CreateReader(input, pool_, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(int64_t position, input->GetPos()); + ASSERT_EQ(reader->GetIndexTypes("embedding"), std::vector({"lumina"})); + ASSERT_EQ(reader->GetIndexTypes("mixed"), + std::vector({"bitmap", "unregistered-index"})); + ASSERT_TRUE(reader->GetIndexTypes("missing").empty()); + ASSERT_OK_AND_ASSIGN(int64_t after, input->GetPos()); + ASSERT_EQ(position, after); + } +} + TEST_F(FileIndexFormatTest, TestWriteAndReadEmptyIndexGoldenBytes) { // the expected bytes are generated from Java Paimon std::vector expected = {0, 5, 78, 78, -48, 26, 53, -82, 0, 0, 0, 1, 0, 0, 0, 47, diff --git a/src/paimon/common/io/cache/cache.cpp b/src/paimon/common/io/cache/cache.cpp index c4c9e5418..231b0d41f 100644 --- a/src/paimon/common/io/cache/cache.cpp +++ b/src/paimon/common/io/cache/cache.cpp @@ -24,20 +24,11 @@ namespace paimon { -CacheValue::CacheValue(const MemorySegment& segment, CacheCallback callback) - : CacheValue(segment, std::move(callback), segment.Size()) {} - -CacheValue::CacheValue(const MemorySegment& segment, CacheCallback callback, int64_t memory_usage) - : segment_(segment), - callback_(std::move(callback)), - memory_usage_(std::max(segment.Size(), memory_usage)) {} +CacheValue::CacheValue(CacheCallback callback, int64_t memory_usage) + : callback_(std::move(callback)), memory_usage_(memory_usage) {} CacheValue::~CacheValue() = default; -const MemorySegment& CacheValue::GetSegment() const { - return segment_; -} - int64_t CacheValue::GetMemoryUsage() const { return memory_usage_; } @@ -48,11 +39,19 @@ void CacheValue::OnEvict(const std::shared_ptr& key) const { } } -bool CacheValue::operator==(const CacheValue& other) const { - if (this == &other) { - return true; - } - return segment_ == other.segment_; +MemorySegmentCacheValue::MemorySegmentCacheValue(const MemorySegment& segment, + CacheCallback callback) + : MemorySegmentCacheValue(segment, std::move(callback), segment.Size()) {} + +MemorySegmentCacheValue::MemorySegmentCacheValue(const MemorySegment& segment, + CacheCallback callback, int64_t memory_usage) + : TypedCacheValue(std::move(callback), + std::max(segment.Size(), memory_usage)), + segment_(segment) {} + +bool MemorySegmentCacheValue::operator==(const CacheValue& other) const { + const auto* rhs = dynamic_cast(&other); + return rhs && GetMemoryUsage() == rhs->GetMemoryUsage() && segment_ == rhs->segment_; } } // namespace paimon diff --git a/src/paimon/common/io/cache/cache_key.cpp b/src/paimon/common/io/cache/cache_key.cpp index 998d387f4..f83d9044f 100644 --- a/src/paimon/common/io/cache/cache_key.cpp +++ b/src/paimon/common/io/cache/cache_key.cpp @@ -18,6 +18,8 @@ #include "paimon/common/io/cache/cache_key.h" +#include "fmt/format.h" + namespace paimon { std::shared_ptr CacheKey::ForPosition(const std::string& file_path, int64_t position, @@ -33,6 +35,15 @@ std::shared_ptr CacheKey::ForKind(const std::string& file_path, int64_ return key; } +std::shared_ptr CacheKey::ForIndexReader(const std::string& uri, + const std::string& column_name, + const std::string& index_type) { + std::string identity = + fmt::format("index-reader:{}:{}:{}:{}:{}:{}", uri.size(), uri, column_name.size(), + column_name, index_type.size(), index_type); + return ForKind(identity, 0, 0, CacheKind::INDEX_READER); +} + std::shared_ptr SnapshotLiveManifestEntriesCacheKey::ForExplicit( const std::string& table_path, const std::string& branch, int32_t bucket) { return std::shared_ptr(new SnapshotLiveManifestEntriesCacheKey( diff --git a/src/paimon/common/io/cache/cache_manager.cpp b/src/paimon/common/io/cache/cache_manager.cpp index 3e5c049b5..b3192b308 100644 --- a/src/paimon/common/io/cache/cache_manager.cpp +++ b/src/paimon/common/io/cache/cache_manager.cpp @@ -28,10 +28,14 @@ Result CacheManager::GetPage( auto supplier = [&](const std::shared_ptr& key) -> Result> { PAIMON_ASSIGN_OR_RAISE(MemorySegment segment, reader(key)); - return std::make_shared(segment, std::move(eviction_callback)); + return std::make_shared(segment, std::move(eviction_callback)); }; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache_value, cache->Get(key, supplier)); - return cache_value->GetSegment(); + auto segment = CacheValue::Get(cache_value); + if (!segment) { + return Status::Invalid("Page cache returned an invalid value type"); + } + return *segment; } void CacheManager::InvalidPage(const std::shared_ptr& key) { diff --git a/src/paimon/common/io/cache/lru_cache_test.cpp b/src/paimon/common/io/cache/lru_cache_test.cpp index 6a9cab5b2..e9770acdc 100644 --- a/src/paimon/common/io/cache/lru_cache_test.cpp +++ b/src/paimon/common/io/cache/lru_cache_test.cpp @@ -49,7 +49,7 @@ class LruCacheTest : public ::testing::Test { CacheCallback callback = {}) const { auto segment = MemorySegment::AllocateHeapMemory(size, pool_.get()); std::memset(segment.MutableData(), fill_byte, size); - return std::make_shared(segment, std::move(callback)); + return std::make_shared(segment, std::move(callback)); } auto MakeSupplier(int32_t size, char fill_byte = 0, CacheCallback callback = {}) const { @@ -77,14 +77,16 @@ TEST_F(LruCacheTest, TestGetCacheHitAndMiss) { // First Get: cache miss, supplier should be called ASSERT_OK_AND_ASSIGN(auto value1, cache.Get(key, supplier)); ASSERT_EQ(supplier_call_count, 1); - ASSERT_EQ(value1->GetSegment().Size(), 64); - ASSERT_EQ(value1->GetSegment().Get(0), 'A'); + auto first_segment = CacheValue::Get(value1); + ASSERT_TRUE(first_segment); + ASSERT_EQ(first_segment->Size(), 64); + ASSERT_EQ(first_segment->Get(0), 'A'); ASSERT_EQ(cache.Size(), 1); // Second Get with same key: cache hit, supplier should NOT be called ASSERT_OK_AND_ASSIGN(auto value2, cache.Get(key, supplier)); ASSERT_EQ(supplier_call_count, 1); - ASSERT_EQ(value2->GetSegment().Get(0), 'A'); + ASSERT_EQ(CacheValue::Get(value2)->Get(0), 'A'); ASSERT_EQ(cache.Size(), 1); ASSERT_EQ(cache.GetMaxWeight(), 1024); } @@ -109,21 +111,24 @@ TEST_F(LruCacheTest, TestPutInsertAndUpdate) { // Verify the updated value is returned ASSERT_OK_AND_ASSIGN(auto result, cache.Get(key, MakeSupplier(0))); - ASSERT_EQ(result->GetSegment().Size(), 128); - ASSERT_EQ(result->GetSegment().Get(0), 'B'); + auto segment = CacheValue::Get(result); + ASSERT_TRUE(segment); + ASSERT_EQ(segment->Size(), 128); + ASSERT_EQ(segment->Get(0), 'B'); } TEST_F(LruCacheTest, TestRetainedMemoryAccounting) { LruCache cache(192); auto segment = MemorySegment::AllocateHeapMemory(64, pool_.get()); - auto value = std::make_shared(segment, CacheCallback(), 128); - ASSERT_EQ(64, value->GetSegment().Size()); + auto value = std::make_shared(segment, CacheCallback(), 128); + ASSERT_EQ(64, CacheValue::Get(value)->Size()); ASSERT_EQ(128, value->GetMemoryUsage()); - ASSERT_EQ(64, CacheValue(segment, CacheCallback()).GetMemoryUsage()); - ASSERT_EQ(64, CacheValue(segment, CacheCallback(), 32).GetMemoryUsage()); + ASSERT_EQ(64, MemorySegmentCacheValue(segment, CacheCallback()).GetMemoryUsage()); + ASSERT_EQ(64, MemorySegmentCacheValue(segment, CacheCallback(), 32).GetMemoryUsage()); + ASSERT_EQ(0, MemorySegmentCacheValue(MemorySegment(), CacheCallback()).GetMemoryUsage()); std::vector> evicted; - auto first = std::make_shared( + auto first = std::make_shared( segment, [&](const std::shared_ptr& key) { evicted.push_back(key); }, 128); auto first_key = MakeKey(0); ASSERT_OK(cache.Put(first_key, first)); @@ -139,14 +144,260 @@ TEST_F(LruCacheTest, TestRetainedMemoryAccounting) { LruCache small_cache(100); ASSERT_OK_AND_ASSIGN( std::shared_ptr loaded, - small_cache.Get(MakeKey(0), - [&](const std::shared_ptr&) - -> Result> { return value; })); + small_cache.Get( + MakeKey(0), + [&](const std::shared_ptr&) -> Result> { + return std::shared_ptr(value); + })); ASSERT_EQ(value, loaded); ASSERT_EQ(0, small_cache.Size()); ASSERT_EQ(0, small_cache.GetCurrentWeight()); } +TEST_F(LruCacheTest, TestObjectCacheReuseAndLifetime) { + LruCache cache(128); + auto key = CacheKey::ForKind("test_file", 0, 64, CacheKind::INDEX_READER); + int32_t loads = 0; + std::weak_ptr weak_object; + std::vector> evicted; + auto supplier = [&](const std::shared_ptr&) -> Result> { + ++loads; + auto object = std::make_shared("loaded metadata"); + weak_object = object; + return std::make_shared>( + object, [&](const std::shared_ptr& removed) { evicted.push_back(removed); }, + 128); + }; + ASSERT_OK_AND_ASSIGN(auto first, cache.Get(key, supplier)); + ASSERT_OK_AND_ASSIGN(auto second, cache.Get(key, supplier)); + auto object = CacheValue::Get(first); + ASSERT_TRUE(object); + std::weak_ptr weak_value = first; + ASSERT_EQ(first, second); + ASSERT_EQ(object, CacheValue::Get(second)); + ASSERT_EQ(1, loads); + ASSERT_EQ(128, cache.GetCurrentWeight()); + first.reset(); + second.reset(); + + // Weight-based eviction releases the cache's ownership, but not an active reader's. + ASSERT_OK(cache.Put(MakeKey(1), MakeValue(128))); + ASSERT_EQ((std::vector>{key}), evicted); + ASSERT_FALSE(weak_object.expired()); + ASSERT_FALSE(weak_value.expired()); + ASSERT_EQ("loaded metadata", *object); + object.reset(); + ASSERT_TRUE(weak_object.expired()); + ASSERT_TRUE(weak_value.expired()); + + ASSERT_OK_AND_ASSIGN(auto reloaded, cache.Get(key, supplier)); + ASSERT_EQ(2, loads); + ASSERT_EQ("loaded metadata", *CacheValue::Get(reloaded)); + cache.InvalidateAll(); +} + +TEST_F(LruCacheTest, TestTypedPayloadAccess) { + ASSERT_EQ(nullptr, CacheValue::Get(nullptr)); + ASSERT_EQ(nullptr, CacheValue::Get(nullptr)); + + auto bytes = MakeValue(16, 'A'); + ASSERT_EQ(nullptr, CacheValue::Get(bytes)); + auto object = std::make_shared>( + std::make_shared("metadata"), CacheCallback(), 16); + ASSERT_EQ(nullptr, CacheValue::Get(bytes)); + ASSERT_EQ(nullptr, CacheValue::Get(object)); + ASSERT_EQ(nullptr, CacheValue::Get(object)); + + auto null_object = std::make_shared>(nullptr, CacheCallback(), 1); + ASSERT_EQ(nullptr, CacheValue::Get(null_object)); + auto empty = std::make_shared(MemorySegment(), CacheCallback()); + auto segment = CacheValue::Get(empty); + ASSERT_TRUE(segment); + ASSERT_EQ(0, segment->Size()); + ASSERT_EQ(nullptr, segment->Data()); +} + +TEST_F(LruCacheTest, TestSegmentViewRetainsAliasedOwnerAfterEviction) { + struct OwnedBuffer { + OwnedBuffer() + : value(MemorySegment::WrapView(data.data(), static_cast(data.size())), + CacheCallback()) {} + + std::string data = "borrowed bytes"; + MemorySegmentCacheValue value; + }; + LruCache cache(64); + auto owner = std::make_shared(); + std::weak_ptr weak_owner = owner; + auto value = std::shared_ptr(owner, &owner->value); + ASSERT_OK(cache.Put(MakeKey(0), value)); + auto segment = CacheValue::Get(value); + ASSERT_TRUE(segment); + owner.reset(); + value.reset(); + + ASSERT_OK(cache.Put(MakeKey(1), MakeValue(64))); + ASSERT_EQ(1, cache.Size()); + ASSERT_EQ(64, cache.GetCurrentWeight()); + ASSERT_FALSE(weak_owner.expired()); + ASSERT_EQ(14, segment->Size()); + ASSERT_EQ("borrowed bytes", std::string(segment->Data(), segment->Size())); + segment.reset(); + ASSERT_TRUE(weak_owner.expired()); +} + +TEST_F(LruCacheTest, TestObjectCacheKindsDoNotCollideWithSegments) { + LruCache cache(1024); + const std::vector kinds = {CacheKind::DATA_FILE_FOOTER, CacheKind::INDEX_READER}; + ASSERT_OK(cache.Put(CacheKey::ForKind("file", 0, 64, kinds[0]), MakeValue(64, 'A'))); + for (size_t i = 1; i < kinds.size(); ++i) { + ASSERT_OK(cache.Put(CacheKey::ForKind("file", 0, 64, kinds[i]), + std::make_shared>( + std::make_shared(i), CacheCallback(), sizeof(size_t)))); + } + ASSERT_EQ(kinds.size(), cache.Size()); + for (size_t i = 0; i < kinds.size(); ++i) { + ASSERT_OK_AND_ASSIGN( + auto value, + cache.Get(CacheKey::ForKind("file", 0, 64, kinds[i]), + [](const std::shared_ptr&) -> Result> { + return Status::Invalid("unexpected cache miss"); + })); + if (i == 0) { + ASSERT_EQ('A', CacheValue::Get(value)->Get(0)); + } else { + ASSERT_EQ(i, *CacheValue::Get(value)); + } + } +} + +TEST_F(LruCacheTest, TestObjectMemoryAccounting) { + LruCache cache(64); + auto object = std::make_shared("metadata"); + auto value = std::make_shared>(object, CacheCallback(), 128); + ASSERT_OK_AND_ASSIGN( + auto loaded, + cache.Get(MakeKey(0), + [&](const std::shared_ptr&) -> Result> { + return std::shared_ptr(value); + })); + ASSERT_EQ(object, CacheValue::Get(loaded)); + ASSERT_EQ(128, loaded->GetMemoryUsage()); + ASSERT_EQ(0, cache.Size()); + ASSERT_EQ(0, cache.GetCurrentWeight()); + ASSERT_EQ(1, ObjectCacheValue(object, CacheCallback(), 0).GetMemoryUsage()); + ASSERT_EQ(1, ObjectCacheValue(object, CacheCallback(), -1).GetMemoryUsage()); + auto replacement = std::make_shared(*object); + ASSERT_OK(cache.Put( + MakeKey(1), std::make_shared>(object, CacheCallback(), 32))); + ASSERT_OK(cache.Put(MakeKey(1), std::make_shared>( + replacement, CacheCallback(), 64))); + ASSERT_OK_AND_ASSIGN(auto replaced, cache.Get(MakeKey(1), MakeSupplier(0))); + ASSERT_EQ(replacement, CacheValue::Get(replaced)); + ASSERT_EQ(64, cache.GetCurrentWeight()); +} + +TEST_F(LruCacheTest, TestPolymorphicValueEquality) { + auto bytes = MakeValue(16, 'A'); + auto equal_bytes = MakeValue(16, 'A'); + auto object = std::make_shared("A"); + std::shared_ptr first = + std::make_shared>(object, CacheCallback(), 16); + std::shared_ptr same = + std::make_shared>(object, CacheCallback(), 16); + std::shared_ptr different = std::make_shared>( + std::make_shared(*object), CacheCallback(), 16); + std::shared_ptr other_type = std::make_shared>( + std::make_shared(1), CacheCallback(), 16); + + ASSERT_TRUE(*bytes == *equal_bytes); + ASSERT_TRUE(*first == *same); + ASSERT_FALSE(*first == *different); + ASSERT_FALSE(*first == *other_type); + ASSERT_FALSE(*other_type == *first); + ASSERT_FALSE(*bytes == *first); + ASSERT_FALSE(*first == *bytes); +} + +TEST_F(LruCacheTest, TestUpdateMemoryUsageForEqualPayloads) { + auto object = std::make_shared("reader"); + auto segment = MemorySegment::AllocateHeapMemory(16, pool_.get()); + for (bool object_value : {false, true}) { + auto make_value = [&](int64_t memory_usage) -> std::shared_ptr { + if (object_value) { + return std::make_shared>(object, CacheCallback(), + memory_usage); + } + return std::make_shared(segment, CacheCallback(), + memory_usage); + }; + LruCache cache(128); + auto key = MakeKey(0); + auto first = make_value(32); + auto larger = make_value(96); + auto smaller = make_value(16); + ASSERT_TRUE(*first == *make_value(32)); + ASSERT_FALSE(*first == *larger); + ASSERT_FALSE(*larger == *first); + ASSERT_OK(cache.Put(key, first)); + ASSERT_OK(cache.Put(MakeKey(1), MakeValue(64))); + ASSERT_EQ(96, cache.GetCurrentWeight()); + ASSERT_EQ(2, cache.Size()); + + // Updating the weight must evict the other entry when the budget is exceeded. + ASSERT_OK(cache.Put(key, larger)); + ASSERT_EQ(96, cache.GetCurrentWeight()); + ASSERT_EQ(1, cache.Size()); + ASSERT_OK_AND_ASSIGN(auto cached, cache.Get(key, MakeSupplier(0))); + ASSERT_EQ(larger, cached); + + ASSERT_OK(cache.Put(key, smaller)); + ASSERT_EQ(16, cache.GetCurrentWeight()); + ASSERT_EQ(1, cache.Size()); + ASSERT_OK_AND_ASSIGN(cached, cache.Get(key, MakeSupplier(0))); + ASSERT_EQ(smaller, cached); + } +} + +TEST_F(LruCacheTest, TestReplaceDifferentValueTypes) { + LruCache cache(128); + auto key = MakeKey(0); + std::vector evicted; + auto bytes = MakeValue(64, 'A', [&](const std::shared_ptr& removed) { + ASSERT_EQ(key, removed); + evicted.push_back(1); + }); + auto object = std::make_shared>( + std::make_shared("reader"), + [&](const std::shared_ptr& removed) { + ASSERT_EQ(key, removed); + evicted.push_back(2); + }, + 32); + auto other_type = std::make_shared>( + std::make_shared(7), + [&](const std::shared_ptr& removed) { + ASSERT_EQ(key, removed); + evicted.push_back(3); + }, + 16); + + ASSERT_OK(cache.Put(key, bytes)); + ASSERT_EQ(64, cache.GetCurrentWeight()); + ASSERT_OK(cache.Put(key, object)); + ASSERT_EQ(32, cache.GetCurrentWeight()); + ASSERT_EQ(std::vector{1}, evicted); + ASSERT_OK(cache.Put(key, other_type)); + ASSERT_EQ(16, cache.GetCurrentWeight()); + ASSERT_EQ(1, cache.Size()); + ASSERT_EQ((std::vector{1, 2}), evicted); + ASSERT_OK_AND_ASSIGN(auto cached, cache.Get(key, MakeSupplier(0))); + ASSERT_EQ(other_type, cached); + cache.InvalidateAll(); + ASSERT_EQ(0, cache.GetCurrentWeight()); + ASSERT_EQ((std::vector{1, 2, 3}), evicted); +} + /// Verifies weight-based eviction: when total weight exceeds max, LRU entries are evicted. TEST_F(LruCacheTest, TestWeightBasedEviction) { // Cache can hold at most 200 bytes @@ -175,7 +426,7 @@ TEST_F(LruCacheTest, TestWeightBasedEviction) { }; ASSERT_OK_AND_ASSIGN(auto result, cache.Get(key0, supplier)); ASSERT_EQ(supplier_called, 1); - ASSERT_EQ(result->GetSegment().Get(0), 'X'); + ASSERT_EQ(CacheValue::Get(result)->Get(0), 'X'); } /// Verifies that eviction invokes the CacheCallback on evicted entries. @@ -216,7 +467,7 @@ TEST_F(LruCacheTest, TestLruOrdering) { // Access key0 via Get to move it to front (most recently used) ASSERT_OK_AND_ASSIGN(auto val, cache.Get(key0, MakeSupplier(0))); - ASSERT_EQ(val->GetSegment().Get(0), 'A'); + ASSERT_EQ(CacheValue::Get(val)->Get(0), 'A'); // Insert key2: should evict key1 (now LRU), NOT key0 std::vector evicted; @@ -410,13 +661,47 @@ TEST_F(LruCacheTest, TestForKindSetsKeyKind) { ASSERT_OK_AND_ASSIGN(auto value, cache.Get(key, supplier)); ASSERT_TRUE(supplier_seen_kind); ASSERT_EQ(CacheKind::MANIFEST, key->GetKind()); - ASSERT_EQ('M', value->GetSegment().Get(0)); + ASSERT_EQ('M', CacheValue::Get(value)->Get(0)); auto put_key = CacheKey::ForKind("test_file", 1, 64, CacheKind::MANIFEST); ASSERT_OK(cache.Put(put_key, MakeValue(64, 'P'))); ASSERT_EQ(CacheKind::MANIFEST, put_key->GetKind()); } +TEST_F(LruCacheTest, TestIndexReaderKeys) { + auto key = CacheKey::ForIndexReader("file.index", "embedding", "lumina"); + auto same = CacheKey::ForIndexReader("file.index", "embedding", "lumina"); + ASSERT_EQ(CacheKind::INDEX_READER, key->GetKind()); + ASSERT_TRUE(key->Equals(*same)); + ASSERT_TRUE(same->Equals(*key)); + ASSERT_EQ(key->HashCode(), same->HashCode()); + ASSERT_FALSE(key->Equals(*CacheKey::ForIndexReader("other.index", "embedding", "lumina"))); + ASSERT_FALSE(key->Equals(*CacheKey::ForIndexReader("file.index", "other", "lumina"))); + auto other_type = CacheKey::ForIndexReader("file.index", "embedding", "bitmap"); + ASSERT_FALSE(key->Equals(*other_type)); + ASSERT_FALSE(other_type->Equals(*key)); + LruCache cache(128); + ASSERT_OK(cache.Put(key, MakeValue(64, 'L'))); + ASSERT_OK(cache.Put(other_type, MakeValue(64, 'B'))); + auto unexpected_load = + [](const std::shared_ptr&) -> Result> { + return Status::Invalid("Expected a cache hit"); + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr lumina_value, + cache.Get(same, unexpected_load)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bitmap_value, + cache.Get(other_type, unexpected_load)); + ASSERT_EQ('L', CacheValue::Get(lumina_value)->Get(0)); + ASSERT_EQ('B', CacheValue::Get(bitmap_value)->Get(0)); + ASSERT_EQ(2, cache.Size()); + + auto uri_with_separator = CacheKey::ForIndexReader("file:index", "column", "type"); + auto column_with_separator = CacheKey::ForIndexReader("file", "index:column", "type"); + auto type_with_separator = CacheKey::ForIndexReader("file", "index", "column:type"); + ASSERT_FALSE(uri_with_separator->Equals(*column_with_separator)); + ASSERT_FALSE(column_with_separator->Equals(*type_with_separator)); +} + TEST_F(LruCacheTest, InferredManifestCacheKeysIncludeBucketCountAndSchema) { auto key = SnapshotLiveManifestEntriesCacheKey::ForInferred("table", "main", 1, 4, 0); auto same = SnapshotLiveManifestEntriesCacheKey::ForInferred("table", "main", 1, 4, 0); diff --git a/src/paimon/core/global_index/live_row_global_index_reader_test.cpp b/src/paimon/core/global_index/live_row_global_index_reader_test.cpp index 53ec5b239..5447ed974 100644 --- a/src/paimon/core/global_index/live_row_global_index_reader_test.cpp +++ b/src/paimon/core/global_index/live_row_global_index_reader_test.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include "paimon/executor.h" +#include "paimon/memory/memory_pool.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" @@ -135,6 +136,7 @@ TEST_F(LiveRowGlobalIndexReaderTest, TestSearchOnlyLiveRowsOfShard) { ASSERT_EQ(captured->include_row_ids.value(), RoaringBitmap64::From({10, 11, 13, 14})); std::shared_ptr vector_search = SearchVector(); + vector_search->pool = GetMemoryPool(); ASSERT_OK_AND_ASSIGN(result, reader.VisitVectorSearch(vector_search)); ASSERT_EQ(result->ToString(), "row ids: {11,13}, scores: {0.90,0.50}"); std::shared_ptr captured_vector = mock_reader->CapturedVectorSearch(); @@ -145,6 +147,7 @@ TEST_F(LiveRowGlobalIndexReaderTest, TestSearchOnlyLiveRowsOfShard) { ASSERT_EQ(captured_vector->predicate, vector_search->predicate); ASSERT_TRUE(captured_vector->distance_type == vector_search->distance_type); ASSERT_EQ(captured_vector->options, vector_search->options); + ASSERT_EQ(captured_vector->pool, vector_search->pool); std::vector row_ids = {1, 9, 10, 11, 12, 13, 14, 15}; ASSERT_EQ(RowsPassingPreFilter(*captured_vector, row_ids), std::vector({10, 11, 13, 14})); diff --git a/src/paimon/core/io/file_index_evaluator.cpp b/src/paimon/core/io/file_index_evaluator.cpp index 796b46e4e..425912b11 100644 --- a/src/paimon/core/io/file_index_evaluator.cpp +++ b/src/paimon/core/io/file_index_evaluator.cpp @@ -31,6 +31,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "fmt/ranges.h" +#include "paimon/cache/cache.h" #include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" @@ -71,28 +72,39 @@ std::string SearchTypeToString(const FileIndexReader::SearchType& search_type) { FileIndexEvaluator::FileIndexEvaluator(std::shared_ptr data_schema, std::shared_ptr file_meta, - std::unique_ptr format_reader) + std::unique_ptr format_reader, + std::shared_ptr cache, std::string uri) : data_schema_(std::move(data_schema)), file_meta_(std::move(file_meta)), - format_reader_(std::move(format_reader)) {} + format_reader_(std::move(format_reader)), + cache_(std::move(cache)), + uri_(std::move(uri)) {} Result> FileIndexEvaluator::Create( const std::shared_ptr& data_schema, const std::map& options, const std::shared_ptr& path_factory, const std::shared_ptr& file_meta, const std::shared_ptr& file_system, - const std::shared_ptr& pool) { + const std::shared_ptr& cache, const std::shared_ptr& pool) { + std::shared_ptr input_stream; std::unique_ptr format_reader; if (file_meta->row_count >= 0 && file_meta->row_count <= RoaringBitmap32::MAX_VALUE) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, + PAIMON_ASSIGN_OR_RAISE(input_stream, ExtractIndexInputStream(path_factory, file_meta, file_system)); if (input_stream) { PAIMON_ASSIGN_OR_RAISE(format_reader, FileIndexFormat::CreateReader(input_stream, pool, options)); } } - return std::unique_ptr( - new FileIndexEvaluator(data_schema, file_meta, std::move(format_reader))); + std::string uri; + if (cache && input_stream) { + auto result = input_stream->GetUri(); + if (result.ok()) { + uri = std::move(result).value(); + } + } + return std::unique_ptr(new FileIndexEvaluator( + data_schema, file_meta, std::move(format_reader), cache, std::move(uri))); } Result>> FileIndexEvaluator::GetReaders( @@ -126,6 +138,37 @@ Result> FileIndexEvaluator::GetSearchReader( return Status::Invalid(fmt::format("Data file '{}' has no File Index for search field '{}'", file_meta_->file_name, field_name)); } + auto unsupported = [&]() { + return Status::NotImplemented( + fmt::format("No File Index reader supports {} for field '{}' in data file '{}'", + SearchTypeToString(search_type), field_name, file_meta_->file_name)); + }; + std::vector index_types; + if (cache_ && !uri_.empty() && search_type == FileIndexReader::SearchType::VECTOR_SEARCH) { + index_types = format_reader_->GetIndexTypes(field_name); + } + // Multiple index types must go through the normal reader selection and validation. + if (index_types.size() == 1) { + auto key = CacheKey::ForIndexReader(uri_, field_name, index_types.front()); + // An error from the supplier prevents insertion, including a null cache entry. + auto value = cache_->Get( + key, [](const std::shared_ptr&) -> Result> { + return Status::NotExist("Vector index reader is not cached"); + }); + if (!value.ok() && !value.status().IsNotExist()) { + return value.status(); + } + if (value.ok() && value.value()) { + auto reader = CacheValue::Get(value.value()); + if (!reader) { + return Status::Invalid("Vector index reader cache returned an invalid value"); + } + if (!reader->SupportsSearch(search_type)) { + return unsupported(); + } + return reader; + } + } PAIMON_ASSIGN_OR_RAISE(std::vector> readers, GetReaders(field_name)); std::shared_ptr selected; @@ -141,9 +184,7 @@ Result> FileIndexEvaluator::GetSearchReader( selected = reader; } if (!selected) { - return Status::NotImplemented( - fmt::format("No File Index reader supports {} for field '{}' in data file '{}'", - SearchTypeToString(search_type), field_name, file_meta_->file_name)); + return unsupported(); } return selected; } diff --git a/src/paimon/core/io/file_index_evaluator.h b/src/paimon/core/io/file_index_evaluator.h index fed6d97a7..0f9e78882 100644 --- a/src/paimon/core/io/file_index_evaluator.h +++ b/src/paimon/core/io/file_index_evaluator.h @@ -36,6 +36,7 @@ class Schema; } // namespace arrow namespace paimon { +class Cache; class LeafPredicate; class CompoundPredicate; class DataFilePathFactory; @@ -59,7 +60,8 @@ class FileIndexEvaluator { const std::map& options, const std::shared_ptr& path_factory, const std::shared_ptr& file_meta, - const std::shared_ptr& file_system, const std::shared_ptr& pool); + const std::shared_ptr& file_system, const std::shared_ptr& cache, + const std::shared_ptr& pool); /// Evaluates a predicate tree for index pruning, reusing the cached field readers. /// This does not perform exact scalar filtering or reorder children by selectivity. @@ -83,7 +85,8 @@ class FileIndexEvaluator { private: FileIndexEvaluator(std::shared_ptr data_schema, std::shared_ptr file_meta, - std::unique_ptr format_reader); + std::unique_ptr format_reader, + std::shared_ptr cache, std::string uri); Result>> GetReaders(const std::string& field_name); @@ -102,6 +105,8 @@ class FileIndexEvaluator { std::shared_ptr data_schema_; std::shared_ptr file_meta_; std::unique_ptr format_reader_; + std::shared_ptr cache_; + std::string uri_; std::map>> field_readers_; }; diff --git a/src/paimon/core/io/file_index_evaluator_test.cpp b/src/paimon/core/io/file_index_evaluator_test.cpp index 55775950c..69d042bc6 100644 --- a/src/paimon/core/io/file_index_evaluator_test.cpp +++ b/src/paimon/core/io/file_index_evaluator_test.cpp @@ -59,6 +59,7 @@ #include "paimon/predicate/vector_search.h" #include "paimon/status.h" #include "paimon/testing/mock/mock_search_file_index.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/testharness.h" #include "paimon/utils/roaring_bitmap32.h" @@ -101,7 +102,7 @@ class FileIndexEvaluatorTest : public ::testing::Test { ASSERT_OK_AND_ASSIGN( auto evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), data_file_path_factory, - data_file_meta, fs_, pool_)); + data_file_meta, fs_, /*cache=*/nullptr, pool_)); { auto predicate = PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); @@ -368,7 +369,7 @@ TEST_F(FileIndexEvaluatorTest, TestTimestampType) { ASSERT_OK_AND_ASSIGN( auto evaluator, FileIndexEvaluator::Create(data_schema, core_options_.ToMap(), data_file_path_factory, - data_file_meta, /*file_system=*/fs_, pool_)); + data_file_meta, /*file_system=*/fs_, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); CheckResult(file_index_result, {0, 6}); } @@ -390,9 +391,9 @@ TEST_F(FileIndexEvaluatorTest, TestCreateRequiresExternalIndexDependencies) { /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); ASSERT_NOK_WITH_MSG( - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - /*path_factory=*/nullptr, data_file_meta, - /*file_system=*/nullptr, pool_), + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), /*path_factory=*/nullptr, + data_file_meta, /*file_system=*/nullptr, /*cache=*/nullptr, + pool_), "read process for FileIndexEvaluator must have data_file_path_factory and file_system"); } @@ -458,9 +459,9 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, void CheckSearchError(SearchIndexType index_type, const std::shared_ptr& meta, const std::string& field, const std::string& message) const { - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); if (index_type == SearchIndexType::VECTOR) { ASSERT_NOK_WITH_MSG(evaluator->SearchVector(MakeVectorSearch(field)), message); } else { @@ -535,9 +536,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestVectorSearchReadsIndexWithScores) { ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator->SearchVector(MakeVectorSearch("embedding"))); ASSERT_TRUE(result); @@ -554,9 +555,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestFullTextSearchReadsIndexWithScores) { MockSearchFileIndex::MakeIndex("title", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, {{3, 0.75f}, {1, 0.25f}}, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator->SearchFullText(MakeFullTextSearch("title"))); ASSERT_TRUE(result); @@ -570,9 +571,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyVectorSearchIndex) { ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr vector_result, evaluator->SearchVector(MakeVectorSearch("embedding"))); ASSERT_TRUE(vector_result); @@ -586,9 +587,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyFullTextSearchIndex) { ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr text_result, evaluator->SearchFullText(MakeFullTextSearch("title"))); ASSERT_TRUE(text_result); @@ -609,17 +610,18 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEvaluateNullPredicateAndMissingIndex) { indexes["title"]["bitmap"] = nullptr; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(nullptr)); ASSERT_EQ(result, FileIndexResult::Remain()); meta->embedded_index.reset(); meta->extra_files.clear(); auto missing = PredicateBuilder::IsNull(2, "missing", FieldType::INT); - ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN( + evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(missing)); ASSERT_EQ(result, FileIndexResult::Remain()); ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(nullptr)); @@ -631,9 +633,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEvaluateValidatesFieldsBeforeAndShortCi indexes["title"]["bitmap"] = nullptr; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); auto empty = PredicateBuilder::IsNotNull(1, "title", FieldType::STRING); ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(empty)); ASSERT_OK_AND_ASSIGN(bool remain, result->IsRemain()); @@ -651,9 +653,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestSessionReturnsQueryScores) { ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::MakeQueryIndex( "title", statistics_index_type_, queries, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_SEARCH)); ASSERT_TRUE(statistics_->executions.empty()); @@ -729,9 +731,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestSessionScalarAndSearchReadersCanCoexist indexes["title"][statistics_index_type_] = search_payload; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::VECTOR_SEARCH), @@ -761,9 +763,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestSessionPrecheckAndMissingScalarIndex) { ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::MakeIndex("title", statistics_index_type_, {{1, 0.5f}}, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("embedding", FileIndexReader::SearchType::VECTOR_SEARCH), "No File Index reader supports"); @@ -782,8 +784,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestSessionPrecheckAndMissingScalarIndex) { meta->embedded_index.reset(); meta->extra_files.clear(); - ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN( + evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH), "has no File Index"); @@ -796,9 +799,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyScalarIndexDoesNotProvideSearchCap indexes["title"]["bitmap"] = nullptr; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH), "No File Index reader supports FULL_TEXT_MATCH for field 'title' in data file " @@ -816,8 +819,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyScalarIndexDoesNotProvideSearchCap MockSearchFileIndex::MakePayload({{1, 0.5f}}, pool_)); ASSERT_OK_AND_ASSIGN(bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN( + evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_OK_AND_ASSIGN(auto result, evaluator->SearchFullText(MakeFullTextSearch("title"))); ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32::From({1})); @@ -830,9 +834,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptySearchIndexPreservesCapabilityAndS indexes["embedding"][MockSearchFileIndex::VECTOR_IDENTIFIER] = nullptr; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - path_factory_, meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*cache=*/nullptr, pool_)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_SEARCH)); ASSERT_NOK_WITH_MSG( @@ -856,6 +860,409 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptySearchIndexPreservesCapabilityAndS ASSERT_TRUE(scored->IsEmpty()); } +namespace { + +struct SearchReaderOpenState { + int32_t count = 0; + bool fail = false; + std::weak_ptr pool; +}; + +class CacheableSearchReader final : public FileIndexReader { + public: + static constexpr int64_t kMemoryUsage = 256; + + CacheableSearchReader(std::shared_ptr delegate, + const std::shared_ptr& pool) + : pool_(pool), delegate_(std::move(delegate)), retained_(kMemoryUsage, pool.get()) {} + + bool SupportsSearch(SearchType search_type) const override { + return delegate_->SupportsSearch(search_type); + } + + Result> VisitVectorSearch( + const std::shared_ptr& search) override { + if (search->query.empty()) { + return Status::Invalid("Mock vector query cannot be empty"); + } + return delegate_->VisitVectorSearch(search); + } + + private: + std::shared_ptr pool_; + std::shared_ptr delegate_; + Bytes retained_; +}; + +class CacheableSearchIndexer final : public FileIndexer { + public: + explicit CacheableSearchIndexer(std::shared_ptr state) + : state_(std::move(state)), delegate_(SearchIndexType::VECTOR, nullptr) {} + + std::shared_ptr CreateEmptyReader() const override { + return delegate_.CreateEmptyReader(); + } + + Result> CreateReader( + ::ArrowSchema* schema, int64_t start, int64_t length, + const std::shared_ptr& input, + const std::shared_ptr& pool) const override { + ++state_->count; + state_->pool = pool; + if (state_->fail) { + return Status::IOError("Cannot open mock vector index"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr reader, + delegate_.CreateReader(schema, start, length, input, pool)); + return std::make_shared(std::move(reader), pool); + } + + Result> CreateWriter( + ::ArrowSchema* schema, const std::shared_ptr& pool) const override { + return delegate_.CreateWriter(schema, pool); + } + + private: + std::shared_ptr state_; + MockSearchFileIndexer delegate_; +}; + +class CacheableSearchFactory final : public FileIndexerFactory { + public: + explicit CacheableSearchFactory(std::shared_ptr state) + : state_(std::move(state)) {} + + const char* Identifier() const override { + return "mock-cacheable-vector"; + } + + Result> Create( + const std::map&) const override { + return std::make_unique(state_); + } + + private: + std::shared_ptr state_; +}; + +class InvalidSearchReaderCache final : public CountingRoutingCache { + public: + InvalidSearchReaderCache() : CountingRoutingCache(CacheKind::INDEX_READER, 4096) {} + + Result> Get( + const std::shared_ptr&, + std::function>(const std::shared_ptr&)>) + override { + return std::make_shared>(std::make_shared(7), nullptr, + sizeof(int32_t)); + } +}; + +} // namespace + +class FileIndexCacheEvaluatorTest : public FileIndexSearchEvaluatorTest { + protected: + void SetUp() override { + FileIndexSearchEvaluatorTest::SetUp(); + open_state_ = std::make_shared(); + auto factory = std::make_unique(open_state_); + identifier_ = factory->Identifier(); + auto* factories = FactoryCreator::GetInstance(); + factories->Register(identifier_, factory.release()); + cache_ = std::make_shared(CacheKind::INDEX_READER, 1024 * 1024); + } + + void TearDown() override { + auto* factories = FactoryCreator::GetInstance(); + factories->TEST_Unregister(identifier_); + FileIndexSearchEvaluatorTest::TearDown(); + } + + Result> MakeCacheableIndex() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, + MockSearchFileIndex::MakeIndex("embedding", identifier_, + {{3, 0.75f}, {1, 0.25f}}, pool_)); + return MakeIndex(bytes); + } + + Result> Search( + const std::shared_ptr& meta, + const std::shared_ptr& search) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, cache_, pool_)); + return evaluator->SearchVector(search); + } + + Status PopulateCache(const std::shared_ptr& meta, const std::string& field) { + if (GetParam()) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), path_factory_, meta, + fs_, /*cache=*/nullptr, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr reader, + evaluator->GetSearchReader(field, FileIndexReader::SearchType::VECTOR_SEARCH)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr input, + FileIndexEvaluator::ExtractIndexInputStream(path_factory_, meta, fs_)); + PAIMON_ASSIGN_OR_RAISE(std::string uri, input->GetUri()); + return cache_->Put(CacheKey::ForIndexReader(uri, field, identifier_), + std::make_shared>( + std::move(reader), nullptr, CacheableSearchReader::kMemoryUsage)); + } + + std::shared_ptr open_state_; + std::string identifier_; + std::shared_ptr cache_; +}; + +TEST_P(FileIndexCacheEvaluatorTest, ReusesReaderWithIndependentQueries) { + ASSERT_OK_AND_ASSIGN(auto meta, MakeCacheableIndex()); + ASSERT_OK(PopulateCache(meta, "embedding")); + ASSERT_OK_AND_ASSIGN(auto first, Search(meta, MakeVectorSearch("embedding"))); + ASSERT_EQ(first->GetRowPositions(), RoaringBitmap32::From({1, 3})); + ASSERT_EQ(first->GetScores(), std::vector({0.25f, 0.75f})); + + auto filtered = MakeVectorSearch("embedding"); + filtered->limit = 1; + filtered->query = {5.0f, 6.0f}; + filtered->pre_filter = [](int64_t row_id) { return row_id != 3; }; + ASSERT_OK_AND_ASSIGN(auto second, Search(meta, filtered)); + ASSERT_EQ(second->GetRowPositions(), RoaringBitmap32::From({1})); + ASSERT_EQ(second->GetScores(), std::vector({0.25f})); + + auto invalid = MakeVectorSearch("embedding"); + invalid->query.clear(); + ASSERT_NOK_WITH_MSG(Search(meta, invalid), "Mock vector query cannot be empty"); + ASSERT_OK_AND_ASSIGN(auto last, Search(meta, MakeVectorSearch("embedding"))); + ASSERT_EQ(last->GetRowPositions(), first->GetRowPositions()); + ASSERT_EQ(last->GetScores(), first->GetScores()); + ASSERT_EQ(open_state_->count, GetParam() ? 4 : 1); + ASSERT_EQ(cache_->GetCount(), GetParam() ? 0 : 4); + ASSERT_EQ(cache_->SupplierCallCount(), 0); + ASSERT_EQ(cache_->Size(), GetParam() ? 0 : 1); +} + +TEST_P(FileIndexCacheEvaluatorTest, DisablesCache) { + ASSERT_OK_AND_ASSIGN(auto meta, MakeCacheableIndex()); + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), path_factory_, meta, + fs_, /*cache=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->SearchVector(MakeVectorSearch("embedding"))); + evaluator.reset(); + ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32::From({1, 3})); + } + ASSERT_EQ(open_state_->count, 2); + ASSERT_EQ(cache_->GetCount(), 0); + ASSERT_EQ(cache_->Size(), 0); +} + +TEST_P(FileIndexCacheEvaluatorTest, CacheMissUsesSuppliedPoolWithoutPopulatingCache) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeCacheableIndex()); + for (int32_t i = 0; i < 2; ++i) { + std::shared_ptr query_pool = GetMemoryPool(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), path_factory_, meta, + fs_, cache_, query_pool)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + evaluator->SearchVector(MakeVectorSearch("embedding"))); + ASSERT_EQ(query_pool, open_state_->pool.lock()); + ASSERT_GE(query_pool->CurrentUsage(), CacheableSearchReader::kMemoryUsage); + evaluator.reset(); + ASSERT_EQ(RoaringBitmap32::From({1, 3}), result->GetRowPositions()); + ASSERT_EQ(std::vector({0.25f, 0.75f}), result->GetScores()); + ASSERT_EQ(0, query_pool->CurrentUsage()); + ASSERT_EQ(0, cache_->Size()); + } + ASSERT_EQ(2, open_state_->count); + ASSERT_EQ(GetParam() ? 0 : 2, cache_->SupplierCallCount()); + ASSERT_TRUE(open_state_->pool.expired()); +} + +TEST_P(FileIndexCacheEvaluatorTest, DoesNotCacheEmptyIndex) { + FileIndexFormat::ColumnIndexes indexes = {{"embedding", {{identifier_, nullptr}}}}; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(auto result, Search(meta, MakeVectorSearch("embedding"))); + ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32()); + ASSERT_EQ(result->GetScores(), std::vector()); + } + ASSERT_EQ(open_state_->count, 0); + ASSERT_EQ(cache_->Size(), 0); + ASSERT_EQ(cache_->SupplierCallCount(), GetParam() ? 0 : 2); +} + +TEST_P(FileIndexCacheEvaluatorTest, BypassesCacheWithMultipleIndexTypes) { + cache_ = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto payload, MockSearchFileIndex::MakePayload({{1, 0.25f}}, pool_)); + FileIndexFormat::ColumnIndexes indexes = { + {"embedding", + {{identifier_, payload}, {"bitmap", nullptr}, {"unregistered-index", payload}}}}; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(auto result, Search(meta, MakeVectorSearch("embedding"))); + ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32::From({1})); + ASSERT_EQ(result->GetScores(), std::vector({0.25f})); + } + ASSERT_EQ(open_state_->count, 2); + ASSERT_EQ(cache_->Size(), 0); + ASSERT_EQ(cache_->SupplierCallCount(), 0); +} + +TEST_P(FileIndexCacheEvaluatorTest, FallsBackOnCacheMissForOtherVectorIndexTypes) { + ASSERT_OK_AND_ASSIGN( + auto bytes, MockSearchFileIndex::MakeIndex( + "embedding", MockSearchFileIndex::VECTOR_IDENTIFIER, {{1, 0.25f}}, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto result, Search(meta, MakeVectorSearch("embedding"))); + ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32::From({1})); + ASSERT_EQ(result->GetScores(), std::vector({0.25f})); + ASSERT_EQ(open_state_->count, 0); + ASSERT_EQ(cache_->Size(), 0); + ASSERT_EQ(cache_->SupplierCallCount(), GetParam() ? 0 : 1); +} + +TEST_P(FileIndexCacheEvaluatorTest, LoadsReaderAfterExternalEntryIsEvicted) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeCacheableIndex()); + ASSERT_OK(PopulateCache(meta, "embedding")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, cache_, pool_)); + ASSERT_OK(evaluator->GetSearchReader("embedding", FileIndexReader::SearchType::VECTOR_SEARCH)); + cache_->InvalidateAll(); + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + evaluator->SearchVector(MakeVectorSearch("embedding"))); + ASSERT_EQ(RoaringBitmap32::From({1, 3}), result->GetRowPositions()); + ASSERT_EQ(std::vector({0.25f, 0.75f}), result->GetScores()); + } + ASSERT_EQ(GetParam() ? 1 : 2, open_state_->count); + ASSERT_EQ(GetParam() ? 0 : 3, cache_->GetCount()); + ASSERT_EQ(GetParam() ? 0 : 2, cache_->SupplierCallCount()); + ASSERT_EQ(0, cache_->Size()); +} + +TEST_P(FileIndexCacheEvaluatorTest, RejectsMultipleReaders) { + cache_ = std::make_shared(); + ASSERT_OK_AND_ASSIGN(auto payload, MockSearchFileIndex::MakePayload({{1, 0.25f}}, pool_)); + FileIndexFormat::ColumnIndexes indexes = { + {"embedding", {{identifier_, payload}, {MockSearchFileIndex::VECTOR_IDENTIFIER, nullptr}}}}; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_NOK_WITH_MSG(Search(meta, MakeVectorSearch("embedding")), + "Multiple File Index readers support VECTOR_SEARCH for field 'embedding'"); + ASSERT_EQ(cache_->Size(), 0); + ASSERT_EQ(cache_->SupplierCallCount(), 0); +} + +TEST_P(FileIndexCacheEvaluatorTest, RetriesFailedLoadsWithoutPopulatingCache) { + ASSERT_OK_AND_ASSIGN(auto meta, MakeCacheableIndex()); + open_state_->fail = true; + ASSERT_NOK_WITH_MSG(Search(meta, MakeVectorSearch("embedding")), + "Cannot open mock vector index"); + ASSERT_EQ(cache_->Size(), 0); + open_state_->fail = false; + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(auto result, Search(meta, MakeVectorSearch("embedding"))); + ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32::From({1, 3})); + ASSERT_EQ(cache_->Size(), 0); + } + ASSERT_EQ(open_state_->count, 3); + ASSERT_EQ(cache_->SupplierCallCount(), GetParam() ? 0 : 3); +} + +TEST_P(FileIndexCacheEvaluatorTest, KeysReadersByFileAndField) { + ASSERT_OK_AND_ASSIGN(auto payload, MockSearchFileIndex::MakePayload({{1, 0.25f}}, pool_)); + FileIndexFormat::ColumnIndexes indexes; + indexes["embedding"][identifier_] = payload; + indexes["title"][identifier_] = payload; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK(PopulateCache(meta, "embedding")); + ASSERT_OK(PopulateCache(meta, "title")); + ASSERT_OK(Search(meta, MakeVectorSearch("embedding"))); + ASSERT_OK(Search(meta, MakeVectorSearch("title"))); + ASSERT_EQ(open_state_->count, 2); + auto options = core_options_.ToMap(); + options["file-index." + identifier_ + ".columns"] = "embedding"; + options["file-index." + identifier_ + ".embedding.variant"] = "second"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr options_evaluator, + FileIndexEvaluator::Create(data_schema_, options, path_factory_, meta, fs_, cache_, pool_)); + ASSERT_OK(options_evaluator->SearchVector(MakeVectorSearch("embedding"))); + options_evaluator.reset(); + ASSERT_EQ(open_state_->count, GetParam() ? 3 : 2); + auto schema = arrow::schema({arrow::field("embedding", arrow::list(arrow::float64()))}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema_evaluator, + FileIndexEvaluator::Create(schema, core_options_.ToMap(), path_factory_, + meta, fs_, cache_, pool_)); + ASSERT_OK(schema_evaluator->SearchVector(MakeVectorSearch("embedding"))); + schema_evaluator.reset(); + ASSERT_EQ(open_state_->count, GetParam() ? 4 : 2); + if (!GetParam()) { + const std::string copy_name = "another-data.orc.index"; + ASSERT_OK(fs_->WriteFile(path_factory_->ToAlignedPath(copy_name, meta), + std::string(bytes->data(), bytes->size()), /*overwrite=*/true)); + meta->extra_files = {copy_name}; + } + ASSERT_OK(Search(meta, MakeVectorSearch("embedding"))); + ASSERT_EQ(open_state_->count, GetParam() ? 5 : 3); + ASSERT_EQ(cache_->Size(), GetParam() ? 0 : 2); +} + +TEST_P(FileIndexCacheEvaluatorTest, RejectsReaderWithoutVectorSearchSupport) { + FileIndexFormat::ColumnIndexes indexes = {{"embedding", {{"bitmap", nullptr}}}}; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, cache_, pool_)); + if (!GetParam()) { + ASSERT_OK_AND_ASSIGN(auto input, + FileIndexEvaluator::ExtractIndexInputStream(path_factory_, meta, fs_)); + ASSERT_OK_AND_ASSIGN(auto uri, input->GetUri()); + ASSERT_OK(cache_->Put(CacheKey::ForIndexReader(uri, "embedding", "bitmap"), + std::make_shared>( + std::make_shared(), nullptr, 1))); + } + auto result = + evaluator->GetSearchReader("embedding", FileIndexReader::SearchType::VECTOR_SEARCH); + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().IsNotImplemented()); + ASSERT_NOK_WITH_MSG(result, + "No File Index reader supports VECTOR_SEARCH for field 'embedding'"); + ASSERT_EQ(cache_->GetCount(), GetParam() ? 0 : 1); + ASSERT_EQ(cache_->SupplierCallCount(), 0); +} + +TEST_P(FileIndexCacheEvaluatorTest, RejectsWrongCacheValue) { + ASSERT_OK_AND_ASSIGN(auto meta, MakeCacheableIndex()); + cache_ = std::make_shared(); + auto result = Search(meta, MakeVectorSearch("embedding")); + if (GetParam()) { + ASSERT_OK(result); + ASSERT_EQ(result.value()->GetRowPositions(), RoaringBitmap32::From({1, 3})); + ASSERT_EQ(open_state_->count, 1); + } else { + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().IsInvalid()); + ASSERT_EQ(open_state_->count, 0); + } +} + +INSTANTIATE_TEST_SUITE_P(IndexStorage, FileIndexCacheEvaluatorTest, ::testing::Bool(), + [](const ::testing::TestParamInfo& info) { + return info.param ? "Embedded" : "External"; + }); + INSTANTIATE_TEST_SUITE_P(IndexStorage, FileIndexSearchEvaluatorTest, ::testing::Bool(), [](const ::testing::TestParamInfo& info) { return info.param ? "Embedded" : "External"; @@ -879,17 +1286,17 @@ TEST_F(FileIndexEvaluatorTest, TestUnsupportedRowCountSkips32BitIndex) { /*column_max_sequence_numbers=*/std::nullopt); auto predicate = PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, - data_file_meta, nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), nullptr, + data_file_meta, nullptr, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(predicate)); ASSERT_TRUE(result->IsRemain().value()); ASSERT_EQ(dynamic_cast(result.get()), nullptr); data_file_meta->row_count = -1; - ASSERT_OK_AND_ASSIGN(evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, - data_file_meta, nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, + data_file_meta, nullptr, /*cache=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(predicate)); ASSERT_EQ(result, FileIndexResult::Remain()); @@ -897,9 +1304,9 @@ TEST_F(FileIndexEvaluatorTest, TestUnsupportedRowCountSkips32BitIndex) { indexes["f2"]["bitmap"] = nullptr; ASSERT_OK_AND_ASSIGN(data_file_meta->embedded_index, MockSearchFileIndex::Serialize(indexes, pool_)); - ASSERT_OK_AND_ASSIGN(evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, - data_file_meta, nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, + data_file_meta, nullptr, /*cache=*/nullptr, pool_)); auto missing = PredicateBuilder::IsNull(4, "missing", FieldType::INT); ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(missing)); ASSERT_EQ(result, FileIndexResult::Remain()); diff --git a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp index 6176900ab..7512c556c 100644 --- a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp +++ b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp @@ -128,9 +128,10 @@ TEST_F(IndexManifestFileHandlerTest, DecodedCacheReusesIpcAcrossReaders) { return Status::Invalid("index manifest was not cached"); })); ASSERT_TRUE(cached); - const auto& segment = cached->GetSegment(); - auto buffer = std::make_shared(reinterpret_cast(segment.Data()), - segment.Size()); + auto segment = CacheValue::Get(cached); + ASSERT_TRUE(segment); + auto buffer = std::make_shared(reinterpret_cast(segment->Data()), + segment->Size()); auto ipc = arrow::ipc::RecordBatchStreamReader::Open( std::make_shared(buffer)); ASSERT_TRUE(ipc.ok()) << ipc.status().ToString(); diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp index 119edde5f..094876e29 100644 --- a/src/paimon/core/manifest/manifest_list_test.cpp +++ b/src/paimon/core/manifest/manifest_list_test.cpp @@ -154,9 +154,10 @@ TEST_F(ManifestListTest, DecodedCacheReusesIpcAcrossReaders) { return Status::Invalid("manifest list was not cached"); })); ASSERT_TRUE(cached); - const auto& segment = cached->GetSegment(); - auto buffer = std::make_shared(reinterpret_cast(segment.Data()), - segment.Size()); + auto segment = CacheValue::Get(cached); + ASSERT_TRUE(segment); + auto buffer = std::make_shared(reinterpret_cast(segment->Data()), + segment->Size()); // The existing cache key contains IPC, not source-format bytes. auto ipc = arrow::ipc::RecordBatchStreamReader::Open( std::make_shared(buffer)); diff --git a/src/paimon/core/manifest/manifest_row_range_test.cpp b/src/paimon/core/manifest/manifest_row_range_test.cpp index 02089e1fb..e0e45c35c 100644 --- a/src/paimon/core/manifest/manifest_row_range_test.cpp +++ b/src/paimon/core/manifest/manifest_row_range_test.cpp @@ -554,7 +554,7 @@ TEST_F(RowRangeManifestFileTest, CacheChargesRetainedArrowCapacity) { return Status::Invalid("manifest was not cached"); })); ASSERT_TRUE(cached); - const int64_t logical_size = cached->GetSegment().Size(); + const int64_t logical_size = CacheValue::Get(cached)->Size(); ASSERT_GT(cached->GetMemoryUsage(), logical_size); ASSERT_EQ(pool->CurrentUsage(), cached->GetMemoryUsage()); cached.reset(); diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index a9c1470bb..9b3c7bfb6 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -242,7 +242,7 @@ Result> AbstractSplitRead::ApplyIndexAndDvReade PAIMON_ASSIGN_OR_RAISE( std::unique_ptr evaluator, FileIndexEvaluator::Create(data_schema, file_index_options, data_file_path_factory, - file, options_.GetFileSystem(), pool_)); + file, options_.GetFileSystem(), /*cache=*/nullptr, pool_)); PAIMON_ASSIGN_OR_RAISE(file_index_result, evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, file_index_result->IsRemain()); if (!is_remain) { diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index 62dc3a45d..6e121c9fc 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -1052,8 +1052,9 @@ TEST(AppendOnlyFileStoreScanTest, SnapshotCacheRebuildsCorruptValues) { 16 * 1024 * 1024); auto scan = BuildScan(table_path, cache, /*bucket=*/0, predicate, lazy_decode); scan->WithSnapshot(snapshot); - ASSERT_OK(cache->Put(scan->CreateSnapshotLiveManifestEntriesCacheKey(0), - std::make_shared(segment, CacheCallback()))); + ASSERT_OK( + cache->Put(scan->CreateSnapshotLiveManifestEntriesCacheKey(0), + std::make_shared(segment, CacheCallback()))); for (int32_t attempt = 0; attempt < 2; ++attempt) { ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); ASSERT_EQ(SortedFileNames(plan->Files()), expected_names); diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 9facbad4c..d13cb9199 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -586,7 +586,7 @@ Result> DataEvolutionSplitRead::SelectByFileIndex PAIMON_ASSIGN_OR_RAISE( std::unique_ptr evaluator, FileIndexEvaluator::Create(written_schema, file_index_options, data_file_path_factory, - file, options_.GetFileSystem(), pool_)); + file, options_.GetFileSystem(), /*cache=*/nullptr, pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, evaluator->Evaluate(data_predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, index_result->IsRemain()); @@ -627,7 +627,7 @@ Result> DataEvolutionSplitRead::ApplyIndexAndDv PAIMON_ASSIGN_OR_RAISE( std::unique_ptr evaluator, FileIndexEvaluator::Create(data_schema, file_index_options, data_file_path_factory, - file, options_.GetFileSystem(), pool_)); + file, options_.GetFileSystem(), /*cache=*/nullptr, pool_)); PAIMON_ASSIGN_OR_RAISE(file_index_result, evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, file_index_result->IsRemain()); if (!is_remain) { diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index 6a1a395b9..a40be6742 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -126,7 +126,7 @@ Result FileStoreScan::TestFileIndex(const std::shared_ptr& pred PAIMON_ASSIGN_OR_RAISE( std::unique_ptr evaluator, FileIndexEvaluator::Create(data_arrow_schema, file_index_options, /*path_factory=*/nullptr, - meta, /*file_system=*/nullptr, pool_)); + meta, /*file_system=*/nullptr, /*cache=*/nullptr, pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, evaluator->Evaluate(data_predicate)); return index_result->IsRemain(); @@ -430,10 +430,11 @@ Result FileStoreScan::ReadManifestEntriesWithCache( [](const std::shared_ptr&) -> Result> { return std::shared_ptr(); }); - if (value.ok() && value.value()) { + auto segment = value.ok() ? CacheValue::Get(value.value()) : nullptr; + if (segment) { Result> selected = - SnapshotLiveManifestEntries::ReadSnapshot(value.value()->GetSegment(), snapshot.Id(), - max_snapshots, row_filter, pool_); + SnapshotLiveManifestEntries::ReadSnapshot(*segment, snapshot.Id(), max_snapshots, + row_filter, pool_); if (predicate_failed) { return selected.status(); } @@ -447,8 +448,7 @@ Result FileStoreScan::ReadManifestEntriesWithCache( // Reuse the same cache value to retain complete historical snapshots on a miss. // Query-filtered entries must never be written back into the cache. Result deserialized = - SnapshotLiveManifestEntries::Deserialize(value.value()->GetSegment(), max_snapshots, - pool_); + SnapshotLiveManifestEntries::Deserialize(*segment, max_snapshots, pool_); if (deserialized.ok()) { cached_entries = std::move(deserialized.value()); } @@ -517,8 +517,8 @@ Status FileStoreScan::StoreSnapshotLiveManifestEntries( if (!bytes_result.ok()) { return Status::OK(); } - auto cache_value = - std::make_shared(MemorySegment::Wrap(bytes_result.value()), CacheCallback()); + auto cache_value = std::make_shared( + MemorySegment::Wrap(bytes_result.value()), CacheCallback()); Status status = core_options_.GetCache()->Put(CreateSnapshotLiveManifestEntriesCacheKey(bucket), cache_value); return status.ok() ? status : Status::OK(); diff --git a/src/paimon/core/operation/precise_file_index_read.cpp b/src/paimon/core/operation/precise_file_index_read.cpp index 8a9204dec..c538c530a 100644 --- a/src/paimon/core/operation/precise_file_index_read.cpp +++ b/src/paimon/core/operation/precise_file_index_read.cpp @@ -179,11 +179,11 @@ Result> PreciseFileIndexRead::Create( PAIMON_ASSIGN_OR_RAISE(index_options, FileIndexOptions::RewriteFieldNames( split_read.context_->GetCoreOptions().ToMap(), field_names)); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr indexes, - FileIndexEvaluator::Create(data_schema, index_options, path_factory, file, - split_read.context_->GetCoreOptions().GetFileSystem(), - split_read.pool_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexes, + FileIndexEvaluator::Create( + data_schema, index_options, path_factory, file, + split_read.context_->GetCoreOptions().GetFileSystem(), + split_read.context_->GetCoreOptions().GetCache(), split_read.pool_)); std::map read_by_name; for (const auto& field : read_fields) { diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 89cdb88b7..f259cf121 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -34,11 +34,14 @@ #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/core/core_options.h" #include "paimon/core/deletionvectors/bitmap64_deletion_vector.h" #include "paimon/core/deletionvectors/bitmap_deletion_vector.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/file_index_evaluator.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/core/schema/schema_manager.h" @@ -51,6 +54,7 @@ #include "paimon/factories/factory_creator.h" #include "paimon/format/file_format.h" #include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/predicate_builder.h" @@ -60,10 +64,60 @@ #include "paimon/table/source/data_split.h" #include "paimon/testing/mock/mock_search_file_index.h" #include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { + +class CacheTrackingFileIndexer : public FileIndexer { + public: + explicit CacheTrackingFileIndexer(std::shared_ptr open_count) + : open_count_(std::move(open_count)), delegate_(SearchIndexType::VECTOR, nullptr) {} + + std::shared_ptr CreateEmptyReader() const override { + return delegate_.CreateEmptyReader(); + } + + Result> CreateReader( + ::ArrowSchema* schema, int64_t start, int64_t length, + const std::shared_ptr& input, + const std::shared_ptr& pool) const override { + ++*open_count_; + return delegate_.CreateReader(schema, start, length, input, pool); + } + + Result> CreateWriter( + ::ArrowSchema* schema, const std::shared_ptr& pool) const override { + return delegate_.CreateWriter(schema, pool); + } + + private: + std::shared_ptr open_count_; + MockSearchFileIndexer delegate_; +}; + +class CacheTrackingFileIndexFactory : public FileIndexerFactory { + public: + explicit CacheTrackingFileIndexFactory(std::shared_ptr open_count) + : open_count_(std::move(open_count)) {} + + const char* Identifier() const override { + return "mock-cache-tracking-vector"; + } + + Result> Create( + const std::map&) const override { + return std::make_unique(open_count_); + } + + private: + std::shared_ptr open_count_; +}; + +} // namespace + class RawFileSplitReadTest : public ::testing::Test { protected: std::vector> PrepareDataSplits() const { @@ -540,6 +594,13 @@ TEST_F(RawFileSplitReadTest, TestCreateReaderWithNonPartitionWithReserveSequence } TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { + auto open_count = std::make_shared(0); + auto factory = std::make_unique(open_count); + const std::string index_type = factory->Identifier(); + auto* factories = FactoryCreator::GetInstance(); + factories->Register(index_type, factory.release()); + ScopeGuard unregister_factory([&]() { factories->TEST_Unregister(index_type); }); + auto cache = std::make_shared(CacheKind::INDEX_READER, 1024 * 1024); std::string path = paimon::test::GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; std::shared_ptr vector_search = std::make_shared( @@ -547,7 +608,9 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { VectorSearch::DistanceType::EUCLIDEAN, std::map{}); ReadContextBuilder context_builder(path); context_builder.EnablePreciseFileIndexEvaluate(true); - context_builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}).SetVectorSearch(vector_search); + context_builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}) + .SetVectorSearch(vector_search) + .WithCache(cache); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); @@ -559,9 +622,8 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { auto data_split = std::dynamic_pointer_cast(data_splits.front()); ASSERT_TRUE(data_split); ASSERT_OK_AND_ASSIGN( - data_split->DataFiles().front()->embedded_index, - MockSearchFileIndex::MakeIndex("f3", MockSearchFileIndex::VECTOR_IDENTIFIER, - {{0, 0.25f}, {2, 0.75f}}, pool_)); + std::shared_ptr index_bytes, + MockSearchFileIndex::MakeIndex("f3", index_type, {{0, 0.25f}, {2, 0.75f}}, pool_)); const CoreOptions& core_options = internal_context->GetCoreOptions(); std::shared_ptr data_schema = @@ -578,6 +640,35 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), /*external_path_weights=*/{}, pool_)); + auto index_dir = UniqueTestDirectory::Create(); + auto file_meta = data_split->DataFiles().front(); + auto fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto data_path_factory, + path_factory->CreateDataFilePathFactory(data_split->Partition(), data_split->Bucket())); + std::string data_bytes; + ASSERT_OK(fs->ReadFile(data_path_factory->ToPath(file_meta), &data_bytes)); + file_meta->external_path = index_dir->Str() + "/" + file_meta->file_name; + ASSERT_OK(fs->WriteFile(file_meta->external_path.value(), data_bytes, /*overwrite=*/true)); + const std::string index_name = file_meta->file_name + ".index"; + ASSERT_OK(fs->WriteFile(data_path_factory->ToAlignedPath(index_name, file_meta), + std::string(index_bytes->data(), index_bytes->size()), + /*overwrite=*/true)); + file_meta->extra_files = {index_name}; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_schema, core_options.ToMap(), data_path_factory, file_meta, + fs, /*cache=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr cached_reader, + evaluator->GetSearchReader("f3", FileIndexReader::SearchType::VECTOR_SEARCH)); + ASSERT_OK_AND_ASSIGN(auto input, + fs->Open(data_path_factory->ToAlignedPath(index_name, file_meta))); + ASSERT_OK_AND_ASSIGN(std::string uri, input->GetUri()); + ASSERT_OK(cache->Put( + CacheKey::ForIndexReader(uri, "f3", index_type), + std::make_shared>(std::move(cached_reader), nullptr, 1))); + evaluator.reset(); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique(path_factory, std::move(internal_context), @@ -594,6 +685,15 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { {R"([[0, 0.25, "Bob"], [0, 0.75, "Tony"]])"}, &expected) .ok()); ASSERT_TRUE(result->Equals(expected)); + ASSERT_EQ(*open_count, 1); + ASSERT_EQ(cache->SupplierCallCount(CacheKind::INDEX_READER), 0); + ASSERT_EQ(cache->Size(), 1); + ASSERT_OK_AND_ASSIGN(reader, split_read->CreateReader(data_split)); + ASSERT_OK_AND_ASSIGN(result, ReadResultCollector::CollectResult(std::move(reader))); + ASSERT_TRUE(result->Equals(expected)); + ASSERT_EQ(*open_count, 1); + ASSERT_EQ(cache->GetCount(CacheKind::INDEX_READER), 4); + ASSERT_EQ(cache->SupplierCallCount(CacheKind::INDEX_READER), 0); ReadContextBuilder without_score_builder(path); without_score_builder.EnablePreciseFileIndexEvaluate(true); @@ -616,6 +716,8 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { {R"([[0, "Bob"], [0, "Tony"]])"}, &without_score_expected) .ok()); ASSERT_TRUE(without_score_result->Equals(without_score_expected)); + ASSERT_EQ(*open_count, 2); + ASSERT_EQ(cache->GetCount(CacheKind::INDEX_READER), 4); } TEST_F(RawFileSplitReadTest, TestVectorSearchFiltersDeletionVectorBeforeTopK) { diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp index 185976387..dd0fff40e 100644 --- a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp +++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp @@ -103,9 +103,10 @@ RealtimeOffsetFileIndexLookup::LookupFilePositions(const RoaringBitmap64& offset std::shared_ptr predicate = PredicateBuilder::In( schema->GetFieldIndex(offset_name), offset_name, FieldType::BIGINT, literals); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr evaluator, - FileIndexEvaluator::Create(schema, options_, path_factory_, file, - file_system_, memory_pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(schema, options_, path_factory_, file, file_system_, + /*cache=*/nullptr, memory_pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool remains, index_result->IsRemain()); diff --git a/src/paimon/core/utils/objects_file.h b/src/paimon/core/utils/objects_file.h index 2a99e98d1..36ba10e05 100644 --- a/src/paimon/core/utils/objects_file.h +++ b/src/paimon/core/utils/objects_file.h @@ -302,7 +302,7 @@ Result> ObjectsFile::SerializeArrowBatches( CacheCallback(), data->capacity()) {} std::shared_ptr pool; std::shared_ptr buffer; - CacheValue value; + MemorySegmentCacheValue value; }; auto owner = std::make_shared(arrow_pool_, buffer); auto* value = &owner->value; @@ -324,7 +324,6 @@ Status ObjectsFile::ReadArrowBatches( auto cached = cache_->Get(key, [&](const std::shared_ptr&) { return SerializeArrowBatches(file_name, file_size, &batches, &read_status); }); - const bool usable = cached.ok() && cached.value() && cached.value()->GetSegment().Data(); if (read_status) { // Publish the query-independent cache before invoking user filters. A filter error // does not invalidate the cache or affect other readers of this file. @@ -334,12 +333,12 @@ Status ObjectsFile::ReadArrowBatches( } return Status::OK(); } - if (!usable) { + auto segment = cached.ok() ? CacheValue::Get(cached.value()) : nullptr; + if (!segment || !segment->Data()) { return ReadUncachedArrowBatches(file_name, file_size, consumer, prepare_reader); } - const auto& segment = cached.value()->GetSegment(); - auto buffer = std::make_shared(reinterpret_cast(segment.Data()), - segment.Size()); + auto buffer = std::make_shared(reinterpret_cast(segment->Data()), + segment->Size()); auto input = std::make_shared(buffer); auto read_options = arrow::ipc::IpcReadOptions::Defaults(); read_options.memory_pool = arrow_pool_.get(); diff --git a/src/paimon/format/parquet/parquet_input_stream.h b/src/paimon/format/parquet/parquet_input_stream.h index fbf45f244..e5650cfd9 100644 --- a/src/paimon/format/parquet/parquet_input_stream.h +++ b/src/paimon/format/parquet/parquet_input_stream.h @@ -105,15 +105,16 @@ class ParquetInputStream : public ArrowInputStreamAdapter { if (size != nbytes) { return Status::IOError("Short read of Parquet page index"); } - return std::make_shared(segment, CacheCallback()); + return std::make_shared(segment, CacheCallback()); }); if (!value.ok()) { return ToArrowStatus(value.status()); } - if (!value.value() || value.value()->GetSegment().Size() != nbytes) { + auto segment = CacheValue::Get(value.value()); + if (!segment || segment->Size() != nbytes) { return arrow::Status::IOError("Invalid Parquet page-index cache value"); } - std::memcpy(out, value.value()->GetSegment().Data(), nbytes); + std::memcpy(out, segment->Data(), nbytes); return nbytes; } diff --git a/src/paimon/format/parquet/parquet_reader_builder.h b/src/paimon/format/parquet/parquet_reader_builder.h index fad961ace..13dd9633e 100644 --- a/src/paimon/format/parquet/parquet_reader_builder.h +++ b/src/paimon/format/parquet/parquet_reader_builder.h @@ -165,15 +165,16 @@ class ParquetReaderBuilder : public ReaderBuilder { PAIMON_ASSIGN_OR_RAISE( MemorySegment segment, SerializeParquetMetadataFooter(input_stream, reader_properties, arrow_pool)); - return std::make_shared(segment, CacheCallback()); + return std::make_shared(segment, CacheCallback()); }; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache_value, cache_->Get(cache_key, supplier)); - if (cache_value == nullptr) { - return Status::Invalid("Parquet metadata cache returned nullptr value"); + auto segment = CacheValue::Get(cache_value); + if (!segment) { + return Status::Invalid("Parquet metadata cache returned an invalid value type"); } - return ParseParquetMetadataFooter(cache_value->GetSegment(), reader_properties); + return ParseParquetMetadataFooter(*segment, reader_properties); } int32_t batch_size_ = -1; diff --git a/src/paimon/indexer/lumina/CMakeLists.txt b/src/paimon/indexer/lumina/CMakeLists.txt index 58ba4daad..280956e7c 100644 --- a/src/paimon/indexer/lumina/CMakeLists.txt +++ b/src/paimon/indexer/lumina/CMakeLists.txt @@ -57,6 +57,7 @@ if(PAIMON_ENABLE_LUMINA) test_utils_static ${PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS} ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} + paimon_mock_file_format_static lumina::interface ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/indexer/lumina/lumina_file_index_test.cpp b/src/paimon/indexer/lumina/lumina_file_index_test.cpp index 91b418aac..3ffae7180 100644 --- a/src/paimon/indexer/lumina/lumina_file_index_test.cpp +++ b/src/paimon/indexer/lumina/lumina_file_index_test.cpp @@ -18,6 +18,9 @@ #include "paimon/indexer/lumina/lumina_file_index.h" +#include +#include +#include #include #include #include @@ -29,12 +32,19 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/io/byte_array_output_stream.h" +#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/file_index_evaluator.h" +#include "paimon/file_index/file_index_format.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/offset_output_stream.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" +#include "paimon/testing/mock/mock_file_format_factory.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/testharness.h" #include "paimon/utils/roaring_bitmap32.h" @@ -286,6 +296,7 @@ TEST(LuminaFileIndexTest, RoundTripWithTagPredicate) { "embedding", /*limit=*/1, std::vector{1.0f, 1.0f, 1.0f, 1.0f}, /*pre_filter=*/nullptr, predicate, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + search->pool = GetMemoryPool(); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, reader->VisitVectorSearch(search)); EXPECT_EQ(RoaringBitmap32::From({3}), result->GetRowPositions()); @@ -296,10 +307,336 @@ TEST(LuminaFileIndexTest, RoundTripWithTagPredicate) { "embedding", /*limit=*/1, std::vector{1.0f, 1.0f, 1.0f, 1.0f}, [](int64_t file_row) { return file_row == 0; }, predicate, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + filtered_search->pool = search->pool; ASSERT_OK_AND_ASSIGN(result, reader->VisitVectorSearch(filtered_search)); + ASSERT_EQ(0, search->pool->CurrentUsage()); + ASSERT_GT(search->pool->MaxMemoryUsage(), 0); EXPECT_EQ(RoaringBitmap32::From({0}), result->GetRowPositions()); ASSERT_EQ(1, result->GetScores().size()); EXPECT_FLOAT_EQ(4.0f, result->GetScores()[0]); } +namespace { + +class LuminaFileIndexCacheTest : public ::testing::Test { + protected: + void SetUp() override { + dir_ = paimon::test::UniqueTestDirectory::Create("local"); + cache_ = std::make_shared(CacheKind::INDEX_READER, + 16 * 1024 * 1024); + path_factory_ = std::make_shared(); + ASSERT_OK(path_factory_->Init(dir_->Str(), /*format_identifier=*/"orc", + /*data_file_prefix=*/"data-", nullptr)); + const std::map index_options = { + {"index.dimension", "4"}, + {"index.type", "bruteforce"}, + {"distance.metric", "l2"}, + {"encoding.type", "rawf32"}, + }; + for (const auto& [key, value] : index_options) { + options_["file-index.lumina.embedding." + key] = value; + } + options_["file-index.lumina.columns"] = "embedding"; + LuminaFileIndexer indexer(index_options); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema_, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + indexer.CreateWriter(&c_schema, pool_)); + ::ArrowArray c_batch; + ASSERT_TRUE(arrow::ExportArray(*CreateVectors(), &c_batch).ok()); + ASSERT_OK(writer->AddBatch(&c_batch)); + auto output = + std::make_shared(std::make_unique( + MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr format_writer, + FileIndexFormat::CreateWriter(output, FileIndexFormat::V_2, pool_)); + ASSERT_OK(format_writer->WriteIndex("embedding", "lumina", [&](OutputStream* payload) { + return writer->WriteTo(payload); + })); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK_AND_ASSIGN(container_, output->Finish(pool_.get())); + ASSERT_OK_AND_ASSIGN(meta_, + DataFileMeta::ForAppend( + "data-vectors.orc", /*file_size=*/0, /*row_count=*/3, + SimpleStats::EmptyStats(), /*min_sequence_number=*/0, + /*max_sequence_number=*/2, /*schema_id=*/0, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt)); + ASSERT_OK(WriteIndex("vectors.index")); + } + + Status WriteIndex(const std::string& name) { + meta_->extra_files = {name}; + return dir_->GetFileSystem()->WriteFile(path_factory_->ToAlignedPath(name, meta_), + std::string(container_->data(), container_->size()), + /*overwrite=*/false); + } + + Result> Search( + const std::shared_ptr& query) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr evaluator, + FileIndexEvaluator::Create(schema_, options_, path_factory_, meta_, + dir_->GetFileSystem(), cache_, pool_)); + return evaluator->SearchVector(query); + } + + Result> OpenReader( + const std::shared_ptr& reader_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr input, + dir_->GetFileSystem()->Open(path_factory_->ToAlignedPath("vectors.index", meta_))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_reader, + FileIndexFormat::CreateReader(input, reader_pool, options_)); + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, &c_schema)); + PAIMON_ASSIGN_OR_RAISE(std::vector> readers, + format_reader->ReadColumnIndex("embedding", &c_schema)); + if (readers.size() != 1) { + return Status::Invalid("Expected one Lumina reader"); + } + return readers.front(); + } + + Result> ReaderKey(const std::string& index_type) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr input, + dir_->GetFileSystem()->Open(path_factory_->ToAlignedPath("vectors.index", meta_))); + PAIMON_ASSIGN_OR_RAISE(std::string uri, input->GetUri()); + return CacheKey::ForIndexReader(uri, "embedding", index_type); + } + + Status PopulateCache() { + std::shared_ptr reader_pool = GetMemoryPool(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr reader, OpenReader(reader_pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key, ReaderKey("lumina")); + return cache_->Put(key, std::make_shared>( + std::move(reader), nullptr, reader_pool->CurrentUsage())); + } + + static std::shared_ptr Query(bool filter) { + auto query = std::make_shared( + "embedding", 1, std::vector{1.0f, 1.0f, 1.0f, 1.0f}, nullptr, nullptr, + VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + if (filter) { + query->pre_filter = [](int64_t row) { return row == 0; }; + } + return query; + } + + static void CheckResult(const std::shared_ptr& result, bool filter) { + ASSERT_TRUE(result); + ASSERT_EQ(RoaringBitmap32::From({filter ? 0 : 2}), result->GetRowPositions()); + ASSERT_EQ(std::vector({filter ? 4.0f : 0.0f}), result->GetScores()); + } + + std::unique_ptr dir_; + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr cache_; + std::shared_ptr path_factory_; + std::shared_ptr meta_; + std::shared_ptr container_; + std::shared_ptr schema_ = + arrow::schema({arrow::field("embedding", arrow::list(arrow::float32()))}); + std::map options_ = { + {"file.format", paimon::test::MockFileFormatFactory::IDENTIFIER}, + {"manifest.format", paimon::test::MockFileFormatFactory::IDENTIFIER}}; +}; + +TEST_F(LuminaFileIndexCacheTest, ReusesExternalReaderWithConcurrentQueryPools) { + ASSERT_OK(PopulateCache()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, Search(Query(false))); + CheckResult(result, false); + ASSERT_EQ(0, cache_->SupplierCallCount()); + + std::promise start; + std::shared_future ready = start.get_future().share(); + std::vector> searches; + constexpr int32_t kThreads = 8; + constexpr int32_t kQueriesPerThread = 32; + for (int32_t i = 0; i < kThreads; ++i) { + searches.push_back(std::async(std::launch::async, [&, i, ready] { + std::shared_ptr query_pool = GetMemoryPool(); + ready.wait(); + for (int32_t j = 0; j < kQueriesPerThread; ++j) { + bool filter = j % 2 == 0; + auto query = Query(filter); + if (i % 2 == 0) { + query->pool = query_pool; + } + query->options["lumina.search.parallel_number"] = "2"; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr found, Search(query)); + if (!(found->GetRowPositions() == RoaringBitmap32::From({filter ? 0 : 2})) || + found->GetScores() != std::vector({filter ? 4.0f : 0.0f}) || + query_pool->CurrentUsage() != 0) { + return Status::Invalid( + "Concurrent query returned wrong results or retained memory"); + } + } + if ((query_pool->MaxMemoryUsage() > 0) != (i % 2 == 0)) { + return Status::Invalid("Search did not use the requested pool"); + } + return Status::OK(); + })); + } + start.set_value(); + for (auto& search : searches) { + ASSERT_OK(search.get()); + } + ASSERT_EQ(0, cache_->SupplierCallCount()); + ASSERT_EQ(1 + kThreads * kQueriesPerThread, cache_->GetCount(CacheKind::INDEX_READER)); + ASSERT_EQ(1, cache_->Size()); +} + +TEST_F(LuminaFileIndexCacheTest, SearchUsesRequestedPoolAndFallsBackToReaderPool) { + std::shared_ptr reader_pool = GetMemoryPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, OpenReader(reader_pool)); + uint64_t retained = reader_pool->CurrentUsage(); + ASSERT_GT(retained, 0); + uint64_t reader_peak = reader_pool->MaxMemoryUsage(); + for (bool filter : {false, true}) { + auto query = Query(filter); + query->pool = GetMemoryPool(); + query->options["lumina.search.parallel_number"] = "2"; + auto copied = query->ReplacePreFilter(query->pre_filter); + ASSERT_EQ(query->pool, copied->pool); + ASSERT_OK_AND_ASSIGN(auto result, reader->VisitVectorSearch(copied)); + CheckResult(result, filter); + ASSERT_GT(query->pool->MaxMemoryUsage(), 0); + ASSERT_EQ(query->pool->CurrentUsage(), 0); + ASSERT_EQ(reader_pool->CurrentUsage(), retained); + ASSERT_EQ(reader_pool->MaxMemoryUsage(), reader_peak); + + copied->query.clear(); + ASSERT_NOK(reader->VisitVectorSearch(copied)); + ASSERT_EQ(query->pool->CurrentUsage(), 0); + copied->query = query->query; + copied->predicate = PredicateBuilder::Equal(0, "tag", FieldType::INT, Literal(1)); + ASSERT_NOK_WITH_MSG(reader->VisitVectorSearch(copied), + "lumina index was not built with tag"); + ASSERT_EQ(query->pool->CurrentUsage(), 0); + ASSERT_OK_AND_ASSIGN(result, reader->VisitVectorSearch(query)); + CheckResult(result, filter); + ASSERT_EQ(query->pool->CurrentUsage(), 0); + } + + // No query pool: temporary search allocations belong to the reader's pool. + std::atomic allocated_from_reader{false}; + auto fallback = Query(true); + fallback->pre_filter = [&](int64_t row) { + if (reader_pool->CurrentUsage() > retained) { + allocated_from_reader.store(true); + } + return row == 0; + }; + ASSERT_OK_AND_ASSIGN(auto result, reader->VisitVectorSearch(fallback)); + CheckResult(result, true); + ASSERT_TRUE(allocated_from_reader.load()); + ASSERT_EQ(reader_pool->CurrentUsage(), retained); + auto invalid = Query(false); + invalid->query.clear(); + ASSERT_NOK(reader->VisitVectorSearch(invalid)); + ASSERT_EQ(reader_pool->CurrentUsage(), retained); + reader.reset(); + ASSERT_EQ(reader_pool->CurrentUsage(), 0); +} + +TEST_F(LuminaFileIndexCacheTest, CacheMissDoesNotPopulateCache) { + pool_ = GetMemoryPool(); + for (int32_t i = 0; i < 2; ++i) { + auto query = Query(i != 0); + query->pool = GetMemoryPool(); + ASSERT_OK_AND_ASSIGN(auto result, Search(query)); + CheckResult(result, i != 0); + ASSERT_EQ(0, pool_->CurrentUsage()); + ASSERT_EQ(0, query->pool->CurrentUsage()); + ASSERT_GT(query->pool->MaxMemoryUsage(), 0); + ASSERT_EQ(0, cache_->Size()); + } + ASSERT_EQ(2, cache_->SupplierCallCount()); +} + +TEST_F(LuminaFileIndexCacheTest, QueryErrorsAndInFlightEvictionPreserveReaderLifetime) { + std::shared_ptr reader_pool = GetMemoryPool(); + ASSERT_OK_AND_ASSIGN(auto reader, OpenReader(reader_pool)); + std::weak_ptr weak_reader = reader; + std::weak_ptr weak_pool = reader_pool; + ASSERT_OK_AND_ASSIGN(auto key, ReaderKey("lumina")); + ASSERT_OK(cache_->Put(key, std::make_shared>( + reader, nullptr, reader_pool->CurrentUsage()))); + reader.reset(); + reader_pool.reset(); + auto invalid = Query(false); + invalid->pool = GetMemoryPool(); + invalid->query.clear(); + ASSERT_NOK(Search(invalid)); + ASSERT_EQ(0, invalid->pool->CurrentUsage()); + ASSERT_EQ(1, cache_->Size()); + + auto query = Query(true); + query->pool = GetMemoryPool(); + std::atomic alive{true}; + query->pre_filter = [&](int64_t row) { + cache_->InvalidateAll(); + if (weak_reader.expired() || weak_pool.expired()) { + alive.store(false); + } + return row == 0; + }; + ASSERT_OK_AND_ASSIGN(auto result, Search(query)); + CheckResult(result, true); + ASSERT_TRUE(alive.load()); + ASSERT_TRUE(weak_reader.expired()); + ASSERT_TRUE(weak_pool.expired()); + ASSERT_EQ(0, query->pool->CurrentUsage()); + ASSERT_EQ(0, cache_->Size()); + ASSERT_EQ(0, cache_->SupplierCallCount()); + ASSERT_OK_AND_ASSIGN(result, Search(Query(false))); + CheckResult(result, false); + ASSERT_EQ(1, cache_->SupplierCallCount()); + ASSERT_EQ(0, cache_->Size()); +} + +TEST_F(LuminaFileIndexCacheTest, IgnoresOtherIndexTypeCacheEntries) { + ASSERT_OK_AND_ASSIGN(auto key, ReaderKey("bitmap")); + ASSERT_OK(cache_->Put(key, std::make_shared>( + std::make_shared(7), nullptr, sizeof(int32_t)))); + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(auto result, Search(Query(false))); + CheckResult(result, false); + } + ASSERT_EQ(2, cache_->SupplierCallCount()); + ASSERT_EQ(1, cache_->Size()); +} + +TEST_F(LuminaFileIndexCacheTest, SeparatesFilesAndDoesNotRetainMisses) { + ASSERT_OK(PopulateCache()); + ASSERT_OK_AND_ASSIGN(auto result, Search(Query(false))); + CheckResult(result, false); + ASSERT_EQ(0, cache_->SupplierCallCount()); + ASSERT_OK(WriteIndex("other.index")); + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(result, Search(Query(false))); + CheckResult(result, false); + } + ASSERT_EQ(2, cache_->SupplierCallCount()); + ASSERT_EQ(1, cache_->Size()); +} + +TEST_F(LuminaFileIndexCacheTest, BypassesEmbeddedIndexesAndDisabledCache) { + meta_->embedded_index = container_; + ASSERT_OK_AND_ASSIGN(auto result, Search(Query(false))); + CheckResult(result, false); + ASSERT_OK_AND_ASSIGN(result, Search(Query(true))); + CheckResult(result, true); + ASSERT_EQ(0, cache_->GetCount()); + ASSERT_EQ(0, cache_->Size()); + + meta_->embedded_index.reset(); + cache_.reset(); + ASSERT_OK_AND_ASSIGN(result, Search(Query(false))); + CheckResult(result, false); +} + +} // namespace + } // namespace paimon::lumina::test diff --git a/src/paimon/indexer/lumina/lumina_index_searcher.cpp b/src/paimon/indexer/lumina/lumina_index_searcher.cpp index 78c025556..33d4e86f6 100644 --- a/src/paimon/indexer/lumina/lumina_index_searcher.cpp +++ b/src/paimon/indexer/lumina/lumina_index_searcher.cpp @@ -79,6 +79,8 @@ Result<::lumina::api::LuminaSearcher::SearchResult> LuminaIndexSearcher::Search( PAIMON_ASSIGN_OR_RAISE(::lumina::api::SearchOptions search_options, LuminaIndexOptions::CreateSearchOptions(*vector_search, index_info_)); ::lumina::api::Query query(vector_search->query.data(), vector_search->query.size()); + LuminaMemoryPool session_pool(vector_search->pool ? vector_search->pool + : pool_->GetPaimonPool()); if (vector_search->predicate) { if (!searcher_with_tag_) { @@ -89,7 +91,7 @@ Result<::lumina::api::LuminaSearcher::SearchResult> LuminaIndexSearcher::Search( if (!vector_search->pre_filter) { PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( ::lumina::api::LuminaSearcher::SearchResult search_result, - searcher_with_tag_->SearchWithTag(query, tag_filter, search_options, *pool_)); + searcher_with_tag_->SearchWithTag(query, tag_filter, search_options, session_pool)); return std::move(search_result); } auto filter = [pre_filter = vector_search->pre_filter]( @@ -97,14 +99,14 @@ Result<::lumina::api::LuminaSearcher::SearchResult> LuminaIndexSearcher::Search( PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( ::lumina::api::LuminaSearcher::SearchResult search_result, searcher_with_tag_->SearchWithTagAndFilter(query, tag_filter, filter, search_options, - *pool_)); + session_pool)); return std::move(search_result); } if (!vector_search->pre_filter) { PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( ::lumina::api::LuminaSearcher::SearchResult search_result, - searcher_->Search(query, search_options, *pool_)); + searcher_->Search(query, search_options, session_pool)); return std::move(search_result); } auto filter = [pre_filter = vector_search->pre_filter](::lumina::core::vector_id_t id) -> bool { @@ -112,7 +114,7 @@ Result<::lumina::api::LuminaSearcher::SearchResult> LuminaIndexSearcher::Search( }; PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( ::lumina::api::LuminaSearcher::SearchResult search_result, - searcher_with_filter_->SearchWithFilter(query, filter, search_options, *pool_)); + searcher_with_filter_->SearchWithFilter(query, filter, search_options, session_pool)); return std::move(search_result); }