diff --git a/include/paimon/global_index/bitmap_global_index_result.h b/include/paimon/global_index/bitmap_global_index_result.h index 4c0c8e9f0..8587459f8 100644 --- a/include/paimon/global_index/bitmap_global_index_result.h +++ b/include/paimon/global_index/bitmap_global_index_result.h @@ -29,11 +29,10 @@ #include "paimon/visibility.h" namespace paimon { -/// Represents a global index query result that **lazily materializes** its matching row ids as a -/// Roaring bitmap. The underlying 64-bit Roaring bitmap is **not constructed during object -/// creation**; instead, it is built on-demand the first time GetBitmap() is called. This design -/// avoids unnecessary computation and memory allocation when the bitmap is not needed (e.g., during -/// early stopping). +/// Represents a global index query result backed by a Roaring bitmap. Supplier-backed results +/// materialize their matching row ids on the first GetBitmap() call, avoiding unnecessary work +/// when the result is not used. OR between bitmap results materializes both operands immediately +/// and returns a cached bitmap without retaining the input results. class PAIMON_EXPORT BitmapGlobalIndexResult : public GlobalIndexResult { public: using BitmapSupplier = std::function()>; @@ -76,11 +75,9 @@ class PAIMON_EXPORT BitmapGlobalIndexResult : public GlobalIndexResult { /// @return A non-owning, const pointer to the bitmap. The returned pointer is valid as long as /// this BitmapGlobalIndexResult object is alive. The caller must not modify the bitmap. - /// @note **Lazy initialization**: The bitmap is constructed only on the first call to this - /// method. - /// Subsequent calls return the cached instance. Construction may involve non-trivial - /// CPU/IO cost (e.g., read indexes or merging bitmap), so avoid calling this if the - /// bitmap is not actually required. **Not thread-safe**. + /// @note Supplier-backed results construct the bitmap on the first call to this method. + /// Precomputed results and subsequent calls return the cached instance. + /// Materialization may involve non-trivial CPU/IO cost. **Not thread-safe**. Result GetBitmap() const; /// Creates `BitmapGlobalIndexResult` for all row ids in the given ranges. diff --git a/include/paimon/utils/roaring_bitmap64.h b/include/paimon/utils/roaring_bitmap64.h index 931f1b528..78124e443 100644 --- a/include/paimon/utils/roaring_bitmap64.h +++ b/include/paimon/utils/roaring_bitmap64.h @@ -118,6 +118,9 @@ class PAIMON_EXPORT RoaringBitmap64 { /// Contain any value in the half-open interval [min, max). bool ContainsAny(int64_t min, int64_t max) const; + /// Optimizes container encodings in place without changing the stored row ids. + void RunOptimize(); + /// Serialize bitmap to bytes. PAIMON_UNIQUE_PTR Serialize(MemoryPool* pool) const; diff --git a/src/paimon/common/global_index/bitmap_global_index_result.cpp b/src/paimon/common/global_index/bitmap_global_index_result.cpp index 2533caca9..ea9a03a18 100644 --- a/src/paimon/common/global_index/bitmap_global_index_result.cpp +++ b/src/paimon/common/global_index/bitmap_global_index_result.cpp @@ -45,18 +45,24 @@ Result> BitmapGlobalIndexResult::Or( const std::shared_ptr& other) { auto typed_result = std::dynamic_pointer_cast(other); if (typed_result) { - auto supplier = [typed_result, result = std::dynamic_pointer_cast( - shared_from_this())]() -> Result { - PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, result->GetBitmap()); - PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, typed_result->GetBitmap()); - return RoaringBitmap64::Or(*r1, *r2); - }; - return std::make_shared(supplier); + // Repeated lazy ORs build deep supplier chains that retain operands and materialized + // intermediate unions. Merge eagerly to avoid deep recursion and let previous results + // be released when callers drop them. AND remains lazy so callers that short-circuit + // can avoid materializing unused intersections. + PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, GetBitmap()); + PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, typed_result->GetBitmap()); + auto result = std::make_shared(BitmapSupplier{}); + result->bitmap_ = RoaringBitmap64::Or(*r1, *r2); + result->initialized_ = true; + return result; } return GlobalIndexResult::Or(other); } Result> BitmapGlobalIndexResult::AddOffset(int64_t offset) { + if (offset == 0) { + return shared_from_this(); + } auto supplier = [offset, result = std::dynamic_pointer_cast( shared_from_this())]() -> Result { PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* bitmap, result->GetBitmap()); @@ -64,6 +70,7 @@ Result> BitmapGlobalIndexResult::AddOffset(in for (auto iter = bitmap->Begin(); iter != bitmap->End(); ++iter) { bitmap64.Add(offset + (*iter)); } + bitmap64.RunOptimize(); return bitmap64; }; return std::make_shared(supplier); diff --git a/src/paimon/common/global_index/bitmap_global_index_result_test.cpp b/src/paimon/common/global_index/bitmap_global_index_result_test.cpp index fedceaf49..e72330434 100644 --- a/src/paimon/common/global_index/bitmap_global_index_result_test.cpp +++ b/src/paimon/common/global_index/bitmap_global_index_result_test.cpp @@ -155,6 +155,22 @@ TEST_F(BitmapGlobalIndexResultTest, TestOr) { "{1,2,3,9223372036854775807}"); } +TEST_F(BitmapGlobalIndexResultTest, TestOrPropagatesSupplierErrorsImmediately) { + for (bool fail_left : {true, false}) { + auto left = + std::make_shared([fail_left]() -> Result { + if (fail_left) { + return Status::Invalid("left supplier failed"); + } + return RoaringBitmap64::From({1}); + }); + auto right = std::make_shared( + []() -> Result { return Status::Invalid("right supplier failed"); }); + const char* expected_error = fail_left ? "left supplier failed" : "right supplier failed"; + ASSERT_NOK_WITH_MSG(left->Or(right), expected_error); + } +} + TEST_F(BitmapGlobalIndexResultTest, TestBitmapResultOrOtherResult) { auto bitmap_supplier1 = [&]() -> Result { return RoaringBitmap64::From({1, 2, 3}); @@ -192,9 +208,18 @@ TEST_F(BitmapGlobalIndexResultTest, TestFromRanges) { TEST_F(BitmapGlobalIndexResultTest, TestAddOffset) { { - auto result = BitmapGlobalIndexResult::FromRanges({Range(0, 5)}); - ASSERT_OK_AND_ASSIGN(auto result_with_offset, result->AddOffset(0)); + int32_t supplier_calls = 0; + auto result = std::make_shared( + [&supplier_calls]() -> Result { + ++supplier_calls; + return RoaringBitmap64::From({0, 1, 2, 3, 4, 5}); + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_with_offset, + result->AddOffset(0)); + ASSERT_EQ(result_with_offset.get(), result.get()); + ASSERT_EQ(supplier_calls, 0); ASSERT_EQ(result_with_offset->ToString(), "{0,1,2,3,4,5}"); + ASSERT_EQ(supplier_calls, 1); ASSERT_OK_AND_ASSIGN(result_with_offset, result->AddOffset(10)); ASSERT_EQ(result_with_offset->ToString(), "{10,11,12,13,14,15}"); diff --git a/src/paimon/common/utils/roaring_bitmap64.cpp b/src/paimon/common/utils/roaring_bitmap64.cpp index 0bd4f8c1e..97f7bf7ca 100644 --- a/src/paimon/common/utils/roaring_bitmap64.cpp +++ b/src/paimon/common/utils/roaring_bitmap64.cpp @@ -285,6 +285,10 @@ bool RoaringBitmap64::IsEmpty() const { return GetRoaringBitmap(roaring_bitmap_).isEmpty(); } +void RoaringBitmap64::RunOptimize() { + GetRoaringBitmap(roaring_bitmap_).runOptimize(); +} + size_t RoaringBitmap64::GetSizeInBytes() const { return GetRoaringBitmap(roaring_bitmap_).getSizeInBytes(); } diff --git a/src/paimon/common/utils/roaring_bitmap64_test.cpp b/src/paimon/common/utils/roaring_bitmap64_test.cpp index e77aba039..da5a2c9c1 100644 --- a/src/paimon/common/utils/roaring_bitmap64_test.cpp +++ b/src/paimon/common/utils/roaring_bitmap64_test.cpp @@ -32,6 +32,31 @@ #include "paimon/utils/range.h" namespace paimon::test { +TEST(RoaringBitmap64Test, TestRunOptimize) { + RoaringBitmap64 bitmap; + constexpr int64_t kBoundary = int64_t{1} << 32; + for (int64_t i = kBoundary - 5000; i < kBoundary + 5000; ++i) { + bitmap.Add(i); + } + bitmap.Add(RoaringBitmap64::MAX_VALUE); + RoaringBitmap64 expected = bitmap; + size_t original_size = bitmap.GetSizeInBytes(); + + bitmap.RunOptimize(); + ASSERT_EQ(bitmap, expected); + ASSERT_EQ(bitmap.Cardinality(), 10001); + ASSERT_LT(bitmap.GetSizeInBytes(), original_size); + + size_t optimized_size = bitmap.GetSizeInBytes(); + bitmap.RunOptimize(); + ASSERT_EQ(bitmap, expected); + ASSERT_EQ(bitmap.GetSizeInBytes(), optimized_size); + + RoaringBitmap64 empty; + empty.RunOptimize(); + ASSERT_TRUE(empty.IsEmpty()); +} + TEST(RoaringBitmap64Test, TestSimple) { RoaringBitmap64 roaring; ASSERT_TRUE(roaring.IsEmpty());