Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 7 additions & 10 deletions include/paimon/global_index/bitmap_global_index_result.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,10 @@
#include "paimon/visibility.h"

namespace paimon {
/// Represents a global index query result that **lazily materializes** its matching row ids as a
/// Roaring bitmap. The underlying 64-bit Roaring bitmap is **not constructed during object
/// creation**; instead, it is built on-demand the first time GetBitmap() is called. This design
/// avoids unnecessary computation and memory allocation when the bitmap is not needed (e.g., during
/// early stopping).
/// Represents a global index query result backed by a Roaring bitmap. Supplier-backed results
/// materialize their matching row ids on the first GetBitmap() call, avoiding unnecessary work
/// when the result is not used. OR between bitmap results materializes both operands immediately
/// and returns a cached bitmap without retaining the input results.
class PAIMON_EXPORT BitmapGlobalIndexResult : public GlobalIndexResult {
public:
using BitmapSupplier = std::function<Result<RoaringBitmap64>()>;
Expand Down Expand Up @@ -76,11 +75,9 @@ class PAIMON_EXPORT BitmapGlobalIndexResult : public GlobalIndexResult {

/// @return A non-owning, const pointer to the bitmap. The returned pointer is valid as long as
/// this BitmapGlobalIndexResult object is alive. The caller must not modify the bitmap.
/// @note **Lazy initialization**: The bitmap is constructed only on the first call to this
/// method.
/// Subsequent calls return the cached instance. Construction may involve non-trivial
/// CPU/IO cost (e.g., read indexes or merging bitmap), so avoid calling this if the
/// bitmap is not actually required. **Not thread-safe**.
/// @note Supplier-backed results construct the bitmap on the first call to this method.
/// Precomputed results and subsequent calls return the cached instance.
/// Materialization may involve non-trivial CPU/IO cost. **Not thread-safe**.
Result<const RoaringBitmap64*> GetBitmap() const;

/// Creates `BitmapGlobalIndexResult` for all row ids in the given ranges.
Expand Down
3 changes: 3 additions & 0 deletions include/paimon/utils/roaring_bitmap64.h
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ class PAIMON_EXPORT RoaringBitmap64 {
/// Contain any value in the half-open interval [min, max).
bool ContainsAny(int64_t min, int64_t max) const;

/// Optimizes container encodings in place without changing the stored row ids.
void RunOptimize();

/// Serialize bitmap to bytes.
PAIMON_UNIQUE_PTR<Bytes> Serialize(MemoryPool* pool) const;

Expand Down
21 changes: 14 additions & 7 deletions src/paimon/common/global_index/bitmap_global_index_result.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,25 +45,32 @@ Result<std::shared_ptr<GlobalIndexResult>> BitmapGlobalIndexResult::Or(
const std::shared_ptr<GlobalIndexResult>& other) {
auto typed_result = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(other);
if (typed_result) {
auto supplier = [typed_result, result = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(
shared_from_this())]() -> Result<RoaringBitmap64> {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, result->GetBitmap());
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, typed_result->GetBitmap());
return RoaringBitmap64::Or(*r1, *r2);
};
return std::make_shared<BitmapGlobalIndexResult>(supplier);
// Repeated lazy ORs build deep supplier chains that retain operands and materialized
// intermediate unions. Merge eagerly to avoid deep recursion and let previous results
// be released when callers drop them. AND remains lazy so callers that short-circuit
// can avoid materializing unused intersections.
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, GetBitmap());
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, typed_result->GetBitmap());
auto result = std::make_shared<BitmapGlobalIndexResult>(BitmapSupplier{});
result->bitmap_ = RoaringBitmap64::Or(*r1, *r2);
result->initialized_ = true;
return result;
}
return GlobalIndexResult::Or(other);
}

Result<std::shared_ptr<GlobalIndexResult>> BitmapGlobalIndexResult::AddOffset(int64_t offset) {
if (offset == 0) {
return shared_from_this();
}
auto supplier = [offset, result = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(
shared_from_this())]() -> Result<RoaringBitmap64> {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* bitmap, result->GetBitmap());
RoaringBitmap64 bitmap64;
for (auto iter = bitmap->Begin(); iter != bitmap->End(); ++iter) {
bitmap64.Add(offset + (*iter));
}
bitmap64.RunOptimize();
return bitmap64;
};
return std::make_shared<BitmapGlobalIndexResult>(supplier);
Expand Down
29 changes: 27 additions & 2 deletions src/paimon/common/global_index/bitmap_global_index_result_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,22 @@ TEST_F(BitmapGlobalIndexResultTest, TestOr) {
"{1,2,3,9223372036854775807}");
}

TEST_F(BitmapGlobalIndexResultTest, TestOrPropagatesSupplierErrorsImmediately) {
for (bool fail_left : {true, false}) {
auto left =
std::make_shared<BitmapGlobalIndexResult>([fail_left]() -> Result<RoaringBitmap64> {
if (fail_left) {
return Status::Invalid("left supplier failed");
}
return RoaringBitmap64::From({1});
});
auto right = std::make_shared<BitmapGlobalIndexResult>(
[]() -> Result<RoaringBitmap64> { return Status::Invalid("right supplier failed"); });
const char* expected_error = fail_left ? "left supplier failed" : "right supplier failed";
ASSERT_NOK_WITH_MSG(left->Or(right), expected_error);
}
}

TEST_F(BitmapGlobalIndexResultTest, TestBitmapResultOrOtherResult) {
auto bitmap_supplier1 = [&]() -> Result<RoaringBitmap64> {
return RoaringBitmap64::From({1, 2, 3});
Expand Down Expand Up @@ -192,9 +208,18 @@ TEST_F(BitmapGlobalIndexResultTest, TestFromRanges) {

TEST_F(BitmapGlobalIndexResultTest, TestAddOffset) {
{
auto result = BitmapGlobalIndexResult::FromRanges({Range(0, 5)});
ASSERT_OK_AND_ASSIGN(auto result_with_offset, result->AddOffset(0));
int32_t supplier_calls = 0;
auto result = std::make_shared<BitmapGlobalIndexResult>(
[&supplier_calls]() -> Result<RoaringBitmap64> {
++supplier_calls;
return RoaringBitmap64::From({0, 1, 2, 3, 4, 5});
});
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexResult> result_with_offset,
result->AddOffset(0));
ASSERT_EQ(result_with_offset.get(), result.get());
ASSERT_EQ(supplier_calls, 0);
ASSERT_EQ(result_with_offset->ToString(), "{0,1,2,3,4,5}");
ASSERT_EQ(supplier_calls, 1);

ASSERT_OK_AND_ASSIGN(result_with_offset, result->AddOffset(10));
ASSERT_EQ(result_with_offset->ToString(), "{10,11,12,13,14,15}");
Expand Down
4 changes: 4 additions & 0 deletions src/paimon/common/utils/roaring_bitmap64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,10 @@ bool RoaringBitmap64::IsEmpty() const {
return GetRoaringBitmap(roaring_bitmap_).isEmpty();
}

void RoaringBitmap64::RunOptimize() {
GetRoaringBitmap(roaring_bitmap_).runOptimize();
}

size_t RoaringBitmap64::GetSizeInBytes() const {
return GetRoaringBitmap(roaring_bitmap_).getSizeInBytes();
}
Expand Down
25 changes: 25 additions & 0 deletions src/paimon/common/utils/roaring_bitmap64_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,31 @@
#include "paimon/utils/range.h"

namespace paimon::test {
TEST(RoaringBitmap64Test, TestRunOptimize) {
RoaringBitmap64 bitmap;
constexpr int64_t kBoundary = int64_t{1} << 32;
for (int64_t i = kBoundary - 5000; i < kBoundary + 5000; ++i) {
bitmap.Add(i);
}
bitmap.Add(RoaringBitmap64::MAX_VALUE);
RoaringBitmap64 expected = bitmap;
size_t original_size = bitmap.GetSizeInBytes();

bitmap.RunOptimize();
ASSERT_EQ(bitmap, expected);
ASSERT_EQ(bitmap.Cardinality(), 10001);
ASSERT_LT(bitmap.GetSizeInBytes(), original_size);

size_t optimized_size = bitmap.GetSizeInBytes();
bitmap.RunOptimize();
ASSERT_EQ(bitmap, expected);
ASSERT_EQ(bitmap.GetSizeInBytes(), optimized_size);

RoaringBitmap64 empty;
empty.RunOptimize();
ASSERT_TRUE(empty.IsEmpty());
}

TEST(RoaringBitmap64Test, TestSimple) {
RoaringBitmap64 roaring;
ASSERT_TRUE(roaring.IsEmpty());
Expand Down
Loading