Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 9 additions & 12 deletions include/paimon/global_index/bitmap_global_index_result.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#pragma once

#include <functional>
#include <memory>
#include <string>
#include <utility>
Expand All @@ -29,15 +30,15 @@
#include "paimon/visibility.h"

namespace paimon {
/// Represents a global index query result backed by a Roaring bitmap. Supplier-backed results
/// materialize their matching row ids on the first GetBitmap() call, avoiding unnecessary work
/// when the result is not used. OR between bitmap results materializes both operands immediately
/// and returns a cached bitmap without retaining the input results.
/// Represents a global index query result backed by a Roaring bitmap.
class PAIMON_EXPORT BitmapGlobalIndexResult : public GlobalIndexResult {
public:
using BitmapSupplier = std::function<Result<RoaringBitmap64>()>;
explicit BitmapGlobalIndexResult(BitmapSupplier bitmap_supplier)
: bitmap_supplier_(bitmap_supplier) {}
explicit BitmapGlobalIndexResult(RoaringBitmap64 bitmap) : bitmap_(std::move(bitmap)) {}

/// Executes the supplier immediately and returns its bitmap, or propagates its error.
/// The returned result does not retain the supplier or its captures.
static Result<std::shared_ptr<BitmapGlobalIndexResult>> Create(BitmapSupplier bitmap_supplier);

class Iterator : public GlobalIndexResult::Iterator {
public:
Expand Down Expand Up @@ -75,18 +76,14 @@ class PAIMON_EXPORT BitmapGlobalIndexResult : public GlobalIndexResult {

/// @return A non-owning, const pointer to the bitmap. The returned pointer is valid as long as
/// this BitmapGlobalIndexResult object is alive. The caller must not modify the bitmap.
/// @note Supplier-backed results construct the bitmap on the first call to this method.
/// Precomputed results and subsequent calls return the cached instance.
/// Materialization may involve non-trivial CPU/IO cost. **Not thread-safe**.
/// @note Returns the stored bitmap without performing computation or I/O.
Result<const RoaringBitmap64*> GetBitmap() const;

/// Creates `BitmapGlobalIndexResult` for all row ids in the given ranges.
/// @note Overlapping or unsorted ranges are accepted.
static std::shared_ptr<BitmapGlobalIndexResult> FromRanges(const std::vector<Range>& ranges);

private:
mutable bool initialized_ = false;
BitmapSupplier bitmap_supplier_;
mutable RoaringBitmap64 bitmap_;
RoaringBitmap64 bitmap_;
};
} // namespace paimon
3 changes: 1 addition & 2 deletions include/paimon/global_index/global_index_result.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,7 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
/// @return A `Result<bool>` where:
/// - `true` indicates the result is empty (no matching rows),
/// - `false` indicates at least one matching row exists,
/// - An error is returned only if internal state is corrupted or I/O fails
/// (e.g., during lazy loading of index data).
/// - An error is returned if the implementation cannot inspect its state.
virtual Result<bool> IsEmpty() const = 0;

/// Creates a new iterator over the selected row ids.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ TEST_F(BitmapGlobalIndexTest, WriteAndReadStringPredicates) {
CreateReader(field->type(), metas[0]));
ASSERT_FALSE(reader->IsThreadSafe());
ASSERT_EQ(reader->GetIndexType(), "bitmap");
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexResult> saved_result,
reader->VisitEqual(StringLiteral("apple")));
CheckResult(reader->VisitIsNull(), {2, 6});
CheckResult(reader->VisitIsNotNull(), {0, 1, 3, 4, 5});
CheckResult(reader->VisitEqual(StringLiteral("apple")), {0, 1});
Expand Down Expand Up @@ -184,6 +186,11 @@ TEST_F(BitmapGlobalIndexTest, WriteAndReadStringPredicates) {
ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(nullptr),
"Full text search is not supported in BitmapIndexReader");
ASSERT_OK(reader->Close());
ASSERT_NOK_WITH_MSG(reader->VisitEqual(StringLiteral("apple")), "closed BitmapIndexReader");
std::weak_ptr<BitmapIndexReader> weak_reader = reader;
reader.reset();
ASSERT_TRUE(weak_reader.expired());
CheckResult(saved_result, {0, 1});
}

TEST_F(BitmapGlobalIndexTest, WriteAndReadLogicalIntegerOrder) {
Expand Down
4 changes: 2 additions & 2 deletions src/paimon/common/global_index/bitmap/bitmap_index_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,9 @@ BitmapIndexReader::~BitmapIndexReader() {
[[maybe_unused]] Status close_status = Close();
}

std::shared_ptr<GlobalIndexResult> BitmapIndexReader::CreateResult(
Result<std::shared_ptr<GlobalIndexResult>> BitmapIndexReader::CreateResult(
std::function<Result<RoaringBitmap64>()> supplier) {
return std::make_shared<BitmapGlobalIndexResult>(std::move(supplier));
return BitmapGlobalIndexResult::Create(std::move(supplier));
}

Result<std::shared_ptr<GlobalIndexResult>> BitmapIndexReader::VisitIsNotNull() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ class BitmapIndexReader : public GlobalIndexReader,
file_size_(file_size),
pool_(std::move(pool)) {}

std::shared_ptr<GlobalIndexResult> CreateResult(
Result<std::shared_ptr<GlobalIndexResult>> CreateResult(
std::function<Result<RoaringBitmap64>()> supplier);

Result<RoaringBitmap64> IsNull();
Expand Down
70 changes: 23 additions & 47 deletions src/paimon/common/global_index/bitmap_global_index_result.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@
#include "paimon/global_index/bitmap_global_index_result.h"

namespace paimon {
Result<std::shared_ptr<BitmapGlobalIndexResult>> BitmapGlobalIndexResult::Create(
BitmapSupplier bitmap_supplier) {
PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 bitmap, bitmap_supplier());
return std::make_shared<BitmapGlobalIndexResult>(std::move(bitmap));
}

Result<std::unique_ptr<GlobalIndexResult::Iterator>> BitmapGlobalIndexResult::CreateIterator()
const {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* bitmap, GetBitmap());
Expand All @@ -30,13 +36,8 @@ Result<std::shared_ptr<GlobalIndexResult>> BitmapGlobalIndexResult::And(
const std::shared_ptr<GlobalIndexResult>& other) {
auto typed_result = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(other);
if (typed_result) {
auto supplier = [typed_result, result = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(
shared_from_this())]() -> Result<RoaringBitmap64> {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, result->GetBitmap());
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, typed_result->GetBitmap());
return RoaringBitmap64::And(*r1, *r2);
};
return std::make_shared<BitmapGlobalIndexResult>(supplier);
return std::make_shared<BitmapGlobalIndexResult>(
RoaringBitmap64::And(bitmap_, typed_result->bitmap_));
}
return GlobalIndexResult::And(other);
}
Expand All @@ -45,16 +46,8 @@ Result<std::shared_ptr<GlobalIndexResult>> BitmapGlobalIndexResult::Or(
const std::shared_ptr<GlobalIndexResult>& other) {
auto typed_result = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(other);
if (typed_result) {
// Repeated lazy ORs build deep supplier chains that retain operands and materialized
// intermediate unions. Merge eagerly to avoid deep recursion and let previous results
// be released when callers drop them. AND remains lazy so callers that short-circuit
// can avoid materializing unused intersections.
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, GetBitmap());
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, typed_result->GetBitmap());
auto result = std::make_shared<BitmapGlobalIndexResult>(BitmapSupplier{});
result->bitmap_ = RoaringBitmap64::Or(*r1, *r2);
result->initialized_ = true;
return result;
return std::make_shared<BitmapGlobalIndexResult>(
RoaringBitmap64::Or(bitmap_, typed_result->bitmap_));
}
return GlobalIndexResult::Or(other);
}
Expand All @@ -63,49 +56,32 @@ Result<std::shared_ptr<GlobalIndexResult>> BitmapGlobalIndexResult::AddOffset(in
if (offset == 0) {
return shared_from_this();
}
auto supplier = [offset, result = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(
shared_from_this())]() -> Result<RoaringBitmap64> {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* bitmap, result->GetBitmap());
RoaringBitmap64 bitmap64;
for (auto iter = bitmap->Begin(); iter != bitmap->End(); ++iter) {
bitmap64.Add(offset + (*iter));
}
bitmap64.RunOptimize();
return bitmap64;
};
return std::make_shared<BitmapGlobalIndexResult>(supplier);
RoaringBitmap64 bitmap64;
for (auto iter = bitmap_.Begin(); iter != bitmap_.End(); ++iter) {
bitmap64.Add(offset + (*iter));
}
bitmap64.RunOptimize();
return std::make_shared<BitmapGlobalIndexResult>(std::move(bitmap64));
}

Result<const RoaringBitmap64*> BitmapGlobalIndexResult::GetBitmap() const {
if (!initialized_) {
PAIMON_ASSIGN_OR_RAISE(bitmap_, bitmap_supplier_());
initialized_ = true;
}
return &bitmap_;
}

Result<bool> BitmapGlobalIndexResult::IsEmpty() const {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* bitmap, GetBitmap());
return bitmap->IsEmpty();
return bitmap_.IsEmpty();
}

std::string BitmapGlobalIndexResult::ToString() const {
auto bitmap = GetBitmap();
if (!bitmap.ok()) {
return bitmap.status().ToString();
}
return bitmap.value()->ToString();
return bitmap_.ToString();
}

std::shared_ptr<BitmapGlobalIndexResult> BitmapGlobalIndexResult::FromRanges(
const std::vector<Range>& ranges) {
BitmapGlobalIndexResult::BitmapSupplier supplier = [ranges]() -> Result<RoaringBitmap64> {
RoaringBitmap64 bitmap;
for (const auto& range : ranges) {
bitmap.AddRange(range.from, range.to + 1);
}
return bitmap;
};
return std::make_shared<BitmapGlobalIndexResult>(supplier);
RoaringBitmap64 bitmap;
for (const auto& range : ranges) {
bitmap.AddRange(range.from, range.to + 1);
}
return std::make_shared<BitmapGlobalIndexResult>(std::move(bitmap));
}
} // namespace paimon
88 changes: 61 additions & 27 deletions src/paimon/common/global_index/bitmap_global_index_result_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ TEST_F(BitmapGlobalIndexResultTest, TestIterator) {
auto bitmap_supplier = [&]() -> Result<RoaringBitmap64> {
return RoaringBitmap64::From(expected_ids);
};
auto index_result = std::make_shared<BitmapGlobalIndexResult>(bitmap_supplier);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<BitmapGlobalIndexResult> index_result,
BitmapGlobalIndexResult::Create(bitmap_supplier));
if (expected_ids.empty()) {
ASSERT_TRUE(index_result->IsEmpty().value());
}
Expand All @@ -101,11 +102,13 @@ TEST_F(BitmapGlobalIndexResultTest, TestAnd) {
auto bitmap_supplier1 = [&]() -> Result<RoaringBitmap64> {
return RoaringBitmap64::From(left);
};
auto index_result1 = std::make_shared<BitmapGlobalIndexResult>(bitmap_supplier1);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<BitmapGlobalIndexResult> index_result1,
BitmapGlobalIndexResult::Create(bitmap_supplier1));
auto bitmap_supplier2 = [&]() -> Result<RoaringBitmap64> {
return RoaringBitmap64::From(right);
};
auto index_result2 = std::make_shared<BitmapGlobalIndexResult>(bitmap_supplier2);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<BitmapGlobalIndexResult> index_result2,
BitmapGlobalIndexResult::Create(bitmap_supplier2));

ASSERT_OK_AND_ASSIGN(auto result, index_result1->And(index_result2));
ASSERT_EQ(result->ToString(), expected_str);
Expand All @@ -123,7 +126,8 @@ TEST_F(BitmapGlobalIndexResultTest, TestBitmapResultAndOtherResult) {
auto bitmap_supplier1 = [&]() -> Result<RoaringBitmap64> {
return RoaringBitmap64::From({1, 2, 3});
};
auto index_result1 = std::make_shared<BitmapGlobalIndexResult>(bitmap_supplier1);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<BitmapGlobalIndexResult> index_result1,
BitmapGlobalIndexResult::Create(bitmap_supplier1));

auto fake_result = std::make_shared<FakeGlobalIndexResult>(std::vector<int64_t>({1l, 2l, 7l}));

Expand All @@ -137,11 +141,13 @@ TEST_F(BitmapGlobalIndexResultTest, TestOr) {
auto bitmap_supplier1 = [&]() -> Result<RoaringBitmap64> {
return RoaringBitmap64::From(left);
};
auto index_result1 = std::make_shared<BitmapGlobalIndexResult>(bitmap_supplier1);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<BitmapGlobalIndexResult> index_result1,
BitmapGlobalIndexResult::Create(bitmap_supplier1));
auto bitmap_supplier2 = [&]() -> Result<RoaringBitmap64> {
return RoaringBitmap64::From(right);
};
auto index_result2 = std::make_shared<BitmapGlobalIndexResult>(bitmap_supplier2);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<BitmapGlobalIndexResult> index_result2,
BitmapGlobalIndexResult::Create(bitmap_supplier2));

ASSERT_OK_AND_ASSIGN(auto result, index_result1->Or(index_result2));
ASSERT_EQ(result->ToString(), expected_str);
Expand All @@ -155,27 +161,29 @@ TEST_F(BitmapGlobalIndexResultTest, TestOr) {
"{1,2,3,9223372036854775807}");
}

TEST_F(BitmapGlobalIndexResultTest, TestOrPropagatesSupplierErrorsImmediately) {
for (bool fail_left : {true, false}) {
auto left =
std::make_shared<BitmapGlobalIndexResult>([fail_left]() -> Result<RoaringBitmap64> {
if (fail_left) {
return Status::Invalid("left supplier failed");
}
return RoaringBitmap64::From({1});
});
auto right = std::make_shared<BitmapGlobalIndexResult>(
[]() -> Result<RoaringBitmap64> { return Status::Invalid("right supplier failed"); });
const char* expected_error = fail_left ? "left supplier failed" : "right supplier failed";
ASSERT_NOK_WITH_MSG(left->Or(right), expected_error);
}
TEST_F(BitmapGlobalIndexResultTest, TestOperationsDoNotRetainOperands) {
auto left = std::make_shared<BitmapGlobalIndexResult>(RoaringBitmap64::From({1, 2}));
auto right = std::make_shared<BitmapGlobalIndexResult>(RoaringBitmap64::From({2, 3}));
std::weak_ptr<GlobalIndexResult> weak_left = left;
std::weak_ptr<GlobalIndexResult> weak_right = right;
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexResult> intersection, left->And(right));
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexResult> union_result, left->Or(right));
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexResult> offset_result, left->AddOffset(10));
left.reset();
right.reset();
ASSERT_TRUE(weak_left.expired());
ASSERT_TRUE(weak_right.expired());
ASSERT_EQ(intersection->ToString(), "{2}");
ASSERT_EQ(union_result->ToString(), "{1,2,3}");
ASSERT_EQ(offset_result->ToString(), "{11,12}");
}

TEST_F(BitmapGlobalIndexResultTest, TestBitmapResultOrOtherResult) {
auto bitmap_supplier1 = [&]() -> Result<RoaringBitmap64> {
return RoaringBitmap64::From({1, 2, 3});
};
auto index_result1 = std::make_shared<BitmapGlobalIndexResult>(bitmap_supplier1);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<BitmapGlobalIndexResult> index_result1,
BitmapGlobalIndexResult::Create(bitmap_supplier1));

auto fake_result = std::make_shared<FakeGlobalIndexResult>(std::vector<int64_t>({1l, 2l, 7l}));

Expand All @@ -187,8 +195,33 @@ TEST_F(BitmapGlobalIndexResultTest, TestInvalidBitmapResult) {
auto bitmap_supplier = [&]() -> Result<RoaringBitmap64> {
return Status::Invalid("invalid supplier");
};
auto result = std::make_shared<BitmapGlobalIndexResult>(bitmap_supplier);
ASSERT_TRUE(result->ToString().find("Invalid: invalid supplier") != std::string::npos);
ASSERT_NOK_WITH_MSG(BitmapGlobalIndexResult::Create(bitmap_supplier), "invalid supplier");
}

TEST_F(BitmapGlobalIndexResultTest, TestCreateMaterializesAndReleasesSupplier) {
auto source = std::make_shared<RoaringBitmap64>(RoaringBitmap64::From({1, 4, 7}));
std::weak_ptr<RoaringBitmap64> weak_source = source;
int32_t supplier_calls = 0;
ASSERT_OK_AND_ASSIGN(
std::shared_ptr<BitmapGlobalIndexResult> result,
BitmapGlobalIndexResult::Create(
[source = std::move(source), &supplier_calls]() -> Result<RoaringBitmap64> {
++supplier_calls;
return *source;
}));
ASSERT_EQ(supplier_calls, 1);
ASSERT_TRUE(weak_source.expired());
ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* first, result->GetBitmap());
ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* second, result->GetBitmap());
ASSERT_EQ(first, second);
ASSERT_EQ(result->ToString(), "{1,4,7}");
ASSERT_OK_AND_ASSIGN(bool empty, result->IsEmpty());
ASSERT_FALSE(empty);
ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalIndexResult::Iterator> iterator,
result->CreateIterator());
ASSERT_TRUE(iterator->HasNext());
ASSERT_EQ(iterator->Next(), 1);
ASSERT_EQ(supplier_calls, 1);
}

TEST_F(BitmapGlobalIndexResultTest, TestFromRanges) {
Expand All @@ -209,15 +242,16 @@ TEST_F(BitmapGlobalIndexResultTest, TestFromRanges) {
TEST_F(BitmapGlobalIndexResultTest, TestAddOffset) {
{
int32_t supplier_calls = 0;
auto result = std::make_shared<BitmapGlobalIndexResult>(
[&supplier_calls]() -> Result<RoaringBitmap64> {
ASSERT_OK_AND_ASSIGN(
std::shared_ptr<BitmapGlobalIndexResult> result,
BitmapGlobalIndexResult::Create([&supplier_calls]() -> Result<RoaringBitmap64> {
++supplier_calls;
return RoaringBitmap64::From({0, 1, 2, 3, 4, 5});
});
}));
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexResult> result_with_offset,
result->AddOffset(0));
ASSERT_EQ(result_with_offset.get(), result.get());
ASSERT_EQ(supplier_calls, 0);
ASSERT_EQ(supplier_calls, 1);
ASSERT_EQ(result_with_offset->ToString(), "{0,1,2,3,4,5}");
ASSERT_EQ(supplier_calls, 1);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ Result<std::shared_ptr<GlobalIndexResult>> BitmapScoredGlobalIndexResult::And(
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, result->GetBitmap());
return RoaringBitmap64::And(*r1, *r2);
};
return std::make_shared<BitmapGlobalIndexResult>(supplier);
return BitmapGlobalIndexResult::Create(std::move(supplier));
}
auto bitmap_other = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(other);
if (bitmap_other) {
Expand Down Expand Up @@ -119,7 +119,7 @@ Result<std::shared_ptr<GlobalIndexResult>> BitmapScoredGlobalIndexResult::Or(
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, result->GetBitmap());
return RoaringBitmap64::Or(*r1, *r2);
};
return std::make_shared<BitmapGlobalIndexResult>(supplier);
return BitmapGlobalIndexResult::Create(std::move(supplier));
}
return GlobalIndexResult::Or(other);
}
Expand Down
Loading
Loading