diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 5fcc3b1e2..8430d3aa9 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -24,6 +24,8 @@ Paimon C++ supports the following global index types: - **Bitmap Index**: An index that maps each distinct scalar value to its matching row IDs. The file format is compatible with the Java bitmap global index. +- **Legacy Bitmap Index**: Wraps the bitmap file index as a global index under the name + ``bitmap-legacy``, accepting unsorted input and storing one index file per shard. - **BTree Index**: An efficient index based on multi-level SST files for scalar column lookups. - **Range Bitmap Index**: A range bitmap index optimized for range predicates on ordered scalar columns. Extends the bitmap approach by encoding value ordering, enabling efficient less-than, greater-than, and range conditions. - **Lucene Index**: A full-text search index powered by Lucene++. Supports tokenized text search with the JSON queries of the full-text index: match, multi-match, phrase and boolean queries. @@ -75,6 +77,18 @@ string predicates can scan selected dictionary files within a configurable fallb - ``bitmap-index.fallback-scan-max-size``: Maximum total selected index file size for fallback scans. Default: ``256MB``; set to ``0`` to disable fallback scans. +Legacy Bitmap Index +------------------- + +Use ``bitmap-legacy`` as the index type to build a global index using the bitmap file index +format. Input values may be unsorted. It supports equality, inequality, ``IN``, ``NOT IN``, +``IS NULL`` and ``IS NOT NULL`` predicates. Other predicates fall back to data filtering. +Each shard uses 32-bit local row IDs; the global index scan maps results to 64-bit global row IDs +and merges results across shards. + +The file index options ``version`` (default: ``2``) and ``index-block-size`` are passed through +to the bitmap file index writer. + BTree Index ----------- diff --git a/src/paimon/common/global_index/CMakeLists.txt b/src/paimon/common/global_index/CMakeLists.txt index a70ec81ca..7f03b9896 100644 --- a/src/paimon/common/global_index/CMakeLists.txt +++ b/src/paimon/common/global_index/CMakeLists.txt @@ -21,6 +21,8 @@ set(PAIMON_GLOBAL_INDEX_SRC bitmap/bitmap_index_reader.cpp bitmap/lazy_filtered_bitmap_reader.cpp bitmap/bitmap_global_index_factory.cpp + bitmap_legacy/bitmap_legacy_global_index.cpp + bitmap_legacy/bitmap_legacy_global_index_factory.cpp btree/btree_file_footer.cpp btree/btree_global_index_factory.cpp btree/btree_global_indexer.cpp diff --git a/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.cpp b/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.cpp new file mode 100644 index 000000000..0ffd587ae --- /dev/null +++ b/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.cpp @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h" + +#include "paimon/common/global_index/wrap/file_index_writer_wrapper.h" + +namespace paimon { + +Result>> BitmapLegacyGlobalIndex::GetExtraFieldNames() + const { + return std::optional>(std::nullopt); +} + +Result> BitmapLegacyGlobalIndex::CreateWriter( + const std::string& /*field_name*/, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, + index_->CreateWriter(arrow_schema, pool)); + return std::make_shared(BitmapLegacyGlobalIndexReader::kIdentifier, + file_writer, writer); +} + +Result> BitmapLegacyGlobalIndex::CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, int64_t /*total_row_count*/, + const std::optional>& /*row_ranges*/, + const std::shared_ptr& pool) const { + if (files.size() != 1) { + return Status::Invalid("BitmapLegacyGlobalIndex requires exactly one index file per shard"); + } + const auto& meta = files[0]; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr in, + file_reader->GetInputStream(meta.file_path)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr reader, + index_->CreateReader(arrow_schema, /*start=*/0, meta.file_size, in, pool)); + return std::make_shared( + reader, FileIndexReaderWrapper::ToGlobalIndexResult); +} + +} // namespace paimon diff --git a/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h b/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h new file mode 100644 index 000000000..e70378d6e --- /dev/null +++ b/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/file_index/bitmap/bitmap_file_index.h" +#include "paimon/common/global_index/wrap/file_index_reader_wrapper.h" +#include "paimon/global_index/global_indexer.h" + +namespace paimon { +/// Exposes the bitmap file index as a global index with one file per shard. +// TODO(xinyu.lxy): Address the following limitations: +// 1. Define dedicated option forwarding for bitmap-legacy. The factory currently passes the full +// global index options map unchanged to BitmapFileIndex, which reads unprefixed "version" and +// "index-block-size" keys. +// 2. Add cache limits or eviction. Queried bitmaps, including empty results, are retained for the +// reader's lifetime. +// 3. Support concurrent reads. Lazy metadata initialization, bitmap caches and shared seek/read +// state are currently unsynchronized. +// 4. Support spilling or streaming writes. All keys and bitmaps for a shard are accumulated in +// memory, and the complete serialized index is also buffered in memory before being written. +class BitmapLegacyGlobalIndex : public GlobalIndexer { + public: + explicit BitmapLegacyGlobalIndex(const std::shared_ptr& index) + : index_(index) {} + + Result>> GetExtraFieldNames() const override; + + Result> CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const override; + + Result> CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, + const std::shared_ptr& pool) const override; + + private: + std::shared_ptr index_; +}; + +class BitmapLegacyGlobalIndexReader : public FileIndexReaderWrapper { + public: + BitmapLegacyGlobalIndexReader(const std::shared_ptr& reader, + const std::function>( + const std::shared_ptr&)>& transform) + : FileIndexReaderWrapper(reader, transform) {} + + static inline const char kIdentifier[] = "bitmap-legacy"; + + bool IsThreadSafe() const override { + return false; + } + + std::string GetIndexType() const override { + return kIdentifier; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index_factory.cpp b/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index_factory.cpp new file mode 100644 index 000000000..9e3979ade --- /dev/null +++ b/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index_factory.cpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index_factory.h" + +#include "paimon/common/file_index/bitmap/bitmap_file_index.h" +#include "paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h" + +namespace paimon { + +const char BitmapLegacyGlobalIndexFactory::IDENTIFIER[] = "bitmap-legacy-global"; + +Result> BitmapLegacyGlobalIndexFactory::Create( + const std::map& options) const { + return std::make_unique(std::make_shared(options)); +} + +REGISTER_PAIMON_FACTORY(BitmapLegacyGlobalIndexFactory); + +} // namespace paimon diff --git a/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index_factory.h b/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index_factory.h new file mode 100644 index 000000000..633c85945 --- /dev/null +++ b/src/paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index_factory.h @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/global_index/global_indexer_factory.h" + +namespace paimon { +/// Factory for creating bitmap-legacy global indexers. +class BitmapLegacyGlobalIndexFactory : public GlobalIndexerFactory { + public: + static const char IDENTIFIER[]; + + const char* Identifier() const override { + return IDENTIFIER; + } + + Result> Create( + const std::map& options) const override; +}; + +} // namespace paimon diff --git a/src/paimon/common/global_index/global_indexer_factory_test.cpp b/src/paimon/common/global_index/global_indexer_factory_test.cpp index 4e94c753f..b79546212 100644 --- a/src/paimon/common/global_index/global_indexer_factory_test.cpp +++ b/src/paimon/common/global_index/global_indexer_factory_test.cpp @@ -23,6 +23,7 @@ #include "gtest/gtest.h" #include "paimon/common/global_index/bitmap/bitmap_global_indexer.h" +#include "paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -34,6 +35,16 @@ TEST(GlobalIndexerFactoryTest, TestBitmap) { ASSERT_FALSE(indexer->SupportsFullTextSearch()); } +TEST(GlobalIndexerFactoryTest, TestBitmapLegacy) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, + GlobalIndexerFactory::Get("bitmap-legacy", {})); + ASSERT_TRUE(dynamic_cast(indexer.get())); + ASSERT_FALSE(indexer->SupportsFullTextSearch()); + ASSERT_OK_AND_ASSIGN(std::optional> extra_fields, + indexer->GetExtraFieldNames()); + ASSERT_FALSE(extra_fields.has_value()); +} + TEST(GlobalIndexerFactoryTest, TestNonExist) { std::map options; ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 25a9eceb3..cc52d8d20 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -4514,6 +4514,120 @@ TEST_P(GlobalIndexTest, TestFilteredRowIdReader) { ASSERT_EQ(rows, RoaringBitmap64::From({1, 3})); } +TEST_P(GlobalIndexTest, TestBitmapLegacyWriteCommitScanReadMultipleShards) { + CreateTable(); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + std::vector write_cols = schema->field_names(); + + // Unsorted values, including duplicates across shards and an all-null final shard. + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Alice", 0, 1, 10.0], +["Bob", 1, 1, 11.0], +[null, 2, 0, 12.0], +["Bob", 3, 0, 13.0], +["Alice", 4, 1, 14.0], +["Lucy", 5, 1, 15.0], +[null, 6, 0, 16.0], +["Bob", 7, 0, 17.0], +[null, 8, 1, 18.0], +[null, 9, 1, 19.0] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + + const std::vector shard_ranges = {Range(0, 3), Range(4, 6), Range(7, 7), Range(8, 9)}; + for (size_t i = 0; i < shard_ranges.size(); ++i) { + // Exercise both legacy file formats through the global index factory. + std::map options = {{"version", i % 2 == 0 ? "1" : "2"}}; + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", "bitmap-legacy", options, + shard_ranges[i])); + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, /*options=*/{}, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(auto readers, + global_index_scan->CreateReaders("f0", /*row_range_index=*/std::nullopt)); + ASSERT_EQ(readers.size(), 1u); + const auto& reader = readers[0]; + ASSERT_FALSE(reader->IsThreadSafe()); + + Literal alice(FieldType::STRING, "Alice", 5); + Literal bob(FieldType::STRING, "Bob", 3); + Literal lucy(FieldType::STRING, "Lucy", 4); + Literal missing(FieldType::STRING, "Missing", 7); + auto check_result = [](const Result>& result, + const std::vector& expected) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr index_result, result); + ASSERT_TRUE(index_result); + auto bitmap_result = std::dynamic_pointer_cast(index_result); + ASSERT_TRUE(bitmap_result); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, bitmap_result->GetBitmap()); + ASSERT_TRUE(bitmap); + ASSERT_EQ(*bitmap, RoaringBitmap64::From(expected)); + }; + check_result(reader->VisitEqual(alice), {0, 4}); + check_result(reader->VisitEqual(bob), {1, 3, 7}); + check_result(reader->VisitEqual(lucy), {5}); + check_result(reader->VisitEqual(missing), {}); + check_result(reader->VisitNotEqual(bob), {0, 4, 5}); + check_result(reader->VisitNotEqual(missing), {0, 1, 3, 4, 5, 7}); + check_result(reader->VisitIn({alice, bob, missing}), {0, 1, 3, 4, 7}); + check_result(reader->VisitNotIn({alice, bob, missing}), {5}); + check_result(reader->VisitIsNull(), {2, 6, 8, 9}); + check_result(reader->VisitIsNotNull(), {0, 1, 3, 4, 5, 7}); + + // Unsupported predicates retain the normal data-filtering fallback. + ASSERT_OK_AND_ASSIGN(auto unsupported, reader->VisitLessThan(bob)); + ASSERT_FALSE(unsupported); + + // Load each shard independently to verify its persisted results and global row-ID offset. + const std::vector> shard_bob_rows = {{1, 3}, {}, {7}, {}}; + const std::vector> shard_null_rows = {{2}, {6}, {}, {8, 9}}; + for (size_t i = 0; i < shard_ranges.size(); ++i) { + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, + RowRangeIndex::Create({shard_ranges[i]})); + ASSERT_OK_AND_ASSIGN(auto shard_readers, + global_index_scan->CreateReaders("f0", row_range_index)); + ASSERT_EQ(shard_readers.size(), 1u); + check_result(shard_readers[0]->VisitEqual(bob), shard_bob_rows[i]); + check_result(shard_readers[0]->VisitIsNull(), shard_null_rows[i]); + } + + // Select multiple shards while excluding another matching shard and the all-null shard. + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(4, 7)})); + ASSERT_OK_AND_ASSIGN(auto range_readers, + global_index_scan->CreateReaders("f0", row_range_index)); + ASSERT_EQ(range_readers.size(), 1u); + check_result(range_readers[0]->VisitEqual(bob), {7}); + check_result(range_readers[0]->VisitIsNull(), {6}); + + ASSERT_OK_AND_ASSIGN(RowRangeIndex outside_range, RowRangeIndex::Create({Range(100, 200)})); + ASSERT_OK_AND_ASSIGN(auto outside_readers, + global_index_scan->CreateReaders("f0", outside_range)); + ASSERT_TRUE(outside_readers.empty()); + + // Evaluate the predicate through the global scan, then read the matching table rows. + auto predicate = PredicateBuilder::Equal(0, "f0", FieldType::STRING, bob); + auto scan_impl = std::dynamic_pointer_cast(global_index_scan); + ASSERT_TRUE(scan_impl); + check_result(scan_impl->Scan(predicate), {1, 3, 7}); + auto result_fields = fields_; + result_fields.insert(result_fields.begin(), SpecialFields::ValueKind().ArrowField()); + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Bob", 1, 1, 11.0], +[0, "Bob", 3, 0, 13.0], +[0, "Bob", 7, 0, 17.0] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); +} + TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndex) { // BTreeGlobalIndexWriter requires keys to be written in monotonically increasing order. // Therefore the source data must be pre-sorted by the indexed column (f0, string).