Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/source/user_guide/global_index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ Paimon C++ supports the following global index types:

- **Bitmap Index**: An index that maps each distinct scalar value to its matching row IDs. The
file format is compatible with the Java bitmap global index.
- **Legacy Bitmap Index**: Wraps the bitmap file index as a global index under the name
``bitmap-legacy``, accepting unsorted input and storing one index file per shard.
- **BTree Index**: An efficient index based on multi-level SST files for scalar column lookups.
- **Range Bitmap Index**: A range bitmap index optimized for range predicates on ordered scalar columns. Extends the bitmap approach by encoding value ordering, enabling efficient less-than, greater-than, and range conditions.
- **Lucene Index**: A full-text search index powered by Lucene++. Supports tokenized text search with the JSON queries of the full-text index: match, multi-match, phrase and boolean queries.
Expand Down Expand Up @@ -75,6 +77,18 @@ string predicates can scan selected dictionary files within a configurable fallb
- ``bitmap-index.fallback-scan-max-size``: Maximum total selected index file size for fallback
scans. Default: ``256MB``; set to ``0`` to disable fallback scans.

Legacy Bitmap Index
-------------------

Use ``bitmap-legacy`` as the index type to build a global index using the bitmap file index
format. Input values may be unsorted. It supports equality, inequality, ``IN``, ``NOT IN``,
``IS NULL`` and ``IS NOT NULL`` predicates. Other predicates fall back to data filtering.
Each shard uses 32-bit local row IDs; the global index scan maps results to 64-bit global row IDs
and merges results across shards.

The file index options ``version`` (default: ``2``) and ``index-block-size`` are passed through
to the bitmap file index writer.

BTree Index
-----------

Expand Down
2 changes: 2 additions & 0 deletions src/paimon/common/global_index/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ set(PAIMON_GLOBAL_INDEX_SRC
bitmap/bitmap_index_reader.cpp
bitmap/lazy_filtered_bitmap_reader.cpp
bitmap/bitmap_global_index_factory.cpp
bitmap_legacy/bitmap_legacy_global_index.cpp
bitmap_legacy/bitmap_legacy_global_index_factory.cpp
btree/btree_file_footer.cpp
btree/btree_global_index_factory.cpp
btree/btree_global_indexer.cpp
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#include "paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h"

#include "paimon/common/global_index/wrap/file_index_writer_wrapper.h"

namespace paimon {

Result<std::optional<std::vector<std::string>>> BitmapLegacyGlobalIndex::GetExtraFieldNames()
const {
return std::optional<std::vector<std::string>>(std::nullopt);
}

Result<std::shared_ptr<GlobalIndexWriter>> BitmapLegacyGlobalIndex::CreateWriter(
const std::string& /*field_name*/, ::ArrowSchema* arrow_schema,
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
const std::shared_ptr<MemoryPool>& pool) const {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexWriter> writer,
index_->CreateWriter(arrow_schema, pool));
return std::make_shared<FileIndexWriterWrapper>(BitmapLegacyGlobalIndexReader::kIdentifier,
file_writer, writer);
}

Result<std::shared_ptr<GlobalIndexReader>> BitmapLegacyGlobalIndex::CreateReader(
::ArrowSchema* arrow_schema, const std::shared_ptr<GlobalIndexFileReader>& file_reader,
const std::vector<GlobalIndexIOMeta>& files, int64_t /*total_row_count*/,
const std::optional<std::vector<Range>>& /*row_ranges*/,
const std::shared_ptr<MemoryPool>& pool) const {
if (files.size() != 1) {
return Status::Invalid("BitmapLegacyGlobalIndex requires exactly one index file per shard");
}
const auto& meta = files[0];
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> in,
file_reader->GetInputStream(meta.file_path));
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<FileIndexReader> reader,
index_->CreateReader(arrow_schema, /*start=*/0, meta.file_size, in, pool));
return std::make_shared<BitmapLegacyGlobalIndexReader>(
reader, FileIndexReaderWrapper::ToGlobalIndexResult);
}

} // namespace paimon
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#pragma once

#include <cstdint>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <vector>

#include "paimon/common/file_index/bitmap/bitmap_file_index.h"
#include "paimon/common/global_index/wrap/file_index_reader_wrapper.h"
#include "paimon/global_index/global_indexer.h"

namespace paimon {
/// Exposes the bitmap file index as a global index with one file per shard.
// TODO(xinyu.lxy): Address the following limitations:
// 1. Define dedicated option forwarding for bitmap-legacy. The factory currently passes the full
// global index options map unchanged to BitmapFileIndex, which reads unprefixed "version" and
// "index-block-size" keys.
// 2. Add cache limits or eviction. Queried bitmaps, including empty results, are retained for the
// reader's lifetime.
// 3. Support concurrent reads. Lazy metadata initialization, bitmap caches and shared seek/read
// state are currently unsynchronized.
// 4. Support spilling or streaming writes. All keys and bitmaps for a shard are accumulated in
// memory, and the complete serialized index is also buffered in memory before being written.
class BitmapLegacyGlobalIndex : public GlobalIndexer {
public:
explicit BitmapLegacyGlobalIndex(const std::shared_ptr<BitmapFileIndex>& index)
: index_(index) {}

Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;

Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
const std::string& field_name, ::ArrowSchema* arrow_schema,
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
const std::shared_ptr<MemoryPool>& pool) const override;

Result<std::shared_ptr<GlobalIndexReader>> CreateReader(
::ArrowSchema* arrow_schema, const std::shared_ptr<GlobalIndexFileReader>& file_reader,
const std::vector<GlobalIndexIOMeta>& files, int64_t total_row_count,
const std::optional<std::vector<Range>>& row_ranges,
const std::shared_ptr<MemoryPool>& pool) const override;

private:
std::shared_ptr<BitmapFileIndex> index_;
};

class BitmapLegacyGlobalIndexReader : public FileIndexReaderWrapper {
public:
BitmapLegacyGlobalIndexReader(const std::shared_ptr<FileIndexReader>& reader,
const std::function<Result<std::shared_ptr<GlobalIndexResult>>(
const std::shared_ptr<FileIndexResult>&)>& transform)
: FileIndexReaderWrapper(reader, transform) {}

static inline const char kIdentifier[] = "bitmap-legacy";

bool IsThreadSafe() const override {
return false;
}

std::string GetIndexType() const override {
return kIdentifier;
}
};

} // namespace paimon
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#include "paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index_factory.h"

#include "paimon/common/file_index/bitmap/bitmap_file_index.h"
#include "paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h"

namespace paimon {

const char BitmapLegacyGlobalIndexFactory::IDENTIFIER[] = "bitmap-legacy-global";

Result<std::unique_ptr<GlobalIndexer>> BitmapLegacyGlobalIndexFactory::Create(
const std::map<std::string, std::string>& options) const {
return std::make_unique<BitmapLegacyGlobalIndex>(std::make_shared<BitmapFileIndex>(options));
}

REGISTER_PAIMON_FACTORY(BitmapLegacyGlobalIndexFactory);

} // namespace paimon
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#pragma once

#include <map>
#include <memory>
#include <string>

#include "paimon/global_index/global_indexer_factory.h"

namespace paimon {
/// Factory for creating bitmap-legacy global indexers.
class BitmapLegacyGlobalIndexFactory : public GlobalIndexerFactory {
public:
static const char IDENTIFIER[];

const char* Identifier() const override {
return IDENTIFIER;
}

Result<std::unique_ptr<GlobalIndexer>> Create(
const std::map<std::string, std::string>& options) const override;
};

} // namespace paimon
11 changes: 11 additions & 0 deletions src/paimon/common/global_index/global_indexer_factory_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

#include "gtest/gtest.h"
#include "paimon/common/global_index/bitmap/bitmap_global_indexer.h"
#include "paimon/common/global_index/bitmap_legacy/bitmap_legacy_global_index.h"
#include "paimon/testing/utils/testharness.h"

namespace paimon::test {
Expand All @@ -34,6 +35,16 @@ TEST(GlobalIndexerFactoryTest, TestBitmap) {
ASSERT_FALSE(indexer->SupportsFullTextSearch());
}

TEST(GlobalIndexerFactoryTest, TestBitmapLegacy) {
ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalIndexer> indexer,
GlobalIndexerFactory::Get("bitmap-legacy", {}));
ASSERT_TRUE(dynamic_cast<BitmapLegacyGlobalIndex*>(indexer.get()));
ASSERT_FALSE(indexer->SupportsFullTextSearch());
ASSERT_OK_AND_ASSIGN(std::optional<std::vector<std::string>> extra_fields,
indexer->GetExtraFieldNames());
ASSERT_FALSE(extra_fields.has_value());
}

TEST(GlobalIndexerFactoryTest, TestNonExist) {
std::map<std::string, std::string> options;
ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalIndexer> indexer,
Expand Down
Loading
Loading