Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/source/user_guide/file_index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ filter rows at the file-index-query stage, before reading all columns requested
the query from each data file. It evaluates the complete predicate to compute
exact matching row positions within that file.

``FileIndexResult::IsExact()`` reports whether a result needs scalar rechecking
within the candidates used for that evaluation. ``Remain`` is not exact, while
``Skip`` is exact. A bitmap alone does not imply exactness. Combining different
predicates preserves an exact guarantee only when justified by both results.

For a root full-text or vector TopK request, predicate filtering, row-range
restrictions and deletion vectors are applied before the TopK limit. Filtering after
TopK truncation could discard selected rows. The final row positions are then passed
Expand Down
2 changes: 1 addition & 1 deletion include/paimon/file_index/bitmap_index_result.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class PAIMON_EXPORT BitmapIndexResult : public FileIndexResult {
public:
using BitmapSupplier = std::function<Result<RoaringBitmap32>()>;

explicit BitmapIndexResult(BitmapSupplier bitmap_supplier);
explicit BitmapIndexResult(BitmapSupplier bitmap_supplier, bool is_exact = false);
~BitmapIndexResult() override;

/// @return Whether the file is remained.
Expand Down
9 changes: 9 additions & 0 deletions include/paimon/file_index/file_index_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
#include "paimon/visibility.h"

namespace paimon {
class LeafPredicate;
class RoaringBitmap32;
struct FullTextMatch;
/// Evaluates filter predicates against a file-level index to determine file eligibility.
///
Expand All @@ -43,13 +45,20 @@ struct FullTextMatch;
/// All `Visit` methods in `FileIndexReader`, will return a non-null result object on success,
/// including when no rows match.
///
/// Readers set FileIndexResult::IsExact() when creating each result; callers use that guarantee.
/// @note Callers of `Visit*` for DECIMAL fields must ensure each literal's scale matches the scale
/// of the indexed data; otherwise, index filtering results may be incorrect.
class PAIMON_EXPORT FileIndexReader : public IndexReaderCapabilities,
public FunctionVisitor<std::shared_ptr<FileIndexResult>> {
public:
~FileIndexReader() override = default;

/// Evaluate a scalar leaf within eligible file-local physical row positions.
/// A null candidates pointer means the entire file. Otherwise, any returned bitmap must
/// be a subset of candidates. Backends may use it to avoid reading irrelevant index data.
virtual Result<std::shared_ptr<FileIndexResult>> Evaluate(
const std::shared_ptr<LeafPredicate>& predicate, const RoaringBitmap32* candidates);

Result<std::shared_ptr<FileIndexResult>> VisitIsNotNull() override;

Result<std::shared_ptr<FileIndexResult>> VisitIsNull() override;
Expand Down
23 changes: 19 additions & 4 deletions include/paimon/file_index/file_index_result.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,20 +43,33 @@ class PAIMON_EXPORT FileIndexResult : public std::enable_shared_from_this<FileIn
/// @return Whether the file is remained.
virtual Result<bool> IsRemain() const = 0;

/// Compute the intersection of the current result with the provided result.
/// Whether these rows exactly match the evaluated predicate within its candidate scope.
Comment thread
lszskye marked this conversation as resolved.
Resolved
bool IsExact() const {
return is_exact_;
}

/// Compute the intersection for two predicates, conservatively propagating precision.
virtual Result<std::shared_ptr<FileIndexResult>> And(
const std::shared_ptr<FileIndexResult>& other);
const std::shared_ptr<FileIndexResult>& other) = 0;

/// Compute the union of the current result with the provided result.
/// Compute the union for two predicates, conservatively propagating precision.
virtual Result<std::shared_ptr<FileIndexResult>> Or(
const std::shared_ptr<FileIndexResult>& other);
const std::shared_ptr<FileIndexResult>& other) = 0;

virtual std::string ToString() const = 0;

protected:
explicit FileIndexResult(bool is_exact) : is_exact_(is_exact) {}

private:
bool is_exact_;
};

/// Concrete implementation of FileIndexResult that always retains the file.
class PAIMON_EXPORT Remain : public FileIndexResult {
public:
Remain() : FileIndexResult(false) {}

Result<bool> IsRemain() const override;
Result<std::shared_ptr<FileIndexResult>> And(
const std::shared_ptr<FileIndexResult>& other) override;
Expand All @@ -68,6 +81,8 @@ class PAIMON_EXPORT Remain : public FileIndexResult {
/// Concrete implementation of FileIndexResult that always skips the file.
class PAIMON_EXPORT Skip : public FileIndexResult {
public:
Skip() : FileIndexResult(true) {}

Result<bool> IsRemain() const override;
Result<std::shared_ptr<FileIndexResult>> And(
const std::shared_ptr<FileIndexResult>& other) override;
Expand Down
12 changes: 10 additions & 2 deletions include/paimon/file_index/scored_file_index_result.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,21 @@ namespace paimon {
class PAIMON_EXPORT ScoredFileIndexResult : public FileIndexResult {
public:
static Result<std::shared_ptr<ScoredFileIndexResult>> Create(RoaringBitmap32&& row_positions,
std::vector<float>&& scores);
std::vector<float>&& scores,
bool is_exact = false);
Comment thread
lszskye marked this conversation as resolved.
Resolved

bool IsEmpty() const {
return row_positions_.IsEmpty();
}

Result<bool> IsRemain() const override;

Result<std::shared_ptr<FileIndexResult>> And(
const std::shared_ptr<FileIndexResult>& other) override;

Result<std::shared_ptr<FileIndexResult>> Or(
const std::shared_ptr<FileIndexResult>& other) override;

const RoaringBitmap32& GetRowPositions() const {
return row_positions_;
}
Expand All @@ -52,7 +59,8 @@ class PAIMON_EXPORT ScoredFileIndexResult : public FileIndexResult {
std::string ToString() const override;

private:
ScoredFileIndexResult(RoaringBitmap32&& row_positions, std::vector<float>&& scores);
ScoredFileIndexResult(RoaringBitmap32&& row_positions, std::vector<float>&& scores,
bool is_exact);

RoaringBitmap32 row_positions_;
std::vector<float> scores_;
Expand Down
7 changes: 7 additions & 0 deletions include/paimon/predicate/predicate_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ class PAIMON_EXPORT PredicateUtils {
template <typename T>
static Result<T> VisitPredicate(const std::shared_ptr<LeafPredicate>& predicate,
const std::shared_ptr<FunctionVisitor<T>>& visitor) {
return VisitPredicate(predicate, visitor.get());
}

/// Dispatches through a borrowed visitor without requiring shared ownership.
template <typename T>
static Result<T> VisitPredicate(const std::shared_ptr<LeafPredicate>& predicate,
FunctionVisitor<T>* visitor) {
auto type = predicate->GetFunction().GetType();
switch (type) {
case Function::Type::IS_NULL:
Expand Down
3 changes: 2 additions & 1 deletion include/paimon/predicate/vector_search.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ struct PAIMON_EXPORT VectorSearch {
/// The distance metric to use for this query, if explicitly specified.
/// If set, this value must match the distance type used by the index (e.g., EUCLIDEAN, COSINE).
/// A mismatch will result in an error during query execution.
/// If not set (std::nullopt), the query will use the distance type configured in the index.
/// Root searches configured through ReadContextBuilder require an explicit supported type.
/// Direct index queries may omit it (std::nullopt) to use the index's configured type.
std::optional<DistanceType> distance_type;
/// A key-value map of query-specific runtime options.
/// Such as the size of candidate list in approximate search or parallelism for this query.
Expand Down
14 changes: 7 additions & 7 deletions include/paimon/read_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -374,9 +374,9 @@ class PAIMON_EXPORT ReadContextBuilder {
/// Configure the root vector TopK request, logically applied after the predicate filter.
/// The search field need not be projected. SetVectorSearch is mutually exclusive with
/// SetFullTextSearch and must have a positive limit.
/// distance_type must be EUCLIDEAN, INNER_PRODUCT or COSINE and match each queried index.
/// Scores are required internally for TopK. Project `_<field>_INDEX_SCORE` to include them
/// in the result.
/// TODO(lisizhuo.lsz): Merge TopK across files.
/// @param vector_search Vector search request, or nullptr to clear it.
/// @return Reference to this builder for method chaining.
/// @note Requires EnablePreciseFileIndexEvaluate(true).
Expand All @@ -388,19 +388,19 @@ class PAIMON_EXPORT ReadContextBuilder {
/// SetVectorSearch and must have a positive limit.
/// Scores are required internally for TopK. Project `_<field>_INDEX_SCORE` to include them
/// in the result.
/// TODO(lisizhuo.lsz): Merge TopK across files.
/// @param full_text_search Full-text search request, or nullptr to clear it.
/// @return Reference to this builder for method chaining.
/// @note Requires EnablePreciseFileIndexEvaluate(true).
ReadContextBuilder& SetFullTextSearch(const std::shared_ptr<FullTextSearch>& full_text_search);

/// Compute exact file-local result row positions during file-index evaluation, using the
/// configured Predicate and optional VectorSearch or FullTextSearch request.
/// Requires file-index.read.enabled=true. Reads available file indexes and rechecks scalar
/// index candidates against the data. If a scalar predicate's field has no index, reads the
/// data to compute exact matching rows. Only scalar predicates may be evaluated without an
/// index. Score output must use named projections such as _<field>_INDEX_SCORE;
/// projecting _INDEX_SCORE by name, field ID or read schema is not supported in this mode.
/// Requires file-index.read.enabled=true. Uses exact scalar index results directly and
/// rechecks the original candidate scope when indexes are inexact. If a scalar predicate's
/// field has no index, reads the data to compute exact matching rows. Only scalar predicates
/// may be evaluated without an index. Score output must use named projections such as
/// _<field>_INDEX_SCORE; projecting _INDEX_SCORE by name, field ID or read schema is not
/// supported in this mode.
/// Real-time append and deduplicate reads use pinned offsets and exact scalar indexes,
/// scanning predicates that have no exact index. Real-time primary-key merge is not supported.
/// The default is false.
Expand Down
4 changes: 4 additions & 0 deletions include/paimon/table/source/table_read.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ class PAIMON_EXPORT TableRead {
/// real-time views before consuming any tickets. The reader retains those views and returns
/// a terminal error if a split fails to open when reached.
///
/// With a root vector or full-text search, returns at most the request's limit across all
/// supplied splits, ordered by search score after predicate, deletion-vector and row-range
/// filtering.
///
/// @param splits The splits to read.
/// @return A Result containing a unique pointer to the `BatchReader` instance.
/// @note `BatchReader`s created by the same `TableRead` are not thread-safe for
Expand Down
1 change: 1 addition & 0 deletions src/paimon/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -504,6 +504,7 @@ set(PAIMON_CORE_SRCS
core/table/source/startup_mode.cpp
core/table/source/table_read.cpp
core/table/source/table_read_utils.cpp
core/table/source/topk_batch_reader.cpp
core/table/source/table_scan.cpp
core/table/source/data_evolution_batch_scan.cpp
core/table/source/data_evolution_full_text_read.cpp
Expand Down
76 changes: 68 additions & 8 deletions src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,8 @@
namespace paimon {
class MemoryPool;

BitmapFileIndex::BitmapFileIndex(const std::map<std::string, std::string>& options)
: options_(options) {}

bool BitmapFileIndex::SupportsExactPredicate(Function::Type function, FieldType field_type) const {
namespace {
bool IsExactBitmapPredicate(Function::Type function, FieldType field_type) {
switch (field_type) {
case FieldType::BOOLEAN:
case FieldType::TINYINT:
Expand Down Expand Up @@ -77,6 +75,15 @@ bool BitmapFileIndex::SupportsExactPredicate(Function::Type function, FieldType
}
}

} // namespace

BitmapFileIndex::BitmapFileIndex(const std::map<std::string, std::string>& options)
: options_(options) {}

bool BitmapFileIndex::SupportsExactPredicate(Function::Type function, FieldType field_type) const {
return IsExactBitmapPredicate(function, field_type);
}

Result<Literal> BitmapFileIndex::ConvertLiteral(
const Literal& literal, const std::shared_ptr<arrow::DataType>& arrow_type) {
if (literal.GetType() != FieldType::TIMESTAMP) {
Expand Down Expand Up @@ -271,6 +278,44 @@ BitmapFileIndexReader::BitmapFileIndexReader(const std::shared_ptr<arrow::DataTy
const std::shared_ptr<MemoryPool>& pool)
: data_type_(data_type), arrow_type_(arrow_type), pool_(pool), input_stream_(input_stream) {}

Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::Evaluate(
const std::shared_ptr<LeafPredicate>& predicate, const RoaringBitmap32* candidates) {
auto function = predicate->GetFunction().GetType();
if (!candidates || (function != Function::Type::IN && function != Function::Type::EQUAL)) {
return FileIndexReader::Evaluate(predicate, candidates);
}
if (predicate->Literals().empty()) {
return Status::Invalid("literals cannot be empty in In predicate");
}
return std::make_shared<BitmapIndexResult>(
[literals = predicate->Literals(), eligible = *candidates,
reader = shared_from_this()]() -> Result<RoaringBitmap32> {
RoaringBitmap32 matched;
if (eligible.IsEmpty()) {
return matched;
}
PAIMON_RETURN_NOT_OK(reader->ReadInternalMeta());
if (eligible.Cardinality() == reader->bitmap_file_index_meta_->GetRowCount()) {
PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 rows,
reader->GetInListResultBitmap(literals));
rows &= eligible;
return rows;
}
for (const auto& literal : literals) {
PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 rows,
reader->GetInListResultBitmap({literal}));
rows &= eligible;
matched |= rows;
// Later postings cannot contribute additional eligible rows.
if (matched.Cardinality() == eligible.Cardinality()) {
break;
}
}
return matched;
},
IsExactBitmapPredicate(function, data_type_));
}

Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitEqual(const Literal& literal) {
return VisitIn({literal});
}
Expand All @@ -289,7 +334,8 @@ Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitIn(
[literals = literals, reader = shared_from_this()]() -> Result<RoaringBitmap32> {
PAIMON_RETURN_NOT_OK(reader->ReadInternalMeta());
return reader->GetInListResultBitmap(literals);
});
},
IsExactBitmapPredicate(Function::Type::IN, data_type_));
}

Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitNotIn(
Expand All @@ -307,15 +353,29 @@ Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitNotIn(
reader->GetInListResultBitmap({Literal(reader->data_type_)}));
bitmap -= null;
return bitmap;
});
},
IsExactBitmapPredicate(Function::Type::NOT_IN, data_type_));
}

Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitIsNull() {
return VisitIn({Literal(data_type_)});
return std::make_shared<BitmapIndexResult>(
[reader = shared_from_this()]() -> Result<RoaringBitmap32> {
PAIMON_RETURN_NOT_OK(reader->ReadInternalMeta());
return reader->GetInListResultBitmap({Literal(reader->data_type_)});
},
IsExactBitmapPredicate(Function::Type::IS_NULL, data_type_));
}

Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitIsNotNull() {
return VisitNotIn({Literal(data_type_)});
return std::make_shared<BitmapIndexResult>(
[reader = shared_from_this()]() -> Result<RoaringBitmap32> {
PAIMON_RETURN_NOT_OK(reader->ReadInternalMeta());
PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 bitmap,
reader->GetInListResultBitmap({Literal(reader->data_type_)}));
bitmap.Flip(/*min=*/0, /*max=*/reader->bitmap_file_index_meta_->GetRowCount());
return bitmap;
},
IsExactBitmapPredicate(Function::Type::IS_NOT_NULL, data_type_));
}

Result<RoaringBitmap32> BitmapFileIndexReader::GetInListResultBitmap(
Expand Down
4 changes: 4 additions & 0 deletions src/paimon/common/file_index/bitmap/bitmap_file_index.h
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,10 @@ class BitmapFileIndexReader : public FileIndexReader,

// TODO(xinyu.lxy): may overwrite VisitGreaterThan... like VisitIsNotNull

Result<std::shared_ptr<FileIndexResult>> Evaluate(
const std::shared_ptr<LeafPredicate>& predicate,
const RoaringBitmap32* candidates) override;

Result<std::shared_ptr<FileIndexResult>> VisitEqual(const Literal& literal) override;
Result<std::shared_ptr<FileIndexResult>> VisitNotEqual(const Literal& literal) override;

Expand Down
Loading
Loading