Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 3 additions & 16 deletions include/paimon/file_index/file_index_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,18 @@

#include <cstdint>
#include <memory>
#include <optional>
#include <vector>

#include "paimon/file_index/file_index_result.h"
#include "paimon/file_index/scored_file_index_result.h"
#include "paimon/index/index_reader_capabilities.h"
#include "paimon/predicate/full_text_search.h"
#include "paimon/predicate/function_visitor.h"
#include "paimon/predicate/vector_search.h"
#include "paimon/result.h"
#include "paimon/visibility.h"

namespace paimon {
class LeafPredicate;
struct FullTextMatch;
/// Evaluates filter predicates against a file-level index to determine file eligibility.
///
Expand All @@ -42,23 +41,11 @@ struct FullTextMatch;
/// whether a given data file may contain rows matching a specific predicate.
/// @note Callers of `Visit*` for DECIMAL fields must ensure each literal's scale matches the scale
/// of the indexed data; otherwise, index filtering results may be incorrect.
class PAIMON_EXPORT FileIndexReader : public FunctionVisitor<std::shared_ptr<FileIndexResult>> {
class PAIMON_EXPORT FileIndexReader : public IndexReaderCapabilities,
public FunctionVisitor<std::shared_ptr<FileIndexResult>> {
public:
enum class SearchType { FULL_TEXT_MATCH, FULL_TEXT_SEARCH, VECTOR_SEARCH };

~FileIndexReader() override = default;

/// Whether this reader can execute the requested search without probing it with a query.
virtual bool SupportsSearch(SearchType search_type) const;

/// Estimates the fraction of rows in the data file expected to match the predicate.
/// Returns a value in [0, 1], or std::nullopt if no estimate is available.
/// Values outside [0, 1] are also treated as unknown.
/// The estimate only guides predicate execution order; it is not used to filter rows.
/// TODO(lisizhuo.lsz): Add selectivity estimates for scalar indexes such as bitmap indexes.
virtual std::optional<double> EstimateSelectivity(
const std::shared_ptr<LeafPredicate>& predicate) const;

Result<std::shared_ptr<FileIndexResult>> VisitIsNotNull() override;

Result<std::shared_ptr<FileIndexResult>> VisitIsNull() override;
Expand Down
51 changes: 51 additions & 0 deletions include/paimon/index/index_reader_capabilities.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

#include <memory>
#include <optional>

#include "paimon/visibility.h"

namespace paimon {
class LeafPredicate;

/// Search capabilities and predicate scheduling hints shared by index readers, independently of
/// their result types and row ID domains.
class PAIMON_EXPORT IndexReaderCapabilities {
public:
enum class SearchType { FULL_TEXT_MATCH, FULL_TEXT_SEARCH, VECTOR_SEARCH };

virtual ~IndexReaderCapabilities() = default;

/// Whether this reader can execute the requested search without probing it with a query.
/// The default implementation returns false.
virtual bool SupportsSearch(SearchType search_type) const;

/// Estimates the fraction of this reader's indexed rows expected to match the predicate.
/// Returns a value in [0, 1], or std::nullopt if no estimate is available.
/// Values outside [0, 1] are also treated as unknown by evaluators.
/// Estimates only guide predicate execution order and must never exclude rows.
/// The default implementation returns std::nullopt.
/// TODO(lisizhuo.lsz): Add selectivity estimates for scalar indexes such as bitmap indexes.
virtual std::optional<double> EstimateSelectivity(
const std::shared_ptr<LeafPredicate>& predicate) const;
};

} // namespace paimon
1 change: 1 addition & 0 deletions include/paimon/predicate/full_text_match.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ struct PAIMON_EXPORT FullTextMatch {
/// JSON DSL query with the same syntax and analyzer semantics as `FullTextSearch::query`.
std::string query;
/// Optional eligible row ids. File readers use file-local physical row positions;
/// real-time index readers use real-time offsets.
/// An empty set matches no rows. If not set, all rows are searched.
std::optional<RoaringBitmap64> include_row_ids;
};
Expand Down
4 changes: 2 additions & 2 deletions include/paimon/predicate/full_text_search.h
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ struct PAIMON_EXPORT FullTextSearch {
int32_t limit;
/// The row ids to search within, aligned with Java `includeRowIds`. The row id domain is
/// determined by the reader: global row ids for GlobalIndexReader and physical file-local row
/// ids for FileIndexReader. Only these rows are ranked, so an empty set matches no rows. If not
/// set, all rows are searched.
/// ids for FileIndexReader, and real-time offsets for RealtimeIndexReader. Only these rows are
/// ranked, so an empty set matches no rows. If not set, all rows are searched.
std::optional<RoaringBitmap64> include_row_ids;
};
} // namespace paimon
9 changes: 5 additions & 4 deletions include/paimon/predicate/vector_search.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,10 @@ struct PAIMON_EXPORT VectorSearch {
/// `PreFilter`: A lightweight pre-filtering function applied **before** similarity scoring.
/// The row id domain is determined by the reader executing the search: `GlobalIndexReader`
/// supplies global row ids, while `FileIndexReader` supplies physical row ids local to the
/// current data file. This filter enables early pruning of irrelevant candidates (e.g., "only
/// consider rows with label X"), significantly reducing the search space. Returns true to
/// include the row in vector search process; false to exclude it.
/// current data file. Real-time index readers supply real-time offsets. This filter enables
/// early pruning of irrelevant candidates (e.g., "only consider rows with label X"),
/// significantly reducing the search space. Returns true to include the row in vector search
/// process; false to exclude it.
///
/// @note Must be thread-safe.
using PreFilter = std::function<bool(int64_t)>;
Expand Down Expand Up @@ -71,7 +72,7 @@ struct PAIMON_EXPORT VectorSearch {
/// The query vector (must match the dimensionality of the indexed vectors).
std::vector<float> query;
/// A pre-filter whose row id domain is determined by the reader: global row ids for Global
/// Index and file-local physical row ids for File Index.
/// Index, file-local physical row ids for File Index, and offsets for Real-time Index.
std::function<bool(int64_t)> pre_filter;
/// A runtime filtering condition that may involve graph traversal of
/// structured attributes. **Using this parameter often yields better
Expand Down
2 changes: 2 additions & 0 deletions include/paimon/read_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,8 @@ class PAIMON_EXPORT ReadContextBuilder {
/// data to compute exact matching rows. Only scalar predicates may be evaluated without an
/// index. Score output must use named projections such as _<field>_INDEX_SCORE;
/// projecting _INDEX_SCORE by name, field ID or read schema is not supported in this mode.
/// Real-time append and deduplicate reads use pinned offsets and exact scalar indexes,
/// scanning predicates that have no exact index. Real-time primary-key merge is not supported.
/// The default is false.
/// @param enabled Whether to compute exact result row positions during file-index evaluation.
/// @return Reference to this builder for method chaining.
Expand Down
63 changes: 63 additions & 0 deletions include/paimon/realtime/index/realtime_index_read_source.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

#include <memory>
#include <string>
#include <vector>

#include "paimon/realtime/index/realtime_index_reader.h"
#include "paimon/realtime/realtime_store.h"

namespace paimon {

/// One immutable query source. Readers and index readers must share its pinned data snapshot.
class PAIMON_EXPORT RealtimeIndexReadSource {
public:
virtual ~RealtimeIndexReadSource() = default;

virtual OffsetRange GetOffsetRange() const = 0;

/// Builds a candidate superset from offset bounds and pinned deletion metadata without
/// reading data. Gaps may be included; index and data readers only return existing rows.
/// Deleted offsets must be excluded before full-text or vector TopK executes.
virtual Result<RoaringBitmap64> CreateCandidates() const = 0;

/// Returns readers bound to this source's offset range. A missing field has no readers.
/// Readers must not expose writes published after the pinned snapshot.
virtual Result<std::vector<std::shared_ptr<RealtimeIndexReader>>> CreateIndexReaders(
const std::string& field_name) const = 0;
};

/// Optional capability for precise real-time filtering and search. Ordinary query reads do not
/// require it. Each visible row in the requested range of the pinned snapshot
/// must belong to exactly one source. Source offset ranges must not overlap, so data readers
/// can use the same pinned view with source-local offset bounds.
class PAIMON_EXPORT RealtimeIndexedReadView : public RealtimeReadView {
public:
~RealtimeIndexedReadView() override = default;

/// Creates sources restricted to the intersection of each segment's range and the query range.
/// @param offset_range Query bounds, which may be narrower than GetOffsetRange() to exclude
/// offsets already covered by the disk snapshot selected after this view was pinned.
virtual Result<std::vector<std::shared_ptr<RealtimeIndexReadSource>>> CreateIndexReadSources(
const OffsetRange& offset_range) const = 0;
};

} // namespace paimon
26 changes: 25 additions & 1 deletion include/paimon/realtime/index/realtime_index_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,43 @@
#include <optional>
#include <vector>

#include "paimon/index/index_reader_capabilities.h"
#include "paimon/predicate/function_visitor.h"
#include "paimon/realtime/index/realtime_index_result.h"
#include "paimon/realtime/index/scored_realtime_index_result.h"
#include "paimon/utils/roaring_bitmap64.h"
#include "paimon/visibility.h"

namespace paimon {
class LeafPredicate;
struct VectorSearch;
struct FullTextSearch;
struct FullTextMatch;

/// Visits predicates on one indexed field, returning realtime offsets.
/// nullopt means no pruning; an empty bitmap means no candidates. False negatives are not allowed.
/// Unsupported predicates default to no pruning. Plugins override only the supported visits.
class PAIMON_EXPORT RealtimeIndexReader : public FunctionVisitor<std::optional<RoaringBitmap64>> {
class PAIMON_EXPORT RealtimeIndexReader : public IndexReaderCapabilities,
public FunctionVisitor<std::optional<RoaringBitmap64>> {
public:
~RealtimeIndexReader() override = default;

/// Search pre-filters and result IDs use real-time offsets, not file-local row positions.
/// Apply filters and pinned visibility before TopK; filtering a truncated result is
/// insufficient. Search results must be stable for the pinned snapshot even while later writes
/// proceed.
virtual Result<std::shared_ptr<ScoredRealtimeIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& search);

virtual Result<std::shared_ptr<ScoredRealtimeIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& search);

/// Returns all matching offsets without TopK truncation. When needs_scores is true,
/// returns ScoredRealtimeIndexResult, including for an empty result; otherwise scores
/// need not be computed.
virtual Result<std::shared_ptr<RealtimeIndexResult>> VisitFullTextMatch(
const std::shared_ptr<FullTextMatch>& match, bool needs_scores);

Result<std::optional<RoaringBitmap64>> VisitIsNotNull() override;
Result<std::optional<RoaringBitmap64>> VisitIsNull() override;
Result<std::optional<RoaringBitmap64>> VisitEqual(const Literal& literal) override;
Expand Down
61 changes: 61 additions & 0 deletions include/paimon/realtime/index/realtime_index_result.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

#include <memory>
#include <utility>

#include "paimon/macros.h"
#include "paimon/result.h"
#include "paimon/utils/roaring_bitmap64.h"
#include "paimon/visibility.h"

namespace paimon {

/// Matching real-time offsets. ScoredRealtimeIndexResult additionally carries scores.
class PAIMON_EXPORT RealtimeIndexResult {
public:
virtual ~RealtimeIndexResult() = default;

static Result<std::shared_ptr<RealtimeIndexResult>> Create(RoaringBitmap64 rows) {
PAIMON_RETURN_NOT_OK(ValidateRowPositions(rows));
return std::shared_ptr<RealtimeIndexResult>(new RealtimeIndexResult(std::move(rows)));
}

const RoaringBitmap64& GetRowPositions() const {
return rows_;
}

protected:
explicit RealtimeIndexResult(RoaringBitmap64 rows) : rows_(std::move(rows)) {}

static Status ValidateRowPositions(const RoaringBitmap64& rows) {
for (auto iter = rows.Begin(); iter != rows.End(); ++iter) {
if (*iter < 0) {
return Status::Invalid("Real-time search returned a negative offset");
}
}
return Status::OK();
}

private:
RoaringBitmap64 rows_;
};

} // namespace paimon
55 changes: 55 additions & 0 deletions include/paimon/realtime/index/scored_realtime_index_result.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

#include <memory>
#include <utility>
#include <vector>

#include "paimon/realtime/index/realtime_index_result.h"
#include "paimon/utils/roaring_bitmap64.h"
#include "paimon/visibility.h"

namespace paimon {

/// Search results keyed by real-time offset. Scores follow ascending offset order, not rank.
class PAIMON_EXPORT ScoredRealtimeIndexResult : public RealtimeIndexResult {
public:
static Result<std::shared_ptr<ScoredRealtimeIndexResult>> Create(RoaringBitmap64 rows,
std::vector<float> scores) {
if (rows.Cardinality() != static_cast<int64_t>(scores.size())) {
return Status::Invalid("Real-time search score count does not match offsets");
}
PAIMON_RETURN_NOT_OK(ValidateRowPositions(rows));
return std::shared_ptr<ScoredRealtimeIndexResult>(
new ScoredRealtimeIndexResult(std::move(rows), std::move(scores)));
}

const std::vector<float>& GetScores() const {
return scores_;
}

private:
ScoredRealtimeIndexResult(RoaringBitmap64 rows, std::vector<float> scores)
: RealtimeIndexResult(std::move(rows)), scores_(std::move(scores)) {}

std::vector<float> scores_;
};

} // namespace paimon
Loading
Loading