Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions include/paimon/file_index/file_index_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ struct FullTextMatch;
/// `std::shared_ptr<FileIndexResult>` objects. It reads pre-built file-level index data
/// (e.g., bitmap, bsi or bloom filters) from index file and evaluates
/// whether a given data file may contain rows matching a specific predicate.
///
/// All `Visit` methods in `FileIndexReader`, will return a non-null result object on success,
/// including when no rows match.
///
/// @note Callers of `Visit*` for DECIMAL fields must ensure each literal's scale matches the scale
/// of the indexed data; otherwise, index filtering results may be incorrect.
class PAIMON_EXPORT FileIndexReader : public FunctionVisitor<std::shared_ptr<FileIndexResult>> {
Expand Down
2 changes: 2 additions & 0 deletions include/paimon/predicate/predicate_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -96,13 +96,15 @@ class PAIMON_EXPORT PredicateBuilder {
/// Create an IN predicate (field IN (literal1, literal2, ...)).
///
/// Tests whether the field value matches any of the provided literal values.
/// @note Empty literal lists will be rejected.
static std::shared_ptr<Predicate> In(int32_t field_index, const std::string& field_name,
const FieldType& field_type,
const std::vector<Literal>& literals);

/// Create a NOT IN predicate (field NOT IN (literal1, literal2, ...)).
///
/// Tests whether the field value does not match any of the provided literal values.
/// @note Empty literal lists will be rejected.
static std::shared_ptr<Predicate> NotIn(int32_t field_index, const std::string& field_name,
const FieldType& field_type,
const std::vector<Literal>& literals);
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/common/file_index/bitmap/bitmap_file_index.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitIn(
Result<std::shared_ptr<FileIndexResult>> BitmapFileIndexReader::VisitNotIn(
const std::vector<Literal>& literals) {
if (literals.empty()) {
return Status::Invalid("literals cannot be empty in In predicate");
return Status::Invalid("literals cannot be empty in NotIn predicate");
}
return std::make_shared<BitmapIndexResult>(
[literals = literals, reader = shared_from_this()]() -> Result<RoaringBitmap32> {
Expand Down
18 changes: 18 additions & 0 deletions src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,24 @@ class BitmapIndexTest : public ::testing::Test {
std::shared_ptr<MemoryPool> pool_;
};

TEST_F(BitmapIndexTest, TestEmptyLiterals) {
ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> array,
(CreateArray<arrow::Int32Builder, int32_t>(arrow::int32(), {1, 2})));
for (int32_t version : {BitmapFileIndex::VERSION_1, BitmapFileIndex::VERSION_2}) {
ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> bytes,
WriteIndex(arrow::int32(), version, array));
auto input_stream = std::make_shared<ByteArrayInputStream>(bytes->data(), bytes->size());
BitmapFileIndex file_index({});
ASSERT_OK_AND_ASSIGN(
std::shared_ptr<FileIndexReader> reader,
file_index.CreateReader(CreateArrowSchema(arrow::int32()).get(),
/*start=*/0, /*length=*/bytes->size(), input_stream, pool_));
ASSERT_TRUE(reader);
ASSERT_NOK_WITH_MSG(reader->VisitIn({}), "literals cannot be empty in In predicate");
ASSERT_NOK_WITH_MSG(reader->VisitNotIn({}), "literals cannot be empty in NotIn predicate");
}
}

TEST_F(BitmapIndexTest, TestStringType) {
// data: a, null, b, null, a
auto type = arrow::utf8();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,24 @@ TEST_F(BloomFilterIndexReaderTest, TestStringType) {
ASSERT_TRUE(reader->VisitEqual(Literal(FieldType::STRING)).value()->IsRemain().value());
}

TEST_F(BloomFilterIndexReaderTest, TestEmptyLiterals) {
const std::vector<std::string> inputs = {"[[1], [null]]", "[[null]]", "[]"};
for (const std::string& json : inputs) {
ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> bytes,
WriteIndex(arrow::int32(), json, {{"items", "10"}}));
auto input_stream = std::make_shared<ByteArrayInputStream>(bytes->data(), bytes->size());
BloomFilterFileIndex file_index({});
ASSERT_OK_AND_ASSIGN(
std::shared_ptr<FileIndexReader> reader,
file_index.CreateReader(CreateArrowSchema(arrow::int32()).get(),
/*start=*/0, /*length=*/bytes->size(), input_stream, pool_));
ASSERT_TRUE(reader);

ASSERT_NOK_WITH_MSG(reader->VisitIn({}), "literals cannot be empty in In predicate");
ASSERT_NOK_WITH_MSG(reader->VisitNotIn({}), "literals cannot be empty in NotIn predicate");
}
}

TEST_F(BloomFilterIndexReaderTest, TestIntegerTypes) {
// data: 1, 2, -1, 123
std::vector<uint8_t> index_bytes = {0, 0, 0, 6, 24, 4, 79, 0, 35, 128, 1, 136, 26, 64, 129};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,9 @@ Result<std::shared_ptr<FileIndexResult>> BitSliceIndexBitmapFileIndexReader::Vis

Result<std::shared_ptr<FileIndexResult>> BitSliceIndexBitmapFileIndexReader::VisitIn(
const std::vector<Literal>& literals) {
if (literals.empty()) {
return Status::Invalid("literals cannot be empty in In predicate");
}
BitmapIndexResult::BitmapSupplier bitmap_supplier =
[literals = literals, reader = shared_from_this()]() -> Result<RoaringBitmap32> {
std::vector<RoaringBitmap32> result_bitmaps;
Expand All @@ -438,6 +441,9 @@ Result<std::shared_ptr<FileIndexResult>> BitSliceIndexBitmapFileIndexReader::Vis

Result<std::shared_ptr<FileIndexResult>> BitSliceIndexBitmapFileIndexReader::VisitNotIn(
const std::vector<Literal>& literals) {
if (literals.empty()) {
return Status::Invalid("literals cannot be empty in NotIn predicate");
}
BitmapIndexResult::BitmapSupplier bitmap_supplier =
[literals = literals, reader = shared_from_this()]() -> Result<RoaringBitmap32> {
auto ebm =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,20 @@ class BitSliceIndexBitmapIndexReaderTest : public ::testing::Test {
std::shared_ptr<MemoryPool> pool_;
};

TEST_F(BitSliceIndexBitmapIndexReaderTest, TestEmptyLiterals) {
ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> bytes,
WriteIndex(arrow::int32(), "[[1], [null]]"));
auto input_stream = std::make_shared<ByteArrayInputStream>(bytes->data(), bytes->size());
BitSliceIndexBitmapFileIndex file_index({});
ASSERT_OK_AND_ASSIGN(
std::shared_ptr<FileIndexReader> reader,
file_index.CreateReader(CreateArrowSchema(arrow::int32()).get(),
/*start=*/0, /*length=*/bytes->size(), input_stream, pool_));
ASSERT_TRUE(reader);
ASSERT_NOK_WITH_MSG(reader->VisitIn({}), "literals cannot be empty in In predicate");
ASSERT_NOK_WITH_MSG(reader->VisitNotIn({}), "literals cannot be empty in NotIn predicate");
}

TEST_F(BitSliceIndexBitmapIndexReaderTest, TestMix) {
// data: 1, 2, null, -2, -2, -1, null, 2, 0, 5, null
std::vector<char> index_bytes = {
Expand Down
4 changes: 4 additions & 0 deletions src/paimon/common/file_index/empty/empty_file_index_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "paimon/file_index/file_index_result.h"
#include "paimon/file_index/scored_file_index_result.h"
#include "paimon/result.h"
#include "paimon/status.h"
#include "paimon/utils/roaring_bitmap32.h"

namespace paimon {
Expand Down Expand Up @@ -99,6 +100,9 @@ class EmptyFileIndexReader : public FileIndexReader {
}
Result<std::shared_ptr<FileIndexResult>> VisitIn(
const std::vector<Literal>& literals) override {
if (literals.empty()) {
return Status::Invalid("literals cannot be empty in In predicate");
}
return FileIndexResult::Skip();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ TEST(EmptyFileIndexReaderTest, TestSimple) {
ASSERT_TRUE(reader.VisitNotIn({lit0}).value()->IsRemain().value());
}

TEST(EmptyFileIndexReaderTest, TestEmptyLiterals) {
EmptyFileIndexReader reader(/*search_types=*/{});
ASSERT_NOK_WITH_MSG(reader.VisitIn({}), "literals cannot be empty in In predicate");
ASSERT_NOK_WITH_MSG(reader.VisitNotIn({}), "literals cannot be empty in NotIn predicate");
}

TEST(EmptyFileIndexReaderTest, TestVectorSearchReturnsNoMatches) {
EmptyFileIndexReader reader(/*search_types=*/{});
auto search = std::make_shared<VectorSearch>(
Expand Down
6 changes: 6 additions & 0 deletions src/paimon/common/file_index/file_index_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitGreaterThan(const

Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitIn(
const std::vector<Literal>& literals) {
if (literals.empty()) {
return Status::Invalid("literals cannot be empty in In predicate");
}
std::shared_ptr<FileIndexResult> file_index_result;
for (const Literal& key : literals) {
if (!file_index_result) {
Expand All @@ -98,6 +101,9 @@ Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitIn(

Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitNotIn(
const std::vector<Literal>& literals) {
if (literals.empty()) {
return Status::Invalid("literals cannot be empty in NotIn predicate");
}
std::shared_ptr<FileIndexResult> file_index_result;
for (const Literal& key : literals) {
if (!file_index_result) {
Expand Down
6 changes: 6 additions & 0 deletions src/paimon/common/file_index/file_index_reader_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,12 @@ TEST(FileIndexReaderTest, TestDefaultIndexReader) {
"does not support complete full-text matching");
}

TEST(FileIndexReaderTest, TestEmptyLiterals) {
FileIndexReader reader;
ASSERT_NOK_WITH_MSG(reader.VisitIn({}), "literals cannot be empty in In predicate");
ASSERT_NOK_WITH_MSG(reader.VisitNotIn({}), "literals cannot be empty in NotIn predicate");
}

TEST(FileIndexReaderTest, TestScalarVisitorAndUnknownSelectivity) {
auto reader = std::make_shared<MockFileIndexReader>();
auto predicate = std::dynamic_pointer_cast<LeafPredicate>(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ Result<std::shared_ptr<FileIndexResult>> RangeBitmapFileIndexReader::VisitNotEqu

Result<std::shared_ptr<FileIndexResult>> RangeBitmapFileIndexReader::VisitIn(
const std::vector<Literal>& literals) {
if (literals.empty()) {
return Status::Invalid("literals cannot be empty in In predicate");
}
PAIMON_ASSIGN_OR_RAISE(std::vector<Literal> converted_literals,
type_adapter_->ToStorageLiterals(literals));
return std::make_shared<BitmapIndexResult>(
Expand All @@ -204,6 +207,9 @@ Result<std::shared_ptr<FileIndexResult>> RangeBitmapFileIndexReader::VisitIn(

Result<std::shared_ptr<FileIndexResult>> RangeBitmapFileIndexReader::VisitNotIn(
const std::vector<Literal>& literals) {
if (literals.empty()) {
return Status::Invalid("literals cannot be empty in NotIn predicate");
}
PAIMON_ASSIGN_OR_RAISE(std::vector<Literal> converted_literals,
type_adapter_->ToStorageLiterals(literals));
return std::make_shared<BitmapIndexResult>(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,15 @@ Result<std::shared_ptr<RangeBitmapFileIndexReader>> RangeBitmapFileIndexTest::Cr
return CreateReaderFromArray(arrow_type, struct_array->field(0), options, serialized_bytes_out);
}

TEST_F(RangeBitmapFileIndexTest, TestEmptyLiterals) {
PAIMON_UNIQUE_PTR<Bytes> bytes;
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RangeBitmapFileIndexReader> reader,
CreateReaderFromJson(arrow::int32(), "[[1], [null]]", {}, &bytes));
ASSERT_TRUE(reader);
ASSERT_NOK_WITH_MSG(reader->VisitIn({}), "literals cannot be empty in In predicate");
ASSERT_NOK_WITH_MSG(reader->VisitNotIn({}), "literals cannot be empty in NotIn predicate");
}

// Test with all NULL values
TEST_F(RangeBitmapFileIndexTest, TestAllNullValues) {
constexpr int32_t num_rows = 10;
Expand Down
9 changes: 9 additions & 0 deletions src/paimon/common/predicate/predicate_validator.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "paimon/data/decimal.h"
#include "paimon/defs.h"
#include "paimon/predicate/compound_predicate.h"
#include "paimon/predicate/function.h"
#include "paimon/predicate/leaf_predicate.h"
#include "paimon/predicate/literal.h"
#include "paimon/predicate/predicate.h"
Expand All @@ -57,6 +58,14 @@ class PredicateValidator {
// check field type (predicate vs. literals)
auto field_type = leaf_predicate->GetFieldType();
const auto& literals = leaf_predicate->Literals();
const Function& function = leaf_predicate->GetFunction();
if ((function.GetType() == Function::Type::IN ||
function.GetType() == Function::Type::NOT_IN) &&
literals.empty()) {
return Status::Invalid(
fmt::format("literals cannot be empty in {} predicate, field name {}",
function.ToString(), field_name));
}
for (const auto& literal : literals) {
if (literal.IsNull()) {
return Status::Invalid(fmt::format(
Expand Down
31 changes: 31 additions & 0 deletions src/paimon/common/predicate/predicate_validator_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,37 @@ TEST(PredicateValidatorTest, TestFullTextMatchAllowsOrAndRepeatedFields) {
ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, disjunction, true));
}

TEST(PredicateValidatorTest, TestRejectEmptyInLiterals) {
auto schema = arrow::schema({arrow::field("f0", arrow::int32())});
auto valid = PredicateBuilder::Equal(0, "f0", FieldType::INT, Literal(1));
for (const auto& predicate : {PredicateBuilder::In(0, "f0", FieldType::INT, {}),
PredicateBuilder::NotIn(0, "f0", FieldType::INT, {})}) {
ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> conjunction,
PredicateBuilder::And({valid, predicate}));
ASSERT_OK_AND_ASSIGN(std::shared_ptr<Predicate> disjunction,
PredicateBuilder::Or({valid, conjunction}));
for (bool validate_field_idx : {false, true}) {
for (const auto& candidate : {predicate, conjunction, disjunction}) {
ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema(
*schema, candidate, validate_field_idx),
"literals cannot be empty");
}
}
}
}

TEST(PredicateValidatorTest, TestAcceptNonEmptyInAndNullChecks) {
auto schema = arrow::schema({arrow::field("f0", arrow::int32())});
for (const auto& predicate : {
PredicateBuilder::In(0, "f0", FieldType::INT, {Literal(1), Literal(2)}),
PredicateBuilder::NotIn(0, "f0", FieldType::INT, {Literal(1), Literal(2)}),
PredicateBuilder::IsNull(0, "f0", FieldType::INT),
PredicateBuilder::IsNotNull(0, "f0", FieldType::INT),
}) {
ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, predicate, true));
}
}

TEST(PredicateValidatorTest, TestValidateLiterals) {
std::string str("apple");
std::shared_ptr<arrow::Schema> schema = arrow::schema(arrow::FieldVector({
Expand Down
Loading