From b90e4efde1202db9e65c2a8d1c8cf024e4b8cda7 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Thu, 19 Nov 2026 03:40:18 +0000 Subject: [PATCH] fix: reject empty IN and NOT IN predicates --- include/paimon/file_index/file_index_reader.h | 4 +++ include/paimon/predicate/predicate_builder.h | 2 ++ .../file_index/bitmap/bitmap_file_index.cpp | 2 +- .../bitmap/bitmap_file_index_test.cpp | 18 +++++++++++ .../bloom_filter_file_index_test.cpp | 18 +++++++++++ .../bsi/bit_slice_index_bitmap_file_index.cpp | 6 ++++ ...bit_slice_index_bitmap_file_index_test.cpp | 14 +++++++++ .../empty/empty_file_index_reader.h | 4 +++ .../empty/empty_file_index_reader_test.cpp | 6 ++++ .../common/file_index/file_index_reader.cpp | 6 ++++ .../file_index/file_index_reader_test.cpp | 6 ++++ .../rangebitmap/range_bitmap_file_index.cpp | 6 ++++ .../range_bitmap_file_index_test.cpp | 9 ++++++ .../common/predicate/predicate_validator.h | 9 ++++++ .../predicate/predicate_validator_test.cpp | 31 +++++++++++++++++++ 15 files changed, 140 insertions(+), 1 deletion(-) diff --git a/include/paimon/file_index/file_index_reader.h b/include/paimon/file_index/file_index_reader.h index 677617bbf..0cdd18da3 100644 --- a/include/paimon/file_index/file_index_reader.h +++ b/include/paimon/file_index/file_index_reader.h @@ -40,6 +40,10 @@ struct FullTextMatch; /// `std::shared_ptr` objects. It reads pre-built file-level index data /// (e.g., bitmap, bsi or bloom filters) from index file and evaluates /// whether a given data file may contain rows matching a specific predicate. +/// +/// All `Visit` methods in `FileIndexReader`, will return a non-null result object on success, +/// including when no rows match. +/// /// @note Callers of `Visit*` for DECIMAL fields must ensure each literal's scale matches the scale /// of the indexed data; otherwise, index filtering results may be incorrect. class PAIMON_EXPORT FileIndexReader : public FunctionVisitor> { diff --git a/include/paimon/predicate/predicate_builder.h b/include/paimon/predicate/predicate_builder.h index 4c829408d..1f2c98856 100644 --- a/include/paimon/predicate/predicate_builder.h +++ b/include/paimon/predicate/predicate_builder.h @@ -96,6 +96,7 @@ class PAIMON_EXPORT PredicateBuilder { /// Create an IN predicate (field IN (literal1, literal2, ...)). /// /// Tests whether the field value matches any of the provided literal values. + /// @note Empty literal lists will be rejected. static std::shared_ptr In(int32_t field_index, const std::string& field_name, const FieldType& field_type, const std::vector& literals); @@ -103,6 +104,7 @@ class PAIMON_EXPORT PredicateBuilder { /// Create a NOT IN predicate (field NOT IN (literal1, literal2, ...)). /// /// Tests whether the field value does not match any of the provided literal values. + /// @note Empty literal lists will be rejected. static std::shared_ptr NotIn(int32_t field_index, const std::string& field_name, const FieldType& field_type, const std::vector& literals); diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp index 2316122e0..c114d8098 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index.cpp @@ -295,7 +295,7 @@ Result> BitmapFileIndexReader::VisitIn( Result> BitmapFileIndexReader::VisitNotIn( const std::vector& literals) { if (literals.empty()) { - return Status::Invalid("literals cannot be empty in In predicate"); + return Status::Invalid("literals cannot be empty in NotIn predicate"); } return std::make_shared( [literals = literals, reader = shared_from_this()]() -> Result { diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp index f2f89718a..61d729ee6 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp @@ -107,6 +107,24 @@ class BitmapIndexTest : public ::testing::Test { std::shared_ptr pool_; }; +TEST_F(BitmapIndexTest, TestEmptyLiterals) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + (CreateArray(arrow::int32(), {1, 2}))); + for (int32_t version : {BitmapFileIndex::VERSION_1, BitmapFileIndex::VERSION_2}) { + ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR bytes, + WriteIndex(arrow::int32(), version, array)); + auto input_stream = std::make_shared(bytes->data(), bytes->size()); + BitmapFileIndex file_index({}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr reader, + file_index.CreateReader(CreateArrowSchema(arrow::int32()).get(), + /*start=*/0, /*length=*/bytes->size(), input_stream, pool_)); + ASSERT_TRUE(reader); + ASSERT_NOK_WITH_MSG(reader->VisitIn({}), "literals cannot be empty in In predicate"); + ASSERT_NOK_WITH_MSG(reader->VisitNotIn({}), "literals cannot be empty in NotIn predicate"); + } +} + TEST_F(BitmapIndexTest, TestStringType) { // data: a, null, b, null, a auto type = arrow::utf8(); diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index_test.cpp b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index_test.cpp index 23fadce28..a3fe8524c 100644 --- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index_test.cpp +++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index_test.cpp @@ -116,6 +116,24 @@ TEST_F(BloomFilterIndexReaderTest, TestStringType) { ASSERT_TRUE(reader->VisitEqual(Literal(FieldType::STRING)).value()->IsRemain().value()); } +TEST_F(BloomFilterIndexReaderTest, TestEmptyLiterals) { + const std::vector inputs = {"[[1], [null]]", "[[null]]", "[]"}; + for (const std::string& json : inputs) { + ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR bytes, + WriteIndex(arrow::int32(), json, {{"items", "10"}})); + auto input_stream = std::make_shared(bytes->data(), bytes->size()); + BloomFilterFileIndex file_index({}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr reader, + file_index.CreateReader(CreateArrowSchema(arrow::int32()).get(), + /*start=*/0, /*length=*/bytes->size(), input_stream, pool_)); + ASSERT_TRUE(reader); + + ASSERT_NOK_WITH_MSG(reader->VisitIn({}), "literals cannot be empty in In predicate"); + ASSERT_NOK_WITH_MSG(reader->VisitNotIn({}), "literals cannot be empty in NotIn predicate"); + } +} + TEST_F(BloomFilterIndexReaderTest, TestIntegerTypes) { // data: 1, 2, -1, 123 std::vector index_bytes = {0, 0, 0, 6, 24, 4, 79, 0, 35, 128, 1, 136, 26, 64, 129}; diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp index 508a4b200..e59bbb5a1 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp @@ -413,6 +413,9 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis Result> BitSliceIndexBitmapFileIndexReader::VisitIn( const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("literals cannot be empty in In predicate"); + } BitmapIndexResult::BitmapSupplier bitmap_supplier = [literals = literals, reader = shared_from_this()]() -> Result { std::vector result_bitmaps; @@ -438,6 +441,9 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis Result> BitSliceIndexBitmapFileIndexReader::VisitNotIn( const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("literals cannot be empty in NotIn predicate"); + } BitmapIndexResult::BitmapSupplier bitmap_supplier = [literals = literals, reader = shared_from_this()]() -> Result { auto ebm = diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp index 8da7ace29..ff064eede 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp @@ -87,6 +87,20 @@ class BitSliceIndexBitmapIndexReaderTest : public ::testing::Test { std::shared_ptr pool_; }; +TEST_F(BitSliceIndexBitmapIndexReaderTest, TestEmptyLiterals) { + ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR bytes, + WriteIndex(arrow::int32(), "[[1], [null]]")); + auto input_stream = std::make_shared(bytes->data(), bytes->size()); + BitSliceIndexBitmapFileIndex file_index({}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr reader, + file_index.CreateReader(CreateArrowSchema(arrow::int32()).get(), + /*start=*/0, /*length=*/bytes->size(), input_stream, pool_)); + ASSERT_TRUE(reader); + ASSERT_NOK_WITH_MSG(reader->VisitIn({}), "literals cannot be empty in In predicate"); + ASSERT_NOK_WITH_MSG(reader->VisitNotIn({}), "literals cannot be empty in NotIn predicate"); +} + TEST_F(BitSliceIndexBitmapIndexReaderTest, TestMix) { // data: 1, 2, null, -2, -2, -1, null, 2, 0, 5, null std::vector index_bytes = { diff --git a/src/paimon/common/file_index/empty/empty_file_index_reader.h b/src/paimon/common/file_index/empty/empty_file_index_reader.h index 5de3bde93..ab6b54b73 100644 --- a/src/paimon/common/file_index/empty/empty_file_index_reader.h +++ b/src/paimon/common/file_index/empty/empty_file_index_reader.h @@ -28,6 +28,7 @@ #include "paimon/file_index/file_index_result.h" #include "paimon/file_index/scored_file_index_result.h" #include "paimon/result.h" +#include "paimon/status.h" #include "paimon/utils/roaring_bitmap32.h" namespace paimon { @@ -99,6 +100,9 @@ class EmptyFileIndexReader : public FileIndexReader { } Result> VisitIn( const std::vector& literals) override { + if (literals.empty()) { + return Status::Invalid("literals cannot be empty in In predicate"); + } return FileIndexResult::Skip(); } diff --git a/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp b/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp index 923d44b07..532704537 100644 --- a/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp +++ b/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp @@ -58,6 +58,12 @@ TEST(EmptyFileIndexReaderTest, TestSimple) { ASSERT_TRUE(reader.VisitNotIn({lit0}).value()->IsRemain().value()); } +TEST(EmptyFileIndexReaderTest, TestEmptyLiterals) { + EmptyFileIndexReader reader(/*search_types=*/{}); + ASSERT_NOK_WITH_MSG(reader.VisitIn({}), "literals cannot be empty in In predicate"); + ASSERT_NOK_WITH_MSG(reader.VisitNotIn({}), "literals cannot be empty in NotIn predicate"); +} + TEST(EmptyFileIndexReaderTest, TestVectorSearchReturnsNoMatches) { EmptyFileIndexReader reader(/*search_types=*/{}); auto search = std::make_shared( diff --git a/src/paimon/common/file_index/file_index_reader.cpp b/src/paimon/common/file_index/file_index_reader.cpp index d51910d2b..f1f2502aa 100644 --- a/src/paimon/common/file_index/file_index_reader.cpp +++ b/src/paimon/common/file_index/file_index_reader.cpp @@ -84,6 +84,9 @@ Result> FileIndexReader::VisitGreaterThan(const Result> FileIndexReader::VisitIn( const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("literals cannot be empty in In predicate"); + } std::shared_ptr file_index_result; for (const Literal& key : literals) { if (!file_index_result) { @@ -98,6 +101,9 @@ Result> FileIndexReader::VisitIn( Result> FileIndexReader::VisitNotIn( const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("literals cannot be empty in NotIn predicate"); + } std::shared_ptr file_index_result; for (const Literal& key : literals) { if (!file_index_result) { diff --git a/src/paimon/common/file_index/file_index_reader_test.cpp b/src/paimon/common/file_index/file_index_reader_test.cpp index 80e70e817..55dd712dc 100644 --- a/src/paimon/common/file_index/file_index_reader_test.cpp +++ b/src/paimon/common/file_index/file_index_reader_test.cpp @@ -91,6 +91,12 @@ TEST(FileIndexReaderTest, TestDefaultIndexReader) { "does not support complete full-text matching"); } +TEST(FileIndexReaderTest, TestEmptyLiterals) { + FileIndexReader reader; + ASSERT_NOK_WITH_MSG(reader.VisitIn({}), "literals cannot be empty in In predicate"); + ASSERT_NOK_WITH_MSG(reader.VisitNotIn({}), "literals cannot be empty in NotIn predicate"); +} + TEST(FileIndexReaderTest, TestScalarVisitorAndUnknownSelectivity) { auto reader = std::make_shared(); auto predicate = std::dynamic_pointer_cast( diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp index b9c7c240b..25dc20f17 100644 --- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp +++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp @@ -193,6 +193,9 @@ Result> RangeBitmapFileIndexReader::VisitNotEqu Result> RangeBitmapFileIndexReader::VisitIn( const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("literals cannot be empty in In predicate"); + } PAIMON_ASSIGN_OR_RAISE(std::vector converted_literals, type_adapter_->ToStorageLiterals(literals)); return std::make_shared( @@ -204,6 +207,9 @@ Result> RangeBitmapFileIndexReader::VisitIn( Result> RangeBitmapFileIndexReader::VisitNotIn( const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("literals cannot be empty in NotIn predicate"); + } PAIMON_ASSIGN_OR_RAISE(std::vector converted_literals, type_adapter_->ToStorageLiterals(literals)); return std::make_shared( diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp index c00c393d9..5007c2725 100644 --- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp @@ -184,6 +184,15 @@ Result> RangeBitmapFileIndexTest::Cr return CreateReaderFromArray(arrow_type, struct_array->field(0), options, serialized_bytes_out); } +TEST_F(RangeBitmapFileIndexTest, TestEmptyLiterals) { + PAIMON_UNIQUE_PTR bytes; + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateReaderFromJson(arrow::int32(), "[[1], [null]]", {}, &bytes)); + ASSERT_TRUE(reader); + ASSERT_NOK_WITH_MSG(reader->VisitIn({}), "literals cannot be empty in In predicate"); + ASSERT_NOK_WITH_MSG(reader->VisitNotIn({}), "literals cannot be empty in NotIn predicate"); +} + // Test with all NULL values TEST_F(RangeBitmapFileIndexTest, TestAllNullValues) { constexpr int32_t num_rows = 10; diff --git a/src/paimon/common/predicate/predicate_validator.h b/src/paimon/common/predicate/predicate_validator.h index b794adc22..00241723f 100644 --- a/src/paimon/common/predicate/predicate_validator.h +++ b/src/paimon/common/predicate/predicate_validator.h @@ -32,6 +32,7 @@ #include "paimon/data/decimal.h" #include "paimon/defs.h" #include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/function.h" #include "paimon/predicate/leaf_predicate.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate.h" @@ -57,6 +58,14 @@ class PredicateValidator { // check field type (predicate vs. literals) auto field_type = leaf_predicate->GetFieldType(); const auto& literals = leaf_predicate->Literals(); + const Function& function = leaf_predicate->GetFunction(); + if ((function.GetType() == Function::Type::IN || + function.GetType() == Function::Type::NOT_IN) && + literals.empty()) { + return Status::Invalid( + fmt::format("literals cannot be empty in {} predicate, field name {}", + function.ToString(), field_name)); + } for (const auto& literal : literals) { if (literal.IsNull()) { return Status::Invalid(fmt::format( diff --git a/src/paimon/common/predicate/predicate_validator_test.cpp b/src/paimon/common/predicate/predicate_validator_test.cpp index 8b10e77b2..487482db9 100644 --- a/src/paimon/common/predicate/predicate_validator_test.cpp +++ b/src/paimon/common/predicate/predicate_validator_test.cpp @@ -74,6 +74,37 @@ TEST(PredicateValidatorTest, TestFullTextMatchAllowsOrAndRepeatedFields) { ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, disjunction, true)); } +TEST(PredicateValidatorTest, TestRejectEmptyInLiterals) { + auto schema = arrow::schema({arrow::field("f0", arrow::int32())}); + auto valid = PredicateBuilder::Equal(0, "f0", FieldType::INT, Literal(1)); + for (const auto& predicate : {PredicateBuilder::In(0, "f0", FieldType::INT, {}), + PredicateBuilder::NotIn(0, "f0", FieldType::INT, {})}) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr conjunction, + PredicateBuilder::And({valid, predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr disjunction, + PredicateBuilder::Or({valid, conjunction})); + for (bool validate_field_idx : {false, true}) { + for (const auto& candidate : {predicate, conjunction, disjunction}) { + ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithSchema( + *schema, candidate, validate_field_idx), + "literals cannot be empty"); + } + } + } +} + +TEST(PredicateValidatorTest, TestAcceptNonEmptyInAndNullChecks) { + auto schema = arrow::schema({arrow::field("f0", arrow::int32())}); + for (const auto& predicate : { + PredicateBuilder::In(0, "f0", FieldType::INT, {Literal(1), Literal(2)}), + PredicateBuilder::NotIn(0, "f0", FieldType::INT, {Literal(1), Literal(2)}), + PredicateBuilder::IsNull(0, "f0", FieldType::INT), + PredicateBuilder::IsNotNull(0, "f0", FieldType::INT), + }) { + ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema, predicate, true)); + } +} + TEST(PredicateValidatorTest, TestValidateLiterals) { std::string str("apple"); std::shared_ptr schema = arrow::schema(arrow::FieldVector({