From e69471a3d5ccbc20d7af7926d009823e324cfce8 Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Thu, 19 Nov 2026 21:53:50 +0800 Subject: [PATCH 1/2] [core] Prune full compaction manifests by individual delete partitions --- .../operation/ManifestFileBlockMerger.java | 3 +- .../operation/ManifestFileLegacyMerger.java | 2 +- .../paimon/partition/PartitionPredicate.java | 72 ++++++++++++++- .../paimon/manifest/ManifestFileMetaTest.java | 49 ++++++++++- .../partition/PartitionPredicateTest.java | 88 +++++++++++++++++++ 5 files changed, 208 insertions(+), 6 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java index 04a4a7d516f9..3d1fbf42f6a7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java @@ -153,7 +153,8 @@ static Optional> tryFullCompaction( if (deletes.isEmpty()) { predicate = PartitionPredicate.ALWAYS_FALSE; } else if (partitionType.getFieldCount() > 0) { - predicate = PartitionPredicate.fromMultiple(partitionType, deletes.partitions()); + predicate = + PartitionPredicate.fromMultiple(partitionType, deletes.partitions(), true); } else { predicate = PartitionPredicate.ALWAYS_TRUE; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java index a016fb0b556e..b094ae43ae30 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java @@ -184,7 +184,7 @@ static Optional> tryFullCompaction( } else if (partitionType.getFieldCount() > 0) { predicate = PartitionPredicate.fromMultiple( - partitionType, computeDeletePartitions(deleteEntries)); + partitionType, computeDeletePartitions(deleteEntries), true); } else { predicate = PartitionPredicate.ALWAYS_TRUE; } diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java index 3a594aa409b6..f5e2faff9168 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java @@ -31,6 +31,7 @@ import org.apache.paimon.predicate.PredicateProjectionConverter; import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.statistics.FullSimpleColStatsCollector; +import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; @@ -51,6 +52,7 @@ import java.util.Optional; import java.util.Set; +import static org.apache.paimon.predicate.CompareUtils.compareLiteral; import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternal; import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternalRow; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -103,12 +105,22 @@ static PartitionPredicate fromMultiple(RowType partitionType, List pa /** Create {@link PartitionPredicate} from multiple partitions. */ @Nullable static PartitionPredicate fromMultiple(RowType partitionType, Set partitions) { + return fromMultiple(partitionType, partitions, false); + } + + /** + * Optionally check individual partition tuples after their combined bounds pass. Full + * compaction uses this to avoid opening base manifests for sparse DELETE partitions. + */ + @Nullable + static PartitionPredicate fromMultiple( + RowType partitionType, Set partitions, boolean checkPartitionTuples) { if (partitionType.getFieldCount() == 0 || partitions.isEmpty()) { return null; } return new MultiplePartitionPredicate( - new RowDataToObjectArrayConverter(partitionType), partitions); + new RowDataToObjectArrayConverter(partitionType), partitions, checkPartitionTuples); } /** Creates {@link PartitionPredicate} that combines multiple predicates using logical AND. */ @@ -221,10 +233,19 @@ class MultiplePartitionPredicate implements PartitionPredicate { private final int fieldNum; private final Predicate[] min; private final Predicate[] max; + @Nullable private final Object[][] tuples; + @Nullable private final RowDataToObjectArrayConverter converter; private MultiplePartitionPredicate( - RowDataToObjectArrayConverter converter, Set partitions) { + RowDataToObjectArrayConverter converter, + Set partitions, + boolean checkPartitionTuples) { this.partitions = partitions; + this.tuples = + checkPartitionTuples && partitions.size() > 1 + ? new Object[partitions.size()][] + : null; + this.converter = tuples == null ? null : converter; RowType partitionType = converter.rowType(); this.fieldNum = partitionType.getFieldCount(); @SuppressWarnings("unchecked") @@ -236,8 +257,12 @@ private MultiplePartitionPredicate( serializers[i] = InternalSerializers.create(partitionType.getTypeAt(i)); collectors[i] = new FullSimpleColStatsCollector(); } + int index = 0; for (BinaryRow part : partitions) { Object[] fields = converter.convert(part); + if (tuples != null) { + tuples[index++] = fields; + } for (int i = 0; i < fields.length; i++) { collectors[i].collect(fields[i], serializers[i]); } @@ -272,7 +297,10 @@ public boolean test( InternalRow minValues, InternalRow maxValues, InternalArray nullCounts) { - if (fieldNum == 0) { + if (fieldNum == 0 + || minValues.getFieldCount() < fieldNum + || maxValues.getFieldCount() < fieldNum + || (nullCounts.size() > 0 && nullCounts.size() < fieldNum)) { return true; } @@ -282,6 +310,44 @@ public boolean test( return false; } } + if (tuples == null || rowCount <= 0) { + return true; + } + Object[] lower = converter.convert(minValues); + Object[] upper = converter.convert(maxValues); + long[] nulls = new long[fieldNum]; + for (int i = 0; i < fieldNum; i++) { + nulls[i] = + nullCounts.size() == 0 || nullCounts.isNullAt(i) + ? -1 + : nullCounts.getLong(i); + } + for (Object[] tuple : tuples) { + if (mayContainTuple(tuple, lower, upper, nulls, rowCount)) { + return true; + } + } + return false; + } + + private boolean mayContainTuple( + Object[] tuple, Object[] lower, Object[] upper, long[] nulls, long rowCount) { + for (int i = 0; i < fieldNum; i++) { + Object value = tuple[i]; + if (value == null) { + if (nulls[i] == 0) { + return false; + } + } else if (nulls[i] == rowCount) { + return false; + } else if (lower[i] != null && upper[i] != null) { + DataType type = converter.rowType().getTypeAt(i); + if (compareLiteral(type, value, lower[i]) < 0 + || compareLiteral(type, value, upper[i]) > 0) { + return false; + } + } + } return true; } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 161591a48e11..70e9df6cfc0d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -718,6 +718,49 @@ public void testMergeFullCompactionWithoutDeleteFile() { .collect(Collectors.toList())); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testFullCompactionWithSparseOverwritePartitions(boolean optimizeMerge) { + CountingReadFileIO io = new CountingReadFileIO(); + RowType type = RowType.of(new IntType(), new IntType(), new IntType()); + manifestFile = createManifestFileForPartitionType(type, io); + List base = new ArrayList<>(); + List delta = new ArrayList<>(); + List expected = new ArrayList<>(); + for (int day = 1; day <= 19; day++) { + for (int event = 0; event < 2; event++) { + String name = day + "-" + event; + // Every base manifest spans two hours, including the ones left unchanged. + base.add( + makeManifest( + makeMultiPartEntry(true, name, day, 13, event), + makeMultiPartEntry(true, name + "-other-hour", day, 14, event))); + expected.add(name + "-other-hour"); + if ((day == 1 && event == 0) || (day == 19 && event == 1)) { + delta.add(makeMultiPartEntry(false, name, day, 13, event)); + delta.add(makeMultiPartEntry(true, name + "-new", day, 13, event)); + expected.add(name + "-new"); + } else { + expected.add(name); + } + } + } + + List input = new ArrayList<>(base); + input.add(makeManifest(delta.toArray(new ManifestEntry[0]))); + Options options = new Options(); + options.set("manifest.full-compaction-threshold-size", "1 b"); + options.set("manifest.target-file-size", "1 b"); + options.set("manifest.merge-optimize.enabled", Boolean.toString(optimizeMerge)); + io.resetReadCounts(); + List output = + ManifestFileMerger.merge(input, manifestFile, type, new CoreOptions(options)); + long opened = base.stream().filter(m -> io.readCount(m.fileName()) > 0).count(); + assertThat(readFileNames(manifestFile, output)) + .containsExactlyInAnyOrderElementsOf(expected); + assertThat(opened).as("base manifests opened during full compaction").isEqualTo(2); + } + @Test public void testFullCompactionReadsSelectedManifestsOnce() throws Exception { CountingReadFileIO fileIO = new CountingReadFileIO(); @@ -3182,7 +3225,11 @@ private ManifestEntry makeMultiPartEntry( private ManifestFile createManifestFileForPartitionType(RowType partitionType) { Path path = new Path(tempDir.toString()); - FileIO fileIO = FileIOFinder.find(path); + return createManifestFileForPartitionType(partitionType, FileIOFinder.find(path)); + } + + private ManifestFile createManifestFileForPartitionType(RowType partitionType, FileIO fileIO) { + Path path = new Path(tempDir.toString()); return new ManifestFile.Factory( fileIO, new FileSystemSchemaManager(fileIO, path), diff --git a/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java b/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java index e89989a32bcd..7d4e8e300095 100644 --- a/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java @@ -27,14 +27,17 @@ import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.Pair; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.apache.paimon.data.BinaryRow.EMPTY_ROW; import static org.apache.paimon.predicate.PredicateBuilder.and; @@ -123,6 +126,91 @@ public void testPartition() { .isFalse(); } + @Test + public void testTupleStatisticsOnRangesAndNulls() throws Exception { + RowType type = DataTypes.ROW(DataTypes.INT(), DataTypes.INT()); + BinaryRow nullPartition = new BinaryRow(2); + BinaryRowWriter writer = new BinaryRowWriter(nullPartition); + writer.setNullAt(0); + writer.writeInt(1, 100); + writer.complete(); + PartitionPredicate predicate = + InstantiationUtil.clone( + PartitionPredicate.fromMultiple( + type, + new HashSet<>( + Arrays.asList( + createPart(1, 100), + createPart(19, 5), + nullPartition)), + true)); + GenericRow lower = GenericRow.of(1, 5); + GenericRow upper = GenericRow.of(4, 8); + BinaryArray noNulls = BinaryArray.fromLongArray(new Long[] {0L, 0L}); + assertThat(predicate.test(10, lower, upper, noNulls)).isFalse(); + assertThat(predicate.test(10, lower, GenericRow.of(19, 8), noNulls)).isTrue(); + // Day 1 no longer matches, but the null partition could match event 100. + lower = GenericRow.of(3, 98); + upper = GenericRow.of(4, 100); + assertThat(predicate.test(10, lower, upper, noNulls)).isFalse(); + for (Long count : Arrays.asList(1L, null)) { + assertThat( + predicate.test( + 10, + lower, + upper, + BinaryArray.fromLongArray(new Long[] {count, 0L}))) + .isTrue(); + } + assertThat(predicate.test(10, lower, upper, BinaryArray.fromLongArray(new Long[0]))) + .isTrue(); + // Without date bounds, the second tuple could match. With all-null dates only + // the null tuple can match, and its event lies outside [5, 8]. + lower = GenericRow.of(null, 5); + upper = GenericRow.of(null, 8); + assertThat( + predicate.test( + 10, lower, upper, BinaryArray.fromLongArray(new Long[] {null, 0L}))) + .isTrue(); + assertThat( + predicate.test( + 10, lower, upper, BinaryArray.fromLongArray(new Long[] {10L, 0L}))) + .isFalse(); + } + + @Test + public void testTupleRangesAgainstPartitionSet() { + Set partitions = new HashSet<>(); + for (int day = 0; day < 40; day++) { + partitions.add(createPart(day, day % 7)); + } + PartitionPredicate predicate = + PartitionPredicate.fromMultiple( + DataTypes.ROW(DataTypes.INT(), DataTypes.INT()), partitions, true); + BinaryArray noNulls = BinaryArray.fromLongArray(new Long[] {0L, 0L}); + for (int day = 0; day < 40; day++) { + for (int event = 0; event < 7; event++) { + int firstDay = day; + int firstEvent = event; + boolean expected = + partitions.stream() + .anyMatch( + p -> + p.getInt(0) >= firstDay + && p.getInt(0) <= firstDay + 2 + && p.getInt(1) >= firstEvent + && p.getInt(1) <= firstEvent + 1); + assertThat( + predicate.test( + 5, + createPart(day, event), + createPart(day + 2, event + 1), + noNulls)) + .isEqualTo(expected); + } + } + } + @Test public void testPartitionWithMultiFields() { RowType type = DataTypes.ROW(DataTypes.INT(), DataTypes.INT()); From e9cdc33b057fb3b1b6af2ee69cbc1bb005cb1b3d Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Fri, 20 Nov 2026 13:21:15 +0800 Subject: [PATCH 2/2] [core] Apply per-partition statistics checks in the shared predicate --- .../operation/ManifestFileBlockMerger.java | 3 +- .../operation/ManifestFileLegacyMerger.java | 2 +- .../paimon/partition/PartitionPredicate.java | 44 +++++++------------ .../paimon/manifest/ManifestFileMetaTest.java | 32 +++++++++++++- .../partition/PartitionPredicateTest.java | 18 +++----- 5 files changed, 55 insertions(+), 44 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java index 3d1fbf42f6a7..04a4a7d516f9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileBlockMerger.java @@ -153,8 +153,7 @@ static Optional> tryFullCompaction( if (deletes.isEmpty()) { predicate = PartitionPredicate.ALWAYS_FALSE; } else if (partitionType.getFieldCount() > 0) { - predicate = - PartitionPredicate.fromMultiple(partitionType, deletes.partitions(), true); + predicate = PartitionPredicate.fromMultiple(partitionType, deletes.partitions()); } else { predicate = PartitionPredicate.ALWAYS_TRUE; } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java index b094ae43ae30..a016fb0b556e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileLegacyMerger.java @@ -184,7 +184,7 @@ static Optional> tryFullCompaction( } else if (partitionType.getFieldCount() > 0) { predicate = PartitionPredicate.fromMultiple( - partitionType, computeDeletePartitions(deleteEntries), true); + partitionType, computeDeletePartitions(deleteEntries)); } else { predicate = PartitionPredicate.ALWAYS_TRUE; } diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java index f5e2faff9168..ec921fe66acc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java @@ -105,22 +105,12 @@ static PartitionPredicate fromMultiple(RowType partitionType, List pa /** Create {@link PartitionPredicate} from multiple partitions. */ @Nullable static PartitionPredicate fromMultiple(RowType partitionType, Set partitions) { - return fromMultiple(partitionType, partitions, false); - } - - /** - * Optionally check individual partition tuples after their combined bounds pass. Full - * compaction uses this to avoid opening base manifests for sparse DELETE partitions. - */ - @Nullable - static PartitionPredicate fromMultiple( - RowType partitionType, Set partitions, boolean checkPartitionTuples) { if (partitionType.getFieldCount() == 0 || partitions.isEmpty()) { return null; } return new MultiplePartitionPredicate( - new RowDataToObjectArrayConverter(partitionType), partitions, checkPartitionTuples); + new RowDataToObjectArrayConverter(partitionType), partitions); } /** Creates {@link PartitionPredicate} that combines multiple predicates using logical AND. */ @@ -233,19 +223,15 @@ class MultiplePartitionPredicate implements PartitionPredicate { private final int fieldNum; private final Predicate[] min; private final Predicate[] max; - @Nullable private final Object[][] tuples; + @Nullable private final Object[][] partitionValues; @Nullable private final RowDataToObjectArrayConverter converter; private MultiplePartitionPredicate( - RowDataToObjectArrayConverter converter, - Set partitions, - boolean checkPartitionTuples) { + RowDataToObjectArrayConverter converter, Set partitions) { this.partitions = partitions; - this.tuples = - checkPartitionTuples && partitions.size() > 1 - ? new Object[partitions.size()][] - : null; - this.converter = tuples == null ? null : converter; + // For a single partition the combined bounds already describe its field values. + this.partitionValues = partitions.size() > 1 ? new Object[partitions.size()][] : null; + this.converter = partitionValues == null ? null : converter; RowType partitionType = converter.rowType(); this.fieldNum = partitionType.getFieldCount(); @SuppressWarnings("unchecked") @@ -260,8 +246,8 @@ private MultiplePartitionPredicate( int index = 0; for (BinaryRow part : partitions) { Object[] fields = converter.convert(part); - if (tuples != null) { - tuples[index++] = fields; + if (partitionValues != null) { + partitionValues[index++] = fields; } for (int i = 0; i < fields.length; i++) { collectors[i].collect(fields[i], serializers[i]); @@ -310,7 +296,7 @@ public boolean test( return false; } } - if (tuples == null || rowCount <= 0) { + if (partitionValues == null || rowCount <= 0) { return true; } Object[] lower = converter.convert(minValues); @@ -322,18 +308,20 @@ public boolean test( ? -1 : nullCounts.getLong(i); } - for (Object[] tuple : tuples) { - if (mayContainTuple(tuple, lower, upper, nulls, rowCount)) { + // At least one requested partition must match on every field for the manifest to + // matter. + for (Object[] partition : partitionValues) { + if (mayContainPartition(partition, lower, upper, nulls, rowCount)) { return true; } } return false; } - private boolean mayContainTuple( - Object[] tuple, Object[] lower, Object[] upper, long[] nulls, long rowCount) { + private boolean mayContainPartition( + Object[] partition, Object[] lower, Object[] upper, long[] nulls, long rowCount) { for (int i = 0; i < fieldNum; i++) { - Object value = tuple[i]; + Object value = partition[i]; if (value == null) { if (nulls[i] == 0) { return false; diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 70e9df6cfc0d..cfa6230c945b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -34,11 +34,13 @@ import org.apache.paimon.operation.ManifestCompactDryRun; import org.apache.paimon.operation.ManifestFileMerger; import org.apache.paimon.operation.ManifestFileMergerTestUtils; +import org.apache.paimon.operation.ManifestsReader; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.stats.StatsTestUtils; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.source.ScanMode; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FailingFileIO; @@ -718,9 +720,32 @@ public void testMergeFullCompactionWithoutDeleteFile() { .collect(Collectors.toList())); } + @Test + public void testScanManifestsForSparsePartitions() { + RowType type = RowType.of(new IntType(), new IntType(), new IntType()); + manifestFile = createManifestFileForPartitionType(type); + ManifestEntry first = makeMultiPartEntry(true, "first", 1, 13, 0); + ManifestEntry last = makeMultiPartEntry(true, "last", 19, 13, 1); + ManifestFileMeta a = makeManifest(first, makeMultiPartEntry(true, "a", 1, 14, 0)); + ManifestFileMeta b = makeManifest(last, makeMultiPartEntry(true, "b", 19, 14, 1)); + ManifestFileMeta gap = + makeManifest( + makeMultiPartEntry(true, "c", 9, 13, 0), + makeMultiPartEntry(true, "d", 11, 14, 1)); + Snapshot snapshot = mock(Snapshot.class); + ManifestList.Factory lists = mock(ManifestList.Factory.class, RETURNS_DEEP_STUBS); + when(lists.create().readDataManifests(snapshot)).thenReturn(Arrays.asList(a, gap, b)); + ManifestsReader.Result result = + new ManifestsReader(type, "default", null, lists) + .withPartitionFilter(Arrays.asList(first.partition(), last.partition())) + .read(snapshot, ScanMode.ALL); + assertThat(result.filteredManifests).containsExactly(a, b); + } + @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testFullCompactionWithSparseOverwritePartitions(boolean optimizeMerge) { + @CsvSource({"true,false", "false,false", "true,true"}) + public void testFullCompactionWithSparseOverwritePartitions( + boolean optimizeMerge, boolean sort) { CountingReadFileIO io = new CountingReadFileIO(); RowType type = RowType.of(new IntType(), new IntType(), new IntType()); manifestFile = createManifestFileForPartitionType(type, io); @@ -752,6 +777,9 @@ public void testFullCompactionWithSparseOverwritePartitions(boolean optimizeMerg options.set("manifest.full-compaction-threshold-size", "1 b"); options.set("manifest.target-file-size", "1 b"); options.set("manifest.merge-optimize.enabled", Boolean.toString(optimizeMerge)); + options.set("manifest-sort.enabled", Boolean.toString(sort)); + // Keep the test focused on partition overlap, independent of discretionary run rewrites. + options.set("manifest-sort.max-rewrite-size", "1 b"); io.resetReadCounts(); List output = ManifestFileMerger.merge(input, manifestFile, type, new CoreOptions(options)); diff --git a/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java b/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java index 7d4e8e300095..fc1baa43fe62 100644 --- a/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java @@ -127,7 +127,7 @@ public void testPartition() { } @Test - public void testTupleStatisticsOnRangesAndNulls() throws Exception { + public void testPartitionValuesAgainstRangeAndNullStatistics() throws Exception { RowType type = DataTypes.ROW(DataTypes.INT(), DataTypes.INT()); BinaryRow nullPartition = new BinaryRow(2); BinaryRowWriter writer = new BinaryRowWriter(nullPartition); @@ -138,12 +138,8 @@ public void testTupleStatisticsOnRangesAndNulls() throws Exception { InstantiationUtil.clone( PartitionPredicate.fromMultiple( type, - new HashSet<>( - Arrays.asList( - createPart(1, 100), - createPart(19, 5), - nullPartition)), - true)); + Arrays.asList( + createPart(1, 100), createPart(19, 5), nullPartition))); GenericRow lower = GenericRow.of(1, 5); GenericRow upper = GenericRow.of(4, 8); BinaryArray noNulls = BinaryArray.fromLongArray(new Long[] {0L, 0L}); @@ -164,8 +160,8 @@ public void testTupleStatisticsOnRangesAndNulls() throws Exception { } assertThat(predicate.test(10, lower, upper, BinaryArray.fromLongArray(new Long[0]))) .isTrue(); - // Without date bounds, the second tuple could match. With all-null dates only - // the null tuple can match, and its event lies outside [5, 8]. + // Without date bounds, the second partition could match. With all-null dates only + // the null partition can match, and its event lies outside [5, 8]. lower = GenericRow.of(null, 5); upper = GenericRow.of(null, 8); assertThat( @@ -179,14 +175,14 @@ public void testTupleStatisticsOnRangesAndNulls() throws Exception { } @Test - public void testTupleRangesAgainstPartitionSet() { + public void testManifestRangesAgainstPartitionSet() { Set partitions = new HashSet<>(); for (int day = 0; day < 40; day++) { partitions.add(createPart(day, day % 7)); } PartitionPredicate predicate = PartitionPredicate.fromMultiple( - DataTypes.ROW(DataTypes.INT(), DataTypes.INT()), partitions, true); + DataTypes.ROW(DataTypes.INT(), DataTypes.INT()), partitions); BinaryArray noNulls = BinaryArray.fromLongArray(new Long[] {0L, 0L}); for (int day = 0; day < 40; day++) { for (int event = 0; event < 7; event++) {