diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java index 3a594aa409b6..ec921fe66acc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java @@ -31,6 +31,7 @@ import org.apache.paimon.predicate.PredicateProjectionConverter; import org.apache.paimon.predicate.PredicateVisitor; import org.apache.paimon.statistics.FullSimpleColStatsCollector; +import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; @@ -51,6 +52,7 @@ import java.util.Optional; import java.util.Set; +import static org.apache.paimon.predicate.CompareUtils.compareLiteral; import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternal; import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternalRow; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -221,10 +223,15 @@ class MultiplePartitionPredicate implements PartitionPredicate { private final int fieldNum; private final Predicate[] min; private final Predicate[] max; + @Nullable private final Object[][] partitionValues; + @Nullable private final RowDataToObjectArrayConverter converter; private MultiplePartitionPredicate( RowDataToObjectArrayConverter converter, Set partitions) { this.partitions = partitions; + // For a single partition the combined bounds already describe its field values. + this.partitionValues = partitions.size() > 1 ? new Object[partitions.size()][] : null; + this.converter = partitionValues == null ? null : converter; RowType partitionType = converter.rowType(); this.fieldNum = partitionType.getFieldCount(); @SuppressWarnings("unchecked") @@ -236,8 +243,12 @@ private MultiplePartitionPredicate( serializers[i] = InternalSerializers.create(partitionType.getTypeAt(i)); collectors[i] = new FullSimpleColStatsCollector(); } + int index = 0; for (BinaryRow part : partitions) { Object[] fields = converter.convert(part); + if (partitionValues != null) { + partitionValues[index++] = fields; + } for (int i = 0; i < fields.length; i++) { collectors[i].collect(fields[i], serializers[i]); } @@ -272,7 +283,10 @@ public boolean test( InternalRow minValues, InternalRow maxValues, InternalArray nullCounts) { - if (fieldNum == 0) { + if (fieldNum == 0 + || minValues.getFieldCount() < fieldNum + || maxValues.getFieldCount() < fieldNum + || (nullCounts.size() > 0 && nullCounts.size() < fieldNum)) { return true; } @@ -282,6 +296,46 @@ public boolean test( return false; } } + if (partitionValues == null || rowCount <= 0) { + return true; + } + Object[] lower = converter.convert(minValues); + Object[] upper = converter.convert(maxValues); + long[] nulls = new long[fieldNum]; + for (int i = 0; i < fieldNum; i++) { + nulls[i] = + nullCounts.size() == 0 || nullCounts.isNullAt(i) + ? -1 + : nullCounts.getLong(i); + } + // At least one requested partition must match on every field for the manifest to + // matter. + for (Object[] partition : partitionValues) { + if (mayContainPartition(partition, lower, upper, nulls, rowCount)) { + return true; + } + } + return false; + } + + private boolean mayContainPartition( + Object[] partition, Object[] lower, Object[] upper, long[] nulls, long rowCount) { + for (int i = 0; i < fieldNum; i++) { + Object value = partition[i]; + if (value == null) { + if (nulls[i] == 0) { + return false; + } + } else if (nulls[i] == rowCount) { + return false; + } else if (lower[i] != null && upper[i] != null) { + DataType type = converter.rowType().getTypeAt(i); + if (compareLiteral(type, value, lower[i]) < 0 + || compareLiteral(type, value, upper[i]) > 0) { + return false; + } + } + } return true; } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 161591a48e11..cfa6230c945b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -34,11 +34,13 @@ import org.apache.paimon.operation.ManifestCompactDryRun; import org.apache.paimon.operation.ManifestFileMerger; import org.apache.paimon.operation.ManifestFileMergerTestUtils; +import org.apache.paimon.operation.ManifestsReader; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.stats.StatsTestUtils; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.source.ScanMode; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FailingFileIO; @@ -718,6 +720,75 @@ public void testMergeFullCompactionWithoutDeleteFile() { .collect(Collectors.toList())); } + @Test + public void testScanManifestsForSparsePartitions() { + RowType type = RowType.of(new IntType(), new IntType(), new IntType()); + manifestFile = createManifestFileForPartitionType(type); + ManifestEntry first = makeMultiPartEntry(true, "first", 1, 13, 0); + ManifestEntry last = makeMultiPartEntry(true, "last", 19, 13, 1); + ManifestFileMeta a = makeManifest(first, makeMultiPartEntry(true, "a", 1, 14, 0)); + ManifestFileMeta b = makeManifest(last, makeMultiPartEntry(true, "b", 19, 14, 1)); + ManifestFileMeta gap = + makeManifest( + makeMultiPartEntry(true, "c", 9, 13, 0), + makeMultiPartEntry(true, "d", 11, 14, 1)); + Snapshot snapshot = mock(Snapshot.class); + ManifestList.Factory lists = mock(ManifestList.Factory.class, RETURNS_DEEP_STUBS); + when(lists.create().readDataManifests(snapshot)).thenReturn(Arrays.asList(a, gap, b)); + ManifestsReader.Result result = + new ManifestsReader(type, "default", null, lists) + .withPartitionFilter(Arrays.asList(first.partition(), last.partition())) + .read(snapshot, ScanMode.ALL); + assertThat(result.filteredManifests).containsExactly(a, b); + } + + @ParameterizedTest + @CsvSource({"true,false", "false,false", "true,true"}) + public void testFullCompactionWithSparseOverwritePartitions( + boolean optimizeMerge, boolean sort) { + CountingReadFileIO io = new CountingReadFileIO(); + RowType type = RowType.of(new IntType(), new IntType(), new IntType()); + manifestFile = createManifestFileForPartitionType(type, io); + List base = new ArrayList<>(); + List delta = new ArrayList<>(); + List expected = new ArrayList<>(); + for (int day = 1; day <= 19; day++) { + for (int event = 0; event < 2; event++) { + String name = day + "-" + event; + // Every base manifest spans two hours, including the ones left unchanged. + base.add( + makeManifest( + makeMultiPartEntry(true, name, day, 13, event), + makeMultiPartEntry(true, name + "-other-hour", day, 14, event))); + expected.add(name + "-other-hour"); + if ((day == 1 && event == 0) || (day == 19 && event == 1)) { + delta.add(makeMultiPartEntry(false, name, day, 13, event)); + delta.add(makeMultiPartEntry(true, name + "-new", day, 13, event)); + expected.add(name + "-new"); + } else { + expected.add(name); + } + } + } + + List input = new ArrayList<>(base); + input.add(makeManifest(delta.toArray(new ManifestEntry[0]))); + Options options = new Options(); + options.set("manifest.full-compaction-threshold-size", "1 b"); + options.set("manifest.target-file-size", "1 b"); + options.set("manifest.merge-optimize.enabled", Boolean.toString(optimizeMerge)); + options.set("manifest-sort.enabled", Boolean.toString(sort)); + // Keep the test focused on partition overlap, independent of discretionary run rewrites. + options.set("manifest-sort.max-rewrite-size", "1 b"); + io.resetReadCounts(); + List output = + ManifestFileMerger.merge(input, manifestFile, type, new CoreOptions(options)); + long opened = base.stream().filter(m -> io.readCount(m.fileName()) > 0).count(); + assertThat(readFileNames(manifestFile, output)) + .containsExactlyInAnyOrderElementsOf(expected); + assertThat(opened).as("base manifests opened during full compaction").isEqualTo(2); + } + @Test public void testFullCompactionReadsSelectedManifestsOnce() throws Exception { CountingReadFileIO fileIO = new CountingReadFileIO(); @@ -3182,7 +3253,11 @@ private ManifestEntry makeMultiPartEntry( private ManifestFile createManifestFileForPartitionType(RowType partitionType) { Path path = new Path(tempDir.toString()); - FileIO fileIO = FileIOFinder.find(path); + return createManifestFileForPartitionType(partitionType, FileIOFinder.find(path)); + } + + private ManifestFile createManifestFileForPartitionType(RowType partitionType, FileIO fileIO) { + Path path = new Path(tempDir.toString()); return new ManifestFile.Factory( fileIO, new FileSystemSchemaManager(fileIO, path), diff --git a/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java b/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java index e89989a32bcd..fc1baa43fe62 100644 --- a/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/partition/PartitionPredicateTest.java @@ -27,14 +27,17 @@ import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.Pair; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.apache.paimon.data.BinaryRow.EMPTY_ROW; import static org.apache.paimon.predicate.PredicateBuilder.and; @@ -123,6 +126,87 @@ public void testPartition() { .isFalse(); } + @Test + public void testPartitionValuesAgainstRangeAndNullStatistics() throws Exception { + RowType type = DataTypes.ROW(DataTypes.INT(), DataTypes.INT()); + BinaryRow nullPartition = new BinaryRow(2); + BinaryRowWriter writer = new BinaryRowWriter(nullPartition); + writer.setNullAt(0); + writer.writeInt(1, 100); + writer.complete(); + PartitionPredicate predicate = + InstantiationUtil.clone( + PartitionPredicate.fromMultiple( + type, + Arrays.asList( + createPart(1, 100), createPart(19, 5), nullPartition))); + GenericRow lower = GenericRow.of(1, 5); + GenericRow upper = GenericRow.of(4, 8); + BinaryArray noNulls = BinaryArray.fromLongArray(new Long[] {0L, 0L}); + assertThat(predicate.test(10, lower, upper, noNulls)).isFalse(); + assertThat(predicate.test(10, lower, GenericRow.of(19, 8), noNulls)).isTrue(); + // Day 1 no longer matches, but the null partition could match event 100. + lower = GenericRow.of(3, 98); + upper = GenericRow.of(4, 100); + assertThat(predicate.test(10, lower, upper, noNulls)).isFalse(); + for (Long count : Arrays.asList(1L, null)) { + assertThat( + predicate.test( + 10, + lower, + upper, + BinaryArray.fromLongArray(new Long[] {count, 0L}))) + .isTrue(); + } + assertThat(predicate.test(10, lower, upper, BinaryArray.fromLongArray(new Long[0]))) + .isTrue(); + // Without date bounds, the second partition could match. With all-null dates only + // the null partition can match, and its event lies outside [5, 8]. + lower = GenericRow.of(null, 5); + upper = GenericRow.of(null, 8); + assertThat( + predicate.test( + 10, lower, upper, BinaryArray.fromLongArray(new Long[] {null, 0L}))) + .isTrue(); + assertThat( + predicate.test( + 10, lower, upper, BinaryArray.fromLongArray(new Long[] {10L, 0L}))) + .isFalse(); + } + + @Test + public void testManifestRangesAgainstPartitionSet() { + Set partitions = new HashSet<>(); + for (int day = 0; day < 40; day++) { + partitions.add(createPart(day, day % 7)); + } + PartitionPredicate predicate = + PartitionPredicate.fromMultiple( + DataTypes.ROW(DataTypes.INT(), DataTypes.INT()), partitions); + BinaryArray noNulls = BinaryArray.fromLongArray(new Long[] {0L, 0L}); + for (int day = 0; day < 40; day++) { + for (int event = 0; event < 7; event++) { + int firstDay = day; + int firstEvent = event; + boolean expected = + partitions.stream() + .anyMatch( + p -> + p.getInt(0) >= firstDay + && p.getInt(0) <= firstDay + 2 + && p.getInt(1) >= firstEvent + && p.getInt(1) <= firstEvent + 1); + assertThat( + predicate.test( + 5, + createPart(day, event), + createPart(day + 2, event + 1), + noNulls)) + .isEqualTo(expected); + } + } + } + @Test public void testPartitionWithMultiFields() { RowType type = DataTypes.ROW(DataTypes.INT(), DataTypes.INT());