Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.paimon.predicate.PredicateProjectionConverter;
import org.apache.paimon.predicate.PredicateVisitor;
import org.apache.paimon.statistics.FullSimpleColStatsCollector;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.Preconditions;
Expand All @@ -51,6 +52,7 @@
import java.util.Optional;
import java.util.Set;

import static org.apache.paimon.predicate.CompareUtils.compareLiteral;
import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternal;
import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternalRow;
import static org.apache.paimon.utils.Preconditions.checkArgument;
Expand Down Expand Up @@ -221,10 +223,15 @@ class MultiplePartitionPredicate implements PartitionPredicate {
private final int fieldNum;
private final Predicate[] min;
private final Predicate[] max;
@Nullable private final Object[][] partitionValues;
@Nullable private final RowDataToObjectArrayConverter converter;

private MultiplePartitionPredicate(
RowDataToObjectArrayConverter converter, Set<BinaryRow> partitions) {
this.partitions = partitions;
// For a single partition the combined bounds already describe its field values.
this.partitionValues = partitions.size() > 1 ? new Object[partitions.size()][] : null;
this.converter = partitionValues == null ? null : converter;
RowType partitionType = converter.rowType();
this.fieldNum = partitionType.getFieldCount();
@SuppressWarnings("unchecked")
Expand All @@ -236,8 +243,12 @@ private MultiplePartitionPredicate(
serializers[i] = InternalSerializers.create(partitionType.getTypeAt(i));
collectors[i] = new FullSimpleColStatsCollector();
}
int index = 0;
for (BinaryRow part : partitions) {
Object[] fields = converter.convert(part);
if (partitionValues != null) {
partitionValues[index++] = fields;
}
for (int i = 0; i < fields.length; i++) {
collectors[i].collect(fields[i], serializers[i]);
}
Expand Down Expand Up @@ -272,7 +283,10 @@ public boolean test(
InternalRow minValues,
InternalRow maxValues,
InternalArray nullCounts) {
if (fieldNum == 0) {
if (fieldNum == 0
|| minValues.getFieldCount() < fieldNum
|| maxValues.getFieldCount() < fieldNum
|| (nullCounts.size() > 0 && nullCounts.size() < fieldNum)) {
return true;
}

Expand All @@ -282,6 +296,46 @@ public boolean test(
return false;
}
}
if (partitionValues == null || rowCount <= 0) {
return true;
}
Object[] lower = converter.convert(minValues);
Object[] upper = converter.convert(maxValues);
long[] nulls = new long[fieldNum];
for (int i = 0; i < fieldNum; i++) {
nulls[i] =
nullCounts.size() == 0 || nullCounts.isNullAt(i)
? -1
: nullCounts.getLong(i);
}
// At least one requested partition must match on every field for the manifest to
// matter.
for (Object[] partition : partitionValues) {
if (mayContainPartition(partition, lower, upper, nulls, rowCount)) {
return true;
}
}
return false;
}

private boolean mayContainPartition(
Object[] partition, Object[] lower, Object[] upper, long[] nulls, long rowCount) {
for (int i = 0; i < fieldNum; i++) {
Object value = partition[i];
if (value == null) {
if (nulls[i] == 0) {
return false;
}
} else if (nulls[i] == rowCount) {
return false;
} else if (lower[i] != null && upper[i] != null) {
DataType type = converter.rowType().getTypeAt(i);
if (compareLiteral(type, value, lower[i]) < 0
|| compareLiteral(type, value, upper[i]) > 0) {
return false;
}
}
}
return true;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,13 @@
import org.apache.paimon.operation.ManifestCompactDryRun;
import org.apache.paimon.operation.ManifestFileMerger;
import org.apache.paimon.operation.ManifestFileMergerTestUtils;
import org.apache.paimon.operation.ManifestsReader;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.schema.FileSystemSchemaManager;
import org.apache.paimon.stats.StatsTestUtils;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.source.ScanMode;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.FailingFileIO;
Expand Down Expand Up @@ -718,6 +720,75 @@ public void testMergeFullCompactionWithoutDeleteFile() {
.collect(Collectors.toList()));
}

@Test
public void testScanManifestsForSparsePartitions() {
RowType type = RowType.of(new IntType(), new IntType(), new IntType());
manifestFile = createManifestFileForPartitionType(type);
ManifestEntry first = makeMultiPartEntry(true, "first", 1, 13, 0);
ManifestEntry last = makeMultiPartEntry(true, "last", 19, 13, 1);
ManifestFileMeta a = makeManifest(first, makeMultiPartEntry(true, "a", 1, 14, 0));
ManifestFileMeta b = makeManifest(last, makeMultiPartEntry(true, "b", 19, 14, 1));
ManifestFileMeta gap =
makeManifest(
makeMultiPartEntry(true, "c", 9, 13, 0),
makeMultiPartEntry(true, "d", 11, 14, 1));
Snapshot snapshot = mock(Snapshot.class);
ManifestList.Factory lists = mock(ManifestList.Factory.class, RETURNS_DEEP_STUBS);
when(lists.create().readDataManifests(snapshot)).thenReturn(Arrays.asList(a, gap, b));
ManifestsReader.Result result =
new ManifestsReader(type, "default", null, lists)
.withPartitionFilter(Arrays.asList(first.partition(), last.partition()))
.read(snapshot, ScanMode.ALL);
assertThat(result.filteredManifests).containsExactly(a, b);
}

@ParameterizedTest
@CsvSource({"true,false", "false,false", "true,true"})
public void testFullCompactionWithSparseOverwritePartitions(
boolean optimizeMerge, boolean sort) {
CountingReadFileIO io = new CountingReadFileIO();
RowType type = RowType.of(new IntType(), new IntType(), new IntType());
manifestFile = createManifestFileForPartitionType(type, io);
List<ManifestFileMeta> base = new ArrayList<>();
List<ManifestEntry> delta = new ArrayList<>();
List<String> expected = new ArrayList<>();
for (int day = 1; day <= 19; day++) {
for (int event = 0; event < 2; event++) {
String name = day + "-" + event;
// Every base manifest spans two hours, including the ones left unchanged.
base.add(
makeManifest(
makeMultiPartEntry(true, name, day, 13, event),
makeMultiPartEntry(true, name + "-other-hour", day, 14, event)));
expected.add(name + "-other-hour");
if ((day == 1 && event == 0) || (day == 19 && event == 1)) {
delta.add(makeMultiPartEntry(false, name, day, 13, event));
delta.add(makeMultiPartEntry(true, name + "-new", day, 13, event));
expected.add(name + "-new");
} else {
expected.add(name);
}
}
}

List<ManifestFileMeta> input = new ArrayList<>(base);
input.add(makeManifest(delta.toArray(new ManifestEntry[0])));
Options options = new Options();
options.set("manifest.full-compaction-threshold-size", "1 b");
options.set("manifest.target-file-size", "1 b");
options.set("manifest.merge-optimize.enabled", Boolean.toString(optimizeMerge));
options.set("manifest-sort.enabled", Boolean.toString(sort));
// Keep the test focused on partition overlap, independent of discretionary run rewrites.
options.set("manifest-sort.max-rewrite-size", "1 b");
io.resetReadCounts();
List<ManifestFileMeta> output =
ManifestFileMerger.merge(input, manifestFile, type, new CoreOptions(options));
long opened = base.stream().filter(m -> io.readCount(m.fileName()) > 0).count();
assertThat(readFileNames(manifestFile, output))
.containsExactlyInAnyOrderElementsOf(expected);
assertThat(opened).as("base manifests opened during full compaction").isEqualTo(2);
}

@Test
public void testFullCompactionReadsSelectedManifestsOnce() throws Exception {
CountingReadFileIO fileIO = new CountingReadFileIO();
Expand Down Expand Up @@ -3182,7 +3253,11 @@ private ManifestEntry makeMultiPartEntry(

private ManifestFile createManifestFileForPartitionType(RowType partitionType) {
Path path = new Path(tempDir.toString());
FileIO fileIO = FileIOFinder.find(path);
return createManifestFileForPartitionType(partitionType, FileIOFinder.find(path));
}

private ManifestFile createManifestFileForPartitionType(RowType partitionType, FileIO fileIO) {
Path path = new Path(tempDir.toString());
return new ManifestFile.Factory(
fileIO,
new FileSystemSchemaManager(fileIO, path),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,17 @@
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.InstantiationUtil;
import org.apache.paimon.utils.Pair;

import org.junit.jupiter.api.Test;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;

import static org.apache.paimon.data.BinaryRow.EMPTY_ROW;
import static org.apache.paimon.predicate.PredicateBuilder.and;
Expand Down Expand Up @@ -123,6 +126,87 @@ public void testPartition() {
.isFalse();
}

@Test
public void testPartitionValuesAgainstRangeAndNullStatistics() throws Exception {
RowType type = DataTypes.ROW(DataTypes.INT(), DataTypes.INT());
BinaryRow nullPartition = new BinaryRow(2);
BinaryRowWriter writer = new BinaryRowWriter(nullPartition);
writer.setNullAt(0);
writer.writeInt(1, 100);
writer.complete();
PartitionPredicate predicate =
InstantiationUtil.clone(
PartitionPredicate.fromMultiple(
type,
Arrays.asList(
createPart(1, 100), createPart(19, 5), nullPartition)));
GenericRow lower = GenericRow.of(1, 5);
GenericRow upper = GenericRow.of(4, 8);
BinaryArray noNulls = BinaryArray.fromLongArray(new Long[] {0L, 0L});
assertThat(predicate.test(10, lower, upper, noNulls)).isFalse();
assertThat(predicate.test(10, lower, GenericRow.of(19, 8), noNulls)).isTrue();
// Day 1 no longer matches, but the null partition could match event 100.
lower = GenericRow.of(3, 98);
upper = GenericRow.of(4, 100);
assertThat(predicate.test(10, lower, upper, noNulls)).isFalse();
for (Long count : Arrays.asList(1L, null)) {
assertThat(
predicate.test(
10,
lower,
upper,
BinaryArray.fromLongArray(new Long[] {count, 0L})))
.isTrue();
}
assertThat(predicate.test(10, lower, upper, BinaryArray.fromLongArray(new Long[0])))
.isTrue();
// Without date bounds, the second partition could match. With all-null dates only
// the null partition can match, and its event lies outside [5, 8].
lower = GenericRow.of(null, 5);
upper = GenericRow.of(null, 8);
assertThat(
predicate.test(
10, lower, upper, BinaryArray.fromLongArray(new Long[] {null, 0L})))
.isTrue();
assertThat(
predicate.test(
10, lower, upper, BinaryArray.fromLongArray(new Long[] {10L, 0L})))
.isFalse();
}

@Test
public void testManifestRangesAgainstPartitionSet() {
Set<BinaryRow> partitions = new HashSet<>();
for (int day = 0; day < 40; day++) {
partitions.add(createPart(day, day % 7));
}
PartitionPredicate predicate =
PartitionPredicate.fromMultiple(
DataTypes.ROW(DataTypes.INT(), DataTypes.INT()), partitions);
BinaryArray noNulls = BinaryArray.fromLongArray(new Long[] {0L, 0L});
for (int day = 0; day < 40; day++) {
for (int event = 0; event < 7; event++) {
int firstDay = day;
int firstEvent = event;
boolean expected =
partitions.stream()
.anyMatch(
p ->
p.getInt(0) >= firstDay
&& p.getInt(0) <= firstDay + 2
&& p.getInt(1) >= firstEvent
&& p.getInt(1) <= firstEvent + 1);
assertThat(
predicate.test(
5,
createPart(day, event),
createPart(day + 2, event + 1),
noNulls))
.isEqualTo(expected);
}
}
}

@Test
public void testPartitionWithMultiFields() {
RowType type = DataTypes.ROW(DataTypes.INT(), DataTypes.INT());
Expand Down
Loading